Actual source code: object_pool.cxx

      1: #include <petsc/private/cpp/object_pool.hpp>

      3: #include <new>       // std::nothrow
      4: #include <limits>    // std::numeric_limits
      5: #include <algorithm> // std::lower_bound()
      6: #include <cstdio>    // std::printf

      8: namespace Petsc
      9: {

     11: namespace memory
     12: {

     14: // ==========================================================================================
     15: // PoolAllocator -- Private API -- AllocationHeader
     16: //
     17: // The header inserted for each allocated pointer. It stores:
     18: //
     19: // - size - the size (in bytes) of the allocation. This includes ONLY the size as requested by
     20: //          the user. i.e. if a user requested 10 bytes but alignment, padding and header
     21: //          overhead results in the actual allocation being 30 bytes, then size = 10.
     22: // - align - the alignment (in bytes) of the allocated pointer.
     23: // ==========================================================================================

     25: struct PoolAllocator::AllocationHeader {
     26:   constexpr AllocationHeader(size_type, align_type) noexcept;

     28:   PETSC_NODISCARD static constexpr align_type max_alignment() noexcept;
     29:   PETSC_NODISCARD static constexpr size_type  header_size() noexcept;
     30:   PETSC_NODISCARD static constexpr size_type  buffer_zone_size() noexcept;

     32:   size_type  size;
     33:   align_type align;
     34: };

     36: // ==========================================================================================
     37: // PoolAllocator -- Private API -- AllocationHeader -- Public API
     38: // ==========================================================================================

     40: /*
     41:   PoolAllocator::AllocationHeader::AllocationHeader
     42: */
     43: constexpr PoolAllocator::AllocationHeader::AllocationHeader(size_type size, align_type align) noexcept : size{size}, align{align} { }

     45: /*
     46:   PoolAllocator::AllocationHeader::max_alignment

     48:   Returns the maximum supported alignment (in bytes) of the memory pool.
     49: */
     50: constexpr PoolAllocator::align_type PoolAllocator::AllocationHeader::max_alignment() noexcept
     51: {
     52: #if PETSC_CPP_VERSION >= 14
     53:   constexpr auto max_align = std::numeric_limits<unsigned char>::max() + 1;
     54:   static_assert(!(max_align & (max_align - 1)), "Maximum alignment must be a power of 2");
     55:   return static_cast<align_type>(max_align);
     56: #else
     57:   return static_cast<align_type>(std::numeric_limits<unsigned char>::max() + 1);
     58: #endif
     59: }

     61: /*
     62:   PoolAllocator::AllocationHeader::buffer_zone_size

     64:   Notes:
     65:   Returns the number of bytes between the allocated pointer and the location where the
     66:   alignment diff is stored. i.e. size of the buffer zone + 1.

     68:   If ASAN is enabled then this buffer zone is poisoned, so any overrun on part of the user is
     69:   potentially caught. The larger the buffer zone, the more likely that the user lands in
     70:   poisoned memory. Turned off in optimized builds.
     71: */
     72: constexpr PoolAllocator::size_type PoolAllocator::AllocationHeader::buffer_zone_size() noexcept
     73: {
     74:   return (PetscDefined(USE_DEBUG) ? 32 : 0) + 1;
     75: }

     77: /*
     78:   PoolAllocator::AllocationHeader::header_size

     80:   Notes:
     81:   Returns the minimum size of the allocation header in bytes. Essentially (literally) the size
     82:   of the header object + size of the buffer zone. Does not include padding due to alignment
     83:   offset itself though.
     84: */
     85: constexpr PoolAllocator::size_type PoolAllocator::AllocationHeader::header_size() noexcept
     86: {
     87:   return sizeof(AllocationHeader) + buffer_zone_size();
     88: }

     90: /*
     91:   PoolAllocator::total_size_ - Compute the maximum total size for an allocation

     93:   Input Parameters:
     94: + size  - the size (in bytes) requested by the user
     95: - align - the alignment (in bytes) requested by the user

     97:   Notes:
     98:   Returns a size so that std::malloc(total_size_(size, align)) allocates enough memory to store
     99:   the allocation header, buffer zone, alignment offset and requested size including any
    100:   potential changes due to alignment.
    101: */
    102: constexpr PoolAllocator::size_type PoolAllocator::total_size_(size_type size, align_type align) noexcept
    103: {
    104:   // align - 1 because aligning the pointer up by align bytes just returns it to its original
    105:   // alignment, so we can save the byte
    106:   return AllocationHeader::header_size() + size + util::to_underlying(align) - 1;
    107: }

    109: /*
    110:   PoolAllocator::delete_ptr_ - deletes a pointer and the corresponding header

    112:   Input Parameter:
    113: + in_ptr - the pointer to user-memory which to delete

    115:   Notes:
    116:   in_ptr may point to nullptr (in which case this does nothing), otherwise it must have been
    117:   allocated by the pool.

    119:   in_ptr may point to poisoned memory, this routine will remove any poisoning before
    120:   deallocating.

    122:   in_ptr is set to nullptr on return.
    123: */
    124: PetscErrorCode PoolAllocator::delete_ptr_(void **in_ptr) noexcept
    125: {
    126:   PetscFunctionBegin;
    127:   PetscAssertPointer(in_ptr, 1);
    128:   if (const auto ptr = util::exchange(*in_ptr, nullptr)) {
    129:     AllocationHeader *header = nullptr;

    131:     PetscCall(extract_header_(ptr, &header, false));
    132:     // must unpoison the header itself before we can access the members
    133:     PetscCall(PetscUnpoisonMemoryRegion(header, sizeof(*header)));
    134:     PetscCall(PetscUnpoisonMemoryRegion(header, total_size_(header->size, header->align)));
    135:     PetscCallCXX(::delete[] reinterpret_cast<unsigned char *>(header));
    136:   }
    137:   PetscFunctionReturn(PETSC_SUCCESS);
    138: }

    140: /*
    141:   PoolAllocator::find_align_ - Return an iterator to the memory pool for a particular alignment

    143:   Input Parameter:
    144: . align - The alignment (in bytes) to search for

    146:   Notes:
    147:   returns pool().end() if alignment not found.
    148: */
    149: PoolAllocator::pool_type::iterator PoolAllocator::find_align_(align_type align) noexcept
    150: {
    151:   return std::lower_bound(this->pool().begin(), this->pool().end(), align, [](const pool_type::value_type &pair, const align_type &align) { return pair.first < align; });
    152: }

    154: PoolAllocator::pool_type::const_iterator PoolAllocator::find_align_(align_type align) const noexcept
    155: {
    156:   return std::lower_bound(this->pool().begin(), this->pool().end(), align, [](const pool_type::value_type &pair, const align_type &align) { return pair.first < align; });
    157: }

    159: /*
    160:   PoolAllocator::clear_ - Clear the memory pool

    162:   Output Parameter:
    163: . remaining - The number of remaining allocations in the pool, nullptr if not needed

    165:   Notes:
    166:   This will clean up the pool, deallocating any memory checked back into the pool. This does
    167:   not delete allocations that were allocated by the pool but not yet returned to it.

    169:   remaining is useful in determining if any allocations were "leaked". Suppose an object
    170:   internally manages a resource and uses the pool to allocate said resource. On destruction the
    171:   object expects to have the pool be empty, i.e. have remaining = 0. This implies all resources
    172:   were returned to the pool.
    173: */
    174: PetscErrorCode PoolAllocator::clear_(size_type *remaining) noexcept
    175: {
    176:   size_type remain = 0;

    178:   PetscFunctionBegin;
    179:   if (remaining) PetscAssertPointer(remaining, 1);
    180:   // clang-format off
    181:   PetscCall(
    182:     this->for_each([&](void *&ptr)
    183:     {
    184:       PetscFunctionBegin;
    185:       ++remain;
    186:       PetscCall(delete_ptr_(&ptr));
    187:       PetscFunctionReturn(PETSC_SUCCESS);
    188:     })
    189:   );
    190:   // clang-format on
    191:   PetscCallCXX(this->pool().clear());
    192:   if (remaining) *remaining = remain;
    193:   PetscFunctionReturn(PETSC_SUCCESS);
    194: }

    196: /*
    197:   PoolAllocator::finalize_ - Routine automatically called during PetscFinalize()

    199:   Notes:
    200:   This will go through and reap any regions that it owns in the underlying container. If it
    201:   owns all regions, it resets the container.

    203:   There currently is no way to ensure that objects remaining in the pool aren't leaked, since
    204:   this routine cannot actually re-register the pool for finalizations without causing an
    205:   infinite loop...

    207:   Thus it is up to the owned object to ensure that the pool is properly finalized.
    208: */
    209: PetscErrorCode PoolAllocator::finalize_() noexcept
    210: {
    211:   PetscFunctionBegin;
    212:   PetscCall(clear_());
    213:   PetscFunctionReturn(PETSC_SUCCESS);
    214: }

    216: // a quick sanity check that the alignment is valid, does nothing in optimized builds
    217: PetscErrorCode PoolAllocator::valid_alignment_(align_type in_align) noexcept
    218: {
    219:   constexpr auto max_align = util::to_underlying(AllocationHeader::max_alignment());
    220:   const auto     align     = util::to_underlying(in_align);

    222:   PetscFunctionBegin;
    223:   PetscAssert((align > 0) && (align <= max_align), PETSC_COMM_SELF, PETSC_ERR_MEMC, "Alignment %zu must be (0, %zu]", align, max_align);
    224:   PetscAssert(!(align & (align - 1)), PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Alignment %zu must be a power of 2", align);
    225:   PetscFunctionReturn(PETSC_SUCCESS);
    226: }

    228: /*
    229:   PoolAllocator::extract_header_ - Extract the header pointer from the aligned pointer

    231:   Input Parameters:
    232: + user_ptr     - the pointer to the aligned user memory
    233: - check_in_ptr - whether to test the validity of aligned_ptr

    235:   Output Parameter:
    236: . header - the pointer to the header

    238:   Notes:
    239:   Setting check_in_ptr to false disabled the PetscAssertPointer() check in the function
    240:   preamble. This allows the method to be used even if the aligned_ptr is poisoned (for example
    241:   when extracting the header from a pointer that is checked into the pool).

    243:   aligned_ptr must have been allocated by the pool.

    245:   The returned header is still poisoned, the user is responsible for unpoisoning it.
    246: */
    247: PetscErrorCode PoolAllocator::extract_header_(void *user_ptr, AllocationHeader **header, bool check_in_ptr) noexcept
    248: {
    249:   PetscFunctionBegin;
    250:   if (check_in_ptr) PetscAssertPointer(user_ptr, 1);
    251:   PetscAssertPointer(header, 2);
    252:   {
    253:     //       AllocationHeader::alignment_offset() (at least 1)
    254:     //                        |
    255:     // header                 |  user_ptr/aligned_ptr
    256:     // |                      |          |
    257:     // v                  v~~~~~~~~~~~~~~v
    258:     // A==============B===C==============D--------------------- ...
    259:     // ^~~~~~~~~~~~~~~^^~~^              ^~~~~~~~~~~~~~~~~~~~~~ ...
    260:     //       |             \______             user memory
    261:     //       |                    buffer_zone_end
    262:     // sizeof(AllocationHeader)
    263:     //
    264:     // ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
    265:     //                poisoned
    266:     //
    267:     const auto aligned_ptr     = reinterpret_cast<unsigned char *>(user_ptr);
    268:     const auto buffer_zone_end = aligned_ptr - AllocationHeader::buffer_zone_size();

    270:     PetscCall(PetscUnpoisonMemoryRegion(buffer_zone_end, sizeof(*buffer_zone_end)));
    271:     {
    272:       // offset added to original pointer due to alignment, B -> C above (may be zero)
    273:       const auto alignment_offset = *buffer_zone_end;

    275:       *header = reinterpret_cast<AllocationHeader *>(buffer_zone_end - alignment_offset - sizeof(AllocationHeader));
    276:     }
    277:     PetscCall(PetscPoisonMemoryRegion(buffer_zone_end, sizeof(*buffer_zone_end)));
    278:   }
    279:   PetscFunctionReturn(PETSC_SUCCESS);
    280: }

    282: /*
    283:   PoolAllocator::allocate_ptr_ - Allocate a pointer and header given requested size and
    284:   alignment

    286:   Input Parameters:
    287: + size  - the size (in bytes) to allocate
    288: - align - the size (in bytes) to align the pointer to

    290:   Output Parameter:
    291: . ret_ptr - the resulting pointer to user-memory

    293:   Notes:
    294:   Both size and align must be > 0. align must be a power of 2.
    295:   This both allocates the user memory and the corresponding metadata region.
    296: */
    297: PetscErrorCode PoolAllocator::allocate_ptr_(size_type size, align_type align, void **ret_ptr) noexcept
    298: {
    299:   constexpr auto header_size = AllocationHeader::header_size();
    300:   const auto     total_size  = total_size_(size, align);
    301:   const auto     size_before = total_size - header_size;
    302:   auto           usable_size = size_before;
    303:   void          *aligned_ptr = nullptr;
    304:   unsigned char *base_ptr    = nullptr;

    306:   PetscFunctionBegin;
    307:   PetscAssertPointer(ret_ptr, 1);
    308:   PetscCall(valid_alignment_(align));
    309:   // memory is laid out as follows:
    310:   //
    311:   //                            aligned_ptr     ret_ptr (and aligned_ptr after std::align())
    312:   // base_ptr     buffer_zone       |     _____/
    313:   // |                      |       |    /     user memory
    314:   // v                  v~~~~~~~~~~~x~~~v~~~~~~~~~~~~~~~~~~~~~ ...
    315:   // A==============B===C===========D===E--------------------- ...
    316:   // ^~~~~~~~~~~~~~~^^~~^
    317:   //        |            \_________
    318:   // sizeof(AllocationHeader)      |
    319:   //                               alignment_offset (may be 0)
    320:   //
    321:   // ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
    322:   //                poisoned
    323:   //                                ^~~~~~~~~~~~~~~~~~~~~~~~~~ ...
    324:   //                                          usable_size
    325:   // ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ...
    326:   //                       total_size_()
    327:   //
    328:   base_ptr = ::new (std::nothrow) unsigned char[total_size];
    329:   PetscAssert(base_ptr, PETSC_COMM_SELF, PETSC_ERR_MEM, "operator new() failed to allocate %zu bytes", total_size);
    330:   PetscCallCXX(base_ptr = reinterpret_cast<unsigned char *>(util::construct_at(reinterpret_cast<AllocationHeader *>(base_ptr), size, align)));
    331:   aligned_ptr = base_ptr + header_size;
    332:   // storing to ret_ptr and not aligned_ptr is deliberate! std::align() returns nullptr if it
    333:   // fails, so we do not want to clobber aligned_ptr
    334:   *ret_ptr = std::align(util::to_underlying(align), size, aligned_ptr, usable_size);
    335:   // note usable_size is has now shrunk by alignment_offset
    336:   PetscAssert(*ret_ptr, PETSC_COMM_SELF, PETSC_ERR_LIB, "std::align() failed to align pointer %p (size %zu, alignment %zu)", aligned_ptr, size, util::to_underlying(align));
    337:   {
    338:     constexpr auto max_align        = util::to_underlying(AllocationHeader::max_alignment());
    339:     const auto     alignment_offset = size_before - usable_size;

    341:     PetscAssert(alignment_offset <= max_align, PETSC_COMM_SELF, PETSC_ERR_MEMC, "Computed alignment offset %zu > maximum allowed alignment %zu", alignment_offset, max_align);
    342:     *(reinterpret_cast<unsigned char *>(aligned_ptr) - AllocationHeader::buffer_zone_size()) = static_cast<unsigned char>(alignment_offset);
    343:     if (PetscDefined(USE_DEBUG)) {
    344:       const auto computed_aligned_ptr = base_ptr + header_size + alignment_offset;

    346:       PetscCheck(computed_aligned_ptr == aligned_ptr, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Base pointer %p + header size %zu + alignment offset %zu = %p != aligned pointer %p", static_cast<void *>(base_ptr), header_size, alignment_offset, static_cast<void *>(computed_aligned_ptr), aligned_ptr);
    347:     }
    348:   }
    349:   // Poison the entire region first, then unpoison only the user region. This ensures that
    350:   // any extra space on *either* ends of the array are poisoned
    351:   PetscCall(PetscPoisonMemoryRegion(base_ptr, total_size));
    352:   PetscCall(PetscUnpoisonMemoryRegion(aligned_ptr, size));
    353:   PetscFunctionReturn(PETSC_SUCCESS);
    354: }

    356: // ==========================================================================================
    357: // PoolAllocator -- Public API
    358: // ==========================================================================================

    360: PoolAllocator::~PoolAllocator() noexcept
    361: {
    362:   size_type leaked{};
    363:   PetscBool init;

    365:   PetscFunctionBegin;
    366:   PetscCallAbort(PETSC_COMM_SELF, clear_(&leaked));
    367:   PetscCallAbort(PETSC_COMM_SELF, PetscInitialized(&init));
    368:   if (init) PetscCheckAbort(leaked == 0, PETSC_COMM_SELF, PETSC_ERR_MEM_LEAK, "%zu objects remaining in the pool are leaked", leaked);
    369:   PetscFunctionReturnVoid();
    370: }

    372: /*
    373:   PoolAllocator::get_attributes - Get the size and alignment of an allocated pointer

    375:   Input Parameter:
    376: . ptr - the pointer to query

    378:   Output Parameters:
    379: + size - the size (in bytes) of the allocated area, nullptr if not needed
    380: - align - the alignment (in bytes) of the allocated, nullptr if not needed

    382:   Note:
    383:   ptr must have been allocated by the pool, and is exactly the pointer returned by either
    384:   allocate() or try_allocate() (if successful).
    385: */
    386: PetscErrorCode PoolAllocator::get_attributes(const void *ptr, size_type *size, align_type *align) noexcept
    387: {
    388:   PetscFunctionBegin;
    389:   // ptr may be poisoned, so cannot check it here
    390:   // PetscAssertPointer(out_ptr, 1);
    391:   if (size) PetscAssertPointer(size, 2);
    392:   if (align) PetscAssertPointer(align, 3);
    393:   if (PetscLikely(size || align)) {
    394:     AllocationHeader *header = nullptr;

    396:     PetscCall(extract_header_(const_cast<void *>(ptr), &header, /* check ptr = */ false));
    397:     PetscCall(PetscUnpoisonMemoryRegion(header, sizeof(*header)));
    398:     if (size) *size = header->size;
    399:     if (align) *align = header->align;
    400:     PetscCall(PetscPoisonMemoryRegion(header, sizeof(*header)));
    401:   }
    402:   PetscFunctionReturn(PETSC_SUCCESS);
    403: }

    405: /*
    406:   PoolAllocator::try_allocate - Attempt to allocate memory from the pool

    408:   Input Parameter:
    409: + size  - the size (in bytes) to attempt to allocate
    410: - align - the alignment (in bytes) of the requested allocation

    412:   Output Parameters:
    413: + out_ptr - the pointer to return the allocated memory in
    414: - success - set to true if out_ptr was successfully is allocated

    416:   Notes:
    417:   Differs from allocate() insofar that this routine does not allocate new memory if it does not
    418:   find a suitable memory chunk in the pool.

    420:   align must be a power of 2, and > 0.

    422:   If size is 0, out_ptr is set to nullptr and success set to false
    423: */
    424: PetscErrorCode PoolAllocator::try_allocate(void **out_ptr, size_type size, align_type align, bool *success) noexcept
    425: {
    426:   void *ptr{};
    427:   bool  found{};

    429:   PetscFunctionBegin;
    430:   PetscAssertPointer(out_ptr, 1);
    431:   PetscAssertPointer(success, 3);
    432:   PetscCall(valid_alignment_(align));
    433:   PetscCall(this->register_finalize());
    434:   if (PetscLikely(size)) {
    435:     const auto align_it = find_align_(align);

    437:     if (align_it != this->pool().end() && align_it->first == align) {
    438:       auto     &&size_map = align_it->second;
    439:       const auto size_it  = size_map.find(size);

    441:       if (size_it != size_map.end()) {
    442:         auto &&ptr_list = size_it->second;

    444:         if (!ptr_list.empty()) {
    445:           found = true;
    446:           ptr   = ptr_list.back();
    447:           PetscCallCXX(ptr_list.pop_back());
    448:           PetscCall(PetscUnpoisonMemoryRegion(ptr, size));
    449:         }
    450:       }
    451:     }
    452:   }
    453:   *out_ptr = ptr;
    454:   *success = found;
    455:   PetscFunctionReturn(PETSC_SUCCESS);
    456: }

    458: /*
    459:   PoolAllocator::allocate - Allocate a chunk of memory from the pool

    461:   Input Parameters:
    462: + size  - The size (in bytes) to allocate
    463: - align - The alignment (in bytes) to align the allocation to

    465:   Output Parameters:
    466: + out_ptr             - A pointer containing the beginning of the allocated region
    467: - allocated_from_pool - True if the region was allocated from the pool, false otherwise

    469:   Notes:
    470:   If size is 0, out_ptr is set to nullptr and was_allocated is set to false.
    471: */
    472: PetscErrorCode PoolAllocator::allocate(void **out_ptr, size_type size, align_type align, bool *allocated_from_pool) noexcept
    473: {
    474:   bool success{};

    476:   PetscFunctionBegin;
    477:   PetscAssertPointer(out_ptr, 1);
    478:   if (allocated_from_pool) PetscAssertPointer(allocated_from_pool, 3);
    479:   PetscCall(try_allocate(out_ptr, size, align, &success));
    480:   if (!success) PetscCall(allocate_ptr_(size, align, out_ptr));
    481:   if (allocated_from_pool) *allocated_from_pool = success;
    482:   PetscFunctionReturn(PETSC_SUCCESS);
    483: }

    485: /*
    486:   PoolAllocate::deallocate - Return a pointer to the pool

    488:   Input Parameter:
    489: . in_ptr - A pointer to the beginning of the allocated region

    491:   Notes:
    492:   On success the region pointed to by in_ptr is poisoned. Any further attempts to access
    493:   the memory pointed to by in_ptr will result in an error.

    495:   in_ptr must have been allocated by the pool, and must point to the beginning of the allocated
    496:   region.

    498:   The value in_ptr points to may be nullptr, in which case this routine does nothing.
    499: */
    500: PetscErrorCode PoolAllocator::deallocate(void **in_ptr, size_type size, align_type align) noexcept
    501: {
    502:   PetscFunctionBegin;
    503:   PetscAssertPointer(in_ptr, 1);
    504:   if (auto ptr = util::exchange(*in_ptr, nullptr)) {
    505:     if (locked_) {
    506:       // This is necessary if an object is "reclaimed" within another PetscFinalize()
    507:       // registered cleanup after this pool has returned from its finalizer. In this case,
    508:       // instead of pushing onto the stack we just delete the pointer directly.
    509:       //
    510:       // However this path is *only* valid if we have already finalized!
    511:       PetscCall(delete_ptr_(&ptr));
    512:     } else {
    513:       auto it = find_align_(align);

    515:       if (it == this->pool().end() || it->first != align) PetscCallCXX(it = this->pool().insert(it, {align, {}}));
    516:       PetscCallCXX(it->second[size].emplace_back(ptr));
    517:       PetscCall(PetscPoisonMemoryRegion(ptr, size));
    518:     }
    519:   }
    520:   PetscFunctionReturn(PETSC_SUCCESS);
    521: }

    523: /*
    524:   PoolAllocator::unpoison - Unpoison a pool-allocated pointer

    526:   Input Parameter:
    527: . ptr - the pointer to poison

    529:   Output Parameter:
    530: . size - the size (in bytes) of the region pointed to by ptr

    532:   Notes:
    533:   ptr must not be nullptr.

    535:   size should be passed to the corresponding repoison() to undo the effects of this
    536:   routine.

    538:   Using this routine in conjunction with unpoison() allows a user to temporarily push and pop
    539:   the poisoning state of a given pointer. The pool does not repoison the pointer for you, so
    540:   use at your own risk!
    541: */
    542: PetscErrorCode PoolAllocator::unpoison(const void *ptr, size_type *size) noexcept
    543: {
    544:   PetscFunctionBegin;
    545:   // ptr may be poisoned, so cannot check it here
    546:   // PetscAssertPointer(ptr, 1);
    547:   PetscAssertPointer(size, 2);
    548:   PetscCall(get_attributes(ptr, size, nullptr));
    549:   PetscCall(PetscUnpoisonMemoryRegion(ptr, *size));
    550:   PetscFunctionReturn(PETSC_SUCCESS);
    551: }

    553: /*
    554:   PoolAllocator::repoison - Poison a pointer previously unpoisoned via unpoison()

    556:   Input Parameters:
    557: + ptr  - the pointer to the unpoisoned region
    558: - size - the size of the region

    560:   Notes:
    561:   size must be exactly the value returned by unpoison().

    563:   ptr cannot be nullptr
    564: */
    565: PetscErrorCode PoolAllocator::repoison(const void *ptr, size_type size) noexcept
    566: {
    567:   PetscFunctionBegin;
    568:   PetscAssertPointer(ptr, 1);
    569:   PetscCall(PetscPoisonMemoryRegion(ptr, size));
    570:   PetscFunctionReturn(PETSC_SUCCESS);
    571: }

    573: PoolAllocator::LockGuard PoolAllocator::lock_guard() noexcept
    574: {
    575:   return LockGuard{this};
    576: }

    578: // ==========================================================================================
    579: // PoolAllocated -- Public API
    580: // ==========================================================================================

    582: void *PoolAllocated::operator new(size_type size) noexcept
    583: {
    584:   void *ptr{};

    586:   PetscFunctionBegin;
    587:   PetscCallAbort(PETSC_COMM_SELF, pool().allocate(&ptr, size, static_cast<align_type>(alignof(std::max_align_t))));
    588:   PetscFunctionReturn(ptr);
    589: }

    591: void PoolAllocated::operator delete(void *ptr) noexcept
    592: {
    593:   PetscFunctionBegin;
    594:   if (PetscLikely(ptr)) {
    595:     size_type       size{};
    596:     align_type      align{};
    597:     allocator_type &allocated = pool();

    599:     PetscCallAbort(PETSC_COMM_SELF, allocated.get_attributes(ptr, &size, &align));
    600:     PetscCallAbort(PETSC_COMM_SELF, allocated.deallocate(&ptr, size, align));
    601:   }
    602:   PetscFunctionReturnVoid();
    603: }

    605: #if PETSC_CPP_VERSION >= 17
    606: void *PoolAllocated::operator new(size_type size, std::align_val_t align) noexcept
    607: {
    608:   void *ptr{};

    610:   PetscFunctionBegin;
    611:   PetscCallAbort(PETSC_COMM_SELF, pool().allocate(&ptr, size, static_cast<align_type>(align)));
    612:   PetscFunctionReturn(ptr);
    613: }

    615: void PoolAllocated::operator delete(void *ptr, std::align_val_t align) noexcept
    616: {
    617:   PetscFunctionBegin;
    618:   if (PetscLikely(ptr)) {
    619:     size_type size{};

    621:     PetscCallAbort(PETSC_COMM_SELF, pool().get_attributes(ptr, &size, nullptr));
    622:     PetscCallAbort(PETSC_COMM_SELF, pool().deallocate(&ptr, size, static_cast<align_type>(align)));
    623:   }
    624:   PetscFunctionReturnVoid();
    625: }
    626: #endif

    628: // ==========================================================================================
    629: // PoolAllocated -- Protected API
    630: // ==========================================================================================

    632: PoolAllocated::allocator_type &PoolAllocated::pool() noexcept
    633: {
    634:   return pool_;
    635: }

    637: } // namespace memory

    639: } // namespace Petsc