AloneGame / Library / PackageCache / com.unity.collections / Unity.Collections / RewindableAllocator.cs

using AOT;
using System;
using System.Runtime.CompilerServices;
using System.Threading;
using UnityEngine.Assertions;
using Unity.Burst;
using Unity.Collections.LowLevel.Unsafe;
using Unity.Jobs.LowLevel.Unsafe;
using Unity.Mathematics;

namespace Unity.Collections
{
    internal struct UnmanagedArray<T> : IDisposable where T : unmanaged
    {
        IntPtr m_pointer;
        int m_length;
        public int Length => m_length;
        AllocatorManager.AllocatorHandle m_allocator;
        public UnmanagedArray(int length, AllocatorManager.AllocatorHandle allocator)
        {
            unsafe
            {
                m_pointer = (IntPtr)Memory.Unmanaged.Array.Allocate<T>(length, allocator);
            }
            m_length = length;
            m_allocator = allocator;
        }
        public void Dispose()
        {
            unsafe
            {
                Memory.Unmanaged.Free((T*)m_pointer, Allocator.Persistent);
            }
        }
        public unsafe T* GetUnsafePointer()
        {
            return (T*)m_pointer;
        }
        public ref T this[int index]
        {
            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            get { unsafe { return ref ((T*)m_pointer)[index]; } }
        }
    }

    /// <summary>
    /// An allocator that is fast like a linear allocator, is threadsafe, and automatically invalidates
    /// all allocations made from it, when "rewound" by the user.
    /// </summary>
    [BurstCompile]
    public struct RewindableAllocator : AllocatorManager.IAllocator
    {
        internal struct Union
        {
            internal long m_long;

            // Number of bits used to store current position in a block to give out memory.
            // This limits the maximum block size to 1TB (2^40).
            const int currentBits = 40;
            // Offset of current position in m_long
            const int currentOffset = 0;
            // Number of bits used to store the allocation count in a block
            const long currentMask = (1L << currentBits) - 1;

            // Number of bits used to store allocation count in a block.
            // This limits the maximum number of allocations per block to 16 millions (2^24)
            const int allocCountBits = 24;
            // Offset of allocation count in m_long
            const int allocCountOffset = currentOffset + currentBits;
            const long allocCountMask = (1L << allocCountBits) - 1;

            // Current position in a block to give out memory
            internal long m_current
            {
                [MethodImpl(MethodImplOptions.AggressiveInlining)]
                get
                {
                    return (m_long >> currentOffset) & currentMask;
                }

                [MethodImpl(MethodImplOptions.AggressiveInlining)]
                set
                {
                    m_long &= ~(currentMask << currentOffset);
                    m_long |= (value & currentMask) << currentOffset;
                }
            }

            // The number of allocations in a block
            internal long m_allocCount
            {
                [MethodImpl(MethodImplOptions.AggressiveInlining)]
                get
                {
                    return (m_long >> allocCountOffset) & allocCountMask;
                }

                [MethodImpl(MethodImplOptions.AggressiveInlining)]
                set
                {
                    m_long &= ~(allocCountMask << allocCountOffset);
                    m_long |= (value & allocCountMask) << allocCountOffset;
                }
            }
        }
        [GenerateTestsForBurstCompatibility]
        internal unsafe struct MemoryBlock : IDisposable
        {
            // can't align any coarser than this many bytes
            public const int kMaximumAlignment = 16384;
            // pointer to contiguous memory
            public byte* m_pointer;
            // how many bytes of contiguous memory it points to
            public long m_bytes;
            // Union of current position to give out memory and allocation counts
            public Union m_union;

            public MemoryBlock(long bytes)
            {
                m_pointer = (byte*)Memory.Unmanaged.Allocate(bytes, kMaximumAlignment, Allocator.Persistent);
                Assert.IsTrue(m_pointer != null, "Memory block allocation failed, system out of memory");
                m_bytes = bytes;
                m_union = default;
            }

            public void Rewind()
            {
                m_union = default;
            }

            public void Dispose()
            {
                Memory.Unmanaged.Free(m_pointer, Allocator.Persistent);
                m_pointer = null;
                m_bytes = 0;
                m_union = default;
            }

            public bool Contains(IntPtr ptr)
            {
                unsafe
                {
                    void* pointer = (void*)ptr;
                    return (pointer >= m_pointer) && (pointer < m_pointer + m_union.m_current);
                }
            }
        };

        // Log2 of Maximum memory block size.  Cannot exceed MemoryBlock.Union.currentBits.
        const int kLog2MaxMemoryBlockSize = 26;

        // Maximum memory block size.  Can exceed maximum memory block size if user requested more.
        const long kMaxMemoryBlockSize = 1L << kLog2MaxMemoryBlockSize;  // 64MB

        /// Minimum memory block size, 128KB.
        const long kMinMemoryBlockSize = 128 * 1024;

        /// Maximum number of memory blocks.
        const int kMaxNumBlocks = 64;

        // Bit mask (bit 31) of the memory block busy flag indicating whether the block is busy rewinding.
        const int kBlockBusyRewindMask = 0x1 << 31;

        // Bit mask of the memory block busy flag indicating whether the block is busy allocating.
        const int kBlockBusyAllocateMask = ~kBlockBusyRewindMask;

        Spinner m_spinner;
        AllocatorManager.AllocatorHandle m_handle;
        UnmanagedArray<MemoryBlock> m_block;
        int m_last;                 // highest-index block that has memory to allocate from
        int m_used;                 // highest-index block that we actually allocated from, since last rewind
        byte m_enableBlockFree;     // flag indicating if allocator enables individual block free
        byte m_reachMaxBlockSize;   // flag indicating if reach maximum block size

        /// <summary>
        /// Initializes the allocator. Must be called before first use.
        /// </summary>
        /// <param name="initialSizeInBytes">The initial capacity of the allocator, in bytes</param>
        /// <param name="enableBlockFree">A flag indicating if allocator enables individual block free</param>
        public void Initialize(int initialSizeInBytes, bool enableBlockFree = false)
        {
            m_spinner = default;
            m_block = new UnmanagedArray<MemoryBlock>(kMaxNumBlocks, Allocator.Persistent);
            // Initial block size should be larger than min block size
            var blockSize = initialSizeInBytes > kMinMemoryBlockSize ? initialSizeInBytes : kMinMemoryBlockSize;
            m_block[0] = new MemoryBlock(blockSize);
            m_last = m_used = 0;
            m_enableBlockFree = enableBlockFree ? (byte)1 : (byte)0;
            m_reachMaxBlockSize = (initialSizeInBytes >= kMaxMemoryBlockSize) ? (byte)1 : (byte)0;
        }

        /// <summary>
        /// Property to get and set enable block free flag, a flag indicating whether the allocator should enable individual block to be freed.
        /// </summary>
        public bool EnableBlockFree
        {
            get => m_enableBlockFree != 0;
            set => m_enableBlockFree = value ? (byte)1 : (byte)0;
        }

        /// <summary>
        /// Retrieves the number of memory blocks that the allocator has requested from the system.
        /// </summary>
        public int BlocksAllocated => (int)(m_last + 1);

        /// <summary>
        /// Retrieves the size of the initial memory block, as requested in the Initialize function.
        /// </summary>
        public int InitialSizeInBytes => (int)(m_block[0].m_bytes);

        /// <summary>
        /// Retrieves the maximum memory block size.
        /// </summary>
        internal long MaxMemoryBlockSize => kMaxMemoryBlockSize;

        /// <summary>
        /// Retrieves the total bytes of the memory blocks allocated by this allocator.
        /// </summary>
        internal long BytesAllocated
        {
            get
            {
                long totalBytes = 0;
                for(int i = 0; i <= m_last; i++)
                {
                    totalBytes += m_block[i].m_bytes;
                }
                return totalBytes;
            }
        }

        /// <summary>
        /// Rewind the allocator; invalidate all allocations made from it, and potentially also free memory blocks
        /// it has allocated from the system.
        /// </summary>
        public void Rewind()
        {
            if (JobsUtility.IsExecutingJob)
                throw new InvalidOperationException("You cannot Rewind a RewindableAllocator from a Job.");
            m_handle.Rewind(); // bump the allocator handle version, invalidate all dependents
            while (m_last > m_used) // *delete* all blocks we didn't even allocate from this time around.
                m_block[m_last--].Dispose();
            while (m_used > 0) // simply *rewind* all blocks we used in this update, to avoid allocating again, every update.
                m_block[m_used--].Rewind();
            m_block[0].Rewind();
        }

        /// <summary>
        /// Dispose the allocator. This must be called to free the memory blocks that were allocated from the system.
        /// </summary>
        public void Dispose()
        {
            if (JobsUtility.IsExecutingJob)
                throw new InvalidOperationException("You cannot Dispose a RewindableAllocator from a Job.");
            m_used = 0; // so that we delete all blocks in Rewind() on the next line
            Rewind();
            m_block[0].Dispose();
            m_block.Dispose();
            m_last = m_used = 0;
        }

        /// <summary>
        /// All allocators must implement this property, in order to be installed in the custom allocator table.
        /// </summary>
        [ExcludeFromBurstCompatTesting("Uses managed delegate")]
        public AllocatorManager.TryFunction Function => Try;

        unsafe int TryAllocate(ref AllocatorManager.Block block, int startIndex, int lastIndex, long alignedSize, long alignmentMask)
        {
            for (int best = startIndex; best <= lastIndex; best++)
            {
                Union oldUnion;
                Union readUnion = default;
                long begin = 0;
                bool skip = false;
                readUnion.m_long = Interlocked.Read(ref m_block[best].m_union.m_long);
                do
                {
                    begin = (readUnion.m_current + alignmentMask) & ~alignmentMask;
                    if (begin + block.Bytes > m_block[best].m_bytes)
                    {
                        skip = true;
                        break;
                    }
                    oldUnion = readUnion;
                    Union newUnion = default;
                    newUnion.m_current = (begin + alignedSize) > m_block[best].m_bytes ? m_block[best].m_bytes : (begin + alignedSize);
                    newUnion.m_allocCount = readUnion.m_allocCount + 1;
                    readUnion.m_long = Interlocked.CompareExchange(ref m_block[best].m_union.m_long, newUnion.m_long, oldUnion.m_long);
                } while (readUnion.m_long != oldUnion.m_long);

                if(skip)
                {
                    continue;
                }

                block.Range.Pointer = (IntPtr)(m_block[best].m_pointer + begin);
                block.AllocatedItems = block.Range.Items;

                Interlocked.MemoryBarrier();
                int oldUsed;
                int readUsed;
                int newUsed;
                readUsed = m_used;
                do
                {
                    oldUsed = readUsed;
                    newUsed = best > oldUsed ? best : oldUsed;
                    readUsed = Interlocked.CompareExchange(ref m_used, newUsed, oldUsed);
                } while (newUsed != oldUsed);

                return AllocatorManager.kErrorNone;
            }

            return AllocatorManager.kErrorBufferOverflow;
        }

        /// <summary>
        /// Try to allocate, free, or reallocate a block of memory. This is an internal function, and
        /// is not generally called by the user.
        /// </summary>
        /// <param name="block">The memory block to allocate, free, or reallocate</param>
        /// <returns>0 if successful. Otherwise, returns the error code from the allocator function.</returns>
        public int Try(ref AllocatorManager.Block block)
        {
            if (block.Range.Pointer == IntPtr.Zero)
            {
                // Make the alignment multiple of cacheline size
                var alignment = math.max(JobsUtility.CacheLineSize, block.Alignment);
                var extra = alignment != JobsUtility.CacheLineSize ? 1 : 0;
                var cachelineMask = JobsUtility.CacheLineSize - 1;
                if (extra == 1)
                {
                    alignment = (alignment + cachelineMask) & ~cachelineMask;
                }

                // Adjust the size to be multiple of alignment, add extra alignment
                // to size if alignment is more than cacheline size
                var mask = alignment - 1L;
                var size = (block.Bytes + extra * alignment + mask) & ~mask;

                // Check all the blocks to see if any of them have enough memory
                var last = m_last;
                int error = TryAllocate(ref block, 0, m_last, size, mask);
                if (error == AllocatorManager.kErrorNone)
                {
                    return error;
                }

                // If that fails, allocate another block that's guaranteed big enough, and allocate from it.
                // Allocate twice as much as last time until it reaches MaxMemoryBlockSize, after that, increase
                // the block size by MaxMemoryBlockSize.
                m_spinner.Acquire();

                // After getting the lock, we must try to allocate again, because if many threads waited at
                // the lock, the first one allocates and when it unlocks, it's likely that there's space for the
                // other threads' allocations in the first thread's block.
                error = TryAllocate(ref block, last, m_last, size, mask);
                if (error == AllocatorManager.kErrorNone)
                {
                    m_spinner.Release();
                    return error;
                }

                long bytes;
                if (m_reachMaxBlockSize == 0)
                {
                    bytes = m_block[m_last].m_bytes << 1;
                }
                else
                {
                    bytes = m_block[m_last].m_bytes + kMaxMemoryBlockSize;
                }
                // if user asks more, skip smaller sizes
                bytes = math.max(bytes, size);
                m_reachMaxBlockSize = (bytes >= kMaxMemoryBlockSize) ? (byte)1 : (byte)0;
                m_block[m_last + 1] = new MemoryBlock(bytes);
                Interlocked.Increment(ref m_last);
                error = TryAllocate(ref block, m_last, m_last, size, mask);
                m_spinner.Release();
                return error;
            }

            // To free memory, no-op unless allocator enables individual block to be freed
            if (block.Range.Items == 0)
            {
                if (m_enableBlockFree != 0)
                {
                    for (int blockIndex = 0; blockIndex <= m_last; ++blockIndex)
                    {
                        if (m_block[blockIndex].Contains(block.Range.Pointer))
                        {
                            Union oldUnion;
                            Union readUnion = default;
                            readUnion.m_long = Interlocked.Read(ref m_block[blockIndex].m_union.m_long);
                            do
                            {
                                oldUnion = readUnion;
                                Union newUnion = readUnion;
                                newUnion.m_allocCount--;
                                if (newUnion.m_allocCount == 0)
                                {
                                    newUnion.m_current = 0;
                                }
                                readUnion.m_long = Interlocked.CompareExchange(ref m_block[blockIndex].m_union.m_long, newUnion.m_long, oldUnion.m_long);
                            } while (readUnion.m_long != oldUnion.m_long);
                        }
                    }
                }
                return 0; // we could check to see if the pointer belongs to us, if we want to be strict about it.
            }

            return -1;
        }

        [BurstCompile]
        [MonoPInvokeCallback(typeof(AllocatorManager.TryFunction))]
        internal static int Try(IntPtr state, ref AllocatorManager.Block block)
        {
            unsafe { return ((RewindableAllocator*)state)->Try(ref block); }
        }

        /// <summary>
        /// Retrieve the AllocatorHandle associated with this allocator. The handle is used as an index into a
        /// global table, for times when a reference to the allocator object isn't available.
        /// </summary>
        /// <value>The AllocatorHandle retrieved.</value>
        public AllocatorManager.AllocatorHandle Handle { get { return m_handle; } set { m_handle = value; } }

        /// <summary>
        /// Retrieve the Allocator associated with this allocator.
        /// </summary>
        /// <value>The Allocator retrieved.</value>
        public Allocator ToAllocator { get { return m_handle.ToAllocator; } }

        /// <summary>
        /// Check whether this AllocatorHandle is a custom allocator.
        /// </summary>
        /// <value>True if this AllocatorHandle is a custom allocator.</value>
        public bool IsCustomAllocator { get { return m_handle.IsCustomAllocator; } }

        /// <summary>
        /// Check whether this allocator will automatically dispose allocations.
        /// </summary>
        /// <remarks>Allocations made by Rewindable allocator are automatically disposed.</remarks>
        /// <value>Always true</value>
        public bool IsAutoDispose { get { return true; } }

        /// <summary>
        /// Allocate a NativeArray of type T from memory that is guaranteed to remain valid until the end of the
        /// next Update of this World. There is no need to Dispose the NativeArray so allocated. It is not possible
        /// to free the memory by Disposing it - it is automatically freed after the end of the next Update for this
        /// World.
        /// </summary>
        /// <typeparam name="T">The element type of the NativeArray to allocate.</typeparam>
        /// <param name="length">The length of the NativeArray to allocate, measured in elements.</param>
        /// <returns>The NativeArray allocated by this function.</returns>
        [GenerateTestsForBurstCompatibility(GenericTypeArguments = new[] { typeof(int) })]
        public NativeArray<T> AllocateNativeArray<T>(int length) where T : unmanaged
        {
            var container = new NativeArray<T>();
            unsafe
            {
                container.m_Buffer = this.AllocateStruct(default(T), length);
            }
            container.m_Length = length;
            container.m_AllocatorLabel = Allocator.None;
#if ENABLE_UNITY_COLLECTIONS_CHECKS
            container.m_MinIndex = 0;
            container.m_MaxIndex = length - 1;
            container.m_Safety = CollectionHelper.CreateSafetyHandle(ToAllocator);
            CollectionHelper.SetStaticSafetyId<NativeArray<T>>(ref container.m_Safety, ref NativeArrayExtensions.NativeArrayStaticId<T>.s_staticSafetyId.Data);
            Handle.AddSafetyHandle(container.m_Safety);
#endif
            return container;
        }

        /// <summary>
        /// Allocate a NativeList of type T from memory that is guaranteed to remain valid until the end of the
        /// next Update of this World. There is no need to Dispose the NativeList so allocated. It is not possible
        /// to free the memory by Disposing it - it is automatically freed after the end of the next Update for this
        /// World. The NativeList must be initialized with its maximum capacity; if it were to dynamically resize,
        /// up to 1/2 of the total final capacity would be wasted, because the memory can't be dynamically freed.
        /// </summary>
        /// <typeparam name="T">The element type of the NativeList to allocate.</typeparam>
        /// <param name="capacity">The capacity of the NativeList to allocate, measured in elements.</param>
        /// <returns>The NativeList allocated by this function.</returns>
        [GenerateTestsForBurstCompatibility(GenericTypeArguments = new[] { typeof(int) })]
        public NativeList<T> AllocateNativeList<T>(int capacity) where T : unmanaged
        {
            var container = new NativeList<T>();
            unsafe
            {
                container.m_ListData = this.Allocate(default(UnsafeList<T>), 1);
                container.m_ListData->Ptr = this.Allocate(default(T), capacity);
                container.m_ListData->m_length = 0;
                container.m_ListData->m_capacity = capacity;
                container.m_ListData->Allocator = Allocator.None;
            }
#if ENABLE_UNITY_COLLECTIONS_CHECKS
            container.m_Safety = CollectionHelper.CreateSafetyHandle(ToAllocator);
            CollectionHelper.SetStaticSafetyId<NativeList<T>>(ref container.m_Safety, ref NativeList<T>.s_staticSafetyId.Data);
            AtomicSafetyHandle.SetBumpSecondaryVersionOnScheduleWrite(container.m_Safety, true);
            Handle.AddSafetyHandle(container.m_Safety);
#endif
            return container;
        }
    }
}