|
|
@@ -13,6 +13,8 @@
|
|
|
|
|
|
namespace anki {
|
|
|
|
|
|
+thread_local DescriptorSetFactory::ThreadLocal* DescriptorSetFactory::m_threadLocal = nullptr;
|
|
|
+
|
|
|
/// Wraps a global descriptor set that is used to store bindless textures.
|
|
|
class DescriptorSetFactory::BindlessDescriptorSet
|
|
|
{
|
|
|
@@ -25,9 +27,9 @@ public:
|
|
|
/// @note It's thread-safe.
|
|
|
U32 bindTexture(const VkImageView view, const VkImageLayout layout);
|
|
|
|
|
|
- /// Bind a storage image.
|
|
|
+ /// Bind a uniform texel buffer.
|
|
|
/// @note It's thread-safe.
|
|
|
- U32 bindImage(const VkImageView view);
|
|
|
+ U32 bindUniformTexelBuffer(VkBufferView view);
|
|
|
|
|
|
/// @note It's thread-safe.
|
|
|
void unbindTexture(U32 idx)
|
|
|
@@ -36,9 +38,9 @@ public:
|
|
|
}
|
|
|
|
|
|
/// @note It's thread-safe.
|
|
|
- void unbindImage(U32 idx)
|
|
|
+ void unbindUniformTexelBuffer(U32 idx)
|
|
|
{
|
|
|
- unbindCommon(idx, m_freeImgIndices, m_freeImgIndexCount);
|
|
|
+ unbindCommon(idx, m_freeTexelBufferIndices, m_freeTexelBufferIndexCount);
|
|
|
}
|
|
|
|
|
|
DescriptorSet getDescriptorSet() const
|
|
|
@@ -64,18 +66,115 @@ private:
|
|
|
Mutex m_mtx;
|
|
|
|
|
|
DynamicArray<U16> m_freeTexIndices;
|
|
|
- DynamicArray<U16> m_freeImgIndices;
|
|
|
+ DynamicArray<U16> m_freeTexelBufferIndices;
|
|
|
|
|
|
- U16 m_freeTexIndexCount ANKI_DEBUG_CODE(= MAX_U16);
|
|
|
- U16 m_freeImgIndexCount ANKI_DEBUG_CODE(= MAX_U16);
|
|
|
+ U16 m_freeTexIndexCount = MAX_U16;
|
|
|
+ U16 m_freeTexelBufferIndexCount = MAX_U16;
|
|
|
|
|
|
void unbindCommon(U32 idx, DynamicArray<U16>& freeIndices, U16& freeIndexCount);
|
|
|
};
|
|
|
|
|
|
+/// Descriptor set internal class.
|
|
|
+class DS : public IntrusiveListEnabled<DS>
|
|
|
+{
|
|
|
+public:
|
|
|
+ VkDescriptorSet m_handle = {};
|
|
|
+ U64 m_lastFrameUsed = MAX_U64;
|
|
|
+ U64 m_hash;
|
|
|
+};
|
|
|
+
|
|
|
+/// Per thread allocator.
|
|
|
+class DescriptorSetFactory::DSAllocator
|
|
|
+{
|
|
|
+public:
|
|
|
+ DSAllocator(const DSAllocator&) = delete; // Non-copyable
|
|
|
+
|
|
|
+ DSAllocator& operator=(const DSAllocator&) = delete; // Non-copyable
|
|
|
+
|
|
|
+ DSAllocator(const DSLayoutCacheEntry* layout)
|
|
|
+ : m_layoutEntry(layout)
|
|
|
+ {
|
|
|
+ ANKI_ASSERT(m_layoutEntry);
|
|
|
+ }
|
|
|
+
|
|
|
+ ~DSAllocator();
|
|
|
+
|
|
|
+ Error init();
|
|
|
+ Error createNewPool();
|
|
|
+
|
|
|
+ Error getOrCreateSet(U64 hash, const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings,
|
|
|
+ StackAllocator<U8>& tmpAlloc, const DS*& out)
|
|
|
+ {
|
|
|
+ out = tryFindSet(hash);
|
|
|
+ if(out == nullptr)
|
|
|
+ {
|
|
|
+ ANKI_CHECK(newSet(hash, bindings, tmpAlloc, out));
|
|
|
+ }
|
|
|
+
|
|
|
+ return Error::NONE;
|
|
|
+ }
|
|
|
+
|
|
|
+private:
|
|
|
+ const DSLayoutCacheEntry* m_layoutEntry; ///< Know your father.
|
|
|
+
|
|
|
+ DynamicArray<VkDescriptorPool> m_pools;
|
|
|
+ U32 m_lastPoolDSCount = 0;
|
|
|
+ U32 m_lastPoolFreeDSCount = 0;
|
|
|
+
|
|
|
+ IntrusiveList<DS> m_list; ///< At the left of the list are the least used sets.
|
|
|
+ HashMap<U64, DS*> m_hashmap;
|
|
|
+
|
|
|
+ [[nodiscard]] const DS* tryFindSet(U64 hash);
|
|
|
+ Error newSet(U64 hash, const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings,
|
|
|
+ StackAllocator<U8>& tmpAlloc, const DS*& out);
|
|
|
+ void writeSet(const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings, const DS& set,
|
|
|
+ StackAllocator<U8>& tmpAlloc);
|
|
|
+};
|
|
|
+
|
|
|
+class alignas(ANKI_CACHE_LINE_SIZE) DescriptorSetFactory::ThreadLocal
|
|
|
+{
|
|
|
+public:
|
|
|
+ DynamicArray<DSAllocator*> m_allocators;
|
|
|
+};
|
|
|
+
|
|
|
+/// Cache entry. It's built around a specific descriptor set layout.
|
|
|
+class DSLayoutCacheEntry
|
|
|
+{
|
|
|
+public:
|
|
|
+ DescriptorSetFactory* m_factory;
|
|
|
+
|
|
|
+ U64 m_hash = 0; ///< Layout hash.
|
|
|
+ VkDescriptorSetLayout m_layoutHandle = {};
|
|
|
+ BitSet<MAX_BINDINGS_PER_DESCRIPTOR_SET, U32> m_activeBindings = {false};
|
|
|
+ Array<U32, MAX_BINDINGS_PER_DESCRIPTOR_SET> m_bindingArraySize = {};
|
|
|
+ Array<DescriptorType, MAX_BINDINGS_PER_DESCRIPTOR_SET> m_bindingType = {};
|
|
|
+ U32 m_minBinding = MAX_U32;
|
|
|
+ U32 m_maxBinding = 0;
|
|
|
+ U32 m_index = 0; ///< Index in DescriptorSetFactory::m_caches
|
|
|
+
|
|
|
+ // Cache the create info
|
|
|
+ Array<VkDescriptorPoolSize, U(DescriptorType::COUNT)> m_poolSizesCreateInf = {};
|
|
|
+ VkDescriptorPoolCreateInfo m_poolCreateInf = {};
|
|
|
+
|
|
|
+ DSLayoutCacheEntry(DescriptorSetFactory* factory, U32 index)
|
|
|
+ : m_factory(factory)
|
|
|
+ , m_index(index)
|
|
|
+ {
|
|
|
+ }
|
|
|
+
|
|
|
+ ~DSLayoutCacheEntry();
|
|
|
+
|
|
|
+ Error init(const DescriptorBinding* bindings, U32 bindingCount, U64 hash);
|
|
|
+
|
|
|
+ /// @note Thread-safe.
|
|
|
+ Error getOrCreateDSAllocator(DescriptorSetFactory::DSAllocator*& alloc);
|
|
|
+};
|
|
|
+
|
|
|
DescriptorSetFactory::BindlessDescriptorSet::~BindlessDescriptorSet()
|
|
|
{
|
|
|
ANKI_ASSERT(m_freeTexIndexCount == m_freeTexIndices.getSize() && "Forgot to unbind some textures");
|
|
|
- ANKI_ASSERT(m_freeImgIndexCount == m_freeImgIndices.getSize() && "Forgot to unbind some images");
|
|
|
+ ANKI_ASSERT(m_freeTexelBufferIndexCount == m_freeTexelBufferIndices.getSize()
|
|
|
+ && "Forgot to unbind some texel buffers");
|
|
|
|
|
|
if(m_pool)
|
|
|
{
|
|
|
@@ -90,12 +189,12 @@ DescriptorSetFactory::BindlessDescriptorSet::~BindlessDescriptorSet()
|
|
|
m_layout = VK_NULL_HANDLE;
|
|
|
}
|
|
|
|
|
|
- m_freeImgIndices.destroy(m_alloc);
|
|
|
m_freeTexIndices.destroy(m_alloc);
|
|
|
+ m_freeTexelBufferIndices.destroy(m_alloc);
|
|
|
}
|
|
|
|
|
|
Error DescriptorSetFactory::BindlessDescriptorSet::init(const GrAllocator<U8>& alloc, VkDevice dev,
|
|
|
- U32 bindlessTextureCount, U32 bindlessImageCount)
|
|
|
+ U32 bindlessTextureCount, U32 bindlessTextureBuffers)
|
|
|
{
|
|
|
ANKI_ASSERT(dev);
|
|
|
m_alloc = alloc;
|
|
|
@@ -110,8 +209,8 @@ Error DescriptorSetFactory::BindlessDescriptorSet::init(const GrAllocator<U8>& a
|
|
|
bindings[0].descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
|
|
|
bindings[1].binding = 1;
|
|
|
bindings[1].stageFlags = VK_SHADER_STAGE_ALL;
|
|
|
- bindings[1].descriptorCount = bindlessImageCount;
|
|
|
- bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
|
+ bindings[1].descriptorCount = bindlessTextureBuffers;
|
|
|
+ bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER;
|
|
|
|
|
|
Array<VkDescriptorBindingFlagsEXT, 2> bindingFlags = {};
|
|
|
bindingFlags[0] = VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT_EXT
|
|
|
@@ -139,8 +238,8 @@ Error DescriptorSetFactory::BindlessDescriptorSet::init(const GrAllocator<U8>& a
|
|
|
Array<VkDescriptorPoolSize, 2> sizes = {};
|
|
|
sizes[0].type = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
|
|
|
sizes[0].descriptorCount = bindlessTextureCount;
|
|
|
- sizes[1].type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
|
- sizes[1].descriptorCount = bindlessImageCount;
|
|
|
+ sizes[1].type = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER;
|
|
|
+ sizes[1].descriptorCount = bindlessTextureBuffers;
|
|
|
|
|
|
VkDescriptorPoolCreateInfo ci = {};
|
|
|
ci.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
|
|
|
@@ -173,12 +272,12 @@ Error DescriptorSetFactory::BindlessDescriptorSet::init(const GrAllocator<U8>& a
|
|
|
m_freeTexIndices[i] = U16(m_freeTexIndices.getSize() - i - 1);
|
|
|
}
|
|
|
|
|
|
- m_freeImgIndices.create(m_alloc, bindlessImageCount);
|
|
|
- m_freeImgIndexCount = U16(m_freeImgIndices.getSize());
|
|
|
+ m_freeTexelBufferIndices.create(m_alloc, bindlessTextureBuffers);
|
|
|
+ m_freeTexelBufferIndexCount = U16(m_freeTexelBufferIndices.getSize());
|
|
|
|
|
|
- for(U32 i = 0; i < m_freeImgIndices.getSize(); ++i)
|
|
|
+ for(U32 i = 0; i < m_freeTexelBufferIndices.getSize(); ++i)
|
|
|
{
|
|
|
- m_freeImgIndices[i] = U16(m_freeImgIndices.getSize() - i - 1);
|
|
|
+ m_freeTexelBufferIndices[i] = U16(m_freeTexelBufferIndices.getSize() - i - 1);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -218,31 +317,27 @@ U32 DescriptorSetFactory::BindlessDescriptorSet::bindTexture(const VkImageView v
|
|
|
return idx;
|
|
|
}
|
|
|
|
|
|
-U32 DescriptorSetFactory::BindlessDescriptorSet::bindImage(const VkImageView view)
|
|
|
+U32 DescriptorSetFactory::BindlessDescriptorSet::bindUniformTexelBuffer(VkBufferView view)
|
|
|
{
|
|
|
ANKI_ASSERT(view);
|
|
|
LockGuard<Mutex> lock(m_mtx);
|
|
|
- ANKI_ASSERT(m_freeImgIndexCount > 0 && "Out of indices");
|
|
|
+ ANKI_ASSERT(m_freeTexelBufferIndexCount > 0 && "Out of indices");
|
|
|
|
|
|
- // Get the index
|
|
|
- --m_freeImgIndexCount;
|
|
|
- const U32 idx = m_freeImgIndices[m_freeImgIndexCount];
|
|
|
- ANKI_ASSERT(idx < m_freeImgIndices.getSize());
|
|
|
+ // Pop the index
|
|
|
+ --m_freeTexelBufferIndexCount;
|
|
|
+ const U16 idx = m_freeTexelBufferIndices[m_freeTexelBufferIndexCount];
|
|
|
+ ANKI_ASSERT(idx < m_freeTexelBufferIndices.getSize());
|
|
|
|
|
|
// Update the set
|
|
|
- VkDescriptorImageInfo imageInf = {};
|
|
|
- imageInf.imageView = view;
|
|
|
- imageInf.imageLayout = VK_IMAGE_LAYOUT_GENERAL; // Storage images are always in general.
|
|
|
-
|
|
|
VkWriteDescriptorSet write = {};
|
|
|
write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
|
|
|
write.pNext = nullptr;
|
|
|
write.dstSet = m_dset;
|
|
|
write.dstBinding = 1;
|
|
|
write.descriptorCount = 1;
|
|
|
- write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
|
+ write.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER;
|
|
|
write.dstArrayElement = idx;
|
|
|
- write.pImageInfo = &imageInf;
|
|
|
+ write.pTexelBufferView = &view;
|
|
|
|
|
|
vkUpdateDescriptorSets(m_dev, 1, &write, 0, nullptr);
|
|
|
|
|
|
@@ -270,101 +365,7 @@ void DescriptorSetFactory::BindlessDescriptorSet::unbindCommon(U32 idx, DynamicA
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-/// Descriptor set internal class.
|
|
|
-class DS : public IntrusiveListEnabled<DS>
|
|
|
-{
|
|
|
-public:
|
|
|
- VkDescriptorSet m_handle = {};
|
|
|
- U64 m_lastFrameUsed = MAX_U64;
|
|
|
- U64 m_hash;
|
|
|
-};
|
|
|
-
|
|
|
-/// Per thread allocator.
|
|
|
-class alignas(ANKI_CACHE_LINE_SIZE) DSThreadAllocator
|
|
|
-{
|
|
|
-public:
|
|
|
- DSThreadAllocator(const DSThreadAllocator&) = delete; // Non-copyable
|
|
|
-
|
|
|
- DSThreadAllocator& operator=(const DSThreadAllocator&) = delete; // Non-copyable
|
|
|
-
|
|
|
- const DSLayoutCacheEntry* m_layoutEntry; ///< Know your father.
|
|
|
-
|
|
|
- ThreadId m_tid;
|
|
|
- DynamicArray<VkDescriptorPool> m_pools;
|
|
|
- U32 m_lastPoolDSCount = 0;
|
|
|
- U32 m_lastPoolFreeDSCount = 0;
|
|
|
-
|
|
|
- IntrusiveList<DS> m_list; ///< At the left of the list are the least used sets.
|
|
|
- HashMap<U64, DS*> m_hashmap;
|
|
|
-
|
|
|
- DSThreadAllocator(const DSLayoutCacheEntry* layout, ThreadId tid)
|
|
|
- : m_layoutEntry(layout)
|
|
|
- , m_tid(tid)
|
|
|
- {
|
|
|
- ANKI_ASSERT(m_layoutEntry);
|
|
|
- }
|
|
|
-
|
|
|
- ~DSThreadAllocator();
|
|
|
-
|
|
|
- ANKI_USE_RESULT Error init();
|
|
|
- ANKI_USE_RESULT Error createNewPool();
|
|
|
-
|
|
|
- ANKI_USE_RESULT Error getOrCreateSet(U64 hash,
|
|
|
- const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings,
|
|
|
- StackAllocator<U8>& tmpAlloc, const DS*& out)
|
|
|
- {
|
|
|
- out = tryFindSet(hash);
|
|
|
- if(out == nullptr)
|
|
|
- {
|
|
|
- ANKI_CHECK(newSet(hash, bindings, tmpAlloc, out));
|
|
|
- }
|
|
|
-
|
|
|
- return Error::NONE;
|
|
|
- }
|
|
|
-
|
|
|
-private:
|
|
|
- ANKI_USE_RESULT const DS* tryFindSet(U64 hash);
|
|
|
- ANKI_USE_RESULT Error newSet(U64 hash, const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings,
|
|
|
- StackAllocator<U8>& tmpAlloc, const DS*& out);
|
|
|
- void writeSet(const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings, const DS& set,
|
|
|
- StackAllocator<U8>& tmpAlloc);
|
|
|
-};
|
|
|
-
|
|
|
-/// Cache entry. It's built around a specific descriptor set layout.
|
|
|
-class DSLayoutCacheEntry
|
|
|
-{
|
|
|
-public:
|
|
|
- DescriptorSetFactory* m_factory;
|
|
|
-
|
|
|
- U64 m_hash = 0; ///< Layout hash.
|
|
|
- VkDescriptorSetLayout m_layoutHandle = {};
|
|
|
- BitSet<MAX_BINDINGS_PER_DESCRIPTOR_SET, U32> m_activeBindings = {false};
|
|
|
- Array<U32, MAX_BINDINGS_PER_DESCRIPTOR_SET> m_bindingArraySize = {};
|
|
|
- Array<DescriptorType, MAX_BINDINGS_PER_DESCRIPTOR_SET> m_bindingType = {};
|
|
|
- U32 m_minBinding = MAX_U32;
|
|
|
- U32 m_maxBinding = 0;
|
|
|
-
|
|
|
- // Cache the create info
|
|
|
- Array<VkDescriptorPoolSize, U(DescriptorType::COUNT)> m_poolSizesCreateInf = {};
|
|
|
- VkDescriptorPoolCreateInfo m_poolCreateInf = {};
|
|
|
-
|
|
|
- DynamicArray<DSThreadAllocator*> m_threadAllocs;
|
|
|
- RWMutex m_threadAllocsMtx;
|
|
|
-
|
|
|
- DSLayoutCacheEntry(DescriptorSetFactory* factory)
|
|
|
- : m_factory(factory)
|
|
|
- {
|
|
|
- }
|
|
|
-
|
|
|
- ~DSLayoutCacheEntry();
|
|
|
-
|
|
|
- ANKI_USE_RESULT Error init(const DescriptorBinding* bindings, U32 bindingCount, U64 hash);
|
|
|
-
|
|
|
- /// @note Thread-safe.
|
|
|
- ANKI_USE_RESULT Error getOrCreateThreadAllocator(ThreadId tid, DSThreadAllocator*& alloc);
|
|
|
-};
|
|
|
-
|
|
|
-DSThreadAllocator::~DSThreadAllocator()
|
|
|
+DescriptorSetFactory::DSAllocator::~DSAllocator()
|
|
|
{
|
|
|
auto alloc = m_layoutEntry->m_factory->m_alloc;
|
|
|
|
|
|
@@ -385,13 +386,13 @@ DSThreadAllocator::~DSThreadAllocator()
|
|
|
m_hashmap.destroy(alloc);
|
|
|
}
|
|
|
|
|
|
-Error DSThreadAllocator::init()
|
|
|
+Error DescriptorSetFactory::DSAllocator::init()
|
|
|
{
|
|
|
ANKI_CHECK(createNewPool());
|
|
|
return Error::NONE;
|
|
|
}
|
|
|
|
|
|
-Error DSThreadAllocator::createNewPool()
|
|
|
+Error DescriptorSetFactory::DSAllocator::createNewPool()
|
|
|
{
|
|
|
m_lastPoolDSCount = (m_lastPoolDSCount != 0) ? U32(F32(m_lastPoolDSCount) * DESCRIPTOR_POOL_SIZE_SCALE)
|
|
|
: DESCRIPTOR_POOL_INITIAL_SIZE;
|
|
|
@@ -424,7 +425,7 @@ Error DSThreadAllocator::createNewPool()
|
|
|
return Error::NONE;
|
|
|
}
|
|
|
|
|
|
-const DS* DSThreadAllocator::tryFindSet(U64 hash)
|
|
|
+const DS* DescriptorSetFactory::DSAllocator::tryFindSet(U64 hash)
|
|
|
{
|
|
|
ANKI_ASSERT(hash > 0);
|
|
|
|
|
|
@@ -446,8 +447,9 @@ const DS* DSThreadAllocator::tryFindSet(U64 hash)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-Error DSThreadAllocator::newSet(U64 hash, const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings,
|
|
|
- StackAllocator<U8>& tmpAlloc, const DS*& out_)
|
|
|
+Error DescriptorSetFactory::DSAllocator::newSet(
|
|
|
+ U64 hash, const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings, StackAllocator<U8>& tmpAlloc,
|
|
|
+ const DS*& out_)
|
|
|
{
|
|
|
DS* out = nullptr;
|
|
|
|
|
|
@@ -495,8 +497,7 @@ Error DSThreadAllocator::newSet(U64 hash, const Array<AnyBindingExtended, MAX_BI
|
|
|
ci.descriptorSetCount = 1;
|
|
|
|
|
|
VkDescriptorSet handle;
|
|
|
- VkResult rez = vkAllocateDescriptorSets(m_layoutEntry->m_factory->m_dev, &ci, &handle);
|
|
|
- (void)rez;
|
|
|
+ [[maybe_unused]] VkResult rez = vkAllocateDescriptorSets(m_layoutEntry->m_factory->m_dev, &ci, &handle);
|
|
|
ANKI_ASSERT(rez == VK_SUCCESS && "That allocation can't fail");
|
|
|
ANKI_TRACE_INC_COUNTER(VK_DESCRIPTOR_SET_CREATE, 1);
|
|
|
|
|
|
@@ -518,13 +519,15 @@ Error DSThreadAllocator::newSet(U64 hash, const Array<AnyBindingExtended, MAX_BI
|
|
|
return Error::NONE;
|
|
|
}
|
|
|
|
|
|
-void DSThreadAllocator::writeSet(const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings,
|
|
|
- const DS& set, StackAllocator<U8>& tmpAlloc)
|
|
|
+void DescriptorSetFactory::DSAllocator::writeSet(
|
|
|
+ const Array<AnyBindingExtended, MAX_BINDINGS_PER_DESCRIPTOR_SET>& bindings, const DS& set,
|
|
|
+ StackAllocator<U8>& tmpAlloc)
|
|
|
{
|
|
|
DynamicArrayAuto<VkWriteDescriptorSet> writeInfos(tmpAlloc);
|
|
|
DynamicArrayAuto<VkDescriptorImageInfo> texInfos(tmpAlloc);
|
|
|
DynamicArrayAuto<VkDescriptorBufferInfo> buffInfos(tmpAlloc);
|
|
|
DynamicArrayAuto<VkWriteDescriptorSetAccelerationStructureKHR> asInfos(tmpAlloc);
|
|
|
+ DynamicArrayAuto<VkBufferView> bufferViews(tmpAlloc);
|
|
|
|
|
|
// First pass: Populate the VkDescriptorImageInfo and VkDescriptorBufferInfo
|
|
|
for(U bindingIdx = m_layoutEntry->m_minBinding; bindingIdx <= m_layoutEntry->m_maxBinding; ++bindingIdx)
|
|
|
@@ -572,6 +575,13 @@ void DSThreadAllocator::writeSet(const Array<AnyBindingExtended, MAX_BINDINGS_PE
|
|
|
info.range = (b.m_buff.m_range == MAX_PTR_SIZE) ? VK_WHOLE_SIZE : b.m_buff.m_range;
|
|
|
break;
|
|
|
}
|
|
|
+ case DescriptorType::READ_TEXTURE_BUFFER:
|
|
|
+ case DescriptorType::READ_WRITE_TEXTURE_BUFFER:
|
|
|
+ {
|
|
|
+ VkBufferView& view = *bufferViews.emplaceBack();
|
|
|
+ view = b.m_textureBuffer.m_buffView;
|
|
|
+ break;
|
|
|
+ }
|
|
|
case DescriptorType::IMAGE:
|
|
|
{
|
|
|
VkDescriptorImageInfo& info = *texInfos.emplaceBack();
|
|
|
@@ -600,8 +610,9 @@ void DSThreadAllocator::writeSet(const Array<AnyBindingExtended, MAX_BINDINGS_PE
|
|
|
U32 texCounter = 0;
|
|
|
U32 buffCounter = 0;
|
|
|
U32 asCounter = 0;
|
|
|
+ U32 buffViewsCounter = 0;
|
|
|
|
|
|
- VkWriteDescriptorSet writeTemplate{};
|
|
|
+ VkWriteDescriptorSet writeTemplate = {};
|
|
|
writeTemplate.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
|
|
|
writeTemplate.pNext = nullptr;
|
|
|
writeTemplate.dstSet = set.m_handle;
|
|
|
@@ -633,6 +644,10 @@ void DSThreadAllocator::writeSet(const Array<AnyBindingExtended, MAX_BINDINGS_PE
|
|
|
case DescriptorType::STORAGE_BUFFER:
|
|
|
writeInfo.pBufferInfo = &buffInfos[buffCounter++];
|
|
|
break;
|
|
|
+ case DescriptorType::READ_TEXTURE_BUFFER:
|
|
|
+ case DescriptorType::READ_WRITE_TEXTURE_BUFFER:
|
|
|
+ writeInfo.pTexelBufferView = &bufferViews[buffViewsCounter++];
|
|
|
+ break;
|
|
|
case DescriptorType::ACCELERATION_STRUCTURE:
|
|
|
writeInfo.pNext = &asInfos[asCounter++];
|
|
|
break;
|
|
|
@@ -652,13 +667,6 @@ DSLayoutCacheEntry::~DSLayoutCacheEntry()
|
|
|
{
|
|
|
auto alloc = m_factory->m_alloc;
|
|
|
|
|
|
- for(DSThreadAllocator* a : m_threadAllocs)
|
|
|
- {
|
|
|
- alloc.deleteInstance(a);
|
|
|
- }
|
|
|
-
|
|
|
- m_threadAllocs.destroy(alloc);
|
|
|
-
|
|
|
if(m_layoutHandle)
|
|
|
{
|
|
|
vkDestroyDescriptorSetLayout(m_factory->m_dev, m_layoutHandle, nullptr);
|
|
|
@@ -740,61 +748,92 @@ Error DSLayoutCacheEntry::init(const DescriptorBinding* bindings, U32 bindingCou
|
|
|
return Error::NONE;
|
|
|
}
|
|
|
|
|
|
-Error DSLayoutCacheEntry::getOrCreateThreadAllocator(ThreadId tid, DSThreadAllocator*& alloc)
|
|
|
+Error DSLayoutCacheEntry::getOrCreateDSAllocator(DescriptorSetFactory::DSAllocator*& alloc)
|
|
|
{
|
|
|
alloc = nullptr;
|
|
|
|
|
|
- class Comp
|
|
|
+ // Get or create thread-local
|
|
|
+ DescriptorSetFactory::ThreadLocal* threadLocal = DescriptorSetFactory::m_threadLocal;
|
|
|
+ if(ANKI_UNLIKELY(threadLocal == nullptr))
|
|
|
{
|
|
|
- public:
|
|
|
- Bool operator()(const DSThreadAllocator* a, ThreadId tid) const
|
|
|
- {
|
|
|
- return a->m_tid < tid;
|
|
|
- }
|
|
|
+ threadLocal = m_factory->m_alloc.newInstance<DescriptorSetFactory::ThreadLocal>();
|
|
|
+ DescriptorSetFactory::m_threadLocal = threadLocal;
|
|
|
|
|
|
- Bool operator()(ThreadId tid, const DSThreadAllocator* a) const
|
|
|
- {
|
|
|
- return tid < a->m_tid;
|
|
|
- }
|
|
|
- };
|
|
|
+ LockGuard<Mutex> lock(m_factory->m_allThreadLocalsMtx);
|
|
|
+ m_factory->m_allThreadLocals.emplaceBack(m_factory->m_alloc, threadLocal);
|
|
|
+ }
|
|
|
|
|
|
- // Find using binary search
|
|
|
+ // Get or create the allocator
|
|
|
+ if(ANKI_UNLIKELY(m_index >= threadLocal->m_allocators.getSize()))
|
|
|
{
|
|
|
- RLockGuard<RWMutex> lock(m_threadAllocsMtx);
|
|
|
- auto it = binarySearch(m_threadAllocs.getBegin(), m_threadAllocs.getEnd(), tid, Comp());
|
|
|
- alloc = (it != m_threadAllocs.getEnd()) ? *it : nullptr;
|
|
|
+ threadLocal->m_allocators.resize(m_factory->m_alloc, m_index + 1, nullptr);
|
|
|
+ alloc = m_factory->m_alloc.newInstance<DescriptorSetFactory::DSAllocator>(this);
|
|
|
+ ANKI_CHECK(alloc->init());
|
|
|
+ threadLocal->m_allocators[m_index] = alloc;
|
|
|
}
|
|
|
-
|
|
|
- if(alloc == nullptr)
|
|
|
+ else if(ANKI_UNLIKELY(threadLocal->m_allocators[m_index] == nullptr))
|
|
|
{
|
|
|
- // Need to create one
|
|
|
+ alloc = m_factory->m_alloc.newInstance<DescriptorSetFactory::DSAllocator>(this);
|
|
|
+ ANKI_CHECK(alloc->init());
|
|
|
+ threadLocal->m_allocators[m_index] = alloc;
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ alloc = threadLocal->m_allocators[m_index];
|
|
|
+ }
|
|
|
|
|
|
- WLockGuard<RWMutex> lock(m_threadAllocsMtx);
|
|
|
+ ANKI_ASSERT(alloc);
|
|
|
+ return Error::NONE;
|
|
|
+}
|
|
|
|
|
|
- // Search again
|
|
|
- auto it = binarySearch(m_threadAllocs.getBegin(), m_threadAllocs.getEnd(), tid, Comp());
|
|
|
- alloc = (it != m_threadAllocs.getEnd()) ? *it : nullptr;
|
|
|
+AnyBinding& DescriptorSetState::getBindingToPopulate(U32 bindingIdx, U32 arrayIdx)
|
|
|
+{
|
|
|
+ ANKI_ASSERT(bindingIdx < MAX_BINDINGS_PER_DESCRIPTOR_SET);
|
|
|
|
|
|
- // Create
|
|
|
- if(alloc == nullptr)
|
|
|
- {
|
|
|
- alloc = m_factory->m_alloc.newInstance<DSThreadAllocator>(this, tid);
|
|
|
- ANKI_CHECK(alloc->init());
|
|
|
+ AnyBindingExtended& extended = m_bindings[bindingIdx];
|
|
|
+ AnyBinding* out;
|
|
|
+ const Bool bindingIsSet = m_bindingSet.get(bindingIdx);
|
|
|
+ m_bindingSet.set(bindingIdx);
|
|
|
+ extended.m_arraySize = (!bindingIsSet) ? 0 : extended.m_arraySize;
|
|
|
|
|
|
- m_threadAllocs.resize(m_factory->m_alloc, m_threadAllocs.getSize() + 1);
|
|
|
- m_threadAllocs[m_threadAllocs.getSize() - 1] = alloc;
|
|
|
+ if(ANKI_LIKELY(arrayIdx == 0 && extended.m_arraySize <= 1))
|
|
|
+ {
|
|
|
+ // Array idx is zero, most common case
|
|
|
+ out = &extended.m_single;
|
|
|
+ extended.m_arraySize = 1;
|
|
|
+ }
|
|
|
+ else if(arrayIdx < extended.m_arraySize)
|
|
|
+ {
|
|
|
+ // It's (or was) an array and there enough space in thar array
|
|
|
+ out = &extended.m_array[arrayIdx];
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ // Need to grow
|
|
|
+ const U32 newSize = max(extended.m_arraySize * 2, arrayIdx + 1);
|
|
|
+ AnyBinding* newArr = m_alloc.newArray<AnyBinding>(newSize);
|
|
|
|
|
|
- // Sort for fast find
|
|
|
- std::sort(m_threadAllocs.getBegin(), m_threadAllocs.getEnd(),
|
|
|
- [](const DSThreadAllocator* a, const DSThreadAllocator* b) {
|
|
|
- return a->m_tid < b->m_tid;
|
|
|
- });
|
|
|
+ if(extended.m_arraySize == 1)
|
|
|
+ {
|
|
|
+ newArr[0] = extended.m_single;
|
|
|
+ }
|
|
|
+ else if(extended.m_arraySize > 1)
|
|
|
+ {
|
|
|
+ // Copy old to new.
|
|
|
+ memcpy(newArr, extended.m_array, sizeof(AnyBinding) * extended.m_arraySize);
|
|
|
}
|
|
|
+
|
|
|
+ // Zero the rest
|
|
|
+ memset(newArr + extended.m_arraySize, 0, sizeof(AnyBinding) * (newSize - extended.m_arraySize));
|
|
|
+ extended.m_arraySize = newSize;
|
|
|
+ extended.m_array = newArr;
|
|
|
+
|
|
|
+ // Return
|
|
|
+ out = &extended.m_array[arrayIdx];
|
|
|
}
|
|
|
|
|
|
- ANKI_ASSERT(alloc);
|
|
|
- ANKI_ASSERT(alloc->m_tid == tid);
|
|
|
- return Error::NONE;
|
|
|
+ ANKI_ASSERT(out);
|
|
|
+ return *out;
|
|
|
}
|
|
|
|
|
|
void DescriptorSetState::flush(U64& hash, Array<PtrSize, MAX_BINDINGS_PER_DESCRIPTOR_SET>& dynamicOffsets,
|
|
|
@@ -885,6 +924,16 @@ void DescriptorSetState::flush(U64& hash, Array<PtrSize, MAX_BINDINGS_PER_DESCRI
|
|
|
dynamicOffsets[dynamicOffsetCount++] = anyBinding.m_buff.m_offset;
|
|
|
dynamicOffsetsDirty = dynamicOffsetsDirty || crntBindingDirty;
|
|
|
break;
|
|
|
+ case DescriptorType::READ_TEXTURE_BUFFER:
|
|
|
+ ANKI_ASSERT(anyBinding.m_type == DescriptorType::READ_TEXTURE_BUFFER
|
|
|
+ && "Have bound the wrong type");
|
|
|
+ toHash[toHashCount++] = anyBinding.m_uuids[1];
|
|
|
+ break;
|
|
|
+ case DescriptorType::READ_WRITE_TEXTURE_BUFFER:
|
|
|
+ ANKI_ASSERT(anyBinding.m_type == DescriptorType::READ_WRITE_TEXTURE_BUFFER
|
|
|
+ && "Have bound the wrong type");
|
|
|
+ toHash[toHashCount++] = anyBinding.m_uuids[1];
|
|
|
+ break;
|
|
|
case DescriptorType::IMAGE:
|
|
|
ANKI_ASSERT(anyBinding.m_type == DescriptorType::IMAGE && "Have bound the wrong type");
|
|
|
break;
|
|
|
@@ -935,21 +984,34 @@ DescriptorSetFactory::~DescriptorSetFactory()
|
|
|
}
|
|
|
|
|
|
Error DescriptorSetFactory::init(const GrAllocator<U8>& alloc, VkDevice dev, U32 bindlessTextureCount,
|
|
|
- U32 bindlessImageCount)
|
|
|
+ U32 bindlessTextureBuffers)
|
|
|
{
|
|
|
m_alloc = alloc;
|
|
|
m_dev = dev;
|
|
|
|
|
|
m_bindless = m_alloc.newInstance<BindlessDescriptorSet>();
|
|
|
- ANKI_CHECK(m_bindless->init(alloc, dev, bindlessTextureCount, bindlessImageCount));
|
|
|
+ ANKI_CHECK(m_bindless->init(alloc, dev, bindlessTextureCount, bindlessTextureBuffers));
|
|
|
m_bindlessTextureCount = bindlessTextureCount;
|
|
|
- m_bindlessImageCount = bindlessImageCount;
|
|
|
+ m_bindlessUniformTexelBufferCount = bindlessTextureBuffers;
|
|
|
|
|
|
return Error::NONE;
|
|
|
}
|
|
|
|
|
|
void DescriptorSetFactory::destroy()
|
|
|
{
|
|
|
+ for(ThreadLocal* threadLocal : m_allThreadLocals)
|
|
|
+ {
|
|
|
+ for(DSAllocator* alloc : threadLocal->m_allocators)
|
|
|
+ {
|
|
|
+ m_alloc.deleteInstance(alloc);
|
|
|
+ }
|
|
|
+
|
|
|
+ threadLocal->m_allocators.destroy(m_alloc);
|
|
|
+ m_alloc.deleteInstance(threadLocal);
|
|
|
+ }
|
|
|
+
|
|
|
+ m_allThreadLocals.destroy(m_alloc);
|
|
|
+
|
|
|
for(DSLayoutCacheEntry* l : m_caches)
|
|
|
{
|
|
|
m_alloc.deleteInstance(l);
|
|
|
@@ -999,8 +1061,8 @@ Error DescriptorSetFactory::newDescriptorSetLayout(const DescriptorSetLayoutInit
|
|
|
{
|
|
|
// All good
|
|
|
}
|
|
|
- else if(binding.m_binding == 1 && binding.m_type == DescriptorType::IMAGE
|
|
|
- && binding.m_arraySize == m_bindlessImageCount)
|
|
|
+ else if(binding.m_binding == 1 && binding.m_type == DescriptorType::READ_TEXTURE_BUFFER
|
|
|
+ && binding.m_arraySize == m_bindlessUniformTexelBufferCount)
|
|
|
{
|
|
|
// All good
|
|
|
}
|
|
|
@@ -1035,11 +1097,10 @@ Error DescriptorSetFactory::newDescriptorSetLayout(const DescriptorSetLayoutInit
|
|
|
|
|
|
if(cache == nullptr)
|
|
|
{
|
|
|
- cache = m_alloc.newInstance<DSLayoutCacheEntry>(this);
|
|
|
+ cache = m_alloc.newInstance<DSLayoutCacheEntry>(this, m_caches.getSize());
|
|
|
ANKI_CHECK(cache->init(bindings.getBegin(), bindingCount, hash));
|
|
|
|
|
|
- m_caches.resize(m_alloc, m_caches.getSize() + 1);
|
|
|
- m_caches[m_caches.getSize() - 1] = cache;
|
|
|
+ m_caches.emplaceBack(m_alloc, cache);
|
|
|
}
|
|
|
|
|
|
// Set the layout
|
|
|
@@ -1050,7 +1111,7 @@ Error DescriptorSetFactory::newDescriptorSetLayout(const DescriptorSetLayoutInit
|
|
|
return Error::NONE;
|
|
|
}
|
|
|
|
|
|
-Error DescriptorSetFactory::newDescriptorSet(ThreadId tid, StackAllocator<U8>& tmpAlloc, DescriptorSetState& state,
|
|
|
+Error DescriptorSetFactory::newDescriptorSet(StackAllocator<U8>& tmpAlloc, DescriptorSetState& state,
|
|
|
DescriptorSet& set, Bool& dirty,
|
|
|
Array<PtrSize, MAX_BINDINGS_PER_DESCRIPTOR_SET>& dynamicOffsets,
|
|
|
U32& dynamicOffsetCount)
|
|
|
@@ -1076,8 +1137,8 @@ Error DescriptorSetFactory::newDescriptorSet(ThreadId tid, StackAllocator<U8>& t
|
|
|
DSLayoutCacheEntry& entry = *layout.m_entry;
|
|
|
|
|
|
// Get thread allocator
|
|
|
- DSThreadAllocator* alloc;
|
|
|
- ANKI_CHECK(entry.getOrCreateThreadAllocator(tid, alloc));
|
|
|
+ DSAllocator* alloc;
|
|
|
+ ANKI_CHECK(entry.getOrCreateDSAllocator(alloc));
|
|
|
|
|
|
// Finally, allocate
|
|
|
const DS* s;
|
|
|
@@ -1100,10 +1161,10 @@ U32 DescriptorSetFactory::bindBindlessTexture(const VkImageView view, const VkIm
|
|
|
return m_bindless->bindTexture(view, layout);
|
|
|
}
|
|
|
|
|
|
-U32 DescriptorSetFactory::bindBindlessImage(const VkImageView view)
|
|
|
+U32 DescriptorSetFactory::bindBindlessUniformTexelBuffer(const VkBufferView view)
|
|
|
{
|
|
|
ANKI_ASSERT(m_bindless);
|
|
|
- return m_bindless->bindImage(view);
|
|
|
+ return m_bindless->bindUniformTexelBuffer(view);
|
|
|
}
|
|
|
|
|
|
void DescriptorSetFactory::unbindBindlessTexture(U32 idx)
|
|
|
@@ -1112,10 +1173,10 @@ void DescriptorSetFactory::unbindBindlessTexture(U32 idx)
|
|
|
m_bindless->unbindTexture(idx);
|
|
|
}
|
|
|
|
|
|
-void DescriptorSetFactory::unbindBindlessImage(U32 idx)
|
|
|
+void DescriptorSetFactory::unbindBindlessUniformTexelBuffer(U32 idx)
|
|
|
{
|
|
|
ANKI_ASSERT(m_bindless);
|
|
|
- m_bindless->unbindImage(idx);
|
|
|
+ m_bindless->unbindUniformTexelBuffer(idx);
|
|
|
}
|
|
|
|
|
|
} // end namespace anki
|