From 1f0ad760e1d8ca475affc5efb4c48f2cd4774124 Mon Sep 17 00:00:00 2001 From: Philip Rebohle Date: Sat, 1 Mar 2025 00:29:51 +0100 Subject: [PATCH] [dxvk] Refactor address range for barrier tracking Allows us to bump the size and offset to 64 bits without increasing the total size of the node. --- src/dxvk/dxvk_barrier.cpp | 40 +++++++++++++++--------------------- src/dxvk/dxvk_barrier.h | 42 +++++++++++++------------------------- src/dxvk/dxvk_buffer.h | 4 ++-- src/dxvk/dxvk_context.cpp | 23 +++++++++++++-------- src/dxvk/dxvk_image.h | 4 ++-- src/dxvk/dxvk_pipelayout.h | 4 +--- 6 files changed, 50 insertions(+), 67 deletions(-) diff --git a/src/dxvk/dxvk_barrier.cpp b/src/dxvk/dxvk_barrier.cpp index 9d9feb191..e643974c6 100644 --- a/src/dxvk/dxvk_barrier.cpp +++ b/src/dxvk/dxvk_barrier.cpp @@ -20,12 +20,11 @@ namespace dxvk { bool DxvkBarrierTracker::findRange( const DxvkAddressRange& range, - DxvkAccess accessType, - DxvkAccessOp accessOp) const { + DxvkAccess accessType) const { uint32_t rootIndex = computeRootIndex(range, accessType); uint32_t nodeIndex = findNode(range, rootIndex); - if (likely(!nodeIndex || accessOp == DxvkAccessOp::None)) + if (likely(!nodeIndex || range.accessOp == DxvkAccessOp::None)) return nodeIndex; // If we are checking for a specific order-invariant store @@ -33,32 +32,33 @@ namespace dxvk { // resource, and the tracked range must cover the requested // range in its entirety so we can rule out that other parts // of the resource have been accessed in a different way. - auto& node = m_nodes[nodeIndex]; + const auto& node = m_nodes[nodeIndex]; - return node.payload.accessOps != DxvkAccessOps(accessOp) - || !node.addressRange.contains(range); + if (node.addressRange.accessOp != range.accessOp) + return true; + + return !node.addressRange.contains(range); } void DxvkBarrierTracker::insertRange( const DxvkAddressRange& range, - DxvkAccess accessType, - DxvkAccessOp accessOp) { - DxvkBarrierPayload payload = { }; - payload.accessOps.set(accessOp); - + DxvkAccess accessType) { // If we can just insert the node with no conflicts, // we don't have to do anything. uint32_t rootIndex = computeRootIndex(range, accessType); - uint32_t nodeIndex = insertNode(range, rootIndex, payload); + uint32_t nodeIndex = insertNode(range, rootIndex); if (likely(!nodeIndex)) return; // If there's an existing node and it contains the entire // range we want to add already, also don't do anything. + // If there are conflicting access ops, reset it. auto& node = m_nodes[nodeIndex]; - node.payload.accessOps.set(payload.accessOps); + + if (node.addressRange.accessOp != range.accessOp) + node.addressRange.accessOp = DxvkAccessOp::None; if (node.addressRange.contains(range)) return; @@ -100,14 +100,15 @@ namespace dxvk { mergedRange.rangeStart = std::min(mergedRange.rangeStart, node.addressRange.rangeStart); mergedRange.rangeEnd = std::max(mergedRange.rangeEnd, node.addressRange.rangeEnd); - payload.accessOps.set(node.payload.accessOps); + if (mergedRange.accessOp != node.addressRange.accessOp) + mergedRange.accessOp = DxvkAccessOp::None; removeNode(nodeIndex, rootIndex); nodeIndex = findNode(range, rootIndex); } - insertNode(mergedRange, rootIndex, payload); + insertNode(mergedRange, rootIndex); } @@ -186,8 +187,7 @@ namespace dxvk { uint32_t DxvkBarrierTracker::insertNode( const DxvkAddressRange& range, - uint32_t rootIndex, - DxvkBarrierPayload payload) { + uint32_t rootIndex) { // Check if the given root is valid at all uint64_t rootBit = uint64_t(1u) << (rootIndex - 1u); @@ -199,7 +199,6 @@ namespace dxvk { auto& node = m_nodes[rootIndex]; node.header = 0; node.addressRange = range; - node.payload = payload; return 0; } else { // Traverse tree and abort if we find any range @@ -231,7 +230,6 @@ namespace dxvk { node.setRed(true); node.setParent(parentIndex); node.addressRange = range; - node.payload = payload; // Only do the fixup to maintain red-black properties if // we haven't marked the root node as red in a deletion. @@ -261,7 +259,6 @@ namespace dxvk { childIndex = m_nodes[childIndex].child(0); node.addressRange = m_nodes[childIndex].addressRange; - node.payload = m_nodes[childIndex].payload; removeNode(childIndex, rootIndex); } else { // Deletion is expected to be exceptionally rare, to the point of @@ -292,7 +289,6 @@ namespace dxvk { node.setRed(child.isRed()); node.addressRange = child.addressRange; - node.payload = child.payload; if (cl) m_nodes[cl].setParent(nodeIndex); if (cr) m_nodes[cr].setParent(nodeIndex); @@ -403,7 +399,6 @@ namespace dxvk { node.setChild(1, rr); std::swap(node.addressRange, m_nodes[r].addressRange); - std::swap(node.payload, m_nodes[r].payload); } @@ -432,7 +427,6 @@ namespace dxvk { node.setChild(1, l); std::swap(node.addressRange, m_nodes[l].addressRange); - std::swap(node.payload, m_nodes[l].payload); } diff --git a/src/dxvk/dxvk_barrier.h b/src/dxvk/dxvk_barrier.h index fb0d0726f..449591207 100644 --- a/src/dxvk/dxvk_barrier.h +++ b/src/dxvk/dxvk_barrier.h @@ -13,43 +13,37 @@ namespace dxvk { * \brief Address range */ struct DxvkAddressRange { - /// Unique resource handle or address - uint64_t resource = 0u; + /// Unique resource handle + bit::uint48_t resource = bit::uint48_t(0u); + /// Access modes used for the given address range + DxvkAccessOp accessOp = DxvkAccessOp::None; /// Range start. For buffers, this shall be a byte offset, /// images can encode the first subresource index here. - uint32_t rangeStart = 0u; + uint64_t rangeStart = 0u; /// Range end. For buffers, this is the offset of the last byte /// included in the range, i.e. offset + size - 1. For images, /// this is the last subresource included in the range. - uint32_t rangeEnd = 0u; + uint64_t rangeEnd = 0u; bool contains(const DxvkAddressRange& other) const { - return resource == other.resource + return uint64_t(resource) == uint64_t(other.resource) && rangeStart <= other.rangeStart && rangeEnd >= other.rangeEnd; } bool overlaps(const DxvkAddressRange& other) const { - return resource == other.resource + return uint64_t(resource) == uint64_t(other.resource) && rangeEnd >= other.rangeStart && rangeStart <= other.rangeEnd; } bool lt(const DxvkAddressRange& other) const { - return (resource < other.resource) - || (resource == other.resource && rangeStart < other.rangeStart); + return (uint64_t(resource) < uint64_t(other.resource)) + || (uint64_t(resource) == uint64_t(other.resource) && rangeStart < other.rangeStart); } }; - /** - * \brief Barrier node payload - */ - struct DxvkBarrierPayload { - DxvkAccessOps accessOps = 0u; - }; - - /** * \brief Barrier tree node * @@ -70,9 +64,6 @@ namespace dxvk { // Address range of the node DxvkAddressRange addressRange = { }; - // Node payload - DxvkBarrierPayload payload = { }; - void setRed(bool red) { header &= ~uint64_t(1u); header |= uint64_t(red); @@ -128,25 +119,21 @@ namespace dxvk { * * \param [in] range Resource range * \param [in] accessType Access type - * \param [in] accessOp Access operation * \returns \c true if the range has a pending access */ bool findRange( const DxvkAddressRange& range, - DxvkAccess accessType, - DxvkAccessOp accessOp) const; + DxvkAccess accessType) const; /** * \brief Inserts address range for a given access type * * \param [in] range Resource range * \param [in] accessType Access type - * \param [in] accessOp Access operation */ void insertRange( const DxvkAddressRange& range, - DxvkAccess accessType, - DxvkAccessOp accessOp); + DxvkAccess accessType); /** * \brief Clears the entire structure @@ -181,8 +168,7 @@ namespace dxvk { uint32_t insertNode( const DxvkAddressRange& range, - uint32_t rootIndex, - DxvkBarrierPayload payload); + uint32_t rootIndex); void removeNode( uint32_t nodeIndex, @@ -205,7 +191,7 @@ namespace dxvk { DxvkAccess access) { // TODO revisit once we use internal allocation // objects or resource cookies here. - size_t hash = size_t(range.resource) * 93887; + size_t hash = uint64_t(range.resource) * 93887; hash ^= (hash >> 16); // Reserve the upper half of the implicit hash table for written diff --git a/src/dxvk/dxvk_buffer.h b/src/dxvk/dxvk_buffer.h index 805cfa76e..e6de10597 100644 --- a/src/dxvk/dxvk_buffer.h +++ b/src/dxvk/dxvk_buffer.h @@ -364,9 +364,9 @@ namespace dxvk { * \brief Retrieves resource ID for barrier tracking * \returns Unique resource ID */ - uint64_t getResourceId() const { + bit::uint48_t getResourceId() const { constexpr static size_t Align = alignof(DxvkResourceAllocation); - return reinterpret_cast(m_storage.ptr()) / (Align & -Align); + return bit::uint48_t(reinterpret_cast(m_storage.ptr()) / (Align & -Align)); } /** diff --git a/src/dxvk/dxvk_context.cpp b/src/dxvk/dxvk_context.cpp index c05b5c281..1013d5b52 100644 --- a/src/dxvk/dxvk_context.cpp +++ b/src/dxvk/dxvk_context.cpp @@ -7945,26 +7945,28 @@ namespace dxvk { if (subresources.levelCount == 1u || subresources.layerCount == layerCount) { DxvkAddressRange range; range.resource = image.getResourceId(); + range.accessOp = accessOp; range.rangeStart = subresources.baseMipLevel * layerCount + subresources.baseArrayLayer; range.rangeEnd = (subresources.baseMipLevel + subresources.levelCount - 1u) * layerCount + (subresources.baseArrayLayer + subresources.layerCount - 1u); if (hasWrite) - m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp); + m_barrierTracker.insertRange(range, DxvkAccess::Write); if (hasRead) - m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp); + m_barrierTracker.insertRange(range, DxvkAccess::Read); } else { DxvkAddressRange range; range.resource = image.getResourceId(); + range.accessOp = accessOp; for (uint32_t i = subresources.baseMipLevel; i < subresources.baseMipLevel + subresources.levelCount; i++) { range.rangeStart = i * layerCount + subresources.baseArrayLayer; range.rangeEnd = range.rangeStart + subresources.layerCount - 1u; if (hasWrite) - m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp); + m_barrierTracker.insertRange(range, DxvkAccess::Write); if (hasRead) - m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp); + m_barrierTracker.insertRange(range, DxvkAccess::Read); } } } @@ -8013,13 +8015,14 @@ namespace dxvk { if (cmdBuffer == DxvkCmdBuffer::ExecBuffer) { DxvkAddressRange range; range.resource = buffer.getResourceId(); + range.accessOp = accessOp; range.rangeStart = offset; range.rangeEnd = offset + size - 1; if (srcAccess & vk::AccessWriteMask) - m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp); + m_barrierTracker.insertRange(range, DxvkAccess::Write); if (srcAccess & vk::AccessReadMask) - m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp); + m_barrierTracker.insertRange(range, DxvkAccess::Read); } } @@ -8188,10 +8191,11 @@ namespace dxvk { DxvkAddressRange range; range.resource = buffer.getResourceId(); + range.accessOp = accessOp; range.rangeStart = offset; range.rangeEnd = offset + size - 1; - return m_barrierTracker.findRange(range, access, accessOp); + return m_barrierTracker.findRange(range, access); } @@ -8218,6 +8222,7 @@ namespace dxvk { // rendering and compute can only access one mip level. DxvkAddressRange range; range.resource = image.getResourceId(); + range.accessOp = accessOp; range.rangeStart = subresources.baseMipLevel * layerCount + subresources.baseArrayLayer; range.rangeEnd = (subresources.baseMipLevel + subresources.levelCount - 1u) * layerCount + (subresources.baseArrayLayer + subresources.layerCount - 1u); @@ -8225,7 +8230,7 @@ namespace dxvk { // Probe all subresources first, only check individual mip levels // if there are overlaps and if we are checking a subset of array // layers of multiple mips. - bool dirty = m_barrierTracker.findRange(range, access, accessOp); + bool dirty = m_barrierTracker.findRange(range, access); if (!dirty || subresources.levelCount == 1u || subresources.layerCount == layerCount) return dirty; @@ -8234,7 +8239,7 @@ namespace dxvk { range.rangeStart = i * layerCount + subresources.baseArrayLayer; range.rangeEnd = range.rangeStart + subresources.layerCount - 1u; - dirty = m_barrierTracker.findRange(range, access, accessOp); + dirty = m_barrierTracker.findRange(range, access); } return dirty; diff --git a/src/dxvk/dxvk_image.h b/src/dxvk/dxvk_image.h index 1bd2730ac..87e3deaf6 100644 --- a/src/dxvk/dxvk_image.h +++ b/src/dxvk/dxvk_image.h @@ -577,9 +577,9 @@ namespace dxvk { * \brief Retrieves resource ID for barrier tracking * \returns Unique resource ID */ - uint64_t getResourceId() const { + bit::uint48_t getResourceId() const { constexpr static size_t Align = alignof(DxvkResourceAllocation); - return reinterpret_cast(m_storage.ptr()) / (Align & -Align); + return bit::uint48_t(reinterpret_cast(m_storage.ptr()) / (Align & -Align)); } /** diff --git a/src/dxvk/dxvk_pipelayout.h b/src/dxvk/dxvk_pipelayout.h index 4f43b60ad..5a3beacfa 100644 --- a/src/dxvk/dxvk_pipelayout.h +++ b/src/dxvk/dxvk_pipelayout.h @@ -17,7 +17,7 @@ namespace dxvk { * Information used to optimize barriers when a resource * is accessed exlusively via order-invariant stores. */ - enum class DxvkAccessOp : uint32_t { + enum class DxvkAccessOp : uint16_t { None = 0, Or = 1, And = 2, @@ -29,8 +29,6 @@ namespace dxvk { UMax = 8, }; - using DxvkAccessOps = Flags; - /** * \brief Descriptor set indices