[dxvk] Refactor address range for barrier tracking

Allows us to bump the size and offset to 64 bits without increasing the
total size of the node.

src/dxvk/dxvk_barrier.cpp

@@ -20,12 +20,11 @@ namespace dxvk {
 
   bool DxvkBarrierTracker::findRange(
     const DxvkAddressRange&           range,
-          DxvkAccess                  accessType,
+          DxvkAccess                  accessType) const {
-          DxvkAccessOp                accessOp) const {
     uint32_t rootIndex = computeRootIndex(range, accessType);
     uint32_t nodeIndex = findNode(range, rootIndex);
 
-    if (likely(!nodeIndex || accessOp == DxvkAccessOp::None))
+    if (likely(!nodeIndex || range.accessOp == DxvkAccessOp::None))
       return nodeIndex;
 
     // If we are checking for a specific order-invariant store
@@ -33,32 +32,33 @@ namespace dxvk {
     // resource, and the tracked range must cover the requested
     // range in its entirety so we can rule out that other parts
     // of the resource have been accessed in a different way.
-    auto& node = m_nodes[nodeIndex];
+    const auto& node = m_nodes[nodeIndex];
 
-    return node.payload.accessOps != DxvkAccessOps(accessOp)
+    if (node.addressRange.accessOp != range.accessOp)
-        || !node.addressRange.contains(range);
+      return true;
+
+    return !node.addressRange.contains(range);
   }
 
 
   void DxvkBarrierTracker::insertRange(
     const DxvkAddressRange&           range,
-          DxvkAccess                  accessType,
+          DxvkAccess                  accessType) {
-          DxvkAccessOp                accessOp) {
-    DxvkBarrierPayload payload = { };
-    payload.accessOps.set(accessOp);
-
     // If we can just insert the node with no conflicts,
     // we don't have to do anything.
     uint32_t rootIndex = computeRootIndex(range, accessType);
-    uint32_t nodeIndex = insertNode(range, rootIndex, payload);
+    uint32_t nodeIndex = insertNode(range, rootIndex);
 
     if (likely(!nodeIndex))
       return;
 
     // If there's an existing node and it contains the entire
     // range we want to add already, also don't do anything.
+    // If there are conflicting access ops, reset it.
     auto& node = m_nodes[nodeIndex];
-    node.payload.accessOps.set(payload.accessOps);
+
+    if (node.addressRange.accessOp != range.accessOp)
+      node.addressRange.accessOp = DxvkAccessOp::None;
 
     if (node.addressRange.contains(range))
       return;
@@ -100,14 +100,15 @@ namespace dxvk {
       mergedRange.rangeStart = std::min(mergedRange.rangeStart, node.addressRange.rangeStart);
       mergedRange.rangeEnd = std::max(mergedRange.rangeEnd, node.addressRange.rangeEnd);
 
-      payload.accessOps.set(node.payload.accessOps);
+      if (mergedRange.accessOp != node.addressRange.accessOp)
+        mergedRange.accessOp = DxvkAccessOp::None;
 
       removeNode(nodeIndex, rootIndex);
 
       nodeIndex = findNode(range, rootIndex);
     }
 
-    insertNode(mergedRange, rootIndex, payload);
+    insertNode(mergedRange, rootIndex);
   }
 
 
@@ -186,8 +187,7 @@ namespace dxvk {
 
   uint32_t DxvkBarrierTracker::insertNode(
     const DxvkAddressRange&           range,
-          uint32_t                    rootIndex,
+          uint32_t                    rootIndex) {
-          DxvkBarrierPayload          payload) {
     // Check if the given root is valid at all
     uint64_t rootBit = uint64_t(1u) << (rootIndex - 1u);
 
@@ -199,7 +199,6 @@ namespace dxvk {
       auto& node = m_nodes[rootIndex];
       node.header = 0;
       node.addressRange = range;
-      node.payload = payload;
       return 0;
     } else {
       // Traverse tree and abort if we find any range
@@ -231,7 +230,6 @@ namespace dxvk {
       node.setRed(true);
       node.setParent(parentIndex);
       node.addressRange = range;
-      node.payload = payload;
 
       // Only do the fixup to maintain red-black properties if
       // we haven't marked the root node as red in a deletion.
@@ -261,7 +259,6 @@ namespace dxvk {
         childIndex = m_nodes[childIndex].child(0);
 
       node.addressRange = m_nodes[childIndex].addressRange;
-      node.payload = m_nodes[childIndex].payload;
       removeNode(childIndex, rootIndex);
     } else {
       // Deletion is expected to be exceptionally rare, to the point of
@@ -292,7 +289,6 @@ namespace dxvk {
           node.setRed(child.isRed());
 
         node.addressRange = child.addressRange;
-        node.payload = child.payload;
 
         if (cl) m_nodes[cl].setParent(nodeIndex);
         if (cr) m_nodes[cr].setParent(nodeIndex);
@@ -403,7 +399,6 @@ namespace dxvk {
     node.setChild(1, rr);
 
     std::swap(node.addressRange, m_nodes[r].addressRange);
-    std::swap(node.payload, m_nodes[r].payload);
   }
 
 
@@ -432,7 +427,6 @@ namespace dxvk {
     node.setChild(1, l);
 
     std::swap(node.addressRange, m_nodes[l].addressRange);
-    std::swap(node.payload, m_nodes[l].payload);
   }
 
 

src/dxvk/dxvk_barrier.h

@@ -13,43 +13,37 @@ namespace dxvk {
    * \brief Address range
    */
   struct DxvkAddressRange {
-    /// Unique resource handle or address
+    /// Unique resource handle
-    uint64_t resource = 0u;
+    bit::uint48_t resource = bit::uint48_t(0u);
+    /// Access modes used for the given address range
+    DxvkAccessOp accessOp = DxvkAccessOp::None;
     /// Range start. For buffers, this shall be a byte offset,
     /// images can encode the first subresource index here.
-    uint32_t rangeStart = 0u;
+    uint64_t rangeStart = 0u;
     /// Range end. For buffers, this is the offset of the last byte
     /// included in the range, i.e. offset + size - 1. For images,
     /// this is the last subresource included in the range.
-    uint32_t rangeEnd = 0u;
+    uint64_t rangeEnd = 0u;
 
     bool contains(const DxvkAddressRange& other) const {
-      return resource == other.resource
+      return uint64_t(resource) == uint64_t(other.resource)
           && rangeStart <= other.rangeStart
           && rangeEnd >= other.rangeEnd;
     }
 
     bool overlaps(const DxvkAddressRange& other) const {
-      return resource == other.resource
+      return uint64_t(resource) == uint64_t(other.resource)
           && rangeEnd >= other.rangeStart
           && rangeStart <= other.rangeEnd;
     }
 
     bool lt(const DxvkAddressRange& other) const {
-      return (resource < other.resource)
+      return (uint64_t(resource) < uint64_t(other.resource))
-          || (resource == other.resource && rangeStart < other.rangeStart);
+          || (uint64_t(resource) == uint64_t(other.resource) && rangeStart < other.rangeStart);
     }
   };
 
 
-  /**
-   * \brief Barrier node payload
-   */
-  struct DxvkBarrierPayload {
-    DxvkAccessOps accessOps = 0u;
-  };
-
-
   /**
    * \brief Barrier tree node
    *
@@ -70,9 +64,6 @@ namespace dxvk {
     // Address range of the node
     DxvkAddressRange addressRange = { };
 
-    // Node payload
-    DxvkBarrierPayload payload = { };
-
     void setRed(bool red) {
       header &= ~uint64_t(1u);
       header |= uint64_t(red);
@@ -128,25 +119,21 @@ namespace dxvk {
      *
      * \param [in] range Resource range
      * \param [in] accessType Access type
-     * \param [in] accessOp Access operation
      * \returns \c true if the range has a pending access
      */
     bool findRange(
       const DxvkAddressRange&           range,
-            DxvkAccess                  accessType,
+            DxvkAccess                  accessType) const;
-            DxvkAccessOp                accessOp) const;
 
     /**
      * \brief Inserts address range for a given access type
      *
      * \param [in] range Resource range
      * \param [in] accessType Access type
-     * \param [in] accessOp Access operation
      */
     void insertRange(
       const DxvkAddressRange&           range,
-            DxvkAccess                  accessType,
+            DxvkAccess                  accessType);
-            DxvkAccessOp                accessOp);
 
     /**
      * \brief Clears the entire structure
@@ -181,8 +168,7 @@ namespace dxvk {
 
     uint32_t insertNode(
       const DxvkAddressRange&           range,
-            uint32_t                    rootIndex,
+            uint32_t                    rootIndex);
-            DxvkBarrierPayload          payload);
 
     void removeNode(
             uint32_t                    nodeIndex,
@@ -205,7 +191,7 @@ namespace dxvk {
             DxvkAccess                  access) {
       // TODO revisit once we use internal allocation
       // objects or resource cookies here.
-      size_t hash = size_t(range.resource) * 93887;
+      size_t hash = uint64_t(range.resource) * 93887;
              hash ^= (hash >> 16);
 
       // Reserve the upper half of the implicit hash table for written

src/dxvk/dxvk_buffer.h

@@ -364,9 +364,9 @@ namespace dxvk {
      * \brief Retrieves resource ID for barrier tracking
      * \returns Unique resource ID
      */
-    uint64_t getResourceId() const {
+    bit::uint48_t getResourceId() const {
       constexpr static size_t Align = alignof(DxvkResourceAllocation);
-      return reinterpret_cast<uintptr_t>(m_storage.ptr()) / (Align & -Align);
+      return bit::uint48_t(reinterpret_cast<uintptr_t>(m_storage.ptr()) / (Align & -Align));
     }
 
     /**

src/dxvk/dxvk_context.cpp

@@ -7945,26 +7945,28 @@ namespace dxvk {
       if (subresources.levelCount == 1u || subresources.layerCount == layerCount) {
         DxvkAddressRange range;
         range.resource = image.getResourceId();
+        range.accessOp = accessOp;
         range.rangeStart = subresources.baseMipLevel * layerCount + subresources.baseArrayLayer;
         range.rangeEnd = (subresources.baseMipLevel + subresources.levelCount - 1u) * layerCount
                        + (subresources.baseArrayLayer + subresources.layerCount - 1u);
 
         if (hasWrite)
-          m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp);
+          m_barrierTracker.insertRange(range, DxvkAccess::Write);
         if (hasRead)
-          m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp);
+          m_barrierTracker.insertRange(range, DxvkAccess::Read);
       } else {
         DxvkAddressRange range;
         range.resource = image.getResourceId();
+        range.accessOp = accessOp;
 
         for (uint32_t i = subresources.baseMipLevel; i < subresources.baseMipLevel + subresources.levelCount; i++) {
           range.rangeStart = i * layerCount + subresources.baseArrayLayer;
           range.rangeEnd = range.rangeStart + subresources.layerCount - 1u;
 
           if (hasWrite)
-            m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp);
+            m_barrierTracker.insertRange(range, DxvkAccess::Write);
           if (hasRead)
-            m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp);
+            m_barrierTracker.insertRange(range, DxvkAccess::Read);
         }
       }
     }
@@ -8013,13 +8015,14 @@ namespace dxvk {
     if (cmdBuffer == DxvkCmdBuffer::ExecBuffer) {
       DxvkAddressRange range;
       range.resource = buffer.getResourceId();
+      range.accessOp = accessOp;
       range.rangeStart = offset;
       range.rangeEnd = offset + size - 1;
 
       if (srcAccess & vk::AccessWriteMask)
-        m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp);
+        m_barrierTracker.insertRange(range, DxvkAccess::Write);
       if (srcAccess & vk::AccessReadMask)
-        m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp);
+        m_barrierTracker.insertRange(range, DxvkAccess::Read);
     }
   }
 
@@ -8188,10 +8191,11 @@ namespace dxvk {
 
     DxvkAddressRange range;
     range.resource = buffer.getResourceId();
+    range.accessOp = accessOp;
     range.rangeStart = offset;
     range.rangeEnd = offset + size - 1;
 
-    return m_barrierTracker.findRange(range, access, accessOp);
+    return m_barrierTracker.findRange(range, access);
   }
 
 
@@ -8218,6 +8222,7 @@ namespace dxvk {
     // rendering and compute can only access one mip level.
     DxvkAddressRange range;
     range.resource = image.getResourceId();
+    range.accessOp = accessOp;
     range.rangeStart = subresources.baseMipLevel * layerCount + subresources.baseArrayLayer;
     range.rangeEnd = (subresources.baseMipLevel + subresources.levelCount - 1u) * layerCount
                    + (subresources.baseArrayLayer + subresources.layerCount - 1u);
@@ -8225,7 +8230,7 @@ namespace dxvk {
     // Probe all subresources first, only check individual mip levels
     // if there are overlaps and if we are checking a subset of array
     // layers of multiple mips.
-    bool dirty = m_barrierTracker.findRange(range, access, accessOp);
+    bool dirty = m_barrierTracker.findRange(range, access);
 
     if (!dirty || subresources.levelCount == 1u || subresources.layerCount == layerCount)
       return dirty;
@@ -8234,7 +8239,7 @@ namespace dxvk {
       range.rangeStart = i * layerCount + subresources.baseArrayLayer;
       range.rangeEnd = range.rangeStart + subresources.layerCount - 1u;
 
-      dirty = m_barrierTracker.findRange(range, access, accessOp);
+      dirty = m_barrierTracker.findRange(range, access);
     }
 
     return dirty;

src/dxvk/dxvk_image.h

@@ -577,9 +577,9 @@ namespace dxvk {
      * \brief Retrieves resource ID for barrier tracking
      * \returns Unique resource ID
      */
-    uint64_t getResourceId() const {
+    bit::uint48_t getResourceId() const {
       constexpr static size_t Align = alignof(DxvkResourceAllocation);
-      return reinterpret_cast<uintptr_t>(m_storage.ptr()) / (Align & -Align);
+      return bit::uint48_t(reinterpret_cast<uintptr_t>(m_storage.ptr()) / (Align & -Align));
     }
 
     /**

src/dxvk/dxvk_pipelayout.h

@@ -17,7 +17,7 @@ namespace dxvk {
    * Information used to optimize barriers when a resource
    * is accessed exlusively via order-invariant stores.
    */
-  enum class DxvkAccessOp : uint32_t {
+  enum class DxvkAccessOp : uint16_t {
     None  = 0,
     Or    = 1,
     And   = 2,
@@ -29,8 +29,6 @@ namespace dxvk {
     UMax  = 8,
   };
 
-  using DxvkAccessOps = Flags<DxvkAccessOp>;
-
 
   /**
    * \brief Descriptor set indices