XDP: Timer synchronization and microcontroller trace changes

src/runtime_src/xdp/profile/database/static_info/aie_constructs.h

@@ -31,6 +31,7 @@ namespace xdp::aie {
     bool broadcast_enable_core;
     bool graph_iterator_event;
     std::string event_trace;
+    bool enable_multi_layer;
   };
 
   struct driver_config

src/runtime_src/xdp/profile/database/static_info/aie_util.cpp

@@ -409,6 +409,28 @@ namespace xdp::aie {
     try {
       xrt::hw_context context = xrt_core::hw_context_int::create_hw_context_from_implementation(hwCtxImpl);
       auto device = xrt_core::hw_context_int::get_core_device(context);
+
+      auto info = xrt_core::device_query_default<xrt_core::query::aie_partition_info>(device.get(), {});
+      for(const auto& e : info) {
+        boost::property_tree::ptree pt;
+        pt.put("start_col", e.start_col);
+        pt.put("num_cols", e.num_cols);
+        infoPt.push_back(std::make_pair("", pt));
+      }
+    }
+    catch(...) {
+      xrt_core::message::send(severity_level::info, "XRT", "Could not retrieve AIE Partition Info.");
+      return infoPt;
+    }
+    return infoPt;
+  }
+
+  boost::property_tree::ptree
+  getAIEPartitionInfo(void* handle)
+  {
+    boost::property_tree::ptree infoPt;
+    try {
+      auto device = xrt_core::get_userpf_device(handle);
 
       auto info = xrt_core::device_query_default<xrt_core::query::aie_partition_info>(device.get(), {});
       for(const auto& e : info) {

src/runtime_src/xdp/profile/database/static_info/aie_util.h

@@ -111,6 +111,10 @@ namespace xdp::aie {
   boost::property_tree::ptree
   getAIEPartitionInfoClient(void* hwCtxImpl);
 
+  XDP_CORE_EXPORT
+  boost::property_tree::ptree
+  getAIEPartitionInfo(void* handle);
+
   XDP_CORE_EXPORT
   void displayColShiftInfo(uint8_t colShift);
 

src/runtime_src/xdp/profile/database/static_info/filetypes/aie_control_config_filetype.cpp

@@ -66,6 +66,8 @@ AIEControlConfigFiletype::getAIECompilerOptions() const
         aie_meta.get("aie_metadata.aiecompiler_options.graph_iterator_event", false);
     aiecompiler_options.event_trace = 
         aie_meta.get("aie_metadata.aiecompiler_options.event_trace", "runtime");
+    aiecompiler_options.enable_multi_layer =
+        aie_meta.get("aie_metadata.aiecompiler_options.enable_multi_layer", false);
     return aiecompiler_options;
 }
 

src/runtime_src/xdp/profile/plugin/aie_trace/CMakeLists.txt

@@ -42,6 +42,7 @@ file(GLOB AIE_TRACE_UTIL_FILES
   "${PROFILE_DIR}/plugin/aie_trace/util/*.cpp"
 )
 
+
 file(GLOB AIE_DRIVER_COMMON_UTIL_FILES
   "${PROFILE_DIR}/device/common/*.h"
   "${PROFILE_DIR}/device/common/*.cpp"

src/runtime_src/xdp/profile/plugin/aie_trace/util/aie_trace_config.cpp

@@ -462,4 +462,86 @@ namespace xdp::aie::trace {
     return true;
   }
 
+  /****************************************************************************
+   * Reset timer for the specified tile range
+   ***************************************************************************/
+  void timerSyncronization(XAie_DevInst* aieDevInst, xaiefal::XAieDev* aieDevice, std::shared_ptr<AieTraceMetadata> metadata, uint8_t startCol, uint8_t numCols)
+  {
+    std::shared_ptr<xaiefal::XAieBroadcast> traceStartBroadcastCh1 = nullptr, traceStartBroadcastCh2 = nullptr;
+    std::vector<XAie_LocType> vL;
+    traceStartBroadcastCh1 = aieDevice->broadcast(vL, XAIE_PL_MOD, XAIE_CORE_MOD);
+    traceStartBroadcastCh1->reserve();
+    traceStartBroadcastCh2 = aieDevice->broadcast(vL, XAIE_PL_MOD, XAIE_CORE_MOD);
+    traceStartBroadcastCh2->reserve();
+
+    uint8_t broadcastId1 = traceStartBroadcastCh1->getBc();
+    uint8_t broadcastId2 = traceStartBroadcastCh2->getBc();
+
+    //build broadcast network
+    aie::trace::build2ChannelBroadcastNetwork(aieDevInst, metadata, broadcastId1, broadcastId2, XAIE_EVENT_USER_EVENT_0_PL, startCol, numCols);
+
+    //set timer control register
+    for (auto& tileMetric : metadata->getConfigMetrics()) {
+      auto tile       = tileMetric.first;
+      auto col        = tile.col + startCol; 
+      auto row        = tile.row;
+      auto type       = aie::getModuleType(row, metadata->getRowOffset());
+      auto loc        = XAie_TileLoc(col, row);
+
+      if(type == module_type::shim) {
+        XAie_Events resetEvent = (XAie_Events)(XAIE_EVENT_BROADCAST_A_0_PL + broadcastId2);
+        if(col == startCol)
+        {
+          resetEvent = XAIE_EVENT_USER_EVENT_0_PL;
+        }
+
+        XAie_SetTimerResetEvent(aieDevInst, loc, XAIE_PL_MOD, resetEvent, XAIE_RESETDISABLE);
+      }
+      else if(type == module_type::mem_tile) {
+        XAie_Events resetEvent = (XAie_Events) (XAIE_EVENT_BROADCAST_0_MEM_TILE + broadcastId1);
+        XAie_SetTimerResetEvent(aieDevInst, loc, XAIE_MEM_MOD, resetEvent, XAIE_RESETDISABLE);
+      }
+      else {
+        XAie_Events resetEvent = (XAie_Events) (XAIE_EVENT_BROADCAST_0_CORE + broadcastId1);
+        XAie_SetTimerResetEvent(aieDevInst, loc, XAIE_CORE_MOD, resetEvent, XAIE_RESETDISABLE);
+        resetEvent = (XAie_Events) (XAIE_EVENT_BROADCAST_0_MEM + broadcastId1);
+        XAie_SetTimerResetEvent(aieDevInst, loc, XAIE_MEM_MOD, resetEvent, XAIE_RESETDISABLE);
+      }
+    }
+
+    //Generate the event to trigger broadcast network to reset timer
+    XAie_EventGenerate(aieDevInst, XAie_TileLoc(startCol, 0), XAIE_PL_MOD, XAIE_EVENT_USER_EVENT_0_PL);
+
+    //reset timer control register so that timer are not reset after this point
+    for (auto& tileMetric : metadata->getConfigMetrics()) {
+      auto tile       = tileMetric.first;
+      auto col        = tile.col + startCol;
+      auto row        = tile.row;
+      auto type       = aie::getModuleType(row, metadata->getRowOffset());
+      auto loc        = XAie_TileLoc(col, row);
+
+      if(type == module_type::shim) {
+        XAie_Events resetEvent = XAIE_EVENT_NONE_PL ;
+        XAie_SetTimerResetEvent(aieDevInst, loc, XAIE_PL_MOD, resetEvent, XAIE_RESETDISABLE);
+      }
+      else if(type == module_type::mem_tile) {
+        XAie_Events resetEvent = XAIE_EVENT_NONE_MEM_TILE;
+        XAie_SetTimerResetEvent(aieDevInst, loc, XAIE_MEM_MOD, resetEvent, XAIE_RESETDISABLE);
+      }
+      else {
+        XAie_Events resetEvent = XAIE_EVENT_NONE_CORE;
+        XAie_SetTimerResetEvent(aieDevInst, loc, XAIE_CORE_MOD, resetEvent, XAIE_RESETDISABLE);
+        resetEvent = XAIE_EVENT_NONE_MEM;
+        XAie_SetTimerResetEvent(aieDevInst, loc, XAIE_MEM_MOD, resetEvent, XAIE_RESETDISABLE);
+      }
+    }
+
+    //reset broadcast network
+    reset2ChannelBroadcastNetwork(aieDevInst, metadata, broadcastId1, broadcastId2, startCol, numCols);
+
+    //release the channels used for timer sync
+    traceStartBroadcastCh1->release();
+    traceStartBroadcastCh2->release();
+  }
+
 }  // namespace xdp

src/runtime_src/xdp/profile/plugin/aie_trace/util/aie_trace_config.h

@@ -20,6 +20,7 @@
 #include <cstdint>
 #include "xaiefal/xaiefal.hpp"
 #include "xdp/profile/database/static_info/aie_constructs.h"
+#include "xdp/profile/plugin/aie_trace/aie_trace_metadata.h"
 
 namespace xdp::aie::trace {  
   /**
@@ -119,6 +120,18 @@ namespace xdp::aie::trace {
    */
   bool configStartIteration(xaiefal::XAieMod& core, uint32_t iteration,
                             XAie_Events& startEvent);
+
+  /**
+   * @brief Reset timers for specified tile range
+   * @param aieDevInst     AIE device Instance
+   * @param aieDevice      AIE device
+   * @param metadata       Trace Metadata
+   * @param startCol       Start column of the partition
+   * @param numCols        Num of columns in the partition
+   */
+  void timerSyncronization(XAie_DevInst* aieDevInst, xaiefal::XAieDev* aieDevice,
+                           std::shared_ptr<AieTraceMetadata> metadata, uint8_t startCol, 
+                           uint8_t numCols);
 }  // namespace xdp::aie::trace
 
 #endif

src/runtime_src/xdp/profile/plugin/aie_trace/util/aie_trace_util.cpp

@@ -632,4 +632,107 @@ namespace xdp::aie::trace {
           XAIE_EVENT_DMA_S2MM_0_MEMORY_BACKPRESSURE_PL, XAIE_EVENT_DMA_S2MM_1_MEMORY_BACKPRESSURE_PL);
     }
   }
+
+  /****************************************************************************
+   * Set up broadcast network 
+   ***************************************************************************/
+  void build2ChannelBroadcastNetwork(XAie_DevInst* aieDevInst, std::shared_ptr<AieTraceMetadata> metadata, uint8_t broadcastId1, uint8_t broadcastId2, XAie_Events event, uint8_t startCol, uint8_t numCols) 
+  {
+    std::vector<uint8_t> maxRowAtCol(startCol + numCols, 0);
+    for (auto& tileMetric : metadata->getConfigMetrics()) {
+      auto tile       = tileMetric.first;
+      auto col        = tile.col;
+      auto row        = tile.row;
+      maxRowAtCol[startCol + col] = std::max(maxRowAtCol[col], (uint8_t)row);
+    }
+
+    XAie_Events bcastEvent2_PL =  (XAie_Events) (XAIE_EVENT_BROADCAST_A_0_PL + broadcastId2);
+    XAie_EventBroadcast(aieDevInst, XAie_TileLoc(startCol, 0), XAIE_PL_MOD, broadcastId2, event);
+
+    for(uint8_t col = startCol; col < startCol + numCols; col++) {
+      for(uint8_t row = 0; row <= maxRowAtCol[col]; row++) {
+        module_type tileType = aie::getModuleType(row, metadata->getRowOffset());
+        auto loc = XAie_TileLoc(col, row);
+
+        if(tileType == module_type::shim) {
+          // first channel is only used to send north
+          if(col == startCol) {
+            XAie_EventBroadcast(aieDevInst, loc, XAIE_PL_MOD, broadcastId1, event);
+          }
+          else {
+            XAie_EventBroadcast(aieDevInst, loc, XAIE_PL_MOD, broadcastId1, bcastEvent2_PL);
+          }
+          if(maxRowAtCol[col] != row) {
+            XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_PL_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_EAST);
+          }
+          else {
+            XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_PL_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_EAST | XAIE_EVENT_BROADCAST_NORTH);
+          }
+
+          // second channel is only used to send east
+          XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_PL_MOD, XAIE_EVENT_SWITCH_A, broadcastId2, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_NORTH);
+
+          if(col != startCol + numCols - 1) {
+            XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_PL_MOD, XAIE_EVENT_SWITCH_B, broadcastId2, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_NORTH);
+          }
+          else {
+            XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_PL_MOD, XAIE_EVENT_SWITCH_B, broadcastId2, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_NORTH | XAIE_EVENT_BROADCAST_EAST);
+          }
+        }
+        else if(tileType == module_type::mem_tile) {
+          if(maxRowAtCol[col] != row) {
+            XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_MEM_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_EAST);
+          }
+          else {
+            XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_MEM_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_EAST | XAIE_EVENT_BROADCAST_NORTH);
+          }
+        }
+        else { //core tile
+          if(maxRowAtCol[col] != row) {
+            XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_CORE_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST);
+          }
+          else {
+            XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_CORE_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_NORTH);
+          }
+          XAie_EventBroadcastBlockDir(aieDevInst, loc, XAIE_MEM_MOD,  XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_SOUTH | XAIE_EVENT_BROADCAST_WEST | XAIE_EVENT_BROADCAST_EAST | XAIE_EVENT_BROADCAST_NORTH);
+        }
+      }
+    }
+  }
+
+  /****************************************************************************
+   * Reset broadcast network
+   ***************************************************************************/
+  void reset2ChannelBroadcastNetwork(XAie_DevInst* aieDevInst, std::shared_ptr<AieTraceMetadata> metadata, uint8_t broadcastId1, uint8_t broadcastId2, uint8_t startCol, uint8_t numCols) {
+    std::vector<uint8_t> maxRowAtCol(startCol + numCols, 0);
+    for (auto& tileMetric : metadata->getConfigMetrics()) {
+      auto tile       = tileMetric.first;
+      auto col        = tile.col;
+      auto row        = tile.row;
+      maxRowAtCol[startCol + col] = std::max(maxRowAtCol[col], (uint8_t)row);
+    }
+
+    XAie_EventBroadcastReset(aieDevInst, XAie_TileLoc(startCol, 0), XAIE_PL_MOD, broadcastId2);
+
+    for(uint8_t col = startCol; col < startCol + numCols; col++) {
+      for(uint8_t row = 0; row <= maxRowAtCol[col]; row++) {
+        module_type tileType = aie::getModuleType(row, metadata->getRowOffset());
+        auto loc = XAie_TileLoc(col, row);
+
+        if(tileType == module_type::shim) {
+          XAie_EventBroadcastReset(aieDevInst, loc, XAIE_PL_MOD, broadcastId1);
+          XAie_EventBroadcastUnblockDir(aieDevInst, loc, XAIE_PL_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_ALL);
+          XAie_EventBroadcastUnblockDir(aieDevInst, loc, XAIE_PL_MOD, XAIE_EVENT_SWITCH_A, broadcastId2, XAIE_EVENT_BROADCAST_ALL);
+          XAie_EventBroadcastUnblockDir(aieDevInst, loc, XAIE_PL_MOD, XAIE_EVENT_SWITCH_B, broadcastId2, XAIE_EVENT_BROADCAST_ALL);
+        }
+        else if(tileType == module_type::mem_tile) {
+          XAie_EventBroadcastUnblockDir(aieDevInst, loc, XAIE_MEM_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_ALL);
+        }
+        else { //core tile
+          XAie_EventBroadcastUnblockDir(aieDevInst, loc, XAIE_CORE_MOD, XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_ALL);
+          XAie_EventBroadcastUnblockDir(aieDevInst, loc, XAIE_MEM_MOD,  XAIE_EVENT_SWITCH_A, broadcastId1, XAIE_EVENT_BROADCAST_ALL);
+        }
+      }
+    }
+  }
 } // namespace xdp::aie

src/runtime_src/xdp/profile/plugin/aie_trace/util/aie_trace_util.h

@@ -20,6 +20,7 @@
 #include <cstdint>
 #include "xaiefal/xaiefal.hpp"
 #include "xdp/profile/database/static_info/aie_constructs.h"
+#include "xdp/profile/plugin/aie_trace/aie_trace_metadata.h"
 
 namespace xdp::aie::trace {
   /**
@@ -178,6 +179,28 @@ namespace xdp::aie::trace {
   void modifyEvents(module_type type, io_type subtype, const std::string metricSet,
                     uint8_t channel, std::vector<XAie_Events>& events);
 
+  /**
+   * @brief Build 2-channel broadcast network for specified tile range
+   * @param aieDevInst     AIE device
+   * @param metadata       Trace Metadata
+   * @param broadcastId1   Broadcast channel 1
+   * @param broadcastId2   Broadcast channel 2
+   * @param event          Event to trigger broadcast network
+   * @param startCol       Start column of the partition
+   * @param numCols        Num of columns in the partition
+   */
+  void build2ChannelBroadcastNetwork(XAie_DevInst* aieDevInst, std::shared_ptr<AieTraceMetadata> metadata, uint8_t broadcastId1, uint8_t broadcastId2, XAie_Events event, uint8_t startCol, uint8_t numCols);
+
+  /**
+   * @brief Reset 2-channel broadcast network for specified tile range
+   * @param aieDevInst     AIE device
+   * @param metadata       Trace Metadata
+   * @param broadcastId1   Broadcast channel 1
+   * @param broadcastId2   Broadcast channel 2
+   * @param startCol       Start column of the partition
+   * @param numCols        Num of columns in the partition
+   */
+  void reset2ChannelBroadcastNetwork(XAie_DevInst* aieDevInst, std::shared_ptr<AieTraceMetadata> metadata, uint8_t broadcastId1, uint8_t broadcastId2, uint8_t startCol, uint8_t numCols);
 }  // namespace xdp::aie::trace
 
 #endif