fix the cases where ncomps > 1 and sendimm == 0

STEllAR-GROUP · Oct 12, 2023 · 1498c35 · 1498c35
1 parent 3265482
commit 1498c35
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 9 deletions.
diff --git a/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp b/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp
@@ -120,8 +120,6 @@ namespace hpx::parcelset {
             // whether the parcelport has been initialized
             // (starting to execute the background works)
             std::atomic<bool> is_initialized = false;
-            // whether the parcelport is sending early parcels
-            std::atomic<bool> is_sending_early_parcel = false;
 
             // LCI objects
             struct completion_manager_t;

diff --git a/libs/full/parcelport_lci/include/hpx/parcelport_lci/sender_connection_base.hpp b/libs/full/parcelport_lci/include/hpx/parcelport_lci/sender_connection_base.hpp
@@ -53,7 +53,7 @@ namespace hpx::parcelset::policies::lci {
           : dst_rank(dst)
           , pp_((lci::parcelport*) pp)
           , there_(parcelset::locality(locality(dst_rank)))
-          , device_p(&pp_->get_tls_device())
+          , device_p(nullptr)
         {
         }
 

diff --git a/libs/full/parcelport_lci/src/parcelport_lci.cpp b/libs/full/parcelport_lci/src/parcelport_lci.cpp
@@ -144,9 +144,7 @@ namespace hpx::parcelset::policies::lci {
     void parcelport::send_early_parcel(
         hpx::parcelset::locality const& dest, parcel p)
     {
-        is_sending_early_parcel = true;
         base_type::send_early_parcel(dest, HPX_MOVE(p));
-        is_sending_early_parcel = false;
     }
 
     bool parcelport::do_background_work(
@@ -454,12 +452,14 @@ namespace hpx::parcelset::policies::lci {
     {
         static thread_local std::size_t tls_device_idx = -1;
 
-        if (hpx::threads::get_self_id() == hpx::threads::invalid_thread_id)
+        if (HPX_UNLIKELY(!is_initialized ||
+                hpx::threads::get_self_id() == hpx::threads::invalid_thread_id))
         {
+            static thread_local std::size_t tls_rr_device_idx = 0;
             util::lci_environment::log(
                 util::lci_environment::log_level_t::debug, "device",
                 "Rank %d unusual phase\n", LCI_RANK);
-            return devices[0];
+            return devices[tls_rr_device_idx++ % devices.size()];
         }
         if (tls_device_idx == std::size_t(-1))
         {

diff --git a/libs/full/parcelport_lci/src/sender_base.cpp b/libs/full/parcelport_lci/src/sender_base.cpp
@@ -30,7 +30,6 @@ namespace hpx::parcelset::policies::lci {
     bool sender_base::background_work(size_t /* num_thread */) noexcept
     {
         bool did_some_work = false;
-        // try to accept a new connection
         auto poll_comp_start = util::lci_environment::pcounter_now();
         auto completion_manager_p = pp_->get_tls_device().completion_manager_p;
         LCI_request_t request = completion_manager_p->send->poll();

diff --git a/libs/full/parcelport_lci/src/sender_connection_base.cpp b/libs/full/parcelport_lci/src/sender_connection_base.cpp
@@ -31,6 +31,7 @@ namespace hpx::parcelset::policies::lci {
         sender_connection_base::postprocess_handler_type&& parcel_postprocess)
     {
         LCT_time_t async_write_start_time = util::lci_environment::pcounter_now();
+        device_p = &pp_->get_tls_device();
         load(HPX_FORWARD(handler_type, handler),
             HPX_FORWARD(postprocess_handler_type, parcel_postprocess));
         return_t ret = send();
@@ -54,7 +55,7 @@ namespace hpx::parcelset::policies::lci {
         return_t ret;
         const int retry_max_spin = 32;
         if (!config_t::enable_lci_backlog_queue ||
-            HPX_UNLIKELY(pp_->is_sending_early_parcel))
+            HPX_UNLIKELY(!pp_->is_initialized))
         {
             // If we are sending early parcels, we should not expect the
             // thread make progress on the backlog queue