From 1498c35e8c7fb781a69da3dc939f733c6655bc9e Mon Sep 17 00:00:00 2001 From: Jiakun Yan Date: Tue, 12 Sep 2023 20:27:20 -0500 Subject: [PATCH] fix the cases where ncomps > 1 and sendimm == 0 --- .../include/hpx/parcelport_lci/parcelport_lci.hpp | 2 -- .../include/hpx/parcelport_lci/sender_connection_base.hpp | 2 +- libs/full/parcelport_lci/src/parcelport_lci.cpp | 8 ++++---- libs/full/parcelport_lci/src/sender_base.cpp | 1 - libs/full/parcelport_lci/src/sender_connection_base.cpp | 3 ++- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp b/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp index d0a3b66d71bf..737af1b966e9 100644 --- a/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp +++ b/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp @@ -120,8 +120,6 @@ namespace hpx::parcelset { // whether the parcelport has been initialized // (starting to execute the background works) std::atomic is_initialized = false; - // whether the parcelport is sending early parcels - std::atomic is_sending_early_parcel = false; // LCI objects struct completion_manager_t; diff --git a/libs/full/parcelport_lci/include/hpx/parcelport_lci/sender_connection_base.hpp b/libs/full/parcelport_lci/include/hpx/parcelport_lci/sender_connection_base.hpp index f28ee7f9fc21..4a5a5ede9842 100644 --- a/libs/full/parcelport_lci/include/hpx/parcelport_lci/sender_connection_base.hpp +++ b/libs/full/parcelport_lci/include/hpx/parcelport_lci/sender_connection_base.hpp @@ -53,7 +53,7 @@ namespace hpx::parcelset::policies::lci { : dst_rank(dst) , pp_((lci::parcelport*) pp) , there_(parcelset::locality(locality(dst_rank))) - , device_p(&pp_->get_tls_device()) + , device_p(nullptr) { } diff --git a/libs/full/parcelport_lci/src/parcelport_lci.cpp b/libs/full/parcelport_lci/src/parcelport_lci.cpp index a203ca9c4f6d..c892940b335c 100644 --- a/libs/full/parcelport_lci/src/parcelport_lci.cpp +++ b/libs/full/parcelport_lci/src/parcelport_lci.cpp @@ -144,9 +144,7 @@ namespace hpx::parcelset::policies::lci { void parcelport::send_early_parcel( hpx::parcelset::locality const& dest, parcel p) { - is_sending_early_parcel = true; base_type::send_early_parcel(dest, HPX_MOVE(p)); - is_sending_early_parcel = false; } bool parcelport::do_background_work( @@ -454,12 +452,14 @@ namespace hpx::parcelset::policies::lci { { static thread_local std::size_t tls_device_idx = -1; - if (hpx::threads::get_self_id() == hpx::threads::invalid_thread_id) + if (HPX_UNLIKELY(!is_initialized || + hpx::threads::get_self_id() == hpx::threads::invalid_thread_id)) { + static thread_local std::size_t tls_rr_device_idx = 0; util::lci_environment::log( util::lci_environment::log_level_t::debug, "device", "Rank %d unusual phase\n", LCI_RANK); - return devices[0]; + return devices[tls_rr_device_idx++ % devices.size()]; } if (tls_device_idx == std::size_t(-1)) { diff --git a/libs/full/parcelport_lci/src/sender_base.cpp b/libs/full/parcelport_lci/src/sender_base.cpp index 90db03aa2738..da2bcfc24b30 100644 --- a/libs/full/parcelport_lci/src/sender_base.cpp +++ b/libs/full/parcelport_lci/src/sender_base.cpp @@ -30,7 +30,6 @@ namespace hpx::parcelset::policies::lci { bool sender_base::background_work(size_t /* num_thread */) noexcept { bool did_some_work = false; - // try to accept a new connection auto poll_comp_start = util::lci_environment::pcounter_now(); auto completion_manager_p = pp_->get_tls_device().completion_manager_p; LCI_request_t request = completion_manager_p->send->poll(); diff --git a/libs/full/parcelport_lci/src/sender_connection_base.cpp b/libs/full/parcelport_lci/src/sender_connection_base.cpp index 5dbadd8c9543..3dc06ad2452c 100644 --- a/libs/full/parcelport_lci/src/sender_connection_base.cpp +++ b/libs/full/parcelport_lci/src/sender_connection_base.cpp @@ -31,6 +31,7 @@ namespace hpx::parcelset::policies::lci { sender_connection_base::postprocess_handler_type&& parcel_postprocess) { LCT_time_t async_write_start_time = util::lci_environment::pcounter_now(); + device_p = &pp_->get_tls_device(); load(HPX_FORWARD(handler_type, handler), HPX_FORWARD(postprocess_handler_type, parcel_postprocess)); return_t ret = send(); @@ -54,7 +55,7 @@ namespace hpx::parcelset::policies::lci { return_t ret; const int retry_max_spin = 32; if (!config_t::enable_lci_backlog_queue || - HPX_UNLIKELY(pp_->is_sending_early_parcel)) + HPX_UNLIKELY(!pp_->is_initialized)) { // If we are sending early parcels, we should not expect the // thread make progress on the backlog queue