Skip to content

Commit 589bb60

Browse files
JDuffeyBQimikejacksongithub-actions[bot]
authored and
nyoungbq
committed
ENH: Improved threadsafe messaging (BlueQuartzSoftware#1340)
* Updated vcpkg with spdlog Signed-off-by: Jared Duffey <[email protected]> * Added spdlog to cmake Signed-off-by: Jared Duffey <[email protected]> * Changed IFilter::MessageHandler to move string Signed-off-by: Jared Duffey <[email protected]> * Added MessageHelper Signed-off-by: Jared Duffey <[email protected]> * Updated filters Signed-off-by: Jared Duffey <[email protected]> * Fixed formatting Signed-off-by: Jared Duffey <[email protected]> * Fixed missing override Signed-off-by: Jared Duffey <[email protected]> * Fixed incomplete type error on gcc/clang Signed-off-by: Jared Duffey <[email protected]> * Fixed incorrect parameter Signed-off-by: Jared Duffey <[email protected]> * Update src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayHistogramByFeature.cpp Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Added doc comments Signed-off-by: Jared Duffey <[email protected]> * Added ThrottleSink to further reduce message output Signed-off-by: Jared Duffey <[email protected]> * Switched percent calculation to return float32 * Fixed progress message template doc comment Signed-off-by: Jared Duffey <[email protected]> * Let thread pool persist for multiple filters Signed-off-by: Jared Duffey <[email protected]> * Fixed formatting Signed-off-by: Jared Duffey <[email protected]> * Fixed shadowing on gcc Signed-off-by: Jared Duffey <[email protected]> * Fixed thread pool race condition * Now use thread pool specifically for filter messaging instead of spdlog's global pool Signed-off-by: Jared Duffey <[email protected]> --------- Signed-off-by: Jared Duffey <[email protected]> Co-authored-by: Michael Jackson <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent c9b4015 commit 589bb60

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+966
-690
lines changed

‎CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ find_package(span-lite CONFIG REQUIRED)
188188
find_package(Eigen3 REQUIRED)
189189
find_package(boost_mp11 CONFIG REQUIRED)
190190
find_package(nod CONFIG REQUIRED)
191+
find_package(spdlog CONFIG REQUIRED)
191192

192193
# -----------------------------------------------------------------------
193194
# Find HDF5 and get the path to the DLL libraries and put that into a
@@ -265,6 +266,7 @@ target_link_libraries(simplnx
265266
HDF5::HDF5
266267
Boost::mp11
267268
nod::nod
269+
spdlog::spdlog
268270
)
269271

270272
if(UNIX)
@@ -542,6 +544,7 @@ set(SIMPLNX_HDRS
542544
${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp
543545
${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.hpp
544546
${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
547+
${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.hpp
545548
${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp
546549
${SIMPLNX_SOURCE_DIR}/Utilities/StringInterpretationUtilities.hpp
547550
${SIMPLNX_SOURCE_DIR}/Utilities/IntersectionUtilities.hpp
@@ -745,6 +748,7 @@ set(SIMPLNX_SRCS
745748
${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.cpp
746749
${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.cpp
747750
${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.cpp
751+
${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.cpp
748752
${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.cpp
749753
${SIMPLNX_SOURCE_DIR}/Utilities/ParallelDataAlgorithm.cpp
750754
${SIMPLNX_SOURCE_DIR}/Utilities/ParallelData2DAlgorithm.cpp

‎src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/AlignSectionsMisorientation.cpp

Lines changed: 3 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -72,16 +72,14 @@ Result<> AlignSectionsMisorientation::findShifts(std::vector<int64_t>& xShifts,
7272

7373
std::vector<LaueOps::Pointer> orientationOps = LaueOps::GetAllOrientationOps();
7474

75-
int32_t progInt = 0;
76-
7775
// Allocate a 2D Array which will be reused from slice to slice
7876
std::vector<bool> misorients(dims[0] * dims[1], false);
7977

8078
const auto halfDim0 = static_cast<int64_t>(dims[0] * 0.5f);
8179
const auto halfDim1 = static_cast<int64_t>(dims[1] * 0.5f);
8280

8381
double deg2Rad = (nx::core::numbers::pi / 180.0);
84-
auto start = std::chrono::steady_clock::now();
82+
ThrottledMessenger throttledMessenger = getMessageHelper().createThrottledMessenger();
8583
if(m_InputValues->StoreAlignmentShifts)
8684
{
8785
auto& slicesStore = m_DataStructure.getDataAs<UInt32Array>(m_InputValues->SlicesArrayPath)->getDataStoreRef();
@@ -94,15 +92,7 @@ Result<> AlignSectionsMisorientation::findShifts(std::vector<int64_t>& xShifts,
9492
{
9593
return {};
9694
}
97-
progInt = static_cast<float>(iter) / static_cast<float>(dims[2]) * 100.0f;
98-
auto now = std::chrono::steady_clock::now();
99-
// Only send updates every 1 second
100-
if(std::chrono::duration_cast<std::chrono::milliseconds>(now - start).count() > 1000)
101-
{
102-
std::string message = fmt::format("Determining Shifts || {}% Complete", progInt);
103-
m_MessageHandler(nx::core::IFilter::ProgressMessage{nx::core::IFilter::Message::Type::Info, message, progInt});
104-
start = std::chrono::steady_clock::now();
105-
}
95+
throttledMessenger.sendThrottledMessage([&]() { return fmt::format("Determining Shifts || {:.2f}% Complete", CalculatePercentComplete(iter, dims[2])); });
10696
if(getCancel())
10797
{
10898
return {};
@@ -210,15 +200,7 @@ Result<> AlignSectionsMisorientation::findShifts(std::vector<int64_t>& xShifts,
210200
// Loop over the Z Direction
211201
for(int64_t iter = 1; iter < dims[2]; iter++)
212202
{
213-
progInt = static_cast<float>(iter) / static_cast<float>(dims[2]) * 100.0f;
214-
auto now = std::chrono::steady_clock::now();
215-
// Only send updates every 1 second
216-
if(std::chrono::duration_cast<std::chrono::milliseconds>(now - start).count() > 1000)
217-
{
218-
std::string message = fmt::format("Determining Shifts || {}% Complete", progInt);
219-
m_MessageHandler(nx::core::IFilter::ProgressMessage{nx::core::IFilter::Message::Type::Info, message, progInt});
220-
start = std::chrono::steady_clock::now();
221-
}
203+
throttledMessenger.sendThrottledMessage([&]() { return fmt::format("Determining Shifts || {:.2f}% Complete", CalculatePercentComplete(iter, dims[2])); });
222204
if(getCancel())
223205
{
224206
return {};

‎src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/BadDataNeighborOrientationCheck.cpp

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "simplnx/DataStructure/DataGroup.hpp"
66
#include "simplnx/DataStructure/Geometry/ImageGeom.hpp"
77
#include "simplnx/Utilities/MaskCompareUtilities.hpp"
8+
#include "simplnx/Utilities/MessageHelper.hpp"
89

910
#include "EbsdLib/Core/Orientation.hpp"
1011
#include "EbsdLib/LaueOps/LaueOps.h"
@@ -80,18 +81,11 @@ Result<> BadDataNeighborOrientationCheck::operator()()
8081

8182
std::vector<int32_t> neighborCount(totalPoints, 0);
8283

83-
int64_t progressInt = 0;
84-
auto start = std::chrono::steady_clock::now();
84+
MessageHelper messageHelper(m_MessageHandler);
85+
ThrottledMessenger throttledMessenger = messageHelper.createThrottledMessenger();
8586
for(size_t i = 0; i < totalPoints; i++)
8687
{
87-
auto now = std::chrono::steady_clock::now();
88-
if(std::chrono::duration_cast<std::chrono::milliseconds>(now - start).count() > 1000)
89-
{
90-
progressInt = static_cast<int64_t>((static_cast<float>(i) / totalPoints) * 100.0f);
91-
std::string ss = fmt::format("Processing Data '{}'% completed", progressInt);
92-
m_MessageHandler({IFilter::Message::Type::Info, ss});
93-
start = std::chrono::steady_clock::now();
94-
}
88+
throttledMessenger.sendThrottledMessage([&]() { return fmt::format("Processing Data {:.2f}% completed", CalculatePercentComplete(i, totalPoints)); });
9589

9690
if(!maskCompare->isTrue(i))
9791
{
@@ -157,19 +151,12 @@ Result<> BadDataNeighborOrientationCheck::operator()()
157151
while(counter > 0)
158152
{
159153
counter = 0;
160-
progressInt = 0;
161-
start = std::chrono::steady_clock::now();
162154
for(size_t i = 0; i < totalPoints; i++)
163155
{
164-
auto now = std::chrono::steady_clock::now();
165-
if(std::chrono::duration_cast<std::chrono::milliseconds>(now - start).count() > 1000)
166-
{
167-
progressInt = static_cast<int64_t>((static_cast<float>(i) / totalPoints) * 100.0f);
168-
std::string ss =
169-
fmt::format("Level '{}' of '{}' || Processing Data ('{}') '{}'% completed", (startLevel - currentLevel) + 1, startLevel - m_InputValues->NumberOfNeighbors, loopNumber, progressInt);
170-
m_MessageHandler({IFilter::Message::Type::Info, ss});
171-
start = std::chrono::steady_clock::now();
172-
}
156+
throttledMessenger.sendThrottledMessage([&]() {
157+
return fmt::format("Level '{}' of '{}' || Processing Data ('{}') {:.2f}% completed", (startLevel - currentLevel) + 1, startLevel - m_InputValues->NumberOfNeighbors, loopNumber,
158+
CalculatePercentComplete(i, totalPoints));
159+
});
173160

174161
if(neighborCount[i] >= currentLevel && !maskCompare->isTrue(i))
175162
{

‎src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeGBCD.cpp

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "simplnx/DataStructure/DataArray.hpp"
55
#include "simplnx/DataStructure/DataGroup.hpp"
66
#include "simplnx/Utilities/Math/MatrixMath.hpp"
7+
#include "simplnx/Utilities/MessageHelper.hpp"
78
#include "simplnx/Utilities/ParallelDataAlgorithm.hpp"
89
#include "simplnx/Utilities/TimeUtilities.hpp"
910

@@ -389,11 +390,13 @@ Result<> ComputeGBCD::operator()()
389390
SizeGBCD sizeGbcd(triangleChunkSize, k_NumMisoReps, m_InputValues->GBCDRes);
390391
int32 totalGBCDBins = sizeGbcd.m_GbcdSizes[0] * sizeGbcd.m_GbcdSizes[1] * sizeGbcd.m_GbcdSizes[2] * sizeGbcd.m_GbcdSizes[3] * sizeGbcd.m_GbcdSizes[4] * 2;
391392

393+
MessageHelper messageHelper(m_MessageHandler);
394+
392395
// create an array to hold the total face area for each phase and initialize the array to 0.0
393396
std::vector<double> totalFaceArea(totalPhases, 0.0);
394-
std::string ss = fmt::format("1/2 Starting GBCD Calculation and Summation Phase");
395-
m_MessageHandler({IFilter::Message::Type::Info, ss});
396-
auto startMillis = std::chrono::steady_clock::now();
397+
auto startTime = std::chrono::steady_clock::now();
398+
messageHelper.sendMessage("1/2 Starting GBCD Calculation and Summation Phase");
399+
ThrottledMessenger throttledMessenger = messageHelper.createThrottledMessenger();
397400

398401
for(usize i = 0; i < totalFaces; i = i + triangleChunkSize)
399402
{
@@ -448,19 +451,17 @@ Result<> ComputeGBCD::operator()()
448451
}
449452
}
450453

451-
auto currentMillis = std::chrono::steady_clock::now();
452-
if(std::chrono::duration_cast<std::chrono::milliseconds>(currentMillis - startMillis).count() > 1000)
453-
{
454+
throttledMessenger.sendThrottledMessage([&]() {
455+
auto currentTime = throttledMessenger.getLastTime();
454456
const usize k_LastTriangleIndex = i + triangleChunkSize;
455-
float32 currentRate = static_cast<float32>(triangleChunkSize) / static_cast<float32>(std::chrono::duration_cast<std::chrono::milliseconds>(currentMillis - startMillis).count());
457+
float32 currentRate = static_cast<float32>(triangleChunkSize) / static_cast<float32>(std::chrono::duration_cast<std::chrono::milliseconds>(currentTime - startTime).count());
456458
uint64 estimatedTime = static_cast<uint64>(totalFaces - k_LastTriangleIndex) / currentRate;
457-
ss = fmt::format("Calculating GBCD || Triangles {}/{} Completed || Est. Time Remain: {}", k_LastTriangleIndex, totalFaces, ConvertMillisToHrsMinSecs(estimatedTime));
458-
startMillis = std::chrono::steady_clock::now();
459-
m_MessageHandler({IFilter::Message::Type::Info, ss});
460-
}
459+
startTime = currentTime;
460+
return fmt::format("Calculating GBCD || Triangles {}/{} Completed || Est. Time Remain: {}", k_LastTriangleIndex, totalFaces, ConvertMillisToHrsMinSecs(estimatedTime));
461+
});
461462
}
462463

463-
m_MessageHandler({IFilter::Message::Type::Info, "2/2 Starting GBCD Normalization Phase"});
464+
messageHelper.sendMessage("2/2 Starting GBCD Normalization Phase");
464465

465466
for(int32 i = 0; i < totalPhases; i++)
466467
{

‎src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeKernelAvgMisorientations.cpp

Lines changed: 15 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "simplnx/DataStructure/DataArray.hpp"
55
#include "simplnx/DataStructure/DataGroup.hpp"
66
#include "simplnx/DataStructure/Geometry/ImageGeom.hpp"
7+
#include "simplnx/Utilities/MessageHelper.hpp"
78
#include "simplnx/Utilities/ParallelData3DAlgorithm.hpp"
89

910
#include "EbsdLib/LaueOps/LaueOps.h"
@@ -17,9 +18,9 @@ namespace
1718
class FindKernelAvgMisorientationsImpl
1819
{
1920
public:
20-
FindKernelAvgMisorientationsImpl(ComputeKernelAvgMisorientations* filter, DataStructure& dataStructure, const ComputeKernelAvgMisorientationsInputValues* inputValues,
21+
FindKernelAvgMisorientationsImpl(ProgressMessageHelper& progressMessenger, DataStructure& dataStructure, const ComputeKernelAvgMisorientationsInputValues* inputValues,
2122
const std::atomic_bool& shouldCancel)
22-
: m_Filter(filter)
23+
: m_ProgressMessageHelper(progressMessenger)
2324
, m_DataStructure(dataStructure)
2425
, m_InputValues(inputValues)
2526
, m_ShouldCancel(shouldCancel)
@@ -54,7 +55,8 @@ class FindKernelAvgMisorientationsImpl
5455
// messenger values
5556
usize counter = 0;
5657
usize increment = (zEnd - zStart) / 100;
57-
auto start = std::chrono::steady_clock::now();
58+
59+
ProgressMessenger progressMessenger = m_ProgressMessageHelper.createProgressMessenger();
5860

5961
auto xPoints = static_cast<int64_t>(udims[0]);
6062
auto yPoints = static_cast<int64_t>(udims[1]);
@@ -68,13 +70,8 @@ class FindKernelAvgMisorientationsImpl
6870

6971
if(counter > increment)
7072
{
71-
auto now = std::chrono::steady_clock::now();
72-
if(std::chrono::duration_cast<std::chrono::milliseconds>(now - start).count() > 1000)
73-
{
74-
m_Filter->sendThreadSafeProgressMessage(counter);
75-
counter = 0;
76-
start = std::chrono::steady_clock::now();
77-
}
73+
progressMessenger.sendProgressMessage(counter);
74+
counter = 0;
7875
}
7976

8077
for(size_t row = yStart; row < yEnd; row++)
@@ -145,7 +142,7 @@ class FindKernelAvgMisorientationsImpl
145142
}
146143
}
147144
}
148-
m_Filter->sendThreadSafeProgressMessage(counter);
145+
progressMessenger.sendProgressMessage(counter);
149146
}
150147

151148
void operator()(const Range3D& range) const
@@ -154,7 +151,7 @@ class FindKernelAvgMisorientationsImpl
154151
}
155152

156153
private:
157-
ComputeKernelAvgMisorientations* m_Filter = nullptr;
154+
ProgressMessageHelper& m_ProgressMessageHelper;
158155
DataStructure& m_DataStructure;
159156
const ComputeKernelAvgMisorientationsInputValues* m_InputValues = nullptr;
160157
const std::atomic_bool& m_ShouldCancel;
@@ -175,40 +172,17 @@ ComputeKernelAvgMisorientations::ComputeKernelAvgMisorientations(DataStructure&
175172
// -----------------------------------------------------------------------------
176173
ComputeKernelAvgMisorientations::~ComputeKernelAvgMisorientations() noexcept = default;
177174

178-
// -----------------------------------------------------------------------------
179-
const std::atomic_bool& ComputeKernelAvgMisorientations::getCancel()
180-
{
181-
return m_ShouldCancel;
182-
}
183-
184-
// -----------------------------------------------------------------------------
185-
void ComputeKernelAvgMisorientations::sendThreadSafeProgressMessage(usize counter)
186-
{
187-
std::lock_guard<std::mutex> guard(m_ProgressMessage_Mutex);
188-
189-
m_ProgressCounter += counter;
190-
auto now = std::chrono::steady_clock::now();
191-
if(std::chrono::duration_cast<std::chrono::milliseconds>(now - m_InitialPoint).count() < 1000)
192-
{
193-
return;
194-
}
195-
196-
auto progressInt = static_cast<usize>((static_cast<float32>(m_ProgressCounter) / static_cast<float32>(m_TotalElements)) * 100.0f);
197-
std::string ss = fmt::format("Finding Kernel Average Misorientations || {}%", progressInt);
198-
m_MessageHandler(IFilter::Message::Type::Info, ss);
199-
200-
m_LastProgressInt = progressInt;
201-
m_InitialPoint = std::chrono::steady_clock::now();
202-
}
203-
204175
// -----------------------------------------------------------------------------
205176
Result<> ComputeKernelAvgMisorientations::operator()()
206177
{
207178
auto* gridGeom = m_DataStructure.getDataAs<ImageGeom>(m_InputValues->InputImageGeometry);
208179
SizeVec3 udims = gridGeom->getDimensions();
209180

210-
// set up threadsafe messenger
211-
m_TotalElements = udims[2] * udims[1] * udims[0];
181+
MessageHelper messageHelper(m_MessageHandler);
182+
ProgressMessageHelper progressMessageHelper = messageHelper.createProgressMessageHelper();
183+
184+
progressMessageHelper.setMaxProgresss(udims[2] * udims[1] * udims[0]);
185+
progressMessageHelper.setProgressMessageTemplate("Finding Kernel Average Misorientations || {:.2f}%");
212186

213187
typename IParallelAlgorithm::AlgorithmArrays algArrays;
214188
algArrays.push_back(m_DataStructure.getDataAs<IDataArray>(m_InputValues->CellPhasesArrayPath));
@@ -220,7 +194,7 @@ Result<> ComputeKernelAvgMisorientations::operator()()
220194
ParallelData3DAlgorithm parallelAlgorithm;
221195
parallelAlgorithm.setRange(Range3D(0, udims[0], 0, udims[1], 0, udims[2]));
222196
parallelAlgorithm.requireArraysInMemory(algArrays);
223-
parallelAlgorithm.execute(FindKernelAvgMisorientationsImpl(this, m_DataStructure, m_InputValues, m_ShouldCancel));
197+
parallelAlgorithm.execute(FindKernelAvgMisorientationsImpl(progressMessageHelper, m_DataStructure, m_InputValues, m_ShouldCancel));
224198

225199
return {};
226200
}

‎src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeKernelAvgMisorientations.hpp

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,22 +40,11 @@ class ORIENTATIONANALYSIS_EXPORT ComputeKernelAvgMisorientations
4040

4141
Result<> operator()();
4242

43-
const std::atomic_bool& getCancel();
44-
45-
void sendThreadSafeProgressMessage(usize counter);
46-
4743
private:
4844
DataStructure& m_DataStructure;
4945
const ComputeKernelAvgMisorientationsInputValues* m_InputValues = nullptr;
5046
const std::atomic_bool& m_ShouldCancel;
5147
const IFilter::MessageHandler& m_MessageHandler;
52-
53-
// Thread safe Progress Message
54-
std::chrono::steady_clock::time_point m_InitialPoint = std::chrono::steady_clock::now();
55-
mutable std::mutex m_ProgressMessage_Mutex;
56-
size_t m_TotalElements = 0;
57-
size_t m_ProgressCounter = 0;
58-
size_t m_LastProgressInt = 0;
5948
};
6049

6150
} // namespace nx::core

0 commit comments

Comments
 (0)