From 17698c2725c5a85fcf7e753bf9ba54ef133421fe Mon Sep 17 00:00:00 2001 From: sfilippone Date: Thu, 30 May 2024 14:43:26 +0200 Subject: [PATCH] Changes to OpenMP MathcBox version needs -DOMP --- amgprec/impl/aggregator/MatchBoxPC.cpp | 2 +- ...mEdgesLinearSearchMesgBndlSmallMateCMP.cpp | 32 +++++--- .../impl/aggregator/processExposedVertex.cpp | 74 ++++++++++--------- amgprec/impl/aggregator/queueTransfer.cpp | 6 +- 4 files changed, 62 insertions(+), 52 deletions(-) diff --git a/amgprec/impl/aggregator/MatchBoxPC.cpp b/amgprec/impl/aggregator/MatchBoxPC.cpp index eb56adce..90b448dc 100644 --- a/amgprec/impl/aggregator/MatchBoxPC.cpp +++ b/amgprec/impl/aggregator/MatchBoxPC.cpp @@ -42,6 +42,7 @@ #include #if !defined(SERIAL_MPI) #include +#endif #include "MatchBoxPC.h" #ifdef __cplusplus @@ -126,4 +127,3 @@ void sMatchBoxPC(MilanLongInt NLVer, MilanLongInt NLEdge, #ifdef __cplusplus } #endif -#endif diff --git a/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp b/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp index 49b366a6..b758dc69 100644 --- a/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp +++ b/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" - +#ifdef OMP // *********************************************************************** // // MatchboxP: A C++ library for approximate weighted matching @@ -126,8 +126,10 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( fflush(stdout); #endif - MilanLongInt StartIndex = verDistance[myRank]; // The starting vertex owned by the current rank - MilanLongInt EndIndex = verDistance[myRank + 1] - 1; // The ending vertex owned by the current rank + // The starting vertex owned by the current rank + MilanLongInt StartIndex = verDistance[myRank]; + // The ending vertex owned by the current rank + MilanLongInt EndIndex = verDistance[myRank + 1] - 1; MPI_Status computeStatus; @@ -145,7 +147,8 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( // only one message will be sent in the initialization phase - // one of: REQUEST/FAILURE/SUCCESS vector QLocalVtx, QGhostVtx, QMsgType; - vector QOwner; // Changed by Fabio to be an integer, addresses needs to be integers! + // Changed by Fabio to be an integer, addresses needs to be integers! + vector QOwner; MilanLongInt *PCounter = new MilanLongInt[numProcs]; for (int i = 0; i < numProcs; i++) @@ -153,7 +156,8 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( MilanLongInt NumMessagesBundled = 0; // TODO when the last computational section will be refactored this could be eliminated - MilanInt ghostOwner = 0; // Changed by Fabio to be an integer, addresses needs to be integers! + // Changed by Fabio to be an integer, addresses needs to be integers! + MilanInt ghostOwner = 0; MilanLongInt *candidateMate = nullptr; #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")NV: " << NLVer << " Edges: " << NLEdge; @@ -168,9 +172,12 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( MilanLongInt myCard = 0; // Build the Ghost Vertex Set: Vg - map Ghost2LocalMap; // Map each ghost vertex to a local vertex - vector Counter; // Store the edge count for each ghost vertex - MilanLongInt numGhostVertices = 0, numGhostEdges = 0; // Number of Ghost vertices + // Map each ghost vertex to a local vertex + map Ghost2LocalMap; + // Store the edge count for each ghost vertex + vector Counter; + // Number of Ghost vertices + MilanLongInt numGhostVertices = 0, numGhostEdges = 0; #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")About to compute Ghost Vertices..."; @@ -222,7 +229,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( cout << myRank << " Finished initialization" << endl; fflush(stdout); #endif - + startTime = MPI_Wtime(); ///////////////////////////////////////////////////////////////////////////////////////// @@ -391,7 +398,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( cout << myRank << " Finished sendBundles" << endl; fflush(stdout); #endif - + *ph1_card = myCard; // Cardinality at the end of Phase-1 startTime = MPI_Wtime(); ///////////////////////////////////////////////////////////////////////////////////////// @@ -422,7 +429,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( /////////////////////////////////////////////////////////////////////////////////// /////////////////////////// PROCESS MATCHED VERTICES ////////////////////////////// /////////////////////////////////////////////////////////////////////////////////// - + processMatchedVerticesAndSendMessages(NLVer, UChunkBeingProcessed, U, @@ -456,7 +463,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( comm, &msgActual, Message); - + ///////////////////////// END OF PROCESS MATCHED VERTICES ///////////////////////// //// BREAK IF NO MESSAGES EXPECTED ///////// @@ -552,3 +559,4 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( #endif #endif +#endif diff --git a/amgprec/impl/aggregator/processExposedVertex.cpp b/amgprec/impl/aggregator/processExposedVertex.cpp index 0336bed5..81ce23eb 100644 --- a/amgprec/impl/aggregator/processExposedVertex.cpp +++ b/amgprec/impl/aggregator/processExposedVertex.cpp @@ -1,6 +1,5 @@ #include "MatchBoxPC.h" -#if !defined(SERIAL_MPI) - +#ifdef OMP void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, MilanLongInt *candidateMate, MilanLongInt *verLocInd, @@ -30,7 +29,7 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, vector &privateQGhostVtx, vector &privateQMsgType, vector &privateQOwner) -{ +{ MilanLongInt v = -1, k = -1, w = -1, adj11 = 0, adj12 = 0, k1 = 0; MilanInt ghostOwner = 0, option, igw; @@ -80,7 +79,7 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, Ghost2LocalMap); candidateMate[v] = w; } - + if (w >= 0) { (*myCard)++; if ((w < StartIndex) || (w > EndIndex)) { // w is a ghost vertex @@ -89,60 +88,63 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, option = 1; Mate[v] = w; GMate[Ghost2LocalMap[w]] = v + StartIndex; // w is a Ghost - + } // End of if CandidateMate[w] = v } // End of if a Ghost Vertex else { // w is a local vertex - + if (candidateMate[w - StartIndex] == (v + StartIndex)) { option = 3; Mate[v] = w; // v is local Mate[w - StartIndex] = v + StartIndex; // w is local - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")MATCH: (" << v + StartIndex << "," << w << ") "; fflush(stdout); #endif - + } // End of if ( candidateMate[w-StartIndex] == (v+StartIndex) ) } // End of Else - + } // End of second if - + } // End critical processExposed - + } // End of if(w >=0) else { - // This piece of code is executed a really small amount of times - adj11 = verLocPtr[v]; - adj12 = verLocPtr[v + 1]; - for (k1 = adj11; k1 < adj12; k1++) { - w = verLocInd[k1]; - if ((w < StartIndex) || (w > EndIndex)) { // A ghost - +#pragma omp critical(adjuse) + { + // This piece of code is executed a really small number of times + adj11 = verLocPtr[v]; + adj12 = verLocPtr[v + 1]; + for (k1 = adj11; k1 < adj12; k1++) { + w = verLocInd[k1]; + if ((w < StartIndex) || (w > EndIndex)) { // A ghost + #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Sending a failure message: "; - cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); - fflush(stdout); + cout << "\n(" << myRank << ")Sending a failure message: "; + cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); + fflush(stdout); #endif - (*msgInd)++; - (*NumMessagesBundled)++; - ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); - // assert(ghostOwner != -1); - // assert(ghostOwner != myRank); - PCounter[ghostOwner]++; - - privateQLocalVtx.push_back(v + StartIndex); - privateQGhostVtx.push_back(w); - privateQMsgType.push_back(FAILURE); - privateQOwner.push_back(ghostOwner); - - } // End of if(GHOST) - } // End of for loop + (*msgInd)++; + (*NumMessagesBundled)++; + ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); + // assert(ghostOwner != -1); + // assert(ghostOwner != myRank); + PCounter[ghostOwner]++; + + privateQLocalVtx.push_back(v + StartIndex); + privateQGhostVtx.push_back(w); + privateQMsgType.push_back(FAILURE); + privateQOwner.push_back(ghostOwner); + + } // End of if(GHOST) + } // End of for loop + } } // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - + switch (option) { case -1: diff --git a/amgprec/impl/aggregator/queueTransfer.cpp b/amgprec/impl/aggregator/queueTransfer.cpp index 33c65749..64a60157 100644 --- a/amgprec/impl/aggregator/queueTransfer.cpp +++ b/amgprec/impl/aggregator/queueTransfer.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" - +#ifdef OMP void queuesTransfer(vector &U, vector &privateU, vector &QLocalVtx, @@ -17,8 +17,6 @@ void queuesTransfer(vector &U, U.insert(U.end(), privateU.begin(), privateU.end()); } - privateU.clear(); - #pragma omp critical(sendMessageTransfer) { @@ -28,9 +26,11 @@ void queuesTransfer(vector &U, QOwner.insert(QOwner.end(), privateQOwner.begin(), privateQOwner.end()); } + privateU.clear(); privateQLocalVtx.clear(); privateQGhostVtx.clear(); privateQMsgType.clear(); privateQOwner.clear(); } +#endif