From dc1675766fe1fc8d75bf1c264176178fa1d2ae85 Mon Sep 17 00:00:00 2001 From: StefanoPetrilli Date: Wed, 13 Jul 2022 16:19:38 -0500 Subject: [PATCH] processMessages.cpp further refactoring --- amgprec/impl/aggregator/MatchBoxPC.h | 43 ++- ...mEdgesLinearSearchMesgBndlSmallMateCMP.cpp | 246 ++++-------------- .../impl/aggregator/computeCandidateMate.cpp | 24 +- amgprec/impl/aggregator/processMessages.cpp | 224 +++++++++++++++- 4 files changed, 304 insertions(+), 233 deletions(-) diff --git a/amgprec/impl/aggregator/MatchBoxPC.h b/amgprec/impl/aggregator/MatchBoxPC.h index 0b3dcd74..eecc57ce 100644 --- a/amgprec/impl/aggregator/MatchBoxPC.h +++ b/amgprec/impl/aggregator/MatchBoxPC.h @@ -345,17 +345,38 @@ extern "C" vector &SRequest, vector &SStatus); - void processMessages(int error_codeC, - MilanInt numProcs, - MilanInt myRank, - int ComputeTag, - int BundleTag, - MPI_Comm comm, - vector &Message, - char *error_message, - int message_length, - vector &ReceiveBuffer, - MilanLongInt *BundleSizePtr); + void processMessages( + MilanLongInt NLVer, + MilanLongInt *Mate, + MilanLongInt *candidateMate, + map &Ghost2LocalMap, + vector &GMate, + vector &Counter, + MilanLongInt StartIndex, + MilanLongInt EndIndex, + MilanLongInt *myCardPtr, + MilanLongInt *msgIndPtr, + MilanLongInt *msgActualPtr, + MilanReal *edgeLocWeight, + MilanLongInt *verDistance, + MilanLongInt *verLocPtr, + MilanLongInt k, + MilanLongInt *verLocInd, + int error_codeC, + MilanInt numProcs, + MilanInt myRank, + int ComputeTag, + int BundleTag, + MPI_Comm comm, + vector &Message, + char *error_message, + int message_length, + vector &ReceiveBuffer, + MilanLongInt u, + MilanLongInt v, + MilanLongInt message_type, + MilanLongInt *SPtr, + staticQueue &U); void extractUChunk( vector &UChunkBeingProcessed, diff --git a/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp b/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp index e45ee792..c5a81c4d 100644 --- a/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp +++ b/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp @@ -598,7 +598,23 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( /////////////////////////// PROCESS MESSAGES ////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////// - processMessages(error_codeC, + processMessages(NLVer, + Mate, + candidateMate, + Ghost2LocalMap, + GMate, + Counter, + StartIndex, + EndIndex, + &myCard, + &msgInd, + &msgActual, + edgeLocWeight, + verDistance, + verLocPtr, + k, + verLocInd, + error_codeC, numProcs, myRank, ComputeTag, @@ -608,205 +624,41 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( error_message, message_length, ReceiveBuffer, - &bundleSize); - - bundleCounter = 0; - while (bundleCounter < bundleSize) - { - u = ReceiveBuffer[bundleCounter]; // GHOST - bundleCounter++; - v = ReceiveBuffer[bundleCounter]; // LOCAL - bundleCounter++; - message_type = ReceiveBuffer[bundleCounter]; // TYPE - bundleCounter++; - - // CASE I: REQUEST - if (message_type == REQUEST) - { -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Message type is REQUEST" << endl; - fflush(stdout); -#endif -#ifdef DEBUG_GHOST_ - if ((v < 0) || (v < StartIndex) || ((v - StartIndex) > NLVer)) - { - cout << "\n(" << myRank << ") case 1 Bad address " << v << " " << StartIndex << " " << v - StartIndex << " " << NLVer << endl; - fflush(stdout); - } + u, + v, + message_type, + &S, + U); -#endif - if (Mate[v - StartIndex] == -1) - { // Process only if not already matched (v is local) - candidateMate[NLVer + Ghost2LocalMap[u]] = v; // Set CandidateMate for the ghost - if (candidateMate[v - StartIndex] == u) - { - GMate[Ghost2LocalMap[u]] = v; // u is ghost - Mate[v - StartIndex] = u; // v is local - // Q.push_back(u); - U.push_back(v); - U.push_back(u); - myCard++; -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")MATCH: (" << v << "," << u << ") " << endl; - fflush(stdout); -#endif - PROCESS_CROSS_EDGE(Counter, Ghost2LocalMap, u, &S); - } // End of if ( candidateMate[v-StartIndex] == u )e - } // End of if ( Mate[v] == -1 ) - } // End of REQUEST - else - { // CASE II: SUCCESS - if (message_type == SUCCESS) - { -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Message type is SUCCESS" << endl; - fflush(stdout); -#endif - GMate[Ghost2LocalMap[u]] = EndIndex + 1; // Set a Dummy Mate to make sure that we do not (u is a ghost) - PROCESS_CROSS_EDGE(Counter, Ghost2LocalMap, u, &S); -#ifdef DEBUG_GHOST_ - if ((v < 0) || (v < StartIndex) || ((v - StartIndex) > NLVer)) - { - cout << "\n(" << myRank << ") case 2 Bad address " << v << " " << StartIndex << " " << v - StartIndex << " " << NLVer << endl; - fflush(stdout); - } -#endif - if (Mate[v - StartIndex] == -1) - { // Process only if not already matched ( v is local) - if (candidateMate[v - StartIndex] == u) - { - // Start: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - w = computeCandidateMate(verLocPtr[v - StartIndex], verLocPtr[v - StartIndex + 1], edgeLocWeight, k, verLocInd, StartIndex, EndIndex, GMate, Mate, Ghost2LocalMap); - candidateMate[v - StartIndex] = w; -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")" << v << " Points to: " << w << endl; - fflush(stdout); -#endif - // If found a dominating edge: - if (w >= 0) - { - if ((w < StartIndex) || (w > EndIndex)) - { // w is a ghost - // Build the Message Packet: - Message[0] = v; // LOCAL - Message[1] = w; // GHOST - Message[2] = REQUEST; // TYPE - // Send a Request (Asynchronous) -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Sending a request message: "; - cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs) << endl; - fflush(stdout); -#endif - ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); - assert(ghostOwner != -1); - assert(ghostOwner != myRank); - MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); - msgInd++; - msgActual++; - if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) - { - Mate[v - StartIndex] = w; // v is local - GMate[Ghost2LocalMap[w]] = v; // w is ghost - U.push_back(v); - U.push_back(w); - myCard++; -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") " << endl; - fflush(stdout); -#endif - // Decrement the counter: - PROCESS_CROSS_EDGE(Counter, Ghost2LocalMap, w, &S); - } // End of if CandidateMate[w] = v - } // End of if a Ghost Vertex - else - { // w is a local vertex - if (candidateMate[w - StartIndex] == v) - { - Mate[v - StartIndex] = w; // v is local - Mate[w - StartIndex] = v; // w is local - // Q.push_back(u); - U.push_back(v); - U.push_back(w); - myCard++; -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") " << endl; - fflush(stdout); -#endif - } // End of if(CandidateMate(w) = v - } // End of Else - } // End of if(w >=0) - else - { // No dominant edge found - adj11 = verLocPtr[v - StartIndex]; - adj12 = verLocPtr[v - StartIndex + 1]; - for (k1 = adj11; k1 < adj12; k1++) - { - w = verLocInd[k1]; - if ((w < StartIndex) || (w > EndIndex)) - { // A ghost - // Build the Message Packet: - Message[0] = v; // LOCAL - Message[1] = w; // GHOST - Message[2] = FAILURE; // TYPE - // Send a Request (Asynchronous) + ///////////////////////// END OF PROCESS MESSAGES ///////////////////////////////// #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Sending a failure message: "; - cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs) << endl; - fflush(stdout); -#endif - ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); - assert(ghostOwner != -1); - assert(ghostOwner != myRank); - MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); - msgInd++; - msgActual++; - } // End of if(GHOST) - } // End of for loop - } // End of Else: w == -1 - // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - } // End of if ( candidateMate[v-StartIndex] == u ) - } // End of if ( Mate[v] == -1 ) - } // End of if ( message_type == SUCCESS ) - else - { // CASE III: FAILURE -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Message type is FAILURE" << endl; - fflush(stdout); -#endif - GMate[Ghost2LocalMap[u]] = EndIndex + 1; // Set a Dummy Mate to make sure that we do not (u is a ghost) - PROCESS_CROSS_EDGE(Counter, Ghost2LocalMap, u, &S); - } // End of else: CASE III - } // End of else: CASE I - } // End of if (!MsgQ.empty()) - ///////////////////////// END OF PROCESS MESSAGES ///////////////////////////////// -#ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Finished Message processing phase: S= " << S; - fflush(stdout); - cout << "\n(" << myRank << ")** SENT : ACTUAL= " << msgActual; - fflush(stdout); - cout << "\n(" << myRank << ")** SENT : INDIVIDUAL= " << msgInd << endl; - fflush(stdout); + cout << "\n(" << myRank << ")Finished Message processing phase: S= " << S; + fflush(stdout); + cout << "\n(" << myRank << ")** SENT : ACTUAL= " << msgActual; + fflush(stdout); + cout << "\n(" << myRank << ")** SENT : INDIVIDUAL= " << msgInd << endl; + fflush(stdout); #endif - } // End of while (true) - - clean(NLVer, - myRank, - MessageIndex, - SRequest, - SStatus, - BufferSize, - Buffer, - msgActual, - msgActualSent, - msgInd, - msgIndSent, - NumMessagesBundled, - msgPercent, - MateLock); - - finishTime = MPI_Wtime(); - *ph2_time = finishTime - startTime; // Time taken for Phase-2 - *ph2_card = myCard; // Cardinality at the end of Phase-2 +} // End of while (true) + +clean(NLVer, + myRank, + MessageIndex, + SRequest, + SStatus, + BufferSize, + Buffer, + msgActual, + msgActualSent, + msgInd, + msgIndSent, + NumMessagesBundled, + msgPercent, + MateLock); + +finishTime = MPI_Wtime(); +*ph2_time = finishTime - startTime; // Time taken for Phase-2 +*ph2_card = myCard; // Cardinality at the end of Phase-2 } // End of algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMate #endif diff --git a/amgprec/impl/aggregator/computeCandidateMate.cpp b/amgprec/impl/aggregator/computeCandidateMate.cpp index 36eaa727..f79fc613 100644 --- a/amgprec/impl/aggregator/computeCandidateMate.cpp +++ b/amgprec/impl/aggregator/computeCandidateMate.cpp @@ -10,9 +10,9 @@ * @return */ MilanLongInt firstComputeCandidateMate(MilanLongInt adj1, - MilanLongInt adj2, - MilanLongInt *verLocInd, - MilanReal *edgeLocWeight) + MilanLongInt adj2, + MilanLongInt *verLocInd, + MilanReal *edgeLocWeight) { MilanInt w = -1; MilanReal heaviestEdgeWt = MilanRealMin; // Assign the smallest Value possible first LDBL_MIN @@ -46,15 +46,15 @@ MilanLongInt firstComputeCandidateMate(MilanLongInt adj1, * @return */ MilanLongInt computeCandidateMate(MilanLongInt adj1, - MilanLongInt adj2, - MilanReal *edgeLocWeight, - MilanLongInt k, - MilanLongInt *verLocInd, - MilanLongInt StartIndex, - MilanLongInt EndIndex, - vector &GMate, - MilanLongInt *Mate, - map &Ghost2LocalMap) + MilanLongInt adj2, + MilanReal *edgeLocWeight, + MilanLongInt k, + MilanLongInt *verLocInd, + MilanLongInt StartIndex, + MilanLongInt EndIndex, + vector &GMate, + MilanLongInt *Mate, + map &Ghost2LocalMap) { // Start: PARALLEL_COMPUTE_CANDIDATE_MATE_B(v) diff --git a/amgprec/impl/aggregator/processMessages.cpp b/amgprec/impl/aggregator/processMessages.cpp index cb43cdb8..7d44b281 100644 --- a/amgprec/impl/aggregator/processMessages.cpp +++ b/amgprec/impl/aggregator/processMessages.cpp @@ -1,21 +1,45 @@ #include "MatchBoxPC.h" -void processMessages(int error_codeC, - MilanInt numProcs, - MilanInt myRank, - int ComputeTag, - int BundleTag, - MPI_Comm comm, - vector &Message, - char *error_message, - int message_length, - vector &ReceiveBuffer, - MilanLongInt *BundleSizePtr) +void processMessages( + MilanLongInt NLVer, + MilanLongInt *Mate, + MilanLongInt *candidateMate, + map &Ghost2LocalMap, + vector &GMate, + vector &Counter, + MilanLongInt StartIndex, + MilanLongInt EndIndex, + MilanLongInt *myCardPtr, + MilanLongInt *msgIndPtr, + MilanLongInt *msgActualPtr, + MilanReal *edgeLocWeight, + MilanLongInt *verDistance, + MilanLongInt *verLocPtr, + MilanLongInt k, + MilanLongInt *verLocInd, + int error_codeC, + MilanInt numProcs, + MilanInt myRank, + int ComputeTag, + int BundleTag, + MPI_Comm comm, + vector &Message, + char *error_message, + int message_length, + vector &ReceiveBuffer, + MilanLongInt u, + MilanLongInt v, + MilanLongInt message_type, + MilanLongInt *SPtr, + staticQueue &U) { MilanInt Sender; MPI_Status computeStatus; - MilanLongInt bundleSize = *BundleSizePtr; + MilanLongInt bundleSize, bundleCounter = 0, myCard = *myCardPtr, msgInd = *msgIndPtr, msgActual = *msgActualPtr, w; + MilanLongInt S = *SPtr; // TODO refactor this + MilanLongInt adj11, adj12, k1; + MilanLongInt ghostOwner; #ifdef PRINT_DEBUG_INFO_ cout @@ -118,6 +142,180 @@ void processMessages(int error_codeC, fflush(stdout); #endif - *BundleSizePtr = bundleSize; + bundleCounter = 0; + while (bundleCounter < bundleSize) + { + u = ReceiveBuffer[bundleCounter]; // GHOST + bundleCounter++; + v = ReceiveBuffer[bundleCounter]; // LOCAL + bundleCounter++; + message_type = ReceiveBuffer[bundleCounter]; // TYPE + bundleCounter++; + + // CASE I: REQUEST + if (message_type == REQUEST) + { +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")Message type is REQUEST" << endl; + fflush(stdout); +#endif +#ifdef DEBUG_GHOST_ + if ((v < 0) || (v < StartIndex) || ((v - StartIndex) > NLVer)) + { + cout << "\n(" << myRank << ") case 1 Bad address " << v << " " << StartIndex << " " << v - StartIndex << " " << NLVer << endl; + fflush(stdout); + } + +#endif + + if (Mate[v - StartIndex] == -1) + { // Process only if not already matched (v is local) + candidateMate[NLVer + Ghost2LocalMap[u]] = v; // Set CandidateMate for the ghost + if (candidateMate[v - StartIndex] == u) + { + GMate[Ghost2LocalMap[u]] = v; // u is ghost + Mate[v - StartIndex] = u; // v is local + // Q.push_back(u); + U.push_back(v); + U.push_back(u); + myCard++; +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")MATCH: (" << v << "," << u << ") " << endl; + fflush(stdout); +#endif + PROCESS_CROSS_EDGE(Counter, Ghost2LocalMap, u, &S); + } // End of if ( candidateMate[v-StartIndex] == u )e + } // End of if ( Mate[v] == -1 ) + } // End of REQUEST + else + { // CASE II: SUCCESS + if (message_type == SUCCESS) + { +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")Message type is SUCCESS" << endl; + fflush(stdout); +#endif + GMate[Ghost2LocalMap[u]] = EndIndex + 1; // Set a Dummy Mate to make sure that we do not (u is a ghost) + PROCESS_CROSS_EDGE(Counter, Ghost2LocalMap, u, &S); +#ifdef DEBUG_GHOST_ + if ((v < 0) || (v < StartIndex) || ((v - StartIndex) > NLVer)) + { + cout << "\n(" << myRank << ") case 2 Bad address " << v << " " << StartIndex << " " << v - StartIndex << " " << NLVer << endl; + fflush(stdout); + } +#endif + if (Mate[v - StartIndex] == -1) + { // Process only if not already matched ( v is local) + if (candidateMate[v - StartIndex] == u) + { + // Start: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) + w = computeCandidateMate(verLocPtr[v - StartIndex], verLocPtr[v - StartIndex + 1], edgeLocWeight, k, verLocInd, StartIndex, EndIndex, GMate, Mate, Ghost2LocalMap); + candidateMate[v - StartIndex] = w; +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")" << v << " Points to: " << w << endl; + fflush(stdout); +#endif + // If found a dominating edge: + if (w >= 0) + { + if ((w < StartIndex) || (w > EndIndex)) + { // w is a ghost + // Build the Message Packet: + Message[0] = v; // LOCAL + Message[1] = w; // GHOST + Message[2] = REQUEST; // TYPE + // Send a Request (Asynchronous) +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")Sending a request message: "; + cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs) << endl; + fflush(stdout); +#endif + ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); + assert(ghostOwner != -1); + assert(ghostOwner != myRank); + + MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); + msgInd++; + msgActual++; + if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) + { + Mate[v - StartIndex] = w; // v is local + GMate[Ghost2LocalMap[w]] = v; // w is ghost + U.push_back(v); + U.push_back(w); + myCard++; +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") " << endl; + fflush(stdout); +#endif + // Decrement the counter: + PROCESS_CROSS_EDGE(Counter, Ghost2LocalMap, w, &S); + } // End of if CandidateMate[w] = v + } // End of if a Ghost Vertex + else + { // w is a local vertex + if (candidateMate[w - StartIndex] == v) + { + Mate[v - StartIndex] = w; // v is local + Mate[w - StartIndex] = v; // w is local + // Q.push_back(u); + U.push_back(v); + U.push_back(w); + myCard++; +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") " << endl; + fflush(stdout); +#endif + } // End of if(CandidateMate(w) = v + } // End of Else + } // End of if(w >=0) + else + { // No dominant edge found + adj11 = verLocPtr[v - StartIndex]; + adj12 = verLocPtr[v - StartIndex + 1]; + for (k1 = adj11; k1 < adj12; k1++) + { + w = verLocInd[k1]; + if ((w < StartIndex) || (w > EndIndex)) + { // A ghost + // Build the Message Packet: + Message[0] = v; // LOCAL + Message[1] = w; // GHOST + Message[2] = FAILURE; // TYPE + // Send a Request (Asynchronous) +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")Sending a failure message: "; + cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs) << endl; + fflush(stdout); +#endif + ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); + assert(ghostOwner != -1); + assert(ghostOwner != myRank); + MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); + msgInd++; + msgActual++; + } // End of if(GHOST) + } // End of for loop + } // End of Else: w == -1 + // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) + } // End of if ( candidateMate[v-StartIndex] == u ) + } // End of if ( Mate[v] == -1 ) + } // End of if ( message_type == SUCCESS ) + else + { // CASE III: FAILURE +#ifdef PRINT_DEBUG_INFO_ + cout << "\n(" << myRank << ")Message type is FAILURE" << endl; + fflush(stdout); +#endif + GMate[Ghost2LocalMap[u]] = EndIndex + 1; // Set a Dummy Mate to make sure that we do not (u is a ghost) process this anymore + PROCESS_CROSS_EDGE(Counter, Ghost2LocalMap, u, &S); // Decrease the counter + } // End of else: CASE III + } // End of else: CASE I + } + + *myCardPtr = myCard; + *SPtr = S; + *msgIndPtr = msgInd; + *msgActualPtr = msgActual; return; } \ No newline at end of file