From b4c60ae409b470658442ef9f74043520bc5492fb Mon Sep 17 00:00:00 2001 From: Cirdans-Home Date: Thu, 3 Oct 2024 13:26:07 +0000 Subject: [PATCH] merge with PolySmooth --- amgprec/impl/aggregator/MatchBoxPC.cpp | 7 +- amgprec/impl/aggregator/MatchBoxPC.h | 19 +---- ...mEdgesLinearSearchMesgBndlSmallMateCMP.cpp | 32 +++----- amgprec/impl/aggregator/clean.cpp | 3 +- .../impl/aggregator/computeCandidateMate.cpp | 8 +- amgprec/impl/aggregator/extractUChunk.cpp | 5 +- amgprec/impl/aggregator/findOwnerOfGhost.cpp | 3 +- amgprec/impl/aggregator/initialize.cpp | 5 +- amgprec/impl/aggregator/isAlreadyMatched.cpp | 7 +- .../parallelComputeCandidateMateB.cpp | 3 +- amgprec/impl/aggregator/processCrossEdge.cpp | 5 +- .../impl/aggregator/processExposedVertex.cpp | 74 +++++++++-------- .../aggregator/processMatchedVertices.cpp | 79 ++++++++++--------- .../processMatchedVerticesAndSendMessages.cpp | 76 +++++++++--------- amgprec/impl/aggregator/processMessages.cpp | 14 ++-- amgprec/impl/aggregator/queueTransfer.cpp | 6 +- .../impl/aggregator/sendBundledMessages.cpp | 8 +- 17 files changed, 165 insertions(+), 189 deletions(-) diff --git a/amgprec/impl/aggregator/MatchBoxPC.cpp b/amgprec/impl/aggregator/MatchBoxPC.cpp index a43fb2f5..eb56adce 100644 --- a/amgprec/impl/aggregator/MatchBoxPC.cpp +++ b/amgprec/impl/aggregator/MatchBoxPC.cpp @@ -42,7 +42,6 @@ #include #if !defined(SERIAL_MPI) #include -#endif #include "MatchBoxPC.h" #ifdef __cplusplus @@ -72,9 +71,8 @@ void dMatchBoxPC(MilanLongInt NLVer, MilanLongInt NLEdge, double tmr = MPI_Wtime(); #endif -// Rimosso per tornare al vecchio matching #define OMP -#ifdef OPENMP - //fprintf(stderr,"Warning: using buggy OpenMP matching!\n"); +#define OMP +#ifdef OMP dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(NLVer, NLEdge, verLocPtr, verLocInd, edgeLocWeight, verDistance, Mate, @@ -128,3 +126,4 @@ void sMatchBoxPC(MilanLongInt NLVer, MilanLongInt NLEdge, #ifdef __cplusplus } #endif +#endif diff --git a/amgprec/impl/aggregator/MatchBoxPC.h b/amgprec/impl/aggregator/MatchBoxPC.h index 24fd3134..67486f2d 100644 --- a/amgprec/impl/aggregator/MatchBoxPC.h +++ b/amgprec/impl/aggregator/MatchBoxPC.h @@ -59,11 +59,7 @@ #include #include #include -#ifdef OPENMP -// OpenMP is included and used if and only if the OpenMP version of the matching -// is required #include "omp.h" -#endif #include "primitiveDataTypeDefinitions.h" #include "dataStrStaticQueue.h" @@ -82,6 +78,8 @@ const int BundleTag = 9; // Predefined tag static vector DEFAULT_VECTOR; +#if !defined(SERIAL_MPI) + // MPI type map template MPI_Datatype TypeMap(); @@ -93,6 +91,7 @@ template <> inline MPI_Datatype TypeMap() { return MPI_DOUBLE; } template <> inline MPI_Datatype TypeMap() { return MPI_FLOAT; } +#endif #ifdef __cplusplus extern "C" @@ -178,10 +177,6 @@ extern "C" #define MilanRealMin MINUS_INFINITY #endif -#ifdef OPENMP -/* These functions are only used in the experimental OMP implementation, if that -is disabled there is no reason to actually compile or reference them. */ - // Function of find the owner of a ghost vertex using binary search: MilanInt findOwnerOfGhost(MilanLongInt vtxIndex, MilanLongInt *mVerDistance, MilanInt myRank, MilanInt numProcs); @@ -428,14 +423,6 @@ is disabled there is no reason to actually compile or reference them. */ MilanLongInt *msgIndSent, MilanLongInt *msgActualSent, MilanReal *msgPercent, MilanReal *ph0_time, MilanReal *ph1_time, MilanReal *ph2_time, MilanLongInt *ph1_card, MilanLongInt *ph2_card); -#endif - - -#ifndef OPENMP - //Function of find the owner of a ghost vertex using binary search: - inline MilanInt findOwnerOfGhost(MilanLongInt vtxIndex, MilanLongInt *mVerDistance, - MilanInt myRank, MilanInt numProcs); -#endif void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateC( MilanLongInt NLVer, MilanLongInt NLEdge, diff --git a/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp b/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp index 668ed626..49b366a6 100644 --- a/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp +++ b/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + // *********************************************************************** // // MatchboxP: A C++ library for approximate weighted matching @@ -126,10 +126,8 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( fflush(stdout); #endif - // The starting vertex owned by the current rank - MilanLongInt StartIndex = verDistance[myRank]; - // The ending vertex owned by the current rank - MilanLongInt EndIndex = verDistance[myRank + 1] - 1; + MilanLongInt StartIndex = verDistance[myRank]; // The starting vertex owned by the current rank + MilanLongInt EndIndex = verDistance[myRank + 1] - 1; // The ending vertex owned by the current rank MPI_Status computeStatus; @@ -147,8 +145,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( // only one message will be sent in the initialization phase - // one of: REQUEST/FAILURE/SUCCESS vector QLocalVtx, QGhostVtx, QMsgType; - // Changed by Fabio to be an integer, addresses needs to be integers! - vector QOwner; + vector QOwner; // Changed by Fabio to be an integer, addresses needs to be integers! MilanLongInt *PCounter = new MilanLongInt[numProcs]; for (int i = 0; i < numProcs; i++) @@ -156,8 +153,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( MilanLongInt NumMessagesBundled = 0; // TODO when the last computational section will be refactored this could be eliminated - // Changed by Fabio to be an integer, addresses needs to be integers! - MilanInt ghostOwner = 0; + MilanInt ghostOwner = 0; // Changed by Fabio to be an integer, addresses needs to be integers! MilanLongInt *candidateMate = nullptr; #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")NV: " << NLVer << " Edges: " << NLEdge; @@ -172,12 +168,9 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( MilanLongInt myCard = 0; // Build the Ghost Vertex Set: Vg - // Map each ghost vertex to a local vertex - map Ghost2LocalMap; - // Store the edge count for each ghost vertex - vector Counter; - // Number of Ghost vertices - MilanLongInt numGhostVertices = 0, numGhostEdges = 0; + map Ghost2LocalMap; // Map each ghost vertex to a local vertex + vector Counter; // Store the edge count for each ghost vertex + MilanLongInt numGhostVertices = 0, numGhostEdges = 0; // Number of Ghost vertices #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")About to compute Ghost Vertices..."; @@ -229,7 +222,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( cout << myRank << " Finished initialization" << endl; fflush(stdout); #endif - + startTime = MPI_Wtime(); ///////////////////////////////////////////////////////////////////////////////////////// @@ -398,7 +391,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( cout << myRank << " Finished sendBundles" << endl; fflush(stdout); #endif - + *ph1_card = myCard; // Cardinality at the end of Phase-1 startTime = MPI_Wtime(); ///////////////////////////////////////////////////////////////////////////////////////// @@ -429,7 +422,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( /////////////////////////////////////////////////////////////////////////////////// /////////////////////////// PROCESS MATCHED VERTICES ////////////////////////////// /////////////////////////////////////////////////////////////////////////////////// - + processMatchedVerticesAndSendMessages(NLVer, UChunkBeingProcessed, U, @@ -463,7 +456,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( comm, &msgActual, Message); - + ///////////////////////// END OF PROCESS MATCHED VERTICES ///////////////////////// //// BREAK IF NO MESSAGES EXPECTED ///////// @@ -559,4 +552,3 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( #endif #endif -#endif diff --git a/amgprec/impl/aggregator/clean.cpp b/amgprec/impl/aggregator/clean.cpp index 479dcce3..87bb699a 100644 --- a/amgprec/impl/aggregator/clean.cpp +++ b/amgprec/impl/aggregator/clean.cpp @@ -1,6 +1,7 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + // TODO comment +#if !defined(SERIAL_MPI) void clean(MilanLongInt NLVer, MilanInt myRank, diff --git a/amgprec/impl/aggregator/computeCandidateMate.cpp b/amgprec/impl/aggregator/computeCandidateMate.cpp index f70b8866..a21db656 100644 --- a/amgprec/impl/aggregator/computeCandidateMate.cpp +++ b/amgprec/impl/aggregator/computeCandidateMate.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + /** * Execute the research fr the Candidate Mate without controlling if the vertices are already matched. * Returns the vertices with the highest weight @@ -9,6 +9,8 @@ * @param edgeLocWeight * @return */ +#if !defined(SERIAL_MPI) + MilanLongInt firstComputeCandidateMate(MilanLongInt adj1, MilanLongInt adj2, MilanLongInt *verLocInd, @@ -60,7 +62,7 @@ MilanLongInt computeCandidateMate(MilanLongInt adj1, for (k = adj1; k < adj2; k++) { if (isAlreadyMatched(verLocInd[k], StartIndex, EndIndex, GMate, Mate, Ghost2LocalMap)) continue; - + if ((edgeLocWeight[k] > heaviestEdgeWt) || ((edgeLocWeight[k] == heaviestEdgeWt) && (w < verLocInd[k]))) { heaviestEdgeWt = edgeLocWeight[k]; @@ -68,7 +70,7 @@ MilanLongInt computeCandidateMate(MilanLongInt adj1, } } // End of for loop // End: PARALLEL_COMPUTE_CANDIDATE_MATE_B(v) - + return w; } #endif diff --git a/amgprec/impl/aggregator/extractUChunk.cpp b/amgprec/impl/aggregator/extractUChunk.cpp index 4e50a4f3..923a0b51 100644 --- a/amgprec/impl/aggregator/extractUChunk.cpp +++ b/amgprec/impl/aggregator/extractUChunk.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + void extractUChunk( vector &UChunkBeingProcessed, vector &U, @@ -28,5 +28,4 @@ void extractUChunk( } } // End of critical U // End of critical U -} -#endif +} \ No newline at end of file diff --git a/amgprec/impl/aggregator/findOwnerOfGhost.cpp b/amgprec/impl/aggregator/findOwnerOfGhost.cpp index 2723a7a3..b9d60614 100644 --- a/amgprec/impl/aggregator/findOwnerOfGhost.cpp +++ b/amgprec/impl/aggregator/findOwnerOfGhost.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + /// Find the owner of a ghost node: MilanInt findOwnerOfGhost(MilanLongInt vtxIndex, MilanLongInt *mVerDistance, MilanInt myRank, MilanInt numProcs) @@ -27,4 +27,3 @@ MilanInt findOwnerOfGhost(MilanLongInt vtxIndex, MilanLongInt *mVerDistance, return Current; } // End of findOwnerOfGhost() -#endif diff --git a/amgprec/impl/aggregator/initialize.cpp b/amgprec/impl/aggregator/initialize.cpp index 2c8f052d..17a4169e 100644 --- a/amgprec/impl/aggregator/initialize.cpp +++ b/amgprec/impl/aggregator/initialize.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + void initialize(MilanLongInt NLVer, MilanLongInt NLEdge, MilanLongInt StartIndex, MilanLongInt EndIndex, MilanLongInt *numGhostEdges, @@ -291,7 +291,7 @@ void initialize(MilanLongInt NLVer, MilanLongInt NLEdge, //new (&U) staticQueue(NLVer + (*numGhostVertices)); U.reserve(NLVer + (*numGhostVertices)); - // Initialize the private vectors + // Initialize the private vectors privateQLocalVtx.reserve(*numGhostVertices); privateQGhostVtx.reserve(*numGhostVertices); privateQMsgType.reserve(*numGhostVertices); @@ -302,4 +302,3 @@ void initialize(MilanLongInt NLVer, MilanLongInt NLEdge, } // End of single region } // End of parallel region } -#endif diff --git a/amgprec/impl/aggregator/isAlreadyMatched.cpp b/amgprec/impl/aggregator/isAlreadyMatched.cpp index 16d47a14..a7d65c15 100644 --- a/amgprec/impl/aggregator/isAlreadyMatched.cpp +++ b/amgprec/impl/aggregator/isAlreadyMatched.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + /** * //TODO documentation * @param k @@ -32,7 +32,7 @@ bool isAlreadyMatched(MilanLongInt node, */ MilanLongInt val; if ((node < StartIndex) || (node > EndIndex)) // if ghost vertex - { + { #pragma omp atomic read val = GMate[Ghost2LocalMap[node]]; return val >= 0; // Already matched @@ -43,5 +43,4 @@ bool isAlreadyMatched(MilanLongInt node, val = Mate[node - StartIndex]; return val >= 0; // Already matched -} -#endif +} \ No newline at end of file diff --git a/amgprec/impl/aggregator/parallelComputeCandidateMateB.cpp b/amgprec/impl/aggregator/parallelComputeCandidateMateB.cpp index 79f253eb..cf340da2 100644 --- a/amgprec/impl/aggregator/parallelComputeCandidateMateB.cpp +++ b/amgprec/impl/aggregator/parallelComputeCandidateMateB.cpp @@ -1,5 +1,6 @@ #include "MatchBoxPC.h" -#ifdef OPENMP +#if !defined(SERIAL_MPI) + void PARALLEL_COMPUTE_CANDIDATE_MATE_B(MilanLongInt NLVer, MilanLongInt *verLocPtr, MilanLongInt *verLocInd, diff --git a/amgprec/impl/aggregator/processCrossEdge.cpp b/amgprec/impl/aggregator/processCrossEdge.cpp index 45cddb44..e844f127 100644 --- a/amgprec/impl/aggregator/processCrossEdge.cpp +++ b/amgprec/impl/aggregator/processCrossEdge.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + void PROCESS_CROSS_EDGE(MilanLongInt *edge, MilanLongInt *S) { @@ -21,5 +21,4 @@ void PROCESS_CROSS_EDGE(MilanLongInt *edge, #endif // End: PARALLEL_PROCESS_CROSS_EDGE_B -} -#endif +} \ No newline at end of file diff --git a/amgprec/impl/aggregator/processExposedVertex.cpp b/amgprec/impl/aggregator/processExposedVertex.cpp index f869d69c..0336bed5 100644 --- a/amgprec/impl/aggregator/processExposedVertex.cpp +++ b/amgprec/impl/aggregator/processExposedVertex.cpp @@ -1,5 +1,6 @@ #include "MatchBoxPC.h" -#ifdef OPENMP +#if !defined(SERIAL_MPI) + void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, MilanLongInt *candidateMate, MilanLongInt *verLocInd, @@ -29,7 +30,7 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, vector &privateQGhostVtx, vector &privateQMsgType, vector &privateQOwner) -{ +{ MilanLongInt v = -1, k = -1, w = -1, adj11 = 0, adj12 = 0, k1 = 0; MilanInt ghostOwner = 0, option, igw; @@ -79,7 +80,7 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, Ghost2LocalMap); candidateMate[v] = w; } - + if (w >= 0) { (*myCard)++; if ((w < StartIndex) || (w > EndIndex)) { // w is a ghost vertex @@ -88,63 +89,60 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, option = 1; Mate[v] = w; GMate[Ghost2LocalMap[w]] = v + StartIndex; // w is a Ghost - + } // End of if CandidateMate[w] = v } // End of if a Ghost Vertex else { // w is a local vertex - + if (candidateMate[w - StartIndex] == (v + StartIndex)) { option = 3; Mate[v] = w; // v is local Mate[w - StartIndex] = v + StartIndex; // w is local - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")MATCH: (" << v + StartIndex << "," << w << ") "; fflush(stdout); #endif - + } // End of if ( candidateMate[w-StartIndex] == (v+StartIndex) ) } // End of Else - + } // End of second if - + } // End critical processExposed - + } // End of if(w >=0) else { -#pragma omp critical(adjuse) - { - // This piece of code is executed a really small number of times - adj11 = verLocPtr[v]; - adj12 = verLocPtr[v + 1]; - for (k1 = adj11; k1 < adj12; k1++) { - w = verLocInd[k1]; - if ((w < StartIndex) || (w > EndIndex)) { // A ghost - + // This piece of code is executed a really small amount of times + adj11 = verLocPtr[v]; + adj12 = verLocPtr[v + 1]; + for (k1 = adj11; k1 < adj12; k1++) { + w = verLocInd[k1]; + if ((w < StartIndex) || (w > EndIndex)) { // A ghost + #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Sending a failure message: "; - cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); - fflush(stdout); + cout << "\n(" << myRank << ")Sending a failure message: "; + cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); + fflush(stdout); #endif - (*msgInd)++; - (*NumMessagesBundled)++; - ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); - // assert(ghostOwner != -1); - // assert(ghostOwner != myRank); - PCounter[ghostOwner]++; - - privateQLocalVtx.push_back(v + StartIndex); - privateQGhostVtx.push_back(w); - privateQMsgType.push_back(FAILURE); - privateQOwner.push_back(ghostOwner); - - } // End of if(GHOST) - } // End of for loop - } + (*msgInd)++; + (*NumMessagesBundled)++; + ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); + // assert(ghostOwner != -1); + // assert(ghostOwner != myRank); + PCounter[ghostOwner]++; + + privateQLocalVtx.push_back(v + StartIndex); + privateQGhostVtx.push_back(w); + privateQMsgType.push_back(FAILURE); + privateQOwner.push_back(ghostOwner); + + } // End of if(GHOST) + } // End of for loop } // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - + switch (option) { case -1: diff --git a/amgprec/impl/aggregator/processMatchedVertices.cpp b/amgprec/impl/aggregator/processMatchedVertices.cpp index 77ec34bb..eadc0531 100644 --- a/amgprec/impl/aggregator/processMatchedVertices.cpp +++ b/amgprec/impl/aggregator/processMatchedVertices.cpp @@ -1,5 +1,6 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + +#if !defined(SERIAL_MPI) void processMatchedVertices( MilanLongInt NLVer, vector &UChunkBeingProcessed, @@ -58,29 +59,29 @@ void processMatchedVertices( { while (!U.empty()) { - + extractUChunk(UChunkBeingProcessed, U, privateU); - + for (MilanLongInt u : UChunkBeingProcessed) { #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")u: " << u; fflush(stdout); #endif if ((u >= StartIndex) && (u <= EndIndex)) { // Process Only the Local Vertices - + #ifdef COUNT_LOCAL_VERTEX localVertices++; #endif - + // Get the Adjacency list for u adj1 = verLocPtr[u - StartIndex]; // Pointer adj2 = verLocPtr[u - StartIndex + 1]; for (k = adj1; k < adj2; k++) { option = -1; v = verLocInd[k]; - + if ((v >= StartIndex) && (v <= EndIndex)) { // If Local Vertex: - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")v: " << v << " c(v)= " << candidateMate[v - StartIndex] << " Mate[v]: " << Mate[v]; fflush(stdout); @@ -102,27 +103,27 @@ void processMatchedVertices( GMate, Mate, Ghost2LocalMap); - + candidateMate[v - StartIndex] = w; - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")" << v << " Points to: " << w; fflush(stdout); #endif // If found a dominating edge: - if (w >= 0) { + if (w >= 0) { if ((w < StartIndex) || (w > EndIndex)) { // A ghost #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Sending a request message:"; cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); #endif option = 2; - + if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) { option = 1; Mate[v - StartIndex] = w; // v is a local vertex GMate[Ghost2LocalMap[w]] = v; // w is a ghost vertex - + } // End of if CandidateMate[w] = v } // End of if a Ghost Vertex else { // w is a local vertex @@ -130,7 +131,7 @@ void processMatchedVertices( option = 3; Mate[v - StartIndex] = w; // v is a local vertex Mate[w - StartIndex] = v; // w is a local vertex - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") "; fflush(stdout); @@ -146,7 +147,7 @@ void processMatchedVertices( } // mateval < 0 } // End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: else { // Neighbor is a ghost vertex - + #pragma omp critical { if (candidateMate[NLVer + Ghost2LocalMap[v]] == u) @@ -155,7 +156,7 @@ void processMatchedVertices( option = 5; // u is local } // End of critical } // End of Else //A Ghost Vertex - + switch (option) { case -1: @@ -165,7 +166,7 @@ void processMatchedVertices( // Found a dominating edge, it is a ghost and candidateMate[NLVer + Ghost2LocalMap[w]] == v privateU.push_back(v); privateU.push_back(w); - + (*myCard)++; #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") "; @@ -174,7 +175,7 @@ void processMatchedVertices( // Decrement the counter: PROCESS_CROSS_EDGE(&Counter[Ghost2LocalMap[w]], SPtr); case 2: - + // Found a dominating edge, it is a ghost ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); // assert(ghostOwner != -1); @@ -182,7 +183,7 @@ void processMatchedVertices( PCounter[ghostOwner]++; (*NumMessagesBundled)++; (*msgInd)++; - + privateQLocalVtx.push_back(v); privateQGhostVtx.push_back(w); privateQMsgType.push_back(REQUEST); @@ -191,7 +192,7 @@ void processMatchedVertices( case 3: privateU.push_back(v); privateU.push_back(w); - + (*myCard)++; break; case 4: @@ -201,94 +202,94 @@ void processMatchedVertices( for (k1 = adj11; k1 < adj12; k1++) { w = verLocInd[k1]; if ((w < StartIndex) || (w > EndIndex)) { // A ghost - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Sending a failure message: "; cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); fflush(stdout); #endif - + ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); // assert(ghostOwner != -1); // assert(ghostOwner != myRank); - + PCounter[ghostOwner]++; (*NumMessagesBundled)++; (*msgInd)++; - + privateQLocalVtx.push_back(v); privateQGhostVtx.push_back(w); privateQMsgType.push_back(FAILURE); privateQOwner.push_back(ghostOwner); - + } // End of if(GHOST) } // End of for loop break; case 5: default: - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Sending a success message: "; cout << "\n(" << myRank << ")Ghost is " << v << " Owner is: " << findOwnerOfGhost(v, verDistance, myRank, numProcs) << "\n"; fflush(stdout); #endif - + ghostOwner = findOwnerOfGhost(v, verDistance, myRank, numProcs); // assert(ghostOwner != -1); // assert(ghostOwner != myRank); - + (*NumMessagesBundled)++; PCounter[ghostOwner]++; (*msgInd)++; - + privateQLocalVtx.push_back(u); privateQGhostVtx.push_back(v); privateQMsgType.push_back(SUCCESS); privateQOwner.push_back(ghostOwner); - + break; } // End of switch - + } // End of inner for } } // End of outer for - + queuesTransfer(U, privateU, QLocalVtx, QGhostVtx, QMsgType, QOwner, privateQLocalVtx, privateQGhostVtx, privateQMsgType, privateQOwner); - + #pragma omp critical(U) { U.insert(U.end(), privateU.begin(), privateU.end()); } - + privateU.clear(); - + #pragma omp critical(sendMessageTransfer) { - + QLocalVtx.insert(QLocalVtx.end(), privateQLocalVtx.begin(), privateQLocalVtx.end()); QGhostVtx.insert(QGhostVtx.end(), privateQGhostVtx.begin(), privateQGhostVtx.end()); QMsgType.insert(QMsgType.end(), privateQMsgType.begin(), privateQMsgType.end()); QOwner.insert(QOwner.end(), privateQOwner.begin(), privateQOwner.end()); } - + privateQLocalVtx.clear(); privateQGhostVtx.clear(); privateQMsgType.clear(); privateQOwner.clear(); - + } // End of while ( !U.empty() ) - + #ifdef COUNT_LOCAL_VERTEX printf("Count local vertexes: %ld for thread %d of processor %d\n", localVertices, omp_get_thread_num(), myRank); - + #endif } // End of parallel region } diff --git a/amgprec/impl/aggregator/processMatchedVerticesAndSendMessages.cpp b/amgprec/impl/aggregator/processMatchedVerticesAndSendMessages.cpp index e02dd9c7..922b5860 100644 --- a/amgprec/impl/aggregator/processMatchedVerticesAndSendMessages.cpp +++ b/amgprec/impl/aggregator/processMatchedVerticesAndSendMessages.cpp @@ -1,6 +1,6 @@ #include "MatchBoxPC.h" -#ifdef OPENMP //#define DEBUG_HANG_ +#if !defined(SERIAL_MPI) void processMatchedVerticesAndSendMessages( MilanLongInt NLVer, vector &UChunkBeingProcessed, @@ -64,29 +64,29 @@ void processMatchedVerticesAndSendMessages( { while (!U.empty()) { - + extractUChunk(UChunkBeingProcessed, U, privateU); - + for (MilanLongInt u : UChunkBeingProcessed) { #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")u: " << u; fflush(stdout); #endif if ((u >= StartIndex) && (u <= EndIndex)) { // Process Only the Local Vertices - + #ifdef COUNT_LOCAL_VERTEX localVertices++; #endif - + // Get the Adjacency list for u adj1 = verLocPtr[u - StartIndex]; // Pointer adj2 = verLocPtr[u - StartIndex + 1]; for (k = adj1; k < adj2; k++) { option = -1; v = verLocInd[k]; - + if ((v >= StartIndex) && (v <= EndIndex)) { // If Local Vertex: - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")v: " << v << " c(v)= " << candidateMate[v - StartIndex] << " Mate[v]: " << Mate[v]; fflush(stdout); @@ -108,28 +108,28 @@ void processMatchedVerticesAndSendMessages( GMate, Mate, Ghost2LocalMap); - + candidateMate[v - StartIndex] = w; - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")" << v << " Points to: " << w; fflush(stdout); #endif // If found a dominating edge: if (w >= 0) { - + if ((w < StartIndex) || (w > EndIndex)) { // A ghost #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Sending a request message:"; cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); #endif option = 2; - + if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) { option = 1; Mate[v - StartIndex] = w; // v is a local vertex GMate[Ghost2LocalMap[w]] = v; // w is a ghost vertex - + } // End of if CandidateMate[w] = v } // End of if a Ghost Vertex else { // w is a local vertex @@ -137,7 +137,7 @@ void processMatchedVerticesAndSendMessages( option = 3; Mate[v - StartIndex] = w; // v is a local vertex Mate[w - StartIndex] = v; // w is a local vertex - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") "; fflush(stdout); @@ -153,7 +153,7 @@ void processMatchedVerticesAndSendMessages( } // mateval < 0 } // End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: else { // Neighbor is a ghost vertex - + #pragma omp critical { if (candidateMate[NLVer + Ghost2LocalMap[v]] == u) @@ -162,7 +162,7 @@ void processMatchedVerticesAndSendMessages( option = 5; // u is local } // End of critical } // End of Else //A Ghost Vertex - + switch (option) { case -1: @@ -180,20 +180,20 @@ void processMatchedVerticesAndSendMessages( // Decrement the counter: PROCESS_CROSS_EDGE(&Counter[Ghost2LocalMap[w]], SPtr); case 2: - + // Found a dominating edge, it is a ghost ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); - + // Build the Message Packet: // Message[0] = v; // LOCAL // Message[1] = w; // GHOST // Message[2] = REQUEST; // TYPE // Send a Request (Asynchronous) // MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); - + (*msgActual)++; (*msgInd)++; - + privateQLocalVtx.push_back(v); privateQGhostVtx.push_back(w); privateQMsgType.push_back(REQUEST); @@ -211,94 +211,94 @@ void processMatchedVerticesAndSendMessages( for (k1 = adj11; k1 < adj12; k1++) { w = verLocInd[k1]; if ((w < StartIndex) || (w > EndIndex)) { // A ghost - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Sending a failure message: "; cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); fflush(stdout); #endif - + ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); - + // Build the Message Packet: // Message[0] = v; // LOCAL // Message[1] = w; // GHOST // Message[2] = FAILURE; // TYPE // Send a Request (Asynchronous) // MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); - + (*msgActual)++; (*msgInd)++; - + privateQLocalVtx.push_back(v); privateQGhostVtx.push_back(w); privateQMsgType.push_back(FAILURE); privateQOwner.push_back(ghostOwner); - + } // End of if(GHOST) } // End of for loop break; case 5: default: - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Sending a success message: "; cout << "\n(" << myRank << ")Ghost is " << v << " Owner is: " << findOwnerOfGhost(v, verDistance, myRank, numProcs) << "\n"; fflush(stdout); #endif - + ghostOwner = findOwnerOfGhost(v, verDistance, myRank, numProcs); - + // Build the Message Packet: // Message[0] = u; // LOCAL // Message[1] = v; // GHOST // Message[2] = SUCCESS; // TYPE // Send a Request (Asynchronous) // MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); - + (*msgActual)++; (*msgInd)++; - + privateQLocalVtx.push_back(u); privateQGhostVtx.push_back(v); privateQMsgType.push_back(SUCCESS); privateQOwner.push_back(ghostOwner); - + break; } // End of switch } // End of inner for } } // End of outer for - + queuesTransfer(U, privateU, QLocalVtx, QGhostVtx, QMsgType, QOwner, privateQLocalVtx, privateQGhostVtx, privateQMsgType, privateQOwner); - + } // End of while ( !U.empty() ) - + #ifdef COUNT_LOCAL_VERTEX printf("Count local vertexes: %ld for thread %d of processor %d\n", localVertices, omp_get_thread_num(), myRank); - + #endif } // End of parallel region - + // Send the messages #ifdef DEBUG_HANG_ cout << myRank<<" Sending: "<(), ghostOwner, ComputeTag, comm); //cout << myRank<<" Sending to "<(), ghostOwner, ComputeTag, comm); diff --git a/amgprec/impl/aggregator/processMessages.cpp b/amgprec/impl/aggregator/processMessages.cpp index dc09cde1..2e3af9db 100644 --- a/amgprec/impl/aggregator/processMessages.cpp +++ b/amgprec/impl/aggregator/processMessages.cpp @@ -1,6 +1,6 @@ #include "MatchBoxPC.h" -#ifdef OPENMP //#define DEBUG_HANG_ +#if !defined(SERIAL_MPI) void processMessages( MilanLongInt NLVer, @@ -139,12 +139,12 @@ void processMessages( if (!ReceiveBuffer.empty()) ReceiveBuffer.clear(); // Empty it out first ReceiveBuffer.resize(bundleSize, -1); // Initialize - + ReceiveBuffer[0] = Message[0]; // u ReceiveBuffer[1] = Message[1]; // v ReceiveBuffer[2] = Message[2]; // message_type } - + #ifdef DEBUG_GHOST_ if ((v < StartIndex) || (v > EndIndex)) { cout << "\n(" << myRank << ") From ReceiveBuffer: This should not happen: u= " << u << " v= " << v << " Type= " << message_type << " StartIndex " << StartIndex << " EndIndex " << EndIndex << endl; @@ -161,7 +161,7 @@ void processMessages( u = ReceiveBuffer[bundleCounter - 3]; // GHOST v = ReceiveBuffer[bundleCounter - 2]; // LOCAL message_type = ReceiveBuffer[bundleCounter - 1]; // TYPE - + // CASE I: REQUEST if (message_type == REQUEST) { #ifdef PRINT_DEBUG_INFO_ @@ -189,7 +189,7 @@ void processMessages( cout << "\n(" << myRank << ")MATCH: (" << v << "," << u << ") " << endl; fflush(stdout); #endif - + PROCESS_CROSS_EDGE(&Counter[Ghost2LocalMap[u]], S); } // End of if ( candidateMate[v-StartIndex] == u )e } // End of if ( Mate[v] == -1 ) @@ -250,7 +250,7 @@ void processMessages( cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") " << endl; fflush(stdout); #endif - + PROCESS_CROSS_EDGE(&Counter[Ghost2LocalMap[w]], S); } // End of if CandidateMate[w] = v } // End of if a Ghost Vertex @@ -311,7 +311,7 @@ void processMessages( } // End of else: CASE III } // End of else: CASE I } - + return; } #endif diff --git a/amgprec/impl/aggregator/queueTransfer.cpp b/amgprec/impl/aggregator/queueTransfer.cpp index 6171b86d..33c65749 100644 --- a/amgprec/impl/aggregator/queueTransfer.cpp +++ b/amgprec/impl/aggregator/queueTransfer.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP + void queuesTransfer(vector &U, vector &privateU, vector &QLocalVtx, @@ -17,6 +17,8 @@ void queuesTransfer(vector &U, U.insert(U.end(), privateU.begin(), privateU.end()); } + privateU.clear(); + #pragma omp critical(sendMessageTransfer) { @@ -26,11 +28,9 @@ void queuesTransfer(vector &U, QOwner.insert(QOwner.end(), privateQOwner.begin(), privateQOwner.end()); } - privateU.clear(); privateQLocalVtx.clear(); privateQGhostVtx.clear(); privateQMsgType.clear(); privateQOwner.clear(); } -#endif diff --git a/amgprec/impl/aggregator/sendBundledMessages.cpp b/amgprec/impl/aggregator/sendBundledMessages.cpp index 3349ce86..07ea5834 100644 --- a/amgprec/impl/aggregator/sendBundledMessages.cpp +++ b/amgprec/impl/aggregator/sendBundledMessages.cpp @@ -1,5 +1,5 @@ #include "MatchBoxPC.h" -#ifdef OPENMP +#if !defined(SERIAL_MPI) void sendBundledMessages(MilanLongInt *numGhostEdges, MilanInt *BufferSize, MilanLongInt *Buffer, @@ -62,7 +62,7 @@ void sendBundledMessages(MilanLongInt *numGhostEdges, for (i = 0; i < numProcs; i++) PCumulative[i + 1] = PCumulative[i] + PCounter[i]; } - + #pragma omp task depend(inout \ : PCounter) { @@ -84,7 +84,7 @@ void sendBundledMessages(MilanLongInt *numGhostEdges, PCounter[QOwner[i]]++; } } - + // Send the Bundled Messages: Use ISend #pragma omp task depend(out \ : SRequest, SStatus) @@ -101,7 +101,7 @@ void sendBundledMessages(MilanLongInt *numGhostEdges, exit(1); } } - + // Send the Messages #pragma omp task depend(inout \ : SRequest, PSizeInfoMessages, PCumulative) depend(out \