diff --git a/amgprec/impl/aggregator/MatchBoxPC.h b/amgprec/impl/aggregator/MatchBoxPC.h index e8a2e2cc..dafe381e 100644 --- a/amgprec/impl/aggregator/MatchBoxPC.h +++ b/amgprec/impl/aggregator/MatchBoxPC.h @@ -76,6 +76,8 @@ const MilanLongInt SIZEINFO = 4; const int ComputeTag = 7; // Predefined tag const int BundleTag = 9; // Predefined tag +static vector DEFAULT_VECTOR; + // MPI type map template MPI_Datatype TypeMap(); @@ -320,27 +322,32 @@ extern "C" staticQueue &privateQLocalVtx, staticQueue &privateQGhostVtx, staticQueue &privateQMsgType, - staticQueue &privateQOwner); - - void sendBundledMessages(MilanLongInt *numGhostEdgesPtr, - MilanInt *BufferSizePtr, - MilanLongInt *Buffer, - vector &PCumulative, - vector &PMessageBundle, - vector &PSizeInfoMessages, - MilanLongInt *PCounter, - MilanLongInt NumMessagesBundled, - MilanLongInt *msgActualPtr, - MilanLongInt *MessageIndexPtr, - MilanInt numProcs, - MilanInt myRank, - MPI_Comm comm, - vector &QLocalVtx, - vector &QGhostVtx, - vector &QMsgType, - vector &QOwner, - vector &SRequest, - vector &SStatus); + staticQueue &privateQOwner, + bool sendMessages = false, + MPI_Comm comm = NULL, + MilanLongInt *msgActual = nullptr, + MilanLongInt *msgInd = nullptr, + vector &Message = DEFAULT_VECTOR); + + void sendBundledMessages(MilanLongInt *numGhostEdgesPtr, + MilanInt *BufferSizePtr, + MilanLongInt *Buffer, + vector &PCumulative, + vector &PMessageBundle, + vector &PSizeInfoMessages, + MilanLongInt *PCounter, + MilanLongInt NumMessagesBundled, + MilanLongInt *msgActualPtr, + MilanLongInt *MessageIndexPtr, + MilanInt numProcs, + MilanInt myRank, + MPI_Comm comm, + vector &QLocalVtx, + vector &QGhostVtx, + vector &QMsgType, + vector &QOwner, + vector &SRequest, + vector &SStatus); void processMessages( MilanLongInt NLVer, diff --git a/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp b/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp index a4fb68e5..99fd57c3 100644 --- a/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp +++ b/amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp @@ -368,9 +368,51 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( /////////////////////////////////////////////////////////////////////////////////// /////////////////////////// PROCESS MATCHED VERTICES ////////////////////////////// /////////////////////////////////////////////////////////////////////////////////// - while (/*!Q.empty()*/ !U.empty()) + ///* + +//#define error +#ifdef error + processMatchedVertices(NLVer, + UChunkBeingProcessed, + U, + privateU, + StartIndex, + EndIndex, + &myCard, + &msgInd, + &NumMessagesBundled, + &S, + verLocPtr, + verLocInd, + verDistance, + PCounter, + Counter, + myRank, + numProcs, + candidateMate, + GMate, + Mate, + Ghost2LocalMap, + edgeLocWeight, + QLocalVtx, + QGhostVtx, + QMsgType, + QOwner, + privateQLocalVtx, + privateQGhostVtx, + privateQMsgType, + privateQOwner, + true, + comm, + &msgActual, + &msgInd, + Message); +#endif +#ifndef error + + while (!U.empty()) { - // Q.pop_front(); + u = U.pop_front(); // Get an element from the queue #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")u: " << u; @@ -385,9 +427,9 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( { v = verLocInd[k]; if ((v >= StartIndex) && (v <= EndIndex)) - { // v is a Local Vertex: - //if (Mate[v - StartIndex] >= 0) // v is already matched - // continue; + { // v is a Local Vertex: + // if (Mate[v - StartIndex] >= 0) // v is already matched + // continue; #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")v: " << v << " c(v)= " << candidateMate[v - StartIndex] << " Mate[v]: " << Mate[v]; fflush(stdout); @@ -500,8 +542,8 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( } // End of Else: w == -1 // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) } // End of If (candidateMate[v-StartIndex] == u) - } - } // End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: + } // if (Mate[v - StartIndex] < 0) + } // End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: else { // Neighbor v is a ghost vertex if (candidateMate[NLVer + Ghost2LocalMap[v]] == u) @@ -536,7 +578,9 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP( } // End of Else //A Ghost Vertex } // End of For Loop adj(u) } // End of if ( (u >= StartIndex) && (u <= EndIndex) ) //Process Only If a Local Vertex - } // End of while ( /*!Q.empty()*/ !U.empty() ) + } // End of while ( !U.empty() ) +#endif + ///////////////////////// END OF PROCESS MATCHED VERTICES ///////////////////////// //// BREAK IF NO MESSAGES EXPECTED ///////// diff --git a/amgprec/impl/aggregator/processMatchedVertices.cpp b/amgprec/impl/aggregator/processMatchedVertices.cpp index 0054ffa2..296bfa15 100644 --- a/amgprec/impl/aggregator/processMatchedVertices.cpp +++ b/amgprec/impl/aggregator/processMatchedVertices.cpp @@ -32,7 +32,12 @@ void processMatchedVertices( staticQueue &privateQLocalVtx, staticQueue &privateQGhostVtx, staticQueue &privateQMsgType, - staticQueue &privateQOwner) + staticQueue &privateQOwner, + bool sendMessages, + MPI_Comm comm, + MilanLongInt *msgActual, + MilanLongInt *msgInd, + vector &Message) { MilanLongInt adj1, adj2, adj11, adj12, k, k1, v = -1, w = -1, ghostOwner; @@ -48,7 +53,7 @@ void processMatchedVertices( #ifdef COUNT_LOCAL_VERTEX MilanLongInt localVertices = 0; #endif -#pragma omp parallel private(k, w, v, k1, adj1, adj2, adj11, adj12, ghostOwner, option) firstprivate(privateU, StartIndex, EndIndex, privateQLocalVtx, privateQGhostVtx, privateQMsgType, privateQOwner, UChunkBeingProcessed) default(shared) num_threads(NUM_THREAD) +#pragma omp parallel private(k, w, v, k1, adj1, adj2, adj11, adj12, ghostOwner, option) firstprivate(Message, privateU, StartIndex, EndIndex, privateQLocalVtx, privateQGhostVtx, privateQMsgType, privateQOwner, UChunkBeingProcessed) default(shared) num_threads(NUM_THREAD) { while (!U.empty()) @@ -144,16 +149,14 @@ void processMatchedVertices( #endif } // End of if(CandidateMate(w) = v } // End of Else - - } // End of if(w >=0) + } // End of if(w >=0) else option = 4; // End of Else: w == -1 // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - } - } // End of task - } // End of If (candidateMate[v-StartIndex] == u - - } // End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: + } // End of If (candidateMate[v-StartIndex] == u + } // End of task + } // mateval < 0 + } // End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: else { // Neighbor is a ghost vertex @@ -184,16 +187,36 @@ void processMatchedVertices( // Decrement the counter: PROCESS_CROSS_EDGE(&Counter[Ghost2LocalMap[w]], SPtr); case 2: + // Found a dominating edge, it is a ghost ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); assert(ghostOwner != -1); assert(ghostOwner != myRank); + if (sendMessages) + { + // Build the Message Packet: + Message[0] = v; // LOCAL + Message[1] = w; // GHOST + Message[2] = REQUEST; // TYPE + // Send a Request (Asynchronous) + //#pragma omp master + // { + MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); +// } #pragma omp atomic - PCounter[ghostOwner]++; + (*msgActual)++; + } + else + { #pragma omp atomic - (*msgIndPtr)++; + PCounter[ghostOwner]++; #pragma omp atomic - (*NumMessagesBundledPtr)++; + (*NumMessagesBundledPtr)++; + } + +#pragma omp atomic + (*msgIndPtr)++; + privateQLocalVtx.push_back(v); privateQGhostVtx.push_back(w); privateQMsgType.push_back(REQUEST); @@ -224,12 +247,30 @@ void processMatchedVertices( ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); assert(ghostOwner != -1); assert(ghostOwner != myRank); + if (sendMessages) + { + // Build the Message Packet: + Message[0] = v; // LOCAL + Message[1] = w; // GHOST + Message[2] = FAILURE; // TYPE + // Send a Request (Asynchronous) + //#pragma omp master + // { + MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); +// } +#pragma omp atomic + (*msgActual)++; + } + else + { #pragma omp atomic - PCounter[ghostOwner]++; + PCounter[ghostOwner]++; #pragma omp atomic - (*msgIndPtr)++; + (*NumMessagesBundledPtr)++; + } + #pragma omp atomic - (*NumMessagesBundledPtr)++; + (*msgIndPtr)++; privateQLocalVtx.push_back(v); privateQGhostVtx.push_back(w); @@ -239,6 +280,7 @@ void processMatchedVertices( } // End of if(GHOST) } // End of for loop break; + case 5: default: #ifdef PRINT_DEBUG_INFO_ @@ -250,12 +292,32 @@ void processMatchedVertices( ghostOwner = findOwnerOfGhost(v, verDistance, myRank, numProcs); assert(ghostOwner != -1); assert(ghostOwner != myRank); + if (sendMessages) + { + // Build the Message Packet: + Message[0] = u; // LOCAL + Message[1] = v; // GHOST + Message[2] = SUCCESS; // TYPE + + // Send a Request (Asynchronous) + //#pragma omp master + // { + MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); +// } #pragma omp atomic - PCounter[ghostOwner]++; + (*msgActual)++; + } + else + { #pragma omp atomic - (*msgIndPtr)++; + (*NumMessagesBundledPtr)++; #pragma omp atomic - (*NumMessagesBundledPtr)++; + PCounter[ghostOwner]++; + } + +#pragma omp atomic + (*msgIndPtr)++; + privateQLocalVtx.push_back(u); privateQGhostVtx.push_back(v); privateQMsgType.push_back(SUCCESS); diff --git a/amgprec/impl/aggregator/processMessages.cpp b/amgprec/impl/aggregator/processMessages.cpp index 43fc5ce9..bb21396b 100644 --- a/amgprec/impl/aggregator/processMessages.cpp +++ b/amgprec/impl/aggregator/processMessages.cpp @@ -104,7 +104,7 @@ void processMessages( ReceiveBuffer.resize(bundleSize, -1); // Initialize #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Message Bundle Before: " << endl; - for (i = 0; i < bundleSize; i++) + for (int i = 0; i < bundleSize; i++) cout << ReceiveBuffer[i] << ","; cout << endl; fflush(stdout); @@ -119,7 +119,7 @@ void processMessages( } #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Message Bundle After: " << endl; - for (i = 0; i < bundleSize; i++) + for (int i = 0; i < bundleSize; i++) cout << ReceiveBuffer[i] << ","; cout << endl; fflush(stdout); diff --git a/amgprec/impl/aggregator/sendBundledMessages.cpp b/amgprec/impl/aggregator/sendBundledMessages.cpp index 8665759c..f7fd2f78 100644 --- a/amgprec/impl/aggregator/sendBundledMessages.cpp +++ b/amgprec/impl/aggregator/sendBundledMessages.cpp @@ -9,7 +9,7 @@ void sendBundledMessages(MilanLongInt *numGhostEdges, MilanLongInt *PCounter, MilanLongInt NumMessagesBundled, MilanLongInt *msgActual, - MilanLongInt *MessageIndex, + MilanLongInt *msgInd, MilanInt numProcs, MilanInt myRank, MPI_Comm comm, @@ -105,7 +105,7 @@ PSizeInfoMessages.resize(numProcs * 3, 0); // Send the Messages #pragma omp task depend(inout \ : SRequest, PSizeInfoMessages, PCumulative) depend(out \ - : *msgActual, *MessageIndex) + : *msgActual, *msgInd) { for (i = 0; i < numProcs; i++) { // Changed by Fabio to be an integer, addresses needs to be integers! @@ -124,9 +124,9 @@ PSizeInfoMessages.resize(numProcs * 3, 0); if (PSizeInfoMessages[i * 3 + 0] > 0) { // Send only if it is a nonempty packet MPI_Isend(&PSizeInfoMessages[i * 3 + 0], 3, TypeMap(), i, ComputeTag, comm, - &SRequest[(*MessageIndex)]); + &SRequest[(*msgInd)]); (*msgActual)++; - (*MessageIndex)++; + (*msgInd)++; // Now Send the message with the data packet: #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")SendiFFng Bundle to : " << i << endl; @@ -136,8 +136,8 @@ PSizeInfoMessages.resize(numProcs * 3, 0); fflush(stdout); #endif MPI_Isend(&PMessageBundle[PCumulative[i] * 3], PSizeInfoMessages[i * 3 + 0], - TypeMap(), i, BundleTag, comm, &SRequest[(*MessageIndex)]); - (*MessageIndex)++; + TypeMap(), i, BundleTag, comm, &SRequest[(*msgInd)]); + (*msgInd)++; } // End of if size > 0 } } @@ -207,9 +207,4 @@ PSizeInfoMessages.resize(numProcs * 3, 0); } } } - -//*MessageIndexPtr = MessageIndex; -//*msgActualPtr = msgActual; -//*numGhostEdgesPtr = numGhostEdges; -//*BufferSizePtr = BufferSize; } \ No newline at end of file diff --git a/amgprec/stZnqhkT b/amgprec/stZnqhkT new file mode 100644 index 00000000..8b277f0d --- /dev/null +++ b/amgprec/stZnqhkT @@ -0,0 +1 @@ +!