From bd2d1e3b2600842bdc5937c37bcd685b15994c8f Mon Sep 17 00:00:00 2001 From: sfilippone Date: Tue, 4 Jun 2024 16:53:52 +0200 Subject: [PATCH 1/2] Additional OpenMP tinkering in matchboxp --- .../impl/aggregator/processExposedVertex.cpp | 109 ++++++++---------- amgprec/impl/aggregator/queueTransfer.cpp | 6 +- 2 files changed, 49 insertions(+), 66 deletions(-) diff --git a/amgprec/impl/aggregator/processExposedVertex.cpp b/amgprec/impl/aggregator/processExposedVertex.cpp index 81ce23eb..915c0ead 100644 --- a/amgprec/impl/aggregator/processExposedVertex.cpp +++ b/amgprec/impl/aggregator/processExposedVertex.cpp @@ -62,58 +62,41 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, fflush(stdout); #endif // If found a dominating edge: - if (w >= 0) - { - -#pragma omp critical(processExposed) - { - if (isAlreadyMatched(verLocInd[k], StartIndex, EndIndex, GMate, Mate, Ghost2LocalMap)) { - w = computeCandidateMate(verLocPtr[v], - verLocPtr[v + 1], - edgeLocWeight, 0, - verLocInd, - StartIndex, - EndIndex, - GMate, - Mate, - Ghost2LocalMap); - candidateMate[v] = w; - } - - if (w >= 0) { - (*myCard)++; - if ((w < StartIndex) || (w > EndIndex)) { // w is a ghost vertex - option = 2; - if (candidateMate[NLVer + Ghost2LocalMap[w]] == v + StartIndex) { - option = 1; - Mate[v] = w; - GMate[Ghost2LocalMap[w]] = v + StartIndex; // w is a Ghost - - } // End of if CandidateMate[w] = v - - } // End of if a Ghost Vertex - else { // w is a local vertex - - if (candidateMate[w - StartIndex] == (v + StartIndex)) { - option = 3; - Mate[v] = w; // v is local - Mate[w - StartIndex] = v + StartIndex; // w is local - + if (w >= 0) { +#pragma omp critical(Matching) + { + if (isAlreadyMatched(verLocInd[k], StartIndex, EndIndex, GMate, Mate, Ghost2LocalMap)) { + w = computeCandidateMate(verLocPtr[v], verLocPtr[v + 1], edgeLocWeight, 0, + verLocInd, StartIndex, EndIndex, + GMate, Mate, Ghost2LocalMap); + candidateMate[v] = w; + } + } + if (w >= 0) { + (*myCard)++; + if ((w < StartIndex) || (w > EndIndex)) { // w is a ghost vertex + option = 2; + if (candidateMate[NLVer + Ghost2LocalMap[w]] == v + StartIndex) { + option = 1; + Mate[v] = w; + GMate[Ghost2LocalMap[w]] = v + StartIndex; // w is a Ghost + } // End of if CandidateMate[w] = v + } // End of if a Ghost Vertex + else { // w is a local vertex + if (candidateMate[w - StartIndex] == (v + StartIndex)) { + option = 3; + Mate[v] = w; // v is local + Mate[w - StartIndex] = v + StartIndex; // w is local #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")MATCH: (" << v + StartIndex << "," << w << ") "; - fflush(stdout); + cout << "\n(" << myRank << ")MATCH: (" << v + StartIndex << "," << w << ") "; + fflush(stdout); #endif - - } // End of if ( candidateMate[w-StartIndex] == (v+StartIndex) ) - } // End of Else - - } // End of second if - - } // End critical processExposed - + } // End of if ( candidateMate[w-StartIndex] == (v+StartIndex) ) + } // End of Else + } // End of second if } // End of if(w >=0) else { -#pragma omp critical(adjuse) + //#pragma omp critical(adjuse) { // This piece of code is executed a really small number of times adj11 = verLocPtr[v]; @@ -132,6 +115,7 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); // assert(ghostOwner != -1); // assert(ghostOwner != myRank); +#pragma omp atomic PCounter[ghostOwner]++; privateQLocalVtx.push_back(v + StartIndex); @@ -144,23 +128,23 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, } } // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - + switch (option) - { - case -1: + { + case -1: break; - case 1: + case 1: privateU.push_back(v + StartIndex); privateU.push_back(w); - + #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")MATCH: (" << v + StartIndex << "," << w << ")"; fflush(stdout); #endif - + // Decrement the counter: PROCESS_CROSS_EDGE(&Counter[Ghost2LocalMap[w]], S); - case 2: + case 2: #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Sending a request message (291):"; cout << "\n(" << myRank << ")Local is: " << v + StartIndex << " Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs) << endl; @@ -171,29 +155,30 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer, ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); // assert(ghostOwner != -1); // assert(ghostOwner != myRank); +#pragma omp atomic PCounter[ghostOwner]++; - + privateQLocalVtx.push_back(v + StartIndex); privateQGhostVtx.push_back(w); privateQMsgType.push_back(REQUEST); privateQOwner.push_back(ghostOwner); break; - case 3: - default: + case 3: + default: privateU.push_back(v + StartIndex); privateU.push_back(w); break; - } - + } + } // End of for ( v=0; v < NLVer; v++ ) - + queuesTransfer(U, privateU, QLocalVtx, QGhostVtx, QMsgType, QOwner, privateQLocalVtx, privateQGhostVtx, privateQMsgType, privateQOwner); - + } // End of parallel region } #endif diff --git a/amgprec/impl/aggregator/queueTransfer.cpp b/amgprec/impl/aggregator/queueTransfer.cpp index 64a60157..67a2e524 100644 --- a/amgprec/impl/aggregator/queueTransfer.cpp +++ b/amgprec/impl/aggregator/queueTransfer.cpp @@ -15,11 +15,9 @@ void queuesTransfer(vector &U, #pragma omp critical(U) { U.insert(U.end(), privateU.begin(), privateU.end()); - } - -#pragma omp critical(sendMessageTransfer) - { + //#pragma omp critical(sendMessageTransfer) + //{ QLocalVtx.insert(QLocalVtx.end(), privateQLocalVtx.begin(), privateQLocalVtx.end()); QGhostVtx.insert(QGhostVtx.end(), privateQGhostVtx.begin(), privateQGhostVtx.end()); QMsgType.insert(QMsgType.end(), privateQMsgType.begin(), privateQMsgType.end()); From e11a134a1fe9d9c5065c94fb2478b4ee749f904b Mon Sep 17 00:00:00 2001 From: sfilippone Date: Tue, 4 Jun 2024 17:39:31 +0200 Subject: [PATCH 2/2] Additional tinkering with OpenMP matchbox --- .../aggregator/processMatchedVertices.cpp | 96 ++++++++-------- .../processMatchedVerticesAndSendMessages.cpp | 106 ++++++++---------- 2 files changed, 93 insertions(+), 109 deletions(-) diff --git a/amgprec/impl/aggregator/processMatchedVertices.cpp b/amgprec/impl/aggregator/processMatchedVertices.cpp index 77ec34bb..457e0de8 100644 --- a/amgprec/impl/aggregator/processMatchedVertices.cpp +++ b/amgprec/impl/aggregator/processMatchedVertices.cpp @@ -91,57 +91,57 @@ void processMatchedVertices( if (mateVal < 0) { #pragma omp critical { - if (candidateMate[v - StartIndex] == u) { +#pragma omp atomic read + mateVal = Mate[v - StartIndex]; + // If the current vertex is pointing to a matched vertex and is not matched + if (mateVal < 0) { + + if (candidateMate[v - StartIndex] == u) { // Start: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - w = computeCandidateMate(verLocPtr[v - StartIndex], - verLocPtr[v - StartIndex + 1], - edgeLocWeight, 0, - verLocInd, - StartIndex, - EndIndex, - GMate, - Mate, - Ghost2LocalMap); - - candidateMate[v - StartIndex] = w; - + w = computeCandidateMate(verLocPtr[v - StartIndex], + verLocPtr[v - StartIndex + 1], + edgeLocWeight, 0, + verLocInd, StartIndex, EndIndex, + GMate, Mate, Ghost2LocalMap); + candidateMate[v - StartIndex] = w; #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")" << v << " Points to: " << w; - fflush(stdout); + cout << "\n(" << myRank << ")" << v << " Points to: " << w; + fflush(stdout); #endif - // If found a dominating edge: - if (w >= 0) { - if ((w < StartIndex) || (w > EndIndex)) { // A ghost + // If found a dominating edge: + if (w >= 0) { + if ((w < StartIndex) || (w > EndIndex)) { // A ghost #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Sending a request message:"; - cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); + cout << "\n(" << myRank << ")Sending a request message:"; + cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); #endif - option = 2; - - if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) { - option = 1; - Mate[v - StartIndex] = w; // v is a local vertex - GMate[Ghost2LocalMap[w]] = v; // w is a ghost vertex - - } // End of if CandidateMate[w] = v - } // End of if a Ghost Vertex - else { // w is a local vertex - if (candidateMate[w - StartIndex] == v) { - option = 3; - Mate[v - StartIndex] = w; // v is a local vertex - Mate[w - StartIndex] = v; // w is a local vertex - + option = 2; + + if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) { + option = 1; + Mate[v - StartIndex] = w; // v is a local vertex + GMate[Ghost2LocalMap[w]] = v; // w is a ghost vertex + + } // End of if CandidateMate[w] = v + } // End of if a Ghost Vertex + else { // w is a local vertex + if (candidateMate[w - StartIndex] == v) { + option = 3; + Mate[v - StartIndex] = w; // v is a local vertex + Mate[w - StartIndex] = v; // w is a local vertex + #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") "; - fflush(stdout); + cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") "; + fflush(stdout); #endif - } // End of if(CandidateMate(w) = v - } // End of Else - } // End of if(w >=0) - else - option = 4; // End of Else: w == -1 - // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - } // End of If (candidateMate[v-StartIndex] == u + } // End of if(CandidateMate(w) = v + } // End of Else + } // End of if(w >=0) + else + option = 4; // End of Else: w == -1 + // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) + } // End of If (candidateMate[v-StartIndex] == u + } } // End of task } // mateval < 0 } // End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: @@ -179,6 +179,7 @@ void processMatchedVertices( ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); // assert(ghostOwner != -1); // assert(ghostOwner != myRank); +#pragma omp atomic PCounter[ghostOwner]++; (*NumMessagesBundled)++; (*msgInd)++; @@ -211,7 +212,7 @@ void processMatchedVertices( ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); // assert(ghostOwner != -1); // assert(ghostOwner != myRank); - +#pragma omp atomic PCounter[ghostOwner]++; (*NumMessagesBundled)++; (*msgInd)++; @@ -248,7 +249,6 @@ void processMatchedVertices( break; } // End of switch - } // End of inner for } } // End of outer for @@ -265,17 +265,15 @@ void processMatchedVertices( U.insert(U.end(), privateU.begin(), privateU.end()); } - privateU.clear(); - #pragma omp critical(sendMessageTransfer) { - QLocalVtx.insert(QLocalVtx.end(), privateQLocalVtx.begin(), privateQLocalVtx.end()); QGhostVtx.insert(QGhostVtx.end(), privateQGhostVtx.begin(), privateQGhostVtx.end()); QMsgType.insert(QMsgType.end(), privateQMsgType.begin(), privateQMsgType.end()); QOwner.insert(QOwner.end(), privateQOwner.begin(), privateQOwner.end()); } + privateU.clear(); privateQLocalVtx.clear(); privateQGhostVtx.clear(); privateQMsgType.clear(); diff --git a/amgprec/impl/aggregator/processMatchedVerticesAndSendMessages.cpp b/amgprec/impl/aggregator/processMatchedVerticesAndSendMessages.cpp index e02dd9c7..e75fa8db 100644 --- a/amgprec/impl/aggregator/processMatchedVerticesAndSendMessages.cpp +++ b/amgprec/impl/aggregator/processMatchedVerticesAndSendMessages.cpp @@ -77,7 +77,6 @@ void processMatchedVerticesAndSendMessages( #ifdef COUNT_LOCAL_VERTEX localVertices++; #endif - // Get the Adjacency list for u adj1 = verLocPtr[u - StartIndex]; // Pointer adj2 = verLocPtr[u - StartIndex + 1]; @@ -97,58 +96,57 @@ void processMatchedVerticesAndSendMessages( if (mateVal < 0) { #pragma omp critical { - if (candidateMate[v - StartIndex] == u) { - // Start: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - w = computeCandidateMate(verLocPtr[v - StartIndex], - verLocPtr[v - StartIndex + 1], - edgeLocWeight, 0, - verLocInd, - StartIndex, - EndIndex, - GMate, - Mate, - Ghost2LocalMap); - - candidateMate[v - StartIndex] = w; - +#pragma omp atomic read + mateVal = Mate[v - StartIndex]; + // If the current vertex is pointing to a matched vertex and is not matched + if (mateVal < 0) { + + if (candidateMate[v - StartIndex] == u) { + // Start: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) + w = computeCandidateMate(verLocPtr[v - StartIndex], + verLocPtr[v - StartIndex + 1], + edgeLocWeight, 0, + verLocInd, StartIndex, EndIndex, + GMate, Mate, Ghost2LocalMap); + candidateMate[v - StartIndex] = w; #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")" << v << " Points to: " << w; - fflush(stdout); + cout << "\n(" << myRank << ")" << v << " Points to: " << w; + fflush(stdout); #endif - // If found a dominating edge: - if (w >= 0) { - - if ((w < StartIndex) || (w > EndIndex)) { // A ghost + // If found a dominating edge: + if (w >= 0) { + + if ((w < StartIndex) || (w > EndIndex)) { // A ghost #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")Sending a request message:"; - cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); + cout << "\n(" << myRank << ")Sending a request message:"; + cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); #endif - option = 2; - - if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) { - option = 1; - Mate[v - StartIndex] = w; // v is a local vertex - GMate[Ghost2LocalMap[w]] = v; // w is a ghost vertex - - } // End of if CandidateMate[w] = v - } // End of if a Ghost Vertex - else { // w is a local vertex - if (candidateMate[w - StartIndex] == v) { - option = 3; - Mate[v - StartIndex] = w; // v is a local vertex - Mate[w - StartIndex] = v; // w is a local vertex - + option = 2; + + if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) { + option = 1; + Mate[v - StartIndex] = w; // v is a local vertex + GMate[Ghost2LocalMap[w]] = v; // w is a ghost vertex + + } // End of if CandidateMate[w] = v + } // End of if a Ghost Vertex + else { // w is a local vertex + if (candidateMate[w - StartIndex] == v) { + option = 3; + Mate[v - StartIndex] = w; // v is a local vertex + Mate[w - StartIndex] = v; // w is a local vertex #ifdef PRINT_DEBUG_INFO_ - cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") "; - fflush(stdout); + cout << "\n(" << myRank << ")MATCH: (" << v << "," << w << ") "; + fflush(stdout); #endif - } // End of if(CandidateMate(w) = v - } // End of Else - } // End of if(w >=0) - else - option = 4; // End of Else: w == -1 - // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) - } // End of If (candidateMate[v-StartIndex] == u + } // End of if(CandidateMate(w) = v + } // End of Else + } // End of if(w >=0) + else + option = 4; // End of Else: w == -1 + // End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) + } // End of If (candidateMate[v-StartIndex] == u + } } // End of task } // mateval < 0 } // End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: @@ -211,15 +209,12 @@ void processMatchedVerticesAndSendMessages( for (k1 = adj11; k1 < adj12; k1++) { w = verLocInd[k1]; if ((w < StartIndex) || (w > EndIndex)) { // A ghost - #ifdef PRINT_DEBUG_INFO_ cout << "\n(" << myRank << ")Sending a failure message: "; cout << "\n(" << myRank << ")Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs); fflush(stdout); #endif - ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); - // Build the Message Packet: // Message[0] = v; // LOCAL // Message[1] = w; // GHOST @@ -229,7 +224,6 @@ void processMatchedVerticesAndSendMessages( (*msgActual)++; (*msgInd)++; - privateQLocalVtx.push_back(v); privateQGhostVtx.push_back(w); privateQMsgType.push_back(FAILURE); @@ -248,17 +242,14 @@ void processMatchedVerticesAndSendMessages( #endif ghostOwner = findOwnerOfGhost(v, verDistance, myRank, numProcs); - // Build the Message Packet: // Message[0] = u; // LOCAL // Message[1] = v; // GHOST // Message[2] = SUCCESS; // TYPE // Send a Request (Asynchronous) // MPI_Bsend(&Message[0], 3, TypeMap(), ghostOwner, ComputeTag, comm); - (*msgActual)++; (*msgInd)++; - privateQLocalVtx.push_back(u); privateQGhostVtx.push_back(v); privateQMsgType.push_back(SUCCESS); @@ -281,10 +272,7 @@ void processMatchedVerticesAndSendMessages( #ifdef COUNT_LOCAL_VERTEX printf("Count local vertexes: %ld for thread %d of processor %d\n", - localVertices, - omp_get_thread_num(), - myRank); - + localVertices, mp_get_thread_num(), myRank); #endif } // End of parallel region @@ -293,12 +281,10 @@ void processMatchedVerticesAndSendMessages( cout << myRank<<" Sending: "<(), ghostOwner, ComputeTag, comm); //cout << myRank<<" Sending to "<(), ghostOwner, ComputeTag, comm);