fdurastante
/
amg4psblas
mirror of https://github.com/sfilippone/amg4psblas.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PROCESS MATCHED VERTICES parallelization draft

Browse Source
omp-walther
StefanoPetrilli 3 years ago
parent f3d7b3ab5e
commit 2c8dc2ffdd
2 changed files with 87 additions and 41 deletions
Show Stats Download Patch File Download Diff File

126
amgprec/impl/aggregator/algoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP.cpp
Unescape Escape View File

@ -210,12 +210,14 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
//Mate array for ghost vertices: //Mate array for ghost vertices:
vector <MilanLongInt> GMate; //Proportional to the number of ghost vertices vector <MilanLongInt> GMate; //Proportional to the number of ghost vertices
MilanLongInt S; MilanLongInt S;
staticQueue U; MilanLongInt privateMyCard = 0;
staticQueue U, privateU;
bool isEmpty;
#ifdef TIME_TRACKER #ifdef TIME_TRACKER
double Ghost2LocalInitialization = MPI_Wtime(); double Ghost2LocalInitialization = MPI_Wtime();
#endif #endif
#pragma omp parallel private(insertMe, k, k1, adj1, adj2, adj11, adj12, heaviestEdgeWt, w, ghostOwner, u) firstprivate(StartIndex, EndIndex) default(shared) num_threads(4) #pragma omp parallel private(insertMe, k, k1, adj1, adj2, adj11, adj12, heaviestEdgeWt, w, ghostOwner, u, privateU, privateMyCard, isEmpty) firstprivate(StartIndex, EndIndex) default(shared) num_threads(4)
{ {
// TODO comments about the reduction // TODO comments about the reduction
@ -451,6 +453,10 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
* The sequential version could be a bit more * The sequential version could be a bit more
* efficient. * efficient.
* *
* TODO: Maybe it is possible to append the values of QLocalVtx, QGhostVtx, QMsgType and QOwner
* first in a local variable and then, only at the end, append them to the real data structure
* to remove the critical sections.
*
* TODO: Test when it's more efficient to execute this code * TODO: Test when it's more efficient to execute this code
* in parallel. * in parallel.
*/ */
@ -508,6 +514,8 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
msgInd++; msgInd++;
NumMessagesBundled++; NumMessagesBundled++;
ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs);
assert(ghostOwner != -1);
assert(ghostOwner != myRank);
PCounter[ghostOwner]++; PCounter[ghostOwner]++;
#pragma omp critical(Mate) #pragma omp critical(Mate)
{ {
@ -590,15 +598,14 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
NumMessagesBundled++; NumMessagesBundled++;
msgInd++; msgInd++;
ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs);
assert(ghostOwner != -1);
assert(ghostOwner != myRank);
PCounter[ghostOwner]++; PCounter[ghostOwner]++;
#pragma omp critical #pragma omp critical
{ {
QLocalVtx.push_back(v + StartIndex); QLocalVtx.push_back(v + StartIndex);
QGhostVtx.push_back(w); QGhostVtx.push_back(w);
QMsgType.push_back(FAILURE); QMsgType.push_back(FAILURE);
//ghostOwner = inputSubGraph.findOwner(w);
assert(ghostOwner != -1);
assert(ghostOwner != myRank);
QOwner.push_back(ghostOwner); QOwner.push_back(ghostOwner);
} }
@ -607,7 +614,6 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
//} // End of Else: w == -1 //} // End of Else: w == -1
//End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) //End: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v)
} //End of for ( v=0; v < NLVer; v++ ) } //End of for ( v=0; v < NLVer; v++ )
} // end of parallel region
tempCounter.clear(); //Do not need this any more tempCounter.clear(); //Do not need this any more
@ -619,19 +625,37 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
/////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////
/////////////////////////// PROCESS MATCHED VERTICES ////////////////////////////// /////////////////////////// PROCESS MATCHED VERTICES //////////////////////////////
/////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////
privateU.~staticQueue();
new(&privateU) staticQueue(1000); //TODO how can I put a meaningfull size?
/*
while ( !U.empty() ) { while ( !U.empty() ) {
u = U.pop_front(); //Get an element from the queue u = U.pop_front(); //Get an element from the queue
*/
isEmpty = false;
while( true )
{
#pragma omp critical(U)
{
if (U.empty()) isEmpty = true;
else u = U.pop_front();
} // End of critical U
if (isEmpty) break;
#ifdef PRINT_DEBUG_INFO_ #ifdef PRINT_DEBUG_INFO_
cout<<"\n("<<myRank<<")u: "<<u; fflush(stdout); cout<<"\n("<<myRank<<")u: "<<u; fflush(stdout);
#endif #endif
if ( (u >= StartIndex) && (u <= EndIndex) ) { //Process Only the Local Vertices if ( (u >= StartIndex) && (u <= EndIndex) ) { //Process Only the Local Vertices
#pragma omp critical
{
//Get the Adjacency list for u //Get the Adjacency list for u
adj1 = verLocPtr[u-StartIndex]; //Pointer adj1 = verLocPtr[u - StartIndex]; //Pointer
adj2 = verLocPtr[u-StartIndex+1]; adj2 = verLocPtr[u - StartIndex + 1];
for( k = adj1; k < adj2; k++ ) { for (k = adj1; k < adj2; k++) {
v = verLocInd[k]; v = verLocInd[k];
if ( (v >= StartIndex) && (v <= EndIndex) ) { //If Local Vertex: if ((v >= StartIndex) && (v <= EndIndex)) { //If Local Vertex:
if (isAlreadyMatched(v, StartIndex, EndIndex, GMate, Mate, Ghost2LocalMap)) continue; if (isAlreadyMatched(v, StartIndex, EndIndex, GMate, Mate, Ghost2LocalMap)) continue;
@ -640,7 +664,7 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
fflush(stdout); fflush(stdout);
#endif #endif
if ( candidateMate[v-StartIndex] == u ) { //Only if pointing to the matched vertex if (candidateMate[v - StartIndex] == u) { //Only if pointing to the matched vertex
//Start: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v) //Start: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v)
//Start: PARALLEL_COMPUTE_CANDIDATE_MATE_B(v) //Start: PARALLEL_COMPUTE_CANDIDATE_MATE_B(v)
w = computeCandidateMate(verLocPtr[v - StartIndex], w = computeCandidateMate(verLocPtr[v - StartIndex],
@ -652,17 +676,16 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
GMate, GMate,
Mate, Mate,
Ghost2LocalMap); Ghost2LocalMap);
#pragma omp critical
{
candidateMate[v - StartIndex] = w; candidateMate[v - StartIndex] = w;
}
//End: PARALLEL_COMPUTE_CANDIDATE_MATE_B(v) //End: PARALLEL_COMPUTE_CANDIDATE_MATE_B(v)
#ifdef PRINT_DEBUG_INFO_ #ifdef PRINT_DEBUG_INFO_
cout<<"\n("<<myRank<<")"<<v<<" Points to: "<<w; fflush(stdout); cout<<"\n("<<myRank<<")"<<v<<" Points to: "<<w; fflush(stdout);
#endif #endif
//If found a dominating edge: //If found a dominating edge:
if ( w >= 0 ) { if (w >= 0) {
if ( (w < StartIndex) || (w > EndIndex) ) { //A ghost if ((w < StartIndex) || (w > EndIndex)) { //A ghost
//Build the Message Packet: //Build the Message Packet:
//Message[0] = v; //LOCAL //Message[0] = v; //LOCAL
//Message[1] = w; //GHOST //Message[1] = w; //GHOST
@ -678,26 +701,28 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
QGhostVtx.push_back(w); QGhostVtx.push_back(w);
QMsgType.push_back(REQUEST); QMsgType.push_back(REQUEST);
//ghostOwner = inputSubGraph.findOwner(w); //ghostOwner = inputSubGraph.findOwner(w);
ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); assert(ghostOwner != -1); assert(ghostOwner != myRank); ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs);
assert(ghostOwner != -1);
assert(ghostOwner != myRank);
QOwner.push_back(ghostOwner); QOwner.push_back(ghostOwner);
PCounter[ghostOwner]++; PCounter[ghostOwner]++;
NumMessagesBundled++; NumMessagesBundled++;
msgInd++; msgInd++;
if ( candidateMate[NLVer+Ghost2LocalMap[w]] == v ) { if (candidateMate[NLVer + Ghost2LocalMap[w]] == v) {
Mate[v-StartIndex] = w; //v is a local vertex Mate[v - StartIndex] = w; //v is a local vertex
GMate[Ghost2LocalMap[w]] = v; //w is a ghost vertex GMate[Ghost2LocalMap[w]] = v; //w is a ghost vertex
//Q.push_back(u); //Q.push_back(u);
U.push_back(v); privateU.push_back(v);
U.push_back(w); privateU.push_back(w);
myCard++; privateMyCard++;
#ifdef PRINT_DEBUG_INFO_ #ifdef PRINT_DEBUG_INFO_
cout<<"\n("<<myRank<<")MATCH: ("<<v<<","<<w<<") "; fflush(stdout); cout<<"\n("<<myRank<<")MATCH: ("<<v<<","<<w<<") "; fflush(stdout);
#endif #endif
//Decrement the counter: //Decrement the counter:
//Start: PARALLEL_PROCESS_CROSS_EDGE_B(v,w) //Start: PARALLEL_PROCESS_CROSS_EDGE_B(v,w)
if ( Counter[Ghost2LocalMap[w]] > 0 ) { if (Counter[Ghost2LocalMap[w]] > 0) {
Counter[Ghost2LocalMap[w]] = Counter[Ghost2LocalMap[w]] - 1; //Decrement Counter[Ghost2LocalMap[w]] = Counter[Ghost2LocalMap[w]] - 1; //Decrement
if ( Counter[Ghost2LocalMap[w]] == 0 ) { if (Counter[Ghost2LocalMap[w]] == 0) {
S--; //Decrement S S--; //Decrement S
#ifdef PRINT_DEBUG_INFO_ #ifdef PRINT_DEBUG_INFO_
cout<<"\n("<<myRank<<")Decrementing S: Ghost vertex "<<w<<" has received all its messages"; cout<<"\n("<<myRank<<")Decrementing S: Ghost vertex "<<w<<" has received all its messages";
@ -709,13 +734,13 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
} //End of if CandidateMate[w] = v } //End of if CandidateMate[w] = v
} //End of if a Ghost Vertex } //End of if a Ghost Vertex
else { //w is a local vertex else { //w is a local vertex
if ( candidateMate[w-StartIndex] == v ) { if (candidateMate[w - StartIndex] == v) {
Mate[v-StartIndex] = w; //v is a local vertex Mate[v - StartIndex] = w; //v is a local vertex
Mate[w-StartIndex] = v; //w is a local vertex Mate[w - StartIndex] = v; //w is a local vertex
//Q.push_back(u); //Q.push_back(u);
U.push_back(v); privateU.push_back(v);
U.push_back(w); privateU.push_back(w);
myCard++; privateMyCard++;
#ifdef PRINT_DEBUG_INFO_ #ifdef PRINT_DEBUG_INFO_
cout<<"\n("<<myRank<<")MATCH: ("<<v<<","<<w<<") "; fflush(stdout); cout<<"\n("<<myRank<<")MATCH: ("<<v<<","<<w<<") "; fflush(stdout);
#endif #endif
@ -723,11 +748,11 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
} //End of Else } //End of Else
} //End of if(w >=0) } //End of if(w >=0)
else { else {
adj11 = verLocPtr[v-StartIndex]; adj11 = verLocPtr[v - StartIndex];
adj12 = verLocPtr[v-StartIndex+1]; adj12 = verLocPtr[v - StartIndex + 1];
for( k1 = adj11; k1 < adj12; k1++ ) { for (k1 = adj11; k1 < adj12; k1++) {
w = verLocInd[k1]; w = verLocInd[k1];
if ( (w < StartIndex) || (w > EndIndex) ) { //A ghost if ((w < StartIndex) || (w > EndIndex)) { //A ghost
//Build the Message Packet: //Build the Message Packet:
//Message[0] = v; //LOCAL //Message[0] = v; //LOCAL
//Message[1] = w; //GHOST //Message[1] = w; //GHOST
@ -744,7 +769,9 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
QGhostVtx.push_back(w); QGhostVtx.push_back(w);
QMsgType.push_back(FAILURE); QMsgType.push_back(FAILURE);
//ghostOwner = inputSubGraph.findOwner(w); //ghostOwner = inputSubGraph.findOwner(w);
ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs); assert(ghostOwner != -1); assert(ghostOwner != myRank); ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs);
assert(ghostOwner != -1);
assert(ghostOwner != myRank);
QOwner.push_back(ghostOwner); QOwner.push_back(ghostOwner);
PCounter[ghostOwner]++; PCounter[ghostOwner]++;
NumMessagesBundled++; NumMessagesBundled++;
@ -756,9 +783,9 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
} //End of If (candidateMate[v-StartIndex] == u) } //End of If (candidateMate[v-StartIndex] == u)
} //End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex: } //End of if ( (v >= StartIndex) && (v <= EndIndex) ) //If Local Vertex:
else { //Neighbor is a ghost vertex else { //Neighbor is a ghost vertex
if ( candidateMate[NLVer+Ghost2LocalMap[v]] == u ) if (candidateMate[NLVer + Ghost2LocalMap[v]] == u)
candidateMate[NLVer+Ghost2LocalMap[v]] = -1; candidateMate[NLVer + Ghost2LocalMap[v]] = -1;
if ( v != Mate[u-StartIndex] ) { //u is local if (v != Mate[u - StartIndex]) { //u is local
//Build the Message Packet: //Build the Message Packet:
//Message[0] = u; //LOCAL //Message[0] = u; //LOCAL
//Message[1] = v; //GHOST //Message[1] = v; //GHOST
@ -774,7 +801,9 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
QGhostVtx.push_back(v); QGhostVtx.push_back(v);
QMsgType.push_back(SUCCESS); QMsgType.push_back(SUCCESS);
//ghostOwner = inputSubGraph.findOwner(v); //ghostOwner = inputSubGraph.findOwner(v);
ghostOwner = findOwnerOfGhost(v, verDistance, myRank, numProcs); assert(ghostOwner != -1); assert(ghostOwner != myRank); ghostOwner = findOwnerOfGhost(v, verDistance, myRank, numProcs);
assert(ghostOwner != -1);
assert(ghostOwner != myRank);
QOwner.push_back(ghostOwner); QOwner.push_back(ghostOwner);
PCounter[ghostOwner]++; PCounter[ghostOwner]++;
NumMessagesBundled++; NumMessagesBundled++;
@ -782,8 +811,24 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
} //End of If( v != Mate[u] ) } //End of If( v != Mate[u] )
} //End of Else //A Ghost Vertex } //End of Else //A Ghost Vertex
} //End of For Loop adj(u) } //End of For Loop adj(u)
}
} //End of if ( (u >= StartIndex) && (u <= EndIndex) ) //Process Only If a Local Vertex } //End of if ( (u >= StartIndex) && (u <= EndIndex) ) //Process Only If a Local Vertex
#pragma omp critical(U)
{
while(!privateU.empty()) {
U.push_back(privateU.pop_front());
}
myCard += privateMyCard;
} //End of critical U
} //End of while ( /*!Q.empty()*/ !U.empty() ) } //End of while ( /*!Q.empty()*/ !U.empty() )
} // end of parallel region
///////////////////////// END OF PROCESS MATCHED VERTICES ///////////////////////// ///////////////////////// END OF PROCESS MATCHED VERTICES /////////////////////////
#ifdef DEBUG_HANG_ #ifdef DEBUG_HANG_
if (myRank == 0) cout<<"\n("<<myRank<<") Send Bundles" <<endl; fflush(stdout); if (myRank == 0) cout<<"\n("<<myRank<<") Send Bundles" <<endl; fflush(stdout);
@ -1097,7 +1142,8 @@ void dalgoDistEdgeApproxDomEdgesLinearSearchMesgBndlSmallMateCMP(
#endif #endif
ghostOwner = findOwnerOfGhost(v, verDistance, myRank, numProcs); assert(ghostOwner != -1); assert(ghostOwner != myRank); ghostOwner = findOwnerOfGhost(v, verDistance, myRank, numProcs); assert(ghostOwner != -1); assert(ghostOwner != myRank);
MPI_Bsend(&Message[0], 3, TypeMap<MilanLongInt>(), ghostOwner, ComputeTag, comm); MPI_Bsend(&Message[0], 3, TypeMap<MilanLongInt>(), ghostOwner, ComputeTag, comm);
msgInd++; msgActual++; msgInd++;
msgActual++;
#ifdef DEBUG_GHOST_ #ifdef DEBUG_GHOST_
if ((u<StartIndex) || (u>EndIndex)) { if ((u<StartIndex) || (u>EndIndex)) {
cout<<"\n("<<myRank<<") "<<__LINE__<<" From Send: should not happen: u= "<<u<<" v= "<<v<< cout<<"\n("<<myRank<<") "<<__LINE__<<" From Send: should not happen: u= "<<u<<" v= "<<v<<

2
samples/advanced/pdegen/runs/amg_pde3d.inp
Unescape Escape View File

@ -1,6 +1,6 @@
%%%%%%%%%%% General arguments % Lines starting with % are ignored. %%%%%%%%%%% General arguments % Lines starting with % are ignored.
CSR ! Storage format CSR COO JAD CSR ! Storage format CSR COO JAD
0123 ! IDIM; domain size. Linear system size is IDIM**3 00030 ! IDIM; domain size. Linear system size is IDIM**3
CONST ! PDECOEFF: CONST, EXP, GAUSS Coefficients of the PDE CONST ! PDECOEFF: CONST, EXP, GAUSS Coefficients of the PDE
BICGSTAB ! Iterative method: BiCGSTAB BiCGSTABL BiCG CG CGS FCG GCR RGMRES BICGSTAB ! Iterative method: BiCGSTAB BiCGSTABL BiCG CG CGS FCG GCR RGMRES
2 ! ISTOPC 2 ! ISTOPC

Write Preview
Loading…
Cancel
Save