Fixed race condition in processExposedVertices

amgprec/impl/aggregator/processExposedVertex.cpp

@@ -32,14 +32,14 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer,
 {
 
     MilanLongInt v = -1, k = -1, w = -1, adj11 = 0, adj12 = 0, k1 = 0;
-    MilanInt ghostOwner = 0;
+    MilanInt ghostOwner = 0, option;
 
-#pragma omp parallel private(k, w, v, k1, adj11, adj12, ghostOwner) firstprivate(privateU, StartIndex, EndIndex, privateQLocalVtx, privateQGhostVtx, privateQMsgType, privateQOwner) default(shared) num_threads(NUM_THREAD)
+#pragma omp parallel private(option, k, w, v, k1, adj11, adj12, ghostOwner) firstprivate(privateU, StartIndex, EndIndex, privateQLocalVtx, privateQGhostVtx, privateQMsgType, privateQOwner) default(shared) num_threads(NUM_THREAD)
     {
-#pragma omp for reduction(+ \
+#pragma omp for reduction(+ : PCounter[:numProcs]) schedule(static)
-                          : PCounter[:numProcs]) schedule(static)
         for (v = 0; v < NLVer; v++)
         {
+            option = -1;
             // Start: PARALLEL_PROCESS_EXPOSED_VERTEX_B(v)
             k = candidateMate[v];
             candidateMate[v] = verLocInd[k];
@@ -58,6 +58,8 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer,
             if (w >= 0)
             {
 
+#pragma omp critical(processExposed)
+                {
                     if (isAlreadyMatched(verLocInd[k], StartIndex, EndIndex, GMate, Mate, Ghost2LocalMap))
                     {
                         w = computeCandidateMate(verLocPtr[v],
@@ -78,42 +80,14 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer,
                         (*myCard)++;
                         if ((w < StartIndex) || (w > EndIndex))
                         { // w is a ghost vertex
-#ifdef PRINT_DEBUG_INFO_
+                            option = 2;
-                        cout << "\n(" << myRank << ")Sending a request message (291):";
-                        cout << "\n(" << myRank << ")Local is: " << v + StartIndex << " Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs) << endl;
-                        fflush(stdout);
-#endif
-#pragma omp atomic
-                        (*msgInd)++;
-#pragma omp atomic
-                        (*NumMessagesBundled)++;
-                        ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs);
-                        assert(ghostOwner != -1);
-                        assert(ghostOwner != myRank);
-                        PCounter[ghostOwner]++;
-
-                        privateQLocalVtx.push_back(v + StartIndex);
-                        privateQGhostVtx.push_back(w);
-                        privateQMsgType.push_back(REQUEST);
-                        privateQOwner.push_back(ghostOwner);
 
                             if (candidateMate[NLVer + Ghost2LocalMap[w]] == v + StartIndex)
                             {
-
+                                option = 1;
-                            privateU.push_back(v + StartIndex);
-                            privateU.push_back(w);
                                 Mate[v] = w;
-                            // FIXME could this instruction create errors?
                                 GMate[Ghost2LocalMap[w]] = v + StartIndex; // w is a Ghost
 
-#ifdef PRINT_DEBUG_INFO_
-                            cout << "\n(" << myRank << ")MATCH: (" << v + StartIndex << "," << w << ")";
-                            fflush(stdout);
-#endif
-
-                            // TODO refactor this!!
-                            //  Decrement the counter:
-                            PROCESS_CROSS_EDGE(&Counter[Ghost2LocalMap[w]], S);
                             } // End of if CandidateMate[w] = v
 
                         } // End of if a Ghost Vertex
@@ -122,11 +96,8 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer,
 
                             if (candidateMate[w - StartIndex] == (v + StartIndex))
                             {
-                            privateU.push_back(v + StartIndex);
+                                option = 3;
-                            privateU.push_back(w);
-
                                 Mate[v] = w;                           // v is local
-                            // FIXME this instruction could create errors
                                 Mate[w - StartIndex] = v + StartIndex; // w is local
 
 #ifdef PRINT_DEBUG_INFO_
@@ -137,13 +108,14 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer,
                             } // End of if ( candidateMate[w-StartIndex] == (v+StartIndex) )
                         }     // End of Else
 
-                    continue;
                     } // End of second if
 
-            } // End of if(w >=0)
+                } // End critical processExposed
 
-            // This piece of code is executed a really small amount of times, I will not allocate a
+            } // End of if(w >=0)
-            // huge amount of memory for the private data structures.
+            else
+            {
+                // This piece of code is executed a really small amount of times
                 adj11 = verLocPtr[v];
                 adj12 = verLocPtr[v + 1];
                 for (k1 = adj11; k1 < adj12; k1++)
@@ -173,7 +145,51 @@ void PARALLEL_PROCESS_EXPOSED_VERTEX_B(MilanLongInt NLVer,
 
                     } // End of if(GHOST)
                 }     // End of for loop
+            }
             // End:   PARALLEL_PROCESS_EXPOSED_VERTEX_B(v)
+
+            switch (option)
+            {
+            case -1:
+                break;
+            case 1:
+                privateU.push_back(v + StartIndex);
+                privateU.push_back(w);
+
+#ifdef PRINT_DEBUG_INFO_
+                cout << "\n(" << myRank << ")MATCH: (" << v + StartIndex << "," << w << ")";
+                fflush(stdout);
+#endif
+
+                //  Decrement the counter:
+                PROCESS_CROSS_EDGE(&Counter[Ghost2LocalMap[w]], S);
+            case 2:
+#ifdef PRINT_DEBUG_INFO_
+                cout << "\n(" << myRank << ")Sending a request message (291):";
+                cout << "\n(" << myRank << ")Local is: " << v + StartIndex << " Ghost is " << w << " Owner is: " << findOwnerOfGhost(w, verDistance, myRank, numProcs) << endl;
+                fflush(stdout);
+#endif
+#pragma omp atomic
+                (*msgInd)++;
+#pragma omp atomic
+                (*NumMessagesBundled)++;
+                ghostOwner = findOwnerOfGhost(w, verDistance, myRank, numProcs);
+                assert(ghostOwner != -1);
+                assert(ghostOwner != myRank);
+                PCounter[ghostOwner]++;
+
+                privateQLocalVtx.push_back(v + StartIndex);
+                privateQGhostVtx.push_back(w);
+                privateQMsgType.push_back(REQUEST);
+                privateQOwner.push_back(ghostOwner);
+                break;
+            case 3:
+            default:
+                privateU.push_back(v + StartIndex);
+                privateU.push_back(w);
+                break;
+            }
+
         } // End of for ( v=0; v < NLVer; v++ )
 
         queuesTransfer(U, privateU, QLocalVtx,