\section{Communication routines}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%       HALO DATA COMMUNICATION 
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The routines in this chapter implement various global communication operators
on vectors associated with a discretization mesh. For auxiliary communication
routines not tied to a discretization space see~\ref{sec:toolsrout}.

\subroutine{psb\_halo}{Halo Data Communication}    

These subroutines gathers the values of the halo
elements, and  (optionally) scale the result:

\[ x \leftarrow \alpha x \]
where:
\begin{description}
\item[$x$] is a global dense  submatrix.
\end{description}

\begin{table}[h]
\begin{center}
\begin{tabular}{ll}
\hline
$\alpha$, $x$ & {\bf Subroutine}\\
\hline
Long Precision Real & psb\_halo \\
Long Precision Complex & psb\_halo \\
\hline
\end{tabular}
\end{center}
\caption{Data types\label{tab:f90halo}}
\end{table}

\syntax{CALL psb\_halo}{x, desc\_a, info}
\syntax*{CALL psb\_halo}{x, desc\_a, info, alpha, work}

\begin{description}
\item[\bf On Entry]
\item[x] global dense matrix $x$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
Specified as:  a rank one or two array with the TARGET attribute
containing numbers of type specified in
Table~\ref{tab:f90halo}.
\item[desc\_a] contains data structures for communications.\\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a structured data of type \descdata.
\item[alpha] the scalar $\alpha$.\\
Scope: {\bf global} \\
Type: {\bf optional} \\
Default: $alpha = 1 $\\	
Specified as: a number of the data type indicated in Table~\ref{tab:f90halo}.
\item[work] the work array. \\
Scope: {\bf local} \\
Type: {\bf optional}\\
Specified as: a rank one array of the same type of $x$ with the
POINTER attribute.

\item[\bf On Return] 
\item[x] global dense result matrix $x$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
Returned as:  a rank one or two array 
containing numbers of type specified in
Table~\ref{tab:f90halo}.
\item[info] the local portion of result submatrix $y$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
An integer value that contains an error code. 
\end{description}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%       OVERLAP UPDATE
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\subroutine{psb\_ovrl}{Overlap Update}    

These subroutines applies an overlap operator to the input vector:

\[ x \leftarrow Q x \]
where:
\begin{description}
\item[$x$] is the global dense submatrix $x$
\item[$Q$] is the overlap operator; it is the composition of two
operators $ P_a$ and $ P^{T}$. 
\end{description}

\begin{table}[h]
\begin{center}
\begin{tabular}{ll}
\hline
$x$ & {\bf Subroutine}\\
\hline
Long Precision Real & psb\_ovrl \\
Long Precision Complex & psb\_ovrl \\
\hline
\end{tabular}
\end{center}
\caption{Data types\label{tab:f90ovrl}}
\end{table}

\syntax{CALL psb\_ovrl}{x, desc\_a, info}
\syntax*{CALL psb\_ovrl}{x, desc\_a, info, update=update\_type, work=work} 

\begin{description}
\item[\bf On Entry]
\item[x] global dense matrix $x$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
Specified as:  a rank one or two array 
containing numbers of type specified in
Table~\ref{tab:f90ovrl}.
\item[desc\_a] contains data structures for communications.\\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a structured data of type \descdata.
\item[update] Update operator. \\
\begin{description}
\item[update = psb\_none\_] Do nothing;
\item[update = psb\_add\_] Sum overlap entries, i.e. apply $P^T$;
\item[update = psb\_avg\_] Average overlap entries, i.e. apply $P_aP^T$;
%% \item[update = psb\_square\_root\_] square root update $\sqrt{P_a}$;
\end{description}
Scope: {\bf global} \\
Default: $update\_type = psb\_avg\_ $\\	
Scope: {\bf global} \\
Specified as: a integer variable.
\item[work] the work array. \\
Scope: {\bf local} \\
Type: {\bf optional}\\
Specified as: a one dimensional array of the same type of $x$.

\item[\bf On Return] 
\item[x] global dense result matrix $x$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
Specified as: an  array of rank one or two
containing numbers of type specified in
Table~\ref{tab:f90ovrl}.
\item[info] the local portion of result submatrix $y$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
An integer value that contains an error code. 
\end{description}


\section*{Usage notes}
\begin{enumerate}
\item If there is no overlap in the data distribution associated with
  the descriptor, no operations are performed;
\item The operator $P^{T}$ performs the reduction sum of overlap
elements; it is a ``prolongation'' operator $P^T$ that
replicates overlap elements, accounting for the physical replication
of data;
\item The operator $P_a$ performs a scaling on the overlap elements by
the amount of replication; thus, when combined with the reduction
operator, it implements the average of replicated elements over all of
their instances. 
%% \item The square root update option makes it possible to applythe
%% following operator: 
%% \[ x\leftarrow \sqrt{P_a} P^{T} K^{-1} P \sqrt{P_a} x\]
%% In the case of a symmetric $K$, this preserves simmetry of the overall
%% preconditioner, which would otherwise be destroyed. 
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%      GATHER GLOBAL DENSE MATRIX
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subroutine{psb\_gather}{Gather Global Dense Matrix}

These subroutines collect the portions of global dense matrix
distributed over all process into one single array stored on one
process.

\[ glob\_x \leftarrow collect(loc\_x_i) \]
where:
\begin{description}
\item[$glob\_x$] is the global submatrix $glob\_x_{iy:iy+m-1,jy:jy+n-1}$
\item[$loc\_x_i$] is the local portion of global dense matrix on
process $i$.
\item[$collect$] is the collect function.
\end{description}

\begin{table}[h]
\begin{center}
\begin{tabular}{ll}
\hline
$x_i, y$ & {\bf Subroutine}\\
\hline
Long Precision Real & psb\_gather \\
Long Precision Complex & psb\_gather \\
\hline
\end{tabular}
\end{center}
\caption{Data types\label{tab:gather}}
\end{table}

\syntax{call psb\_gather}{glob\_x, loc\_x, desc\_a, info, 
  root, iglobx, jglobx, ilocx, jlocx, k}
\syntax{call psb\_gather}{glob\_x, loc\_x, desc\_a, info, 
  root, iglobx, ilocx}

\begin{description}
\item[\bf On Entry]
\item[loc\_x] the local portion of global dense matrix
$glob\_x$. \\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a rank one or two array containing numbers of the type
indicated in Table~\ref{tab:gather}.
\item[desc\_a] contains data structures for communications.\\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a structured data of type \descdata.
\item[root]  The process that holds the global copy. If $root=-1$ all
  the processes will have a copy of the global vector.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable $0\le ix\le np$. 
\item[iglobx]  Row index to define a submatrix in glob\_x into which
  gather the local pieces.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable $1\le ix\le matrix\_data(psb\_m\_)$. 
\item[jglobx]  Column index to define a submatrix in glob\_x into which
  gather the local pieces.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[ilocx]  Row index to define a submatrix in loc\_x that has to
  be gathered into glob\_x.\\
Scope: {\bf local} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[jlocx]  Columns index to define a submatrix in loc\_x that has
  to be gathered into glob\_x.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable.
\item[k]  The number of columns to gather.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[\bf On Return] 
\item[glob\_x] The array where the local parts must be gathered.\\
Scope: {\bf global} \\
Type: {\bf required}\\
Specified as: a rank one or two array.
\item[info] the local portion of result submatrix $y$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
An integer value that contains an error code. 
\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%      SCATTER GLOBAL DENSE MATRIX
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subroutine{psb\_scatter}{Scatter Global Dense Matrix}

These subroutines scatters the portions of global dense matrix owned
by a process to all the processes in the processes grid.

\[ loc\_x_i \leftarrow scatter(glob\_x_i) \]
where:
\begin{description}
\item[$glob\_x$] is the global submatrix $glob\_x_{iy:iy+m-1,jy:jy+n-1}$
\item[$loc\_x_i$] is the local portion of global dense matrix on
process $i$.
\item[$scatter$] is the scatter function.
\end{description}

\begin{table}[h]
\begin{center}
\begin{tabular}{ll}
\hline
$x_i, y$ & {\bf Subroutine}\\
\hline
Long Precision Real & psb\_scatter \\
Long Precision Complex & psb\_scatter \\
\hline
\end{tabular}
\end{center}
\caption{Data types\label{tab:scatter}}
\end{table}

\syntax{call psb\_scatter}{glob\_x, loc\_x, desc\_a, info, 
  root, iglobx, jglobx, ilocx, jlocx, k}
\syntax{call psb\_scatter}{glob\_x, loc\_x, desc\_a, info, 
  root, iglobx, ilocx}

\begin{description}
\item[\bf On Entry]
\item[glob\_x] The array that must be scattered into local pieces.\\
Scope: {\bf global} \\
Type: {\bf required}\\
Specified as: a rank one or two array.
\item[desc\_a] contains data structures for communications.\\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a structured data of type \descdata.
\item[root]  The process that holds the global copy. If $root=-1$ all
  the processes have a copy of the global vector.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable $0\le ix\le np$. 
\item[iglobx]  Row index to define a submatrix in glob\_x that has to
  be scattered into local pieces.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable $1\le ix\le matrix\_data(psb\_m\_)$. 
\item[jglobx]  Column index to define a submatrix in glob\_x that has to
  be scattered into local pieces.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[ilocx]  Row index to define a submatrix in loc\_x into which
  scatter the local piece of glob\_x.\\
Scope: {\bf local} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[jlocx]  Columns index to define a submatrix in loc\_x into which
  scatter the local piece of glob\_x.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable.
\item[k]  The number of columns to scatter.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[\bf On Return] 
\item[loc\_x] the local portion of global dense matrix
$glob\_x$. \\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a rank one or two array containing numbers of the type
indicated in Table~\ref{tab:scatter}.
\item[info] the local portion of result submatrix $y$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
An integer value that contains an error code. 
\end{description}

%%% Local Variables: 
%%% mode: latex
%%% TeX-master: "userguide"
%%% End: