\section{Communication routines}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%       HALO DATA COMMUNICATION 
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\subroutine{psb\_halo}{Halo Data Communication}    

These subroutines restore a consistent status for the halo
elements, and  (optionally) scale the result:

\[ x \leftarrow \alpha x \]
where:
\begin{description}
\item[$x$] is a global dense  submatrix.
\end{description}

\begin{table}[h]
\begin{center}
\begin{tabular}{ll}
\hline
$\alpha$, $x$ & {\bf Subroutine}\\
\hline
Single Precision Real & psb\_halo\\
Long Precision Real & psb\_halo \\
Long Precision Complex & psb\_halo \\
\hline
\end{tabular}
\end{center}
\caption{Data types\label{tab:f90halo}}
\end{table}

\syntax{CALL psb\_halo}{x, desc\_a, info}
\syntax*{CALL psb\_halo}{x, desc\_a, info, alpha, work}

\begin{description}
\item[\bf On Entry]
\item[x] global dense matrix $x$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
Specified as:  a rank one or two array with the POINTER attribute
containing numbers of type specified in
Table~\ref{tab:f90halo}.
\item[desc\_a] contains data structures for communications.\\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a structured data of type \descdata.
\item[alpha] the scalar $\alpha$.\\
Scope: {\bf global} \\
Type: {\bf optional} \\
Default: $alpha = 1 $\\	
Specified as: a number of the data type indicated in Table~\ref{tab:f90halo}.
\item[work] the work array. \\
Scope: {\bf local} \\
Type: {\bf optional}\\
Specified as: a rank one array of the same type of $x$ with the
POINTER attribute.

\item[\bf On Return] 
\item[x] global dense result matrix $x$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
Returned as:  a rank one or two array with the POINTER attribute
containing numbers of type specified in
Table~\ref{tab:f90halo}.
\item[info] the local portion of result submatrix $y$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
An integer value that contains an error code. 
\end{description}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%       OVERLAP UPDATE
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\subroutine{psb\_ovrl}{Overlap Update}    

These subroutines restore a consistent status for the overlap
elements:

\[ x \leftarrow Q x \]
where:
\begin{description}
\item[$x$] is the global dense submatrix $x$
\item[$Q$] is the overlap operator; it is the composition of two
operators $ P_a$ and $ P^{T}$. 
\end{description}

\begin{table}[h]
\begin{center}
\begin{tabular}{ll}
\hline
$x$ & {\bf Subroutine}\\
\hline
Single Precision Real & psb\_ovrl\\
Long Precision Real & psb\_ovrl \\
Long Precision Complex & psb\_ovrl \\
\hline
\end{tabular}
\end{center}
\caption{Data types\label{tab:f90ovrl}}
\end{table}

\syntax{CALL psb\_ovrl}{x, desc\_a, info}
\syntax*{CALL psb\_ovrl}{x, desc\_a, info, choice=choice,
update\_type=update\_type, work=work} 

\begin{description}
\item[\bf On Entry]
\item[x] global dense matrix $x$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
Specified as:  a rank one or two array with the POINTER attribute
containing numbers of type specified in
Table~\ref{tab:f90ovrl}.
\item[desc\_a] contains data structures for communications.\\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a structured data of type \descdata.
\item[choice] specify if exchange overlap elements.
\begin{description}
\item[choice = .true.] exchange overlap elements, i.e. apply operator
$P^{T}$;
\item[choice = .false.] don't exchange overlap elements
\end{description}
Scope: {\bf global} \\
Type: {\bf optional} \\
Default: $choice = .true. $\\	
Specified as: a logical variable.
\begin{description}
\item[update\_type = 1] normal update $P_a$;
\item[update\_type = 2] square root update $\sqrt{P_a}$;
\end{description}
Scope: {\bf global} \\
Default: $update\_type = .true. $\\	
Scope: {\bf global} \\
Specified as: a integer variable.
\item[work] the work array. \\
Scope: {\bf local} \\
Type: {\bf optional}\\
Specified as: a one dimensional array of the same type of $x$.

\item[\bf On Return] 
\item[x] global dense result matrix $x$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
Specified as: a pointer to array of rank one or two
containing numbers of type specified in
Table~\ref{tab:f90ovrl}.
\item[info] the local portion of result submatrix $y$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
An integer value that contains an error code. 
\end{description}


\section*{Usage notes}
\begin{enumerate}
\item If there is no overlap in the data distribution, no operations
are performed;
\item The operator $P^{T}$ performs the reduction sum of overlap
elements; it is the inverse of a ``stretch'' operator $P$ that
replicates overlap elements, accounting for the physical replication
of data;
\item The operator $P_a$ performs a scaling on the overlap elements by
the amount of replication; thus, when combined with the reduction
operator, it implements the average of replicated elements over all of
their instances. 
\item The square root update option makes it possible to applythe
following operator: 
\[ x\leftarrow \sqrt{P_a} P^{T} K^{-1} P \sqrt{P_a} x\]
In the case of a symmetric $K$, this preserves simmetry of the overall
preconditioner, which would otherwise be destroyed. 
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%      GATHER GLOBAL DENSE MATRIX
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subroutine{psb\_gather}{Gather Global Dense Matrix}

These subroutines collect the portions of global dense matrix
distributed over all process into one single array stored on one
process.

\[ glob\_x \leftarrow collect(loc\_x_i) \]
where:
\begin{description}
\item[$glob\_x$] is the global submatrix $glob\_x_{iy:iy+m-1,jy:jy+n-1}$
\item[$loc\_x_i$] is the local portion of global dense matrix on
process $i$.
\item[$collect$] is the collect function.
\end{description}

\begin{table}[h]
\begin{center}
\begin{tabular}{ll}
\hline
$x_i, y$ & {\bf Subroutine}\\
\hline
Single Precision Real & psb\_gather\\
Long Precision Real & psb\_gather \\
Long Precision Complex & psb\_gather \\
\hline
\end{tabular}
\end{center}
\caption{Data types\label{tab:gather}}
\end{table}

\syntax{call psb\_gather}{glob\_x, loc\_x, desc\_a, info, 
  root, iglobx, jglobx, ilocx, jlocx, k}
\syntax{call psb\_gather}{glob\_x, loc\_x, desc\_a, info, 
  root, iglobx, ilocx}

\begin{description}
\item[\bf On Entry]
\item[loc\_x] the local portion of global dense matrix
$glob\_x$. \\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a rank one or two array containing numbers of the type
indicated in Table~\ref{tab:gather}.
\item[desc\_a] contains data structures for communications.\\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a structured data of type \descdata.
\item[root]  The process that holds the global copy. If $root=-1$ all
  the processes will have a copy of the global vector.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable $0\le ix\le np$. 
\item[iglobx]  Row index to define a submatrix in glob\_x into which
  gather the local pieces.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable $1\le ix\le matrix\_data(psb\_m\_)$. 
\item[jglobx]  Column index to define a submatrix in glob\_x into which
  gather the local pieces.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[ilocx]  Row index to define a submatrix in loc\_x that has to
  be gathered into glob\_x.\\
Scope: {\bf local} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[jlocx]  Columns index to define a submatrix in loc\_x that has
  to be gathered into glob\_x.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable.
\item[k]  The number of columns to gather.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[\bf On Return] 
\item[glob\_x] The array where the local parts must be gathered.\\
Scope: {\bf global} \\
Type: {\bf required}\\
Specified as: a rank one or two array.
\item[info] the local portion of result submatrix $y$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
An integer value that contains an error code. 
\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%      SCATTER GLOBAL DENSE MATRIX
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subroutine{psb\_scatter}{Scatter Global Dense Matrix}

These subroutines scatters the portions of global dense matrix owned
by a process to all the processes in the processes grid.

\[ loc\_x_i \leftarrow scatter(glob\_x_i) \]
where:
\begin{description}
\item[$glob\_x$] is the global submatrix $glob\_x_{iy:iy+m-1,jy:jy+n-1}$
\item[$loc\_x_i$] is the local portion of global dense matrix on
process $i$.
\item[$scatter$] is the scatter function.
\end{description}

\begin{table}[h]
\begin{center}
\begin{tabular}{ll}
\hline
$x_i, y$ & {\bf Subroutine}\\
\hline
Single Precision Real & psb\_scatter\\
Long Precision Real & psb\_scatter \\
Long Precision Complex & psb\_scatter \\
\hline
\end{tabular}
\end{center}
\caption{Data types\label{tab:scatter}}
\end{table}

\syntax{call psb\_scatter}{glob\_x, loc\_x, desc\_a, info, 
  root, iglobx, jglobx, ilocx, jlocx, k}
\syntax{call psb\_scatter}{glob\_x, loc\_x, desc\_a, info, 
  root, iglobx, ilocx}

\begin{description}
\item[\bf On Entry]
\item[glob\_x] The array that must be scattered into local pieces.\\
Scope: {\bf global} \\
Type: {\bf required}\\
Specified as: a rank one or two array.
\item[desc\_a] contains data structures for communications.\\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a structured data of type \descdata.
\item[root]  The process that holds the global copy. If $root=-1$ all
  the processes have a copy of the global vector.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable $0\le ix\le np$. 
\item[iglobx]  Row index to define a submatrix in glob\_x that has to
  be scattered into local pieces.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable $1\le ix\le matrix\_data(psb\_m\_)$. 
\item[jglobx]  Column index to define a submatrix in glob\_x that has to
  be scattered into local pieces.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[ilocx]  Row index to define a submatrix in loc\_x into which
  scatter the local piece of glob\_x.\\
Scope: {\bf local} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[jlocx]  Columns index to define a submatrix in loc\_x into which
  scatter the local piece of glob\_x.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable.
\item[k]  The number of columns to scatter.\\
Scope: {\bf global} \\
Type: {\bf optional}\\
Specified as: an integer variable. 
\item[\bf On Return] 
\item[loc\_x] the local portion of global dense matrix
$glob\_x$. \\
Scope: {\bf local} \\
Type: {\bf required}\\
Specified as: a rank one or two array containing numbers of the type
indicated in Table~\ref{tab:scatter}.
\item[info] the local portion of result submatrix $y$.\\
Scope: {\bf local} \\
Type: {\bf required} \\
An integer value that contains an error code. 
\end{description}

%%% Local Variables: 
%%% mode: latex
%%% TeX-master: "userguide"
%%% End: