You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
amg4psblas/examples/gpu/amg_dexample_gpu.f90

354 lines
11 KiB
Fortran

4 years ago
!
!
! AMG4PSBLAS version 1.0
! Algebraic Multigrid Package
! based on PSBLAS (Parallel Sparse BLAS version 3.7)
!
! (C) Copyright 2021
!
! Salvatore Filippone
! Pasqua D'Ambra
! Fabio Durastante
!
! Redistribution and use in source and binary forms, with or without
! modification, are permitted provided that the following conditions
! are met:
! 1. Redistributions of source code must retain the above copyright
! notice, this list of conditions and the following disclaimer.
! 2. Redistributions in binary form must reproduce the above copyright
! notice, this list of conditions, and the following disclaimer in the
! documentation and/or other materials provided with the distribution.
! 3. The name of the AMG4PSBLAS group or the names of its contributors may
! not be used to endorse or promote products derived from this
! software without specific written permission.
!
! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
! ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
! TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
! PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AMG4PSBLAS GROUP OR ITS CONTRIBUTORS
! BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
! CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
! SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
! INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
! CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
! ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
! POSSIBILITY OF SUCH DAMAGE.
!
!
! File: amg_dexample_gpu.f90
!
! This sample program solves a linear system obtained by discretizing a
! PDE with Dirichlet BCs. The solver is CG, coupled with one of the
! following multi-level preconditioner, as explained in Section 4.1 of
! the AMG4PSBLAS User's and Reference Guide:
!
! - choice = 1, the default multi-level preconditioner solver, i.e.,
! V-cycle with decoupled smoothed aggregation, 1 hybrid forward/backward
! GS sweep as pre/post-smoother and UMFPACK as coarsest-level
! solver (Sec. 4.1, Listing 1)
!
! - choice = 2, a V-cycle preconditioner with 1 block-Jacobi sweep
! (with ILU(0) on the blocks) as pre- and post-smoother, and 8 block-Jacobi
! sweeps (with ILU(0) on the blocks) as coarsest-level solver (Sec. 4.1, Listing 2)
!
! - choice = 3, W-cycle preconditioner based on the coupled aggregation relying
! on matching, with maximum size of aggregates equal to 8 and smoothed prolongators,
! 2 hybrid forward/backward GS sweeps as pre/post-smoother, a distributed coarsest
! matrix, and preconditioned Flexible Conjugate Gradient as coarsest-level solver
! (Sec. 4.1, Listing 3)
!
! The matrix and the rhs are read from files (if an rhs is not available, the
! unit rhs is set).
!
!
! The PDE is a general second order equation in 3d
!
! a1 dd(u) a2 dd(u) a3 dd(u) b1 d(u) b2 d(u) b3 d(u)
! - ------ - ------ - ------ + ----- + ------ + ------ + c u = f
! dxdx dydy dzdz dx dy dz
!
! with Dirichlet boundary conditions
! u = g
!
! on the unit cube 0<=x,y,z<=1.
!
!
! Note that if b1=b2=b3=c=0., the PDE is the Laplace equation.
!
! In this sample program the index space of the discretized
! computational domain is first numbered sequentially in a standard way,
! then the corresponding vector is distributed according to a BLOCK
! data distribution.
!
program amg_dexample_gpu
use psb_base_mod
use amg_prec_mod
use psb_krylov_mod
use psb_util_mod
use psb_gpu_mod
use data_input
use amg_d_pde_mod
implicit none
! input parameters
! sparse matrices
type(psb_dspmat_type) :: A
! sparse matrices descriptor
type(psb_desc_type):: desc_A
! preconditioner
type(amg_dprec_type) :: P
! right-hand side, solution and residual vectors
type(psb_d_vect_type) :: x, b, r
! GPU variables
type(psb_d_hlg_sparse_mat) :: agmold
type(psb_d_vect_gpu) :: vgmold
type(psb_i_vect_gpu) :: igmold
! solver and preconditioner parameters
real(psb_dpk_) :: tol, err
integer :: itmax, iter, istop
integer :: nlev
! parallel environment parameters
type(psb_ctxt_type) :: ctxt
integer :: iam, np
! other variables
integer :: choice
integer :: i,info,j
integer(psb_epk_) :: amatsize, precsize, descsize
integer(psb_epk_) :: system_size
integer :: idim, ierr, ircode
real(psb_dpk_) :: resmx, resmxp
real(psb_dpk_) :: t1, t2, tprec
character(len=5) :: afmt='CSR'
character(len=20) :: name, kmethod
! initialize the parallel environment
call psb_init(ctxt)
call psb_info(ctxt,iam,np)
!
! BEWARE: if you have NGPUS per node, the default is to
! attach to mod(IAM,NGPUS)
!
call psb_gpu_init(ictxt)
if (iam < 0) then
! This should not happen, but just in case
call psb_exit(ctxt)
stop
endif
name='amg_dexample_gpu'
if(psb_get_errstatus() /= 0) goto 9999
info=psb_success_
call psb_set_errverbosity(2)
!
! Hello world
!
if (iam == psb_root_) then
write(*,*) 'Welcome to AMG4PSBLAS version: ',amg_version_string_
write(*,*) 'This is the ',trim(name),' sample program'
end if
write(*,*) 'Process ',iam,' running on device: ', psb_cuda_getDevice(),' out of', psb_cuda_getDeviceCount()
write(*,*) 'Process ',iam,' device ', psb_cuda_getDevice(),' is a: ', trim(psb_gpu_DeviceName())
! get parameters
call get_parms(ctxt,choice,idim,itmax,tol)
! allocate and fill in the coefficient matrix, rhs and initial guess
call psb_barrier(ctxt)
t1 = psb_wtime()
call amg_gen_pde3d(ctxt,idim,a,b,x,desc_a,afmt,&
& a1,a2,a3,b1,b2,b3,c,g,info)
call psb_barrier(ctxt)
t2 = psb_wtime() - t1
if(info /= psb_success_) then
info=psb_err_from_subroutine_
call psb_errpush(info,name)
goto 9999
end if
if (iam == psb_root_) write(*,'("Overall matrix creation time : ",es12.5)')t2
if (iam == psb_root_) write(*,'(" ")')
select case(choice)
case(1)
! initialize a V-cycle preconditioner with 4 Jacobi sweep
! and 8 Jacobi sweeps as coarsest-level solver
call P%init(ctxt,'ML',info)
call P%set('SMOOTHER_TYPE','JACOBI',info)
call P%set('SMOOTHER_SWEEPS',4,info)
call P%set('COARSE_SOLVE','JACOBI',info)
call P%set('COARSE_SWEEPS',8,info)
kmethod = 'CG'
case(2)
! initialize a V-cycle preconditioner based on the coupled aggregation relying on matching,
! with maximum size of aggregates equal to 8 and smoothed prolongators,
! Block-Jacobi smoother using approximate inverse INVK and
! and 4 sweeps of INVK on he coarsest level
call P%init(ctxt,'ML',info)
call P%set('PAR_AGGR_ALG','COUPLED',info)
call P%set('AGGR_TYPE','MATCHBOXP',info)
call P%set('AGGR_SIZE',8,info)
call P%set('ML_CYCLE','WCYCLE',info)
call P%set('SMOOTHER_SWEEPS',2,info)
call P%set('SUB_SOLVE','INVK',info)
call P%set('COARSE_SOLVE','INVK',info)
call P%set('COARSE_MAT','DIST',info)
kmethod = 'CG'
end select
call psb_barrier(ctxt)
t1 = psb_wtime()
! build the preconditioner
call P%hierarchy_build(A,desc_A,info)
call P%smoothers_build(A,desc_A,info, amold=agmold, vmold=vgmold, imold=igmold)
tprec = psb_wtime()-t1
call psb_amx(ctxt, tprec)
if (info /= psb_success_) then
call psb_errpush(psb_err_from_subroutine_,name,a_err='amg_precbld')
goto 9999
end if
! set the solver parameters and the initial guess
call psb_geall(x,desc_A,info)
call x%zero()
call psb_geasb(x,desc_A,info)
! Convert A, DESC_A,X,B to a GPU-enabled format
call desc_a%cnv(mold=igmold)
call a%cscnv(info,mold=agmold)
call psb_geasb(x,desc_a,info,mold=vgmold)
call psb_geasb(b,desc_a,info,mold=vgmold)
! solve Ax=b with preconditioned Krylov method
call psb_barrier(ctxt)
call prec%allocate_wrk(info)
t1 = psb_wtime()
call psb_krylov(kmethod,A,P,b,x,tol,desc_A,info,itmax,iter,err,itrace=1,istop=2)
t2 = psb_wtime() - t1
call psb_amx(ctxt,t2)
call prec%deallocate_wrk(info)
call psb_geall(r,desc_A,info)
call r%zero()
call psb_geasb(r,desc_A,info)
call psb_geaxpby(done,b,dzero,r,desc_A,info)
call psb_spmm(-done,A,x,done,r,desc_A,info)
resmx = psb_genrm2(r,desc_A,info)
resmxp = psb_geamax(r,desc_A,info)
amatsize = a%sizeof()
descsize = desc_a%sizeof()
precsize = p%sizeof()
system_size = desc_a%get_global_rows()
call psb_sum(ctxt,amatsize)
call psb_sum(ctxt,descsize)
call psb_sum(ctxt,precsize)
call P%descr(info)
if (iam == psb_root_) then
write(*,'(" ")')
write(*,'("Matrix from PDE example")')
write(*,'("Computed solution on ",i8," processors")')np
write(*,'("Linear system size : ",i12)') system_size
write(*,'("Krylov method : ",a)') kmethod
write(*,'("Iterations to convergence : ",i6)')iter
write(*,'("Error estimate on exit : ",es12.5)')err
write(*,'("Time to build prec. : ",es12.5)')tprec
write(*,'("Time to solve system : ",es12.5)')t2
write(*,'("Time per iteration : ",es12.5)')t2/(iter)
write(*,'("Total time : ",es12.5)')t2+tprec
write(*,'("Residual 2-norm : ",es12.5)')resmx
write(*,'("Residual inf-norm : ",es12.5)')resmxp
write(*,'("Total memory occupation for A : ",i12)')amatsize
write(*,'("Total memory occupation for DESC_A : ",i12)')descsize
write(*,'("Total memory occupation for PREC : ",i12)')precsize
end if
call psb_gefree(b, desc_A,info)
call psb_gefree(x, desc_A,info)
call psb_spfree(A, desc_A,info)
call P%free(info)
call psb_cdfree(desc_A,info)
call psb_gpu_exit()
call psb_exit(ctxt)
stop
9999 continue
call psb_error(ctxt)
contains
!
! get parameters from standard input
!
subroutine get_parms(ctxt,choice,idim,itmax,tol)
implicit none
type(psb_ctxt_type) :: ctxt
integer :: choice, idim, itmax
real(psb_dpk_) :: tol
integer :: iam, np, inp_unit
character(len=1024) :: filename
call psb_info(ctxt,iam,np)
if (iam == psb_root_) then
if (command_argument_count()>0) then
call get_command_argument(1,filename)
inp_unit = 30
open(inp_unit,file=filename,action='read',iostat=info)
if (info /= 0) then
write(psb_err_unit,*) 'Could not open file ',filename,' for input'
call psb_abort(ctxt)
stop
else
write(psb_err_unit,*) 'Opened file ',trim(filename),' for input'
end if
else
inp_unit=psb_inp_unit
end if
! read input parameters
call read_data(choice,inp_unit)
call read_data(idim,inp_unit)
call read_data(itmax,inp_unit)
call read_data(tol,inp_unit)
if (inp_unit /= psb_inp_unit) then
close(inp_unit)
end if
end if
call psb_bcast(ctxt,choice)
call psb_bcast(ctxt,idim)
call psb_bcast(ctxt,itmax)
call psb_bcast(ctxt,tol)
end subroutine get_parms
end program amg_dexample_gpu