From 4291f08cc7561ac144c12639970c0842f1a712ba Mon Sep 17 00:00:00 2001 From: Salvatore Filippone Date: Wed, 5 Feb 2020 16:34:06 +0000 Subject: [PATCH] Streamline new smoothed implementation, first step --- .../impl/aggregator/mld_d_spmm_bld_inner.f90 | 185 +++++++++++++++++- .../impl/aggregator/mld_daggrmat_smth_bld.f90 | 56 ++++-- mlprec/mld_d_base_aggregator_mod.f90 | 17 +- 3 files changed, 234 insertions(+), 24 deletions(-) diff --git a/mlprec/impl/aggregator/mld_d_spmm_bld_inner.f90 b/mlprec/impl/aggregator/mld_d_spmm_bld_inner.f90 index 742acf6e..3b0bf582 100644 --- a/mlprec/impl/aggregator/mld_d_spmm_bld_inner.f90 +++ b/mlprec/impl/aggregator/mld_d_spmm_bld_inner.f90 @@ -35,6 +35,186 @@ ! File: mld_daggrmat_nosmth_bld.F90 ! ! +subroutine mld_d_new_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& + & coo_prol,desc_cprol,coo_restr,info) + use psb_base_mod + use mld_d_inner_mod + use mld_d_base_aggregator_mod, mld_protect_name => mld_d_new_spmm_bld_inner + implicit none + + ! Arguments + type(psb_ld_csr_sparse_mat), intent(inout) :: a_csr + type(psb_desc_type), intent(in) :: desc_a + integer(psb_lpk_), intent(inout) :: nlaggr(:) + type(mld_dml_parms), intent(inout) :: parms + type(psb_ld_coo_sparse_mat), intent(inout) :: coo_prol, coo_restr + type(psb_desc_type), intent(inout) :: desc_cprol + type(psb_ldspmat_type), intent(out) :: ac + integer(psb_ipk_), intent(out) :: info + + ! Local variables + integer(psb_ipk_) :: err_act + integer(psb_ipk_) :: ictxt,np,me, icomm, ndx, minfo + character(len=40) :: name + integer(psb_ipk_) :: ierr(5) + type(psb_ld_coo_sparse_mat) :: ac_coo, tmpcoo + type(psb_ld_csr_sparse_mat) :: acsr3, csr_prol, ac_csr, csr_restr + integer(psb_ipk_) :: debug_level, debug_unit, naggr + integer(psb_lpk_) :: nrow, nglob, ncol, ntaggr, nzl, ip, & + & nzt, naggrm1, naggrp1, i, k + integer(psb_lpk_) :: nrsave, ncsave, nzsave, nza, nrpsave, ncpsave, nzpsave + logical, parameter :: do_timings=.true., oldstyle=.false., debug=.true. + integer(psb_ipk_), save :: idx_spspmm=-1 + + name='mld_spmm_bld_inner' + if(psb_get_errstatus().ne.0) return + info=psb_success_ + call psb_erractionsave(err_act) + + + ictxt = desc_a%get_context() + icomm = desc_a%get_mpic() + call psb_info(ictxt, me, np) + debug_unit = psb_get_debug_unit() + debug_level = psb_get_debug_level() + nglob = desc_a%get_global_rows() + nrow = desc_a%get_local_rows() + ncol = desc_a%get_local_cols() + + if ((do_timings).and.(idx_spspmm==-1)) & + & idx_spspmm = psb_get_timer_idx("SPMM_BLD: par_spspmm") + + naggr = nlaggr(me+1) + ntaggr = sum(nlaggr) + naggrm1 = sum(nlaggr(1:me)) + naggrp1 = sum(nlaggr(1:me+1)) + !write(0,*)me,' ',name,' input sizes',nlaggr(:),':',naggr + nrpsave = coo_prol%get_nrows() + ncpsave = coo_prol%get_ncols() + nzpsave = coo_prol%get_nzeros() + write(0,*)me,' ',name,' input coo_prol ',nrpsave,ncpsave,nzpsave + + ! + ! Here COO_PROL should be with GLOBAL indices on the cols + ! and LOCAL indices on the rows. + ! + if (debug) write(0,*) me,' ',trim(name),' Size check on entry New: ',& + & coo_prol%get_fmt(),coo_prol%get_nrows(),coo_prol%get_ncols(),coo_prol%get_nzeros(),& + & nrow,ntaggr,naggr + + call coo_prol%cp_to_fmt(csr_prol,info) + + if (debug) write(0,*) me,trim(name),' Product AxPROL ',& + & a_csr%get_nrows(),a_csr%get_ncols(), csr_prol%get_nrows(), & + & desc_a%get_local_rows(),desc_a%get_local_cols(),& + & desc_cprol%get_local_rows(),desc_a%get_local_cols() + if (debug) flush(0) + + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(a_csr,desc_a,csr_prol,acsr3,desc_cprol,info) + if (do_timings) call psb_toc(idx_spspmm) + + if (debug) write(0,*) me,trim(name),' Done AxPROL ',& + & acsr3%get_nrows(),acsr3%get_ncols(), acsr3%get_nzeros(),& + & desc_cprol%get_local_rows(),desc_cprol%get_local_cols() + + ! + ! Ok first product done. + ! + ! Remember that RESTR must be built from PROL after halo extension, + ! which is done above in psb_par_spspmm + if (debug) write(0,*)me,' ',name,' No inp_restr, transposing prol ',& + & csr_prol%get_nrows(),csr_prol%get_ncols(),csr_prol%get_nzeros() + call csr_prol%cp_to_fmt(coo_restr,info) +!!$ write(0,*)me,' ',name,' new into transposition ',coo_restr%get_nrows(),& +!!$ & coo_restr%get_ncols(),coo_restr%get_nzeros() + call coo_restr%transp() + nzl = coo_restr%get_nzeros() + call desc_cprol%l2gip(coo_restr%ia(1:nzl),info) + i=0 + ! + ! Now we have to fix this. The only rows of the restrictor that are correct + ! are those corresponding to "local" aggregates, i.e. indices in ilaggr(:) + ! + do k=1, nzl + if ((naggrm1 < coo_restr%ia(k)) .and.(coo_restr%ia(k) <= naggrp1)) then + i = i+1 + coo_restr%val(i) = coo_restr%val(k) + coo_restr%ia(i) = coo_restr%ia(k) + coo_restr%ja(i) = coo_restr%ja(k) + end if + end do + call coo_restr%set_nzeros(i) + call coo_restr%fix(info) + call coo_restr%cp_to_coo(tmpcoo,info) +!!$ write(0,*)me,' ',name,' after transposition ',coo_restr%get_nrows(),coo_restr%get_ncols(),coo_restr%get_nzeros() + + if (info /= psb_success_) then + call psb_errpush(psb_err_from_subroutine_,name,a_err='spcnv coo_restr') + goto 9999 + end if + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'starting sphalo/ rwxtd' + nzl = tmpcoo%get_nzeros() + call psb_glob_to_loc(tmpcoo%ia(1:nzl),desc_cprol,info,iact='I',owned=.true.) + call tmpcoo%clean_negidx(info) + nzl = tmpcoo%get_nzeros() + call tmpcoo%set_nrows(desc_cprol%get_local_rows()) + call tmpcoo%set_ncols(desc_a%get_local_cols()) +!!$ write(0,*)me,' ',name,' after G2L on rows ',tmpcoo%get_nrows(),tmpcoo%get_ncols(),tmpcoo%get_nzeros() + call csr_restr%mv_from_coo(tmpcoo,info) + + if (debug) write(0,*) me,trim(name),' Product RESTRxAP ',& + & csr_restr%get_nrows(),csr_restr%get_ncols(), & + & desc_cprol%get_local_rows(),desc_a%get_local_cols(),& + & acsr3%get_nrows(),acsr3%get_ncols() + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(csr_restr,desc_a,acsr3,ac_csr,desc_cprol,info) + if (do_timings) call psb_toc(idx_spspmm) + call csr_restr%free() + call ac_csr%mv_to_coo(ac_coo,info) + nza = ac_coo%get_nzeros() + if (debug) write(0,*) me,trim(name),' Fixing ac ',& + & ac_coo%get_nrows(),ac_coo%get_ncols(), nza + call ac_coo%fix(info) + call desc_cprol%indxmap%l2gip(ac_coo%ia(1:nza),info) + call desc_cprol%indxmap%l2gip(ac_coo%ja(1:nza),info) + call ac_coo%set_nrows(ntaggr) + call ac_coo%set_ncols(ntaggr) + if (debug) write(0,*) me,' ',trim(name),' Before mv_from',psb_get_errstatus() + if (info == 0) call ac%mv_from(ac_coo) + if (debug) write(0,*) me,' ',trim(name),' After mv_from',psb_get_errstatus() + if (debug) write(0,*) me,' ',trim(name),' ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros(),naggr,ntaggr + ! write(0,*) me,' ',trim(name),' Final AC newstyle ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros() + + nza = coo_prol%get_nzeros() + call desc_cprol%indxmap%l2gip(coo_prol%ja(1:nza),info) + + if (debug) then + write(0,*) me,' ',trim(name),' Checkpoint at exit' + call psb_barrier(ictxt) + write(0,*) me,' ',trim(name),' Checkpoint through' + end if + + if (info /= psb_success_) then + call psb_errpush(psb_err_internal_error_,name,a_err='Build ac = coo_restr x am3') + goto 9999 + end if + + + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'Done smooth_aggregate ' + + call psb_erractionrestore(err_act) + return + +9999 call psb_error_handler(err_act) + + return + +end subroutine mld_d_new_spmm_bld_inner subroutine mld_d_spmm_bld_inner(a_csr,desc_a,ilaggr,nlaggr,parms,ac,& & op_prol,op_restr,info) use psb_base_mod @@ -63,7 +243,7 @@ subroutine mld_d_spmm_bld_inner(a_csr,desc_a,ilaggr,nlaggr,parms,ac,& integer(psb_lpk_) :: nrow, nglob, ncol, ntaggr, nzl, ip, & & nzt, naggrm1, naggrp1, i, k integer(psb_lpk_) :: nrsave, ncsave, nzsave, nza, nrpsave, ncpsave, nzpsave - logical, parameter :: do_timings=.true., oldstyle=.false., debug=.false. + logical, parameter :: do_timings=.true., oldstyle=.false., debug=.true. integer(psb_ipk_), save :: idx_spspmm=-1 name='mld_spmm_bld_inner' @@ -92,7 +272,7 @@ subroutine mld_d_spmm_bld_inner(a_csr,desc_a,ilaggr,nlaggr,parms,ac,& nrpsave = op_prol%get_nrows() ncpsave = op_prol%get_ncols() nzpsave = op_prol%get_nzeros() - !write(0,*)me,' ',name,' input op_prol ',nrpsave,ncpsave,nzpsave + write(0,*)me,' ',name,' input op_prol ',nrpsave,ncpsave,nzpsave ! ! Here OP_PROL should be with GLOBAL indices on the cols @@ -219,3 +399,4 @@ subroutine mld_d_spmm_bld_inner(a_csr,desc_a,ilaggr,nlaggr,parms,ac,& return end subroutine mld_d_spmm_bld_inner + diff --git a/mlprec/impl/aggregator/mld_daggrmat_smth_bld.f90 b/mlprec/impl/aggregator/mld_daggrmat_smth_bld.f90 index ff4495b7..1d0dd693 100644 --- a/mlprec/impl/aggregator/mld_daggrmat_smth_bld.f90 +++ b/mlprec/impl/aggregator/mld_daggrmat_smth_bld.f90 @@ -126,7 +126,7 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,ac,op_prol,op_rest integer(psb_ipk_) :: ictxt, np, me character(len=20) :: name type(psb_desc_type) :: tmp_desc - type(psb_ld_coo_sparse_mat) :: tmpcoo + type(psb_ld_coo_sparse_mat) :: coo_prol, coo_restr, tmpcoo type(psb_ld_csr_sparse_mat) :: acsr1, acsrf, csr_prol, acsr real(psb_dpk_), allocatable :: adiag(:) real(psb_dpk_), allocatable :: arwsum(:) @@ -176,7 +176,7 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,ac,op_prol,op_rest if (info == psb_success_) & & call psb_halo(adiag,desc_a,info) if (info == psb_success_) call a%cp_to(acsr) - call op_prol%cp_to(tmpcoo) + call op_prol%mv_to(coo_prol) if(info /= psb_success_) then call psb_errpush(psb_err_from_subroutine_,name,a_err='sp_getdiag') @@ -186,10 +186,10 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,ac,op_prol,op_rest if (debug_level >= psb_debug_outer_) & & write(debug_unit,*) me,' ',trim(name),& & ' Initial copies done.' - + call acsr%cp_to_fmt(acsrf,info) - + if (filter_mat) then ! ! Build the filtered matrix Af from A @@ -251,16 +251,16 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,ac,op_prol,op_rest goto 9999 end if - + call acsrf%scal(adiag,info) if (info /= psb_success_) goto 9999 inaggr = naggr call psb_cdall(ictxt,tmp_desc,info,nl=inaggr) - nzlp = tmpcoo%get_nzeros() - call tmp_desc%indxmap%g2lip_ins(tmpcoo%ja(1:nzlp),info) - call tmpcoo%set_ncols(tmp_desc%get_local_cols()) - call tmpcoo%mv_to_fmt(csr_prol,info) + nzlp = coo_prol%get_nzeros() + call tmp_desc%indxmap%g2lip_ins(coo_prol%ja(1:nzlp),info) + call coo_prol%set_ncols(tmp_desc%get_local_cols()) + call coo_prol%mv_to_fmt(csr_prol,info) ! ! Build the smoothed prolongator using either A or Af ! acsr1 = (I-w*D*A) Prol acsr1 = (I-w*D*Af) Prol @@ -273,21 +273,37 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,ac,op_prol,op_rest call psb_errpush(psb_err_from_subroutine_,name,a_err='spspmm 1') goto 9999 end if - - + + if (debug_level >= psb_debug_outer_) & & write(debug_unit,*) me,' ',trim(name),& & 'Done SPSPMM 1' nzl = acsr1%get_nzeros() - call tmp_desc%l2gip(acsr1%ja(1:nzl),info) - call acsr1%set_dupl(psb_dupl_add_) - call acsr1%set_ncols(ntaggr) - call op_prol%cp_from(acsr1) - - call mld_spmm_bld_inner(acsr,desc_a,ilaggr,nlaggr,parms,ac,& - & op_prol,op_restr,info) - + write(0,*) acsr1%get_nrows(),acsr1%get_ncols(),acsr1%get_nzeros() + if (.false.) then + + call tmp_desc%l2gip(acsr1%ja(1:nzl),info) + call acsr1%set_dupl(psb_dupl_add_) + call acsr1%set_ncols(ntaggr) + call op_prol%cp_from(acsr1) + + call mld_spmm_bld_inner(acsr,desc_a,ilaggr,nlaggr,parms,ac,& + & op_prol,op_restr,info) + call op_prol%print(0) + else + + !call tmp_desc%l2gip(acsr1%ja(1:nzl),info) + + call acsr1%mv_to_coo(coo_prol,info) + + call mld_spmm_bld_inner(acsr,desc_a,nlaggr,parms,ac,& + & coo_prol,tmp_desc,coo_restr,info) + + call op_prol%mv_from(coo_prol) + call op_prol%print(0) + call op_restr%mv_from(coo_restr) + end if if (debug_level >= psb_debug_outer_) & & write(debug_unit,*) me,' ',trim(name),& @@ -301,7 +317,7 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,ac,op_prol,op_rest return contains - + subroutine omega_smooth(omega,acsr) implicit none real(psb_dpk_),intent(in) :: omega diff --git a/mlprec/mld_d_base_aggregator_mod.f90 b/mlprec/mld_d_base_aggregator_mod.f90 index 46a1dec5..c4acbf5f 100644 --- a/mlprec/mld_d_base_aggregator_mod.f90 +++ b/mlprec/mld_d_base_aggregator_mod.f90 @@ -44,6 +44,7 @@ module mld_d_base_aggregator_mod use mld_base_prec_type, only : mld_dml_parms, mld_daggr_data use psb_base_mod, only : psb_dspmat_type, psb_ldspmat_type, psb_d_vect_type, & & psb_d_base_vect_type, psb_dlinmap_type, psb_dpk_, psb_ld_csr_sparse_mat, & + & psb_ld_coo_sparse_mat, & & psb_ipk_, psb_epk_, psb_lpk_, psb_desc_type, psb_i_base_vect_type, & & psb_erractionsave, psb_error_handler, psb_success_, psb_toupper ! @@ -124,8 +125,6 @@ module mld_d_base_aggregator_mod import :: psb_ld_csr_sparse_mat, psb_ldspmat_type, psb_desc_type, & & mld_dml_parms, psb_dpk_, psb_ipk_, psb_lpk_ implicit none - - ! Arguments type(psb_ld_csr_sparse_mat), intent(inout) :: a_csr type(psb_desc_type), intent(in) :: desc_a integer(psb_lpk_), intent(inout) :: ilaggr(:), nlaggr(:) @@ -134,6 +133,20 @@ module mld_d_base_aggregator_mod type(psb_ldspmat_type), intent(out) :: ac integer(psb_ipk_), intent(out) :: info end subroutine mld_d_spmm_bld_inner + subroutine mld_d_new_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& + & coo_prol,desc_cprol,coo_restr,info) + import :: psb_ld_csr_sparse_mat, psb_ldspmat_type, psb_desc_type, & + & psb_ld_coo_sparse_mat, mld_dml_parms, psb_dpk_, psb_ipk_, psb_lpk_ + implicit none + type(psb_ld_csr_sparse_mat), intent(inout) :: a_csr + type(psb_desc_type), intent(in) :: desc_a + integer(psb_lpk_), intent(inout) :: nlaggr(:) + type(mld_dml_parms), intent(inout) :: parms + type(psb_ld_coo_sparse_mat), intent(inout) :: coo_prol, coo_restr + type(psb_desc_type), intent(inout) :: desc_cprol + type(psb_ldspmat_type), intent(out) :: ac + integer(psb_ipk_), intent(out) :: info + end subroutine mld_d_new_spmm_bld_inner end interface mld_spmm_bld_inner contains