From 653160a71434cbf5eee4ef4f403d43ec97b2c07b Mon Sep 17 00:00:00 2001 From: Salvatore Filippone Date: Fri, 12 Jun 2020 14:27:15 +0200 Subject: [PATCH] Better handshaking mat_bld/mat_asb --- .../mld_d_dec_aggregator_mat_asb.f90 | 243 ++++--- .../aggregator/mld_d_dec_aggregator_tprol.f90 | 2 +- .../impl/aggregator/mld_d_spmm_bld_inner.f90 | 642 ++++++++++++------ .../aggregator/mld_daggrmat_nosmth_bld.f90 | 42 +- .../impl/aggregator/mld_daggrmat_smth_bld.f90 | 15 +- 5 files changed, 630 insertions(+), 314 deletions(-) diff --git a/mlprec/impl/aggregator/mld_d_dec_aggregator_mat_asb.f90 b/mlprec/impl/aggregator/mld_d_dec_aggregator_mat_asb.f90 index 6fab2ee9..6e00f74c 100644 --- a/mlprec/impl/aggregator/mld_d_dec_aggregator_mat_asb.f90 +++ b/mlprec/impl/aggregator/mld_d_dec_aggregator_mat_asb.f90 @@ -99,7 +99,7 @@ subroutine mld_d_dec_aggregator_mat_asb(ag,parms,a,desc_a,ilaggr,nlaggr,& integer(psb_ipk_), intent(out) :: info ! integer(psb_ipk_) :: ictxt, np, me - type(psb_ld_coo_sparse_mat) :: lacoo, lbcoo + type(psb_ld_coo_sparse_mat) :: tmpcoo type(psb_d_coo_sparse_mat) :: acoo type(psb_ld_csr_sparse_mat) :: acsr1 type(psb_dspmat_type) :: tmp_ac @@ -117,102 +117,152 @@ subroutine mld_d_dec_aggregator_mat_asb(ag,parms,a,desc_a,ilaggr,nlaggr,& ictxt = desc_a%get_context() call psb_info(ictxt,me,np) + ntaggr = sum(nlaggr) + if (.false.) then + select case(parms%coarse_mat) - ntaggr = sum(nlaggr) + case(mld_distr_mat_) - select case(parms%coarse_mat) - - case(mld_distr_mat_) - - call ac%mv_to(lbcoo) - nzl = lbcoo%get_nzeros() - i_nl = nlaggr(me+1) - if (info == psb_success_) call psb_cdall(ictxt,desc_ac,info,nl=i_nl) - if (info == psb_success_) call psb_cdins(nzl,lbcoo%ia,lbcoo%ja,desc_ac,info) - if (info == psb_success_) call psb_cdasb(desc_ac,info) - if (info == psb_success_) call psb_glob_to_loc(lbcoo%ia(1:nzl),desc_ac,info,iact='I') - if (info == psb_success_) call psb_glob_to_loc(lbcoo%ja(1:nzl),desc_ac,info,iact='I') - if (info /= psb_success_) then - call psb_errpush(psb_err_internal_error_,name,& - & a_err='Creating desc_ac and converting ac') - goto 9999 - end if - if (debug_level >= psb_debug_outer_) & - & write(debug_unit,*) me,' ',trim(name),& - & 'Assembld aux descr. distr.' - call ac%mv_from(lbcoo) - call ac%set_nrows(desc_ac%get_local_rows()) - call ac%set_ncols(desc_ac%get_local_cols()) - call ac%set_asb() - - if (info /= psb_success_) then - call psb_errpush(psb_err_from_subroutine_,name,a_err='psb_sp_free') - goto 9999 - end if - - if (np>1) then - call op_prol%mv_to(acsr1) - nzl = acsr1%get_nzeros() - call psb_glob_to_loc(acsr1%ja(1:nzl),desc_ac,info,'I') - if(info /= psb_success_) then - call psb_errpush(psb_err_from_subroutine_,name,a_err='psb_glob_to_loc') + call ac%mv_to(tmpcoo) + nzl = tmpcoo%get_nzeros() + i_nl = nlaggr(me+1) + if (info == psb_success_) call psb_cdall(ictxt,desc_ac,info,nl=i_nl) + if (info == psb_success_) call psb_cdins(nzl,tmpcoo%ia,tmpcoo%ja,desc_ac,info) + if (info == psb_success_) call psb_cdasb(desc_ac,info) + if (info == psb_success_) call psb_glob_to_loc(tmpcoo%ia(1:nzl),desc_ac,info,iact='I') + if (info == psb_success_) call psb_glob_to_loc(tmpcoo%ja(1:nzl),desc_ac,info,iact='I') + if (info /= psb_success_) then + call psb_errpush(psb_err_internal_error_,name,& + & a_err='Creating desc_ac and converting ac') goto 9999 end if - call op_prol%mv_from(acsr1) - endif - call op_prol%set_ncols(desc_ac%get_local_cols()) - - if (np>1) then - !call op_restr%cscnv(info,type='coo',dupl=psb_dupl_add_) - call op_restr%mv_to(lacoo) - nzl = lacoo%get_nzeros() - if (info == psb_success_) call psb_glob_to_loc(lacoo%ia(1:nzl),desc_ac,info,'I') - call lacoo%set_dupl(psb_dupl_add_) - if (info == psb_success_) call op_restr%mv_from(lacoo) - if (info == psb_success_) call op_restr%cscnv(info,type='csr') - if(info /= psb_success_) then - call psb_errpush(psb_err_internal_error_,name,& - & a_err='Converting op_restr to local') + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'Assembld aux descr. distr.' + call ac%mv_from(tmpcoo) + call ac%set_nrows(desc_ac%get_local_rows()) + call ac%set_ncols(desc_ac%get_local_cols()) + call ac%set_asb() + + if (info /= psb_success_) then + call psb_errpush(psb_err_from_subroutine_,name,a_err='psb_sp_free') goto 9999 end if - end if - ! - ! Clip to local rows. - ! - call op_restr%set_nrows(desc_ac%get_local_rows()) - - if (debug_level >= psb_debug_outer_) & - & write(debug_unit,*) me,' ',trim(name),& - & 'Done ac ' - - case(mld_repl_mat_) - ! - ! - ! If we are here, it means we assume that an IPK version of the - ! coarse matrix can hold all indices. User beware! - ! - - ! - ! op_prol/op_restr come from par_spmm_bld with local sizes - ! suitable for DIST option, fix relevant sizes - ! - call op_prol%set_ncols(ntaggr) - call op_restr%set_nrows(ntaggr) - call psb_cdall(ictxt,desc_ac,info,mg=ntaggr,repl=.true.) - if (info == psb_success_) call psb_cdasb(desc_ac,info) - if (info == psb_success_) call ac%mv_to(acoo) - if (info == psb_success_) call tmp_ac%mv_from(acoo) - if (info == psb_success_) & - & call psb_gather(ac,tmp_ac,desc_ac,info,dupl=psb_dupl_add_,keeploc=.false.) - if (info /= psb_success_) goto 9999 - - case default - info = psb_err_internal_error_ - call psb_errpush(info,name,a_err='invalid mld_coarse_mat_') - goto 9999 - end select + if (np>1) then + call op_prol%mv_to(acsr1) + nzl = acsr1%get_nzeros() + call psb_glob_to_loc(acsr1%ja(1:nzl),desc_ac,info,'I') + if(info /= psb_success_) then + call psb_errpush(psb_err_from_subroutine_,name,a_err='psb_glob_to_loc') + goto 9999 + end if + call op_prol%mv_from(acsr1) + endif + call op_prol%set_ncols(desc_ac%get_local_cols()) + + if (np>1) then + !call op_restr%cscnv(info,type='coo',dupl=psb_dupl_add_) + call op_restr%mv_to(tmpcoo) + nzl = tmpcoo%get_nzeros() + if (info == psb_success_) call psb_glob_to_loc(tmpcoo%ia(1:nzl),desc_ac,info,'I') + call tmpcoo%set_dupl(psb_dupl_add_) + if (info == psb_success_) call op_restr%mv_from(tmpcoo) + if (info == psb_success_) call op_restr%cscnv(info,type='csr') + if(info /= psb_success_) then + call psb_errpush(psb_err_internal_error_,name,& + & a_err='Converting op_restr to local') + goto 9999 + end if + end if + ! + ! Clip to local rows. + ! + call op_restr%set_nrows(desc_ac%get_local_rows()) + + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'Done ac ' + + case(mld_repl_mat_) + ! + ! + ! If we are here, it means we assume that an IPK version of the + ! coarse matrix can hold all indices. User beware! + ! + + ! + ! op_prol/op_restr come from par_spmm_bld with local sizes + ! suitable for DIST option, fix relevant sizes + ! + call op_prol%set_ncols(ntaggr) + call op_restr%set_nrows(ntaggr) + call psb_cdall(ictxt,desc_ac,info,mg=ntaggr,repl=.true.) + if (info == psb_success_) call psb_cdasb(desc_ac,info) + if (info == psb_success_) call ac%mv_to(acoo) + if (info == psb_success_) call tmp_ac%mv_from(acoo) + if (info == psb_success_) & + & call psb_gather(ac,tmp_ac,desc_ac,info,dupl=psb_dupl_add_,keeploc=.false.) + if (info /= psb_success_) goto 9999 + + case default + info = psb_err_internal_error_ + call psb_errpush(info,name,a_err='invalid mld_coarse_mat_') + goto 9999 + end select + + else + + select case(parms%coarse_mat) + + case(mld_distr_mat_) + + call ac%cscnv(info,type='csr') + call op_prol%cscnv(info,type='csr') + call op_restr%cscnv(info,type='csr') + + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'Done ac ' + + case(mld_repl_mat_) + ! + ! + call op_prol%mv_to(tmpcoo) + nzl = tmpcoo%get_nzeros() + call psb_loc_to_glob(tmpcoo%ja(1:nzl),desc_ac,info,'I') + call op_prol%mv_from(tmpcoo) + + call op_restr%mv_to(tmpcoo) + nzl = tmpcoo%get_nzeros() + call psb_loc_to_glob(tmpcoo%ia(1:nzl),desc_ac,info,'I') + call op_restr%mv_from(tmpcoo) + + call op_prol%set_ncols(ntaggr) + call op_restr%set_nrows(ntaggr) + + call ac%mv_to(tmpcoo) + call tmp_ac%mv_from(tmpcoo) + call psb_gather(ac,tmp_ac,desc_ac,info,root=-ione,dupl=psb_dupl_add_,keeploc=.false.) + + call psb_cdall(ictxt,desc_ac,info,mg=ntaggr,repl=.true.) + if (info == psb_success_) call psb_cdasb(desc_ac,info) + ! + ! Now that we have the descriptors and the restrictor, we should + ! update the W. But we don't, because REPL is only valid + ! at the coarsest level, so no need to carry over. + ! + + if (info /= psb_success_) goto 9999 + + case default + info = psb_err_internal_error_ + call psb_errpush(info,name,a_err='invalid mld_coarse_mat_') + goto 9999 + end select + + end if call psb_erractionrestore(err_act) return @@ -221,4 +271,19 @@ subroutine mld_d_dec_aggregator_mat_asb(ag,parms,a,desc_a,ilaggr,nlaggr,& return +contains + subroutine check_coo(me,string,coo) + implicit none + integer(psb_ipk_) :: me + type(psb_ld_coo_sparse_mat) :: coo + character(len=*) :: string + integer(psb_lpk_) :: nr,nc,nz + nr = coo%get_nrows() + nc = coo%get_ncols() + nz = coo%get_nzeros() + write(0,*) me,string,nr,nc,& + & minval(coo%ia(1:nz)),maxval(coo%ia(1:nz)),& + & minval(coo%ja(1:nz)),maxval(coo%ja(1:nz)) + + end subroutine check_coo end subroutine mld_d_dec_aggregator_mat_asb diff --git a/mlprec/impl/aggregator/mld_d_dec_aggregator_tprol.f90 b/mlprec/impl/aggregator/mld_d_dec_aggregator_tprol.f90 index 4f6a88a8..6a963b00 100644 --- a/mlprec/impl/aggregator/mld_d_dec_aggregator_tprol.f90 +++ b/mlprec/impl/aggregator/mld_d_dec_aggregator_tprol.f90 @@ -122,7 +122,7 @@ subroutine mld_d_dec_aggregator_build_tprol(ag,parms,ag_data,& ! clean_zeros = ag%do_clean_zeros call ag%soc_map_bld(parms%aggr_ord,parms%aggr_thresh,clean_zeros,a,desc_a,nlaggr,ilaggr,info) - + if (info==psb_success_) call mld_map_to_tprol(desc_a,ilaggr,nlaggr,op_prol,info) if (info /= psb_success_) then info=psb_err_from_subroutine_ diff --git a/mlprec/impl/aggregator/mld_d_spmm_bld_inner.f90 b/mlprec/impl/aggregator/mld_d_spmm_bld_inner.f90 index 585a5775..0eed142b 100644 --- a/mlprec/impl/aggregator/mld_d_spmm_bld_inner.f90 +++ b/mlprec/impl/aggregator/mld_d_spmm_bld_inner.f90 @@ -36,7 +36,7 @@ ! ! subroutine mld_d_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& - & coo_prol,desc_cprol,coo_restr,info) + & coo_prol,desc_ac,coo_restr,info) use psb_base_mod use mld_d_inner_mod use mld_d_base_aggregator_mod, mld_protect_name => mld_d_spmm_bld_inner @@ -48,7 +48,7 @@ subroutine mld_d_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& integer(psb_lpk_), intent(inout) :: nlaggr(:) type(mld_dml_parms), intent(inout) :: parms type(psb_ld_coo_sparse_mat), intent(inout) :: coo_prol, coo_restr - type(psb_desc_type), intent(inout) :: desc_cprol + type(psb_desc_type), intent(inout) :: desc_ac type(psb_ldspmat_type), intent(out) :: ac integer(psb_ipk_), intent(out) :: info @@ -60,7 +60,7 @@ subroutine mld_d_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& type(psb_ld_coo_sparse_mat) :: ac_coo, tmpcoo type(psb_d_csr_sparse_mat) :: acsr3, csr_prol, ac_csr, csr_restr integer(psb_ipk_) :: debug_level, debug_unit, naggr - integer(psb_lpk_) :: nrow, nglob, ncol, ntaggr, nzl, ip, & + integer(psb_lpk_) :: nrow, nglob, ncol, ntaggr, nrl, nzl, ip, & & nzt, naggrm1, naggrp1, i, k integer(psb_lpk_) :: nrsave, ncsave, nzsave, nza logical, parameter :: do_timings=.false., oldstyle=.false., debug=.false. @@ -90,114 +90,210 @@ subroutine mld_d_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& naggrp1 = sum(nlaggr(1:me+1)) !write(0,*)me,' ',name,' input sizes',nlaggr(:),':',naggr - ! - ! COO_PROL should arrive here with local numbering - ! - if (debug) write(0,*) me,' ',trim(name),' Size check on entry New: ',& - & coo_prol%get_fmt(),coo_prol%get_nrows(),coo_prol%get_ncols(),coo_prol%get_nzeros(),& - & nrow,ntaggr,naggr - - call coo_prol%cp_to_ifmt(csr_prol,info) - - if (debug) write(0,*) me,trim(name),' Product AxPROL ',& - & a_csr%get_nrows(),a_csr%get_ncols(), csr_prol%get_nrows(), & - & desc_a%get_local_rows(),desc_a%get_local_cols(),& - & desc_cprol%get_local_rows(),desc_a%get_local_cols() - if (debug) flush(0) - - if (do_timings) call psb_tic(idx_spspmm) - call psb_par_spspmm(a_csr,desc_a,csr_prol,acsr3,desc_cprol,info) - if (do_timings) call psb_toc(idx_spspmm) - - if (debug) write(0,*) me,trim(name),' Done AxPROL ',& - & acsr3%get_nrows(),acsr3%get_ncols(), acsr3%get_nzeros(),& - & desc_cprol%get_local_rows(),desc_cprol%get_local_cols() - - ! - ! Ok first product done. - ! - ! Remember that RESTR must be built from PROL after halo extension, - ! which is done above in psb_par_spspmm - if (debug) write(0,*)me,' ',name,' No inp_restr, transposing prol ',& - & csr_prol%get_nrows(),csr_prol%get_ncols(),csr_prol%get_nzeros() - call csr_prol%cp_to_lcoo(coo_restr,info) + if (.false.) then + ! + ! COO_PROL should arrive here with local numbering + ! + if (debug) write(0,*) me,' ',trim(name),' Size check on entry New: ',& + & coo_prol%get_fmt(),coo_prol%get_nrows(),coo_prol%get_ncols(),coo_prol%get_nzeros(),& + & nrow,ntaggr,naggr + + call coo_prol%cp_to_ifmt(csr_prol,info) + + if (debug) write(0,*) me,trim(name),' Product AxPROL ',& + & a_csr%get_nrows(),a_csr%get_ncols(), csr_prol%get_nrows(), & + & desc_a%get_local_rows(),desc_a%get_local_cols(),& + & desc_ac%get_local_rows(),desc_a%get_local_cols() + if (debug) flush(0) + + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(a_csr,desc_a,csr_prol,acsr3,desc_ac,info) + if (do_timings) call psb_toc(idx_spspmm) + + if (debug) write(0,*) me,trim(name),' Done AxPROL ',& + & acsr3%get_nrows(),acsr3%get_ncols(), acsr3%get_nzeros(),& + & desc_ac%get_local_rows(),desc_ac%get_local_cols() + + ! + ! Ok first product done. + ! + ! Remember that RESTR must be built from PROL after halo extension, + ! which is done above in psb_par_spspmm + if (debug) write(0,*)me,' ',name,' No inp_restr, transposing prol ',& + & csr_prol%get_nrows(),csr_prol%get_ncols(),csr_prol%get_nzeros() + call csr_prol%cp_to_lcoo(coo_restr,info) !!$ write(0,*)me,' ',name,' new into transposition ',coo_restr%get_nrows(),& !!$ & coo_restr%get_ncols(),coo_restr%get_nzeros() - call coo_restr%transp() - nzl = coo_restr%get_nzeros() - call desc_cprol%l2gip(coo_restr%ia(1:nzl),info) - i=0 - ! - ! Now we have to fix this. The only rows of the restrictor that are correct - ! are those corresponding to "local" aggregates, i.e. indices in ilaggr(:) - ! - do k=1, nzl - if ((naggrm1 < coo_restr%ia(k)) .and.(coo_restr%ia(k) <= naggrp1)) then - i = i+1 - coo_restr%val(i) = coo_restr%val(k) - coo_restr%ia(i) = coo_restr%ia(k) - coo_restr%ja(i) = coo_restr%ja(k) - end if - end do - call coo_restr%set_nzeros(i) - call coo_restr%fix(info) - call coo_restr%cp_to_coo(tmpcoo,info) + call coo_restr%transp() + nzl = coo_restr%get_nzeros() + call desc_ac%l2gip(coo_restr%ia(1:nzl),info) + i=0 + ! + ! Now we have to fix this. The only rows of the restrictor that are correct + ! are those corresponding to "local" aggregates, i.e. indices in ilaggr(:) + ! + do k=1, nzl + if ((naggrm1 < coo_restr%ia(k)) .and.(coo_restr%ia(k) <= naggrp1)) then + i = i+1 + coo_restr%val(i) = coo_restr%val(k) + coo_restr%ia(i) = coo_restr%ia(k) + coo_restr%ja(i) = coo_restr%ja(k) + end if + end do + call coo_restr%set_nzeros(i) + call coo_restr%fix(info) + call coo_restr%cp_to_coo(tmpcoo,info) !!$ write(0,*)me,' ',name,' after transposition ',coo_restr%get_nrows(),coo_restr%get_ncols(),coo_restr%get_nzeros() - if (info /= psb_success_) then - call psb_errpush(psb_err_from_subroutine_,name,a_err='spcnv coo_restr') - goto 9999 - end if - if (debug_level >= psb_debug_outer_) & - & write(debug_unit,*) me,' ',trim(name),& - & 'starting sphalo/ rwxtd' - nzl = tmpcoo%get_nzeros() - call psb_glob_to_loc(tmpcoo%ia(1:nzl),desc_cprol,info,iact='I',owned=.true.) - call tmpcoo%clean_negidx(info) - nzl = tmpcoo%get_nzeros() - call tmpcoo%set_nrows(desc_cprol%get_local_rows()) - call tmpcoo%set_ncols(desc_a%get_local_cols()) + if (info /= psb_success_) then + call psb_errpush(psb_err_from_subroutine_,name,a_err='spcnv coo_restr') + goto 9999 + end if + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'starting sphalo/ rwxtd' + nzl = tmpcoo%get_nzeros() + call psb_glob_to_loc(tmpcoo%ia(1:nzl),desc_ac,info,iact='I',owned=.true.) + call tmpcoo%clean_negidx(info) + nzl = tmpcoo%get_nzeros() + call tmpcoo%set_nrows(desc_ac%get_local_rows()) + call tmpcoo%set_ncols(desc_a%get_local_cols()) !!$ write(0,*)me,' ',name,' after G2L on rows ',tmpcoo%get_nrows(),tmpcoo%get_ncols(),tmpcoo%get_nzeros() - call csr_restr%mv_from_lcoo(tmpcoo,info) - - if (debug) write(0,*) me,trim(name),' Product RESTRxAP ',& - & csr_restr%get_nrows(),csr_restr%get_ncols(), & - & desc_cprol%get_local_rows(),desc_a%get_local_cols(),& - & acsr3%get_nrows(),acsr3%get_ncols() - if (do_timings) call psb_tic(idx_spspmm) - call psb_par_spspmm(csr_restr,desc_a,acsr3,ac_csr,desc_cprol,info) - if (do_timings) call psb_toc(idx_spspmm) - call csr_restr%free() - call acsr3%free() - call ac_csr%mv_to_lcoo(ac_coo,info) - call ac_coo%fix(info) - nza = ac_coo%get_nzeros() - if (debug) write(0,*) me,trim(name),' Fixed ac ',& - & ac_coo%get_nrows(),ac_coo%get_ncols(), nza - call desc_cprol%indxmap%l2gip(ac_coo%ia(1:nza),info) - call desc_cprol%indxmap%l2gip(ac_coo%ja(1:nza),info) - call ac_coo%set_nrows(ntaggr) - call ac_coo%set_ncols(ntaggr) - if (debug) write(0,*) me,' ',trim(name),' Before mv_from',psb_get_errstatus() - if (info == 0) call ac%mv_from(ac_coo) - if (debug) write(0,*) me,' ',trim(name),' After mv_from',psb_get_errstatus() - if (debug) write(0,*) me,' ',trim(name),' ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros(),naggr,ntaggr - ! write(0,*) me,' ',trim(name),' Final AC newstyle ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros() - - nza = coo_prol%get_nzeros() - call desc_cprol%indxmap%l2gip(coo_prol%ja(1:nza),info) - - if (debug) then - write(0,*) me,' ',trim(name),' Checkpoint at exit' - call psb_barrier(ictxt) - write(0,*) me,' ',trim(name),' Checkpoint through' - end if + call csr_restr%mv_from_lcoo(tmpcoo,info) + + if (debug) write(0,*) me,trim(name),' Product RESTRxAP ',& + & csr_restr%get_nrows(),csr_restr%get_ncols(), & + & desc_ac%get_local_rows(),desc_a%get_local_cols(),& + & acsr3%get_nrows(),acsr3%get_ncols() + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(csr_restr,desc_a,acsr3,ac_csr,desc_ac,info) + if (do_timings) call psb_toc(idx_spspmm) + call csr_restr%free() + call acsr3%free() + call ac_csr%mv_to_lcoo(ac_coo,info) + call ac_coo%fix(info) + nza = ac_coo%get_nzeros() + if (debug) write(0,*) me,trim(name),' Fixed ac ',& + & ac_coo%get_nrows(),ac_coo%get_ncols(), nza + call desc_ac%indxmap%l2gip(ac_coo%ia(1:nza),info) + call desc_ac%indxmap%l2gip(ac_coo%ja(1:nza),info) + call ac_coo%set_nrows(ntaggr) + call ac_coo%set_ncols(ntaggr) + if (debug) write(0,*) me,' ',trim(name),' Before mv_from',psb_get_errstatus() + if (info == 0) call ac%mv_from(ac_coo) + if (debug) write(0,*) me,' ',trim(name),' After mv_from',psb_get_errstatus() + if (debug) write(0,*) me,' ',trim(name),' ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros(),naggr,ntaggr + ! write(0,*) me,' ',trim(name),' Final AC newstyle ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros() + + nza = coo_prol%get_nzeros() + call desc_ac%indxmap%l2gip(coo_prol%ja(1:nza),info) + + if (debug) then + write(0,*) me,' ',trim(name),' Checkpoint at exit' + call psb_barrier(ictxt) + write(0,*) me,' ',trim(name),' Checkpoint through' + end if - if (info /= psb_success_) then - call psb_errpush(psb_err_internal_error_,name,a_err='Build ac = coo_restr x am3') - goto 9999 - end if + if (info /= psb_success_) then + call psb_errpush(psb_err_internal_error_,name,a_err='Build ac = coo_restr x am3') + goto 9999 + end if + else + ! + ! COO_PROL should arrive here with local numbering + ! + if (debug) write(0,*) me,' ',trim(name),' Size check on entry New: ',& + & coo_prol%get_fmt(),coo_prol%get_nrows(),coo_prol%get_ncols(),coo_prol%get_nzeros(),& + & nrow,ntaggr,naggr + + call coo_prol%cp_to_ifmt(csr_prol,info) + + if (debug) write(0,*) me,trim(name),' Product AxPROL ',& + & a_csr%get_nrows(),a_csr%get_ncols(), csr_prol%get_nrows(), & + & desc_a%get_local_rows(),desc_a%get_local_cols(),& + & desc_ac%get_local_rows(),desc_a%get_local_cols() + if (debug) flush(0) + + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(a_csr,desc_a,csr_prol,acsr3,desc_ac,info) + if (do_timings) call psb_toc(idx_spspmm) + + if (debug) write(0,*) me,trim(name),' Done AxPROL ',& + & acsr3%get_nrows(),acsr3%get_ncols(), acsr3%get_nzeros(),& + & desc_ac%get_local_rows(),desc_ac%get_local_cols() + + ! + ! Ok first product done. + ! + ! Remember that RESTR must be built from PROL after halo extension, + ! which is done above in psb_par_spspmm + if (debug) write(0,*)me,' ',name,' No inp_restr, transposing prol ',& + & csr_prol%get_nrows(),csr_prol%get_ncols(),csr_prol%get_nzeros() + call csr_prol%mv_to_lcoo(coo_restr,info) +!!$ write(0,*)me,' ',name,' new into transposition ',coo_restr%get_nrows(),& +!!$ & coo_restr%get_ncols(),coo_restr%get_nzeros() + if (debug) call check_coo(me,trim(name)//' Check 1 (before transp) on coo_restr:',coo_restr) + + call coo_restr%transp() + nzl = coo_restr%get_nzeros() + nrl = desc_ac%get_local_rows() + i=0 + ! + ! Only keep local rows + ! + do k=1, nzl + if ((1 <= coo_restr%ia(k)) .and.(coo_restr%ia(k) <= nrl)) then + i = i+1 + coo_restr%val(i) = coo_restr%val(k) + coo_restr%ia(i) = coo_restr%ia(k) + coo_restr%ja(i) = coo_restr%ja(k) + end if + end do + call coo_restr%set_nzeros(i) + call coo_restr%fix(info) + nzl = coo_restr%get_nzeros() + call coo_restr%set_nrows(desc_ac%get_local_rows()) + call coo_restr%set_ncols(desc_a%get_local_cols()) + if (debug) call check_coo(me,trim(name)//' Check 2 on coo_restr:',coo_restr) + call csr_restr%cp_from_lcoo(coo_restr,info) + +!!$ write(0,*)me,' ',name,' after transposition ',coo_restr%get_nrows(),coo_restr%get_ncols(),coo_restr%get_nzeros() + + if (info /= psb_success_) then + call psb_errpush(psb_err_from_subroutine_,name,a_err='spcnv coo_restr') + goto 9999 + end if + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'starting sphalo/ rwxtd' + + if (debug) write(0,*) me,trim(name),' Product RESTRxAP ',& + & csr_restr%get_nrows(),csr_restr%get_ncols(), & + & desc_ac%get_local_rows(),desc_a%get_local_cols(),& + & acsr3%get_nrows(),acsr3%get_ncols() + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(csr_restr,desc_a,acsr3,ac_csr,desc_ac,info) + if (do_timings) call psb_toc(idx_spspmm) + call acsr3%free() + + call psb_cdasb(desc_ac,info) + + call ac_csr%set_nrows(desc_ac%get_local_rows()) + call ac_csr%set_ncols(desc_ac%get_local_cols()) + call ac%mv_from(ac_csr) + call ac%set_asb() + + if (debug) write(0,*) me,' ',trim(name),' After mv_from',psb_get_errstatus() + if (debug) write(0,*) me,' ',trim(name),' ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros(),naggr,ntaggr + ! write(0,*) me,' ',trim(name),' Final AC newstyle ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros() + + call coo_prol%set_ncols(desc_ac%get_local_cols()) + !call coo_restr%mv_from_ifmt(csr_restr,info) +!!$ call coo_restr%set_nrows(desc_ac%get_local_rows()) +!!$ call coo_restr%set_ncols(desc_a%get_local_cols()) + if (debug) call check_coo(me,trim(name)//' Check 3 on coo_restr:',coo_restr) + end if if (debug_level >= psb_debug_outer_) & & write(debug_unit,*) me,' ',trim(name),& @@ -209,11 +305,26 @@ subroutine mld_d_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& 9999 call psb_error_handler(err_act) return - +contains + subroutine check_coo(me,string,coo) + implicit none + integer(psb_ipk_) :: me + type(psb_ld_coo_sparse_mat) :: coo + character(len=*) :: string + integer(psb_lpk_) :: nr,nc,nz + nr = coo%get_nrows() + nc = coo%get_ncols() + nz = coo%get_nzeros() + write(0,*) me,string,nr,nc,& + & minval(coo%ia(1:nz)),maxval(coo%ia(1:nz)),& + & minval(coo%ja(1:nz)),maxval(coo%ja(1:nz)) + + end subroutine check_coo + end subroutine mld_d_spmm_bld_inner subroutine mld_ld_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& - & coo_prol,desc_cprol,coo_restr,info) + & coo_prol,desc_ac,coo_restr,info) use psb_base_mod use mld_d_inner_mod use mld_d_base_aggregator_mod, mld_protect_name => mld_ld_spmm_bld_inner @@ -225,7 +336,7 @@ subroutine mld_ld_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& integer(psb_lpk_), intent(inout) :: nlaggr(:) type(mld_dml_parms), intent(inout) :: parms type(psb_ld_coo_sparse_mat), intent(inout) :: coo_prol, coo_restr - type(psb_desc_type), intent(inout) :: desc_cprol + type(psb_desc_type), intent(inout) :: desc_ac type(psb_ldspmat_type), intent(out) :: ac integer(psb_ipk_), intent(out) :: info @@ -237,7 +348,7 @@ subroutine mld_ld_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& type(psb_ld_coo_sparse_mat) :: ac_coo, tmpcoo type(psb_ld_csr_sparse_mat) :: acsr3, csr_prol, ac_csr, csr_restr integer(psb_ipk_) :: debug_level, debug_unit, naggr - integer(psb_lpk_) :: nrow, nglob, ncol, ntaggr, nzl, ip, & + integer(psb_lpk_) :: nrow, nglob, ncol, ntaggr, nrl, nzl, ip, & & nzt, naggrm1, naggrp1, i, k integer(psb_lpk_) :: nrsave, ncsave, nzsave, nza logical, parameter :: do_timings=.true., oldstyle=.false., debug=.false. @@ -266,113 +377,208 @@ subroutine mld_ld_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& naggrm1 = sum(nlaggr(1:me)) naggrp1 = sum(nlaggr(1:me+1)) !write(0,*)me,' ',name,' input sizes',nlaggr(:),':',naggr - - ! - ! Here COO_PROL should be with GLOBAL indices on the cols - ! and LOCAL indices on the rows. - ! - if (debug) write(0,*) me,' ',trim(name),' Size check on entry New: ',& - & coo_prol%get_fmt(),coo_prol%get_nrows(),coo_prol%get_ncols(),coo_prol%get_nzeros(),& - & nrow,ntaggr,naggr - - call coo_prol%cp_to_fmt(csr_prol,info) - - if (debug) write(0,*) me,trim(name),' Product AxPROL ',& - & a_csr%get_nrows(),a_csr%get_ncols(), csr_prol%get_nrows(), & - & desc_a%get_local_rows(),desc_a%get_local_cols(),& - & desc_cprol%get_local_rows(),desc_a%get_local_cols() - if (debug) flush(0) - - if (do_timings) call psb_tic(idx_spspmm) - call psb_par_spspmm(a_csr,desc_a,csr_prol,acsr3,desc_cprol,info) - if (do_timings) call psb_toc(idx_spspmm) - - if (debug) write(0,*) me,trim(name),' Done AxPROL ',& - & acsr3%get_nrows(),acsr3%get_ncols(), acsr3%get_nzeros(),& - & desc_cprol%get_local_rows(),desc_cprol%get_local_cols() - - ! - ! Ok first product done. - ! - ! Remember that RESTR must be built from PROL after halo extension, - ! which is done above in psb_par_spspmm - if (debug) write(0,*)me,' ',name,' No inp_restr, transposing prol ',& - & csr_prol%get_nrows(),csr_prol%get_ncols(),csr_prol%get_nzeros() - call csr_prol%cp_to_fmt(coo_restr,info) + if (.false.) then + ! + ! Here COO_PROL should be with GLOBAL indices on the cols + ! and LOCAL indices on the rows. + ! + if (debug) write(0,*) me,' ',trim(name),' Size check on entry New: ',& + & coo_prol%get_fmt(),coo_prol%get_nrows(),coo_prol%get_ncols(),coo_prol%get_nzeros(),& + & nrow,ntaggr,naggr + + call coo_prol%cp_to_fmt(csr_prol,info) + + if (debug) write(0,*) me,trim(name),' Product AxPROL ',& + & a_csr%get_nrows(),a_csr%get_ncols(), csr_prol%get_nrows(), & + & desc_a%get_local_rows(),desc_a%get_local_cols(),& + & desc_ac%get_local_rows(),desc_a%get_local_cols() + if (debug) flush(0) + + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(a_csr,desc_a,csr_prol,acsr3,desc_ac,info) + if (do_timings) call psb_toc(idx_spspmm) + + if (debug) write(0,*) me,trim(name),' Done AxPROL ',& + & acsr3%get_nrows(),acsr3%get_ncols(), acsr3%get_nzeros(),& + & desc_ac%get_local_rows(),desc_ac%get_local_cols() + + ! + ! Ok first product done. + ! + ! Remember that RESTR must be built from PROL after halo extension, + ! which is done above in psb_par_spspmm + if (debug) write(0,*)me,' ',name,' No inp_restr, transposing prol ',& + & csr_prol%get_nrows(),csr_prol%get_ncols(),csr_prol%get_nzeros() + call csr_prol%cp_to_fmt(coo_restr,info) !!$ write(0,*)me,' ',name,' new into transposition ',coo_restr%get_nrows(),& !!$ & coo_restr%get_ncols(),coo_restr%get_nzeros() - call coo_restr%transp() - nzl = coo_restr%get_nzeros() - call desc_cprol%l2gip(coo_restr%ia(1:nzl),info) - i=0 - ! - ! Now we have to fix this. The only rows of the restrictor that are correct - ! are those corresponding to "local" aggregates, i.e. indices in ilaggr(:) - ! - do k=1, nzl - if ((naggrm1 < coo_restr%ia(k)) .and.(coo_restr%ia(k) <= naggrp1)) then - i = i+1 - coo_restr%val(i) = coo_restr%val(k) - coo_restr%ia(i) = coo_restr%ia(k) - coo_restr%ja(i) = coo_restr%ja(k) - end if - end do - call coo_restr%set_nzeros(i) - call coo_restr%fix(info) - call coo_restr%cp_to_coo(tmpcoo,info) + call coo_restr%transp() + nzl = coo_restr%get_nzeros() + call desc_ac%l2gip(coo_restr%ia(1:nzl),info) + i=0 + ! + ! Now we have to fix this. The only rows of the restrictor that are correct + ! are those corresponding to "local" aggregates, i.e. indices in ilaggr(:) + ! + do k=1, nzl + if ((naggrm1 < coo_restr%ia(k)) .and.(coo_restr%ia(k) <= naggrp1)) then + i = i+1 + coo_restr%val(i) = coo_restr%val(k) + coo_restr%ia(i) = coo_restr%ia(k) + coo_restr%ja(i) = coo_restr%ja(k) + end if + end do + call coo_restr%set_nzeros(i) + call coo_restr%fix(info) + call coo_restr%cp_to_coo(tmpcoo,info) !!$ write(0,*)me,' ',name,' after transposition ',coo_restr%get_nrows(),coo_restr%get_ncols(),coo_restr%get_nzeros() - if (info /= psb_success_) then - call psb_errpush(psb_err_from_subroutine_,name,a_err='spcnv coo_restr') - goto 9999 - end if - if (debug_level >= psb_debug_outer_) & - & write(debug_unit,*) me,' ',trim(name),& - & 'starting sphalo/ rwxtd' - nzl = tmpcoo%get_nzeros() - call psb_glob_to_loc(tmpcoo%ia(1:nzl),desc_cprol,info,iact='I',owned=.true.) - call tmpcoo%clean_negidx(info) - nzl = tmpcoo%get_nzeros() - call tmpcoo%set_nrows(desc_cprol%get_local_rows()) - call tmpcoo%set_ncols(desc_a%get_local_cols()) + if (info /= psb_success_) then + call psb_errpush(psb_err_from_subroutine_,name,a_err='spcnv coo_restr') + goto 9999 + end if + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'starting sphalo/ rwxtd' + nzl = tmpcoo%get_nzeros() + call psb_glob_to_loc(tmpcoo%ia(1:nzl),desc_ac,info,iact='I',owned=.true.) + call tmpcoo%clean_negidx(info) + nzl = tmpcoo%get_nzeros() + call tmpcoo%set_nrows(desc_ac%get_local_rows()) + call tmpcoo%set_ncols(desc_a%get_local_cols()) !!$ write(0,*)me,' ',name,' after G2L on rows ',tmpcoo%get_nrows(),tmpcoo%get_ncols(),tmpcoo%get_nzeros() - call csr_restr%mv_from_coo(tmpcoo,info) - - if (debug) write(0,*) me,trim(name),' Product RESTRxAP ',& - & csr_restr%get_nrows(),csr_restr%get_ncols(), & - & desc_cprol%get_local_rows(),desc_a%get_local_cols(),& - & acsr3%get_nrows(),acsr3%get_ncols() - if (do_timings) call psb_tic(idx_spspmm) - call psb_par_spspmm(csr_restr,desc_a,acsr3,ac_csr,desc_cprol,info) - if (do_timings) call psb_toc(idx_spspmm) - call csr_restr%free() - call ac_csr%mv_to_coo(ac_coo,info) - nza = ac_coo%get_nzeros() - if (debug) write(0,*) me,trim(name),' Fixing ac ',& - & ac_coo%get_nrows(),ac_coo%get_ncols(), nza - call ac_coo%fix(info) - call desc_cprol%indxmap%l2gip(ac_coo%ia(1:nza),info) - call desc_cprol%indxmap%l2gip(ac_coo%ja(1:nza),info) - call ac_coo%set_nrows(ntaggr) - call ac_coo%set_ncols(ntaggr) - if (debug) write(0,*) me,' ',trim(name),' Before mv_from',psb_get_errstatus() - if (info == 0) call ac%mv_from(ac_coo) - if (debug) write(0,*) me,' ',trim(name),' After mv_from',psb_get_errstatus() - if (debug) write(0,*) me,' ',trim(name),' ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros(),naggr,ntaggr - ! write(0,*) me,' ',trim(name),' Final AC newstyle ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros() - - nza = coo_prol%get_nzeros() - call desc_cprol%indxmap%l2gip(coo_prol%ja(1:nza),info) - - if (debug) then - write(0,*) me,' ',trim(name),' Checkpoint at exit' - call psb_barrier(ictxt) - write(0,*) me,' ',trim(name),' Checkpoint through' - end if + call csr_restr%mv_from_coo(tmpcoo,info) + + if (debug) write(0,*) me,trim(name),' Product RESTRxAP ',& + & csr_restr%get_nrows(),csr_restr%get_ncols(), & + & desc_ac%get_local_rows(),desc_a%get_local_cols(),& + & acsr3%get_nrows(),acsr3%get_ncols() + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(csr_restr,desc_a,acsr3,ac_csr,desc_ac,info) + if (do_timings) call psb_toc(idx_spspmm) + call csr_restr%free() + call ac_csr%mv_to_coo(ac_coo,info) + nza = ac_coo%get_nzeros() + if (debug) write(0,*) me,trim(name),' Fixing ac ',& + & ac_coo%get_nrows(),ac_coo%get_ncols(), nza + call ac_coo%fix(info) + call desc_ac%indxmap%l2gip(ac_coo%ia(1:nza),info) + call desc_ac%indxmap%l2gip(ac_coo%ja(1:nza),info) + call ac_coo%set_nrows(ntaggr) + call ac_coo%set_ncols(ntaggr) + if (debug) write(0,*) me,' ',trim(name),' Before mv_from',psb_get_errstatus() + if (info == 0) call ac%mv_from(ac_coo) + if (debug) write(0,*) me,' ',trim(name),' After mv_from',psb_get_errstatus() + if (debug) write(0,*) me,' ',trim(name),' ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros(),naggr,ntaggr + ! write(0,*) me,' ',trim(name),' Final AC newstyle ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros() + + nza = coo_prol%get_nzeros() + call desc_ac%indxmap%l2gip(coo_prol%ja(1:nza),info) + + if (debug) then + write(0,*) me,' ',trim(name),' Checkpoint at exit' + call psb_barrier(ictxt) + write(0,*) me,' ',trim(name),' Checkpoint through' + end if - if (info /= psb_success_) then - call psb_errpush(psb_err_internal_error_,name,a_err='Build ac = coo_restr x am3') - goto 9999 + if (info /= psb_success_) then + call psb_errpush(psb_err_internal_error_,name,a_err='Build ac = coo_restr x am3') + goto 9999 + end if + else + ! + ! COO_PROL should arrive here with local numbering + ! + if (debug) write(0,*) me,' ',trim(name),' Size check on entry New: ',& + & coo_prol%get_fmt(),coo_prol%get_nrows(),coo_prol%get_ncols(),coo_prol%get_nzeros(),& + & nrow,ntaggr,naggr + + call coo_prol%cp_to_fmt(csr_prol,info) + + if (debug) write(0,*) me,trim(name),' Product AxPROL ',& + & a_csr%get_nrows(),a_csr%get_ncols(), csr_prol%get_nrows(), & + & desc_a%get_local_rows(),desc_a%get_local_cols(),& + & desc_ac%get_local_rows(),desc_a%get_local_cols() + if (debug) flush(0) + + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(a_csr,desc_a,csr_prol,acsr3,desc_ac,info) + if (do_timings) call psb_toc(idx_spspmm) + + if (debug) write(0,*) me,trim(name),' Done AxPROL ',& + & acsr3%get_nrows(),acsr3%get_ncols(), acsr3%get_nzeros(),& + & desc_ac%get_local_rows(),desc_ac%get_local_cols() + + ! + ! Ok first product done. + ! + ! Remember that RESTR must be built from PROL after halo extension, + ! which is done above in psb_par_spspmm + if (debug) write(0,*)me,' ',name,' No inp_restr, transposing prol ',& + & csr_prol%get_nrows(),csr_prol%get_ncols(),csr_prol%get_nzeros() + call csr_prol%mv_to_coo(coo_restr,info) +!!$ write(0,*)me,' ',name,' new into transposition ',coo_restr%get_nrows(),& +!!$ & coo_restr%get_ncols(),coo_restr%get_nzeros() + if (debug) call check_coo(me,trim(name)//' Check 1 (before transp) on coo_restr:',coo_restr) + + call coo_restr%transp() + nzl = coo_restr%get_nzeros() + nrl = desc_ac%get_local_rows() + i=0 + ! + ! Only keep local rows + ! + do k=1, nzl + if ((1 <= coo_restr%ia(k)) .and.(coo_restr%ia(k) <= nrl)) then + i = i+1 + coo_restr%val(i) = coo_restr%val(k) + coo_restr%ia(i) = coo_restr%ia(k) + coo_restr%ja(i) = coo_restr%ja(k) + end if + end do + call coo_restr%set_nzeros(i) + call coo_restr%fix(info) + nzl = coo_restr%get_nzeros() + call coo_restr%set_nrows(desc_ac%get_local_rows()) + call coo_restr%set_ncols(desc_a%get_local_cols()) + if (debug) call check_coo(me,trim(name)//' Check 2 on coo_restr:',coo_restr) + call csr_restr%cp_from_coo(coo_restr,info) + +!!$ write(0,*)me,' ',name,' after transposition ',coo_restr%get_nrows(),coo_restr%get_ncols(),coo_restr%get_nzeros() + + if (info /= psb_success_) then + call psb_errpush(psb_err_from_subroutine_,name,a_err='spcnv coo_restr') + goto 9999 + end if + if (debug_level >= psb_debug_outer_) & + & write(debug_unit,*) me,' ',trim(name),& + & 'starting sphalo/ rwxtd' + + if (debug) write(0,*) me,trim(name),' Product RESTRxAP ',& + & csr_restr%get_nrows(),csr_restr%get_ncols(), & + & desc_ac%get_local_rows(),desc_a%get_local_cols(),& + & acsr3%get_nrows(),acsr3%get_ncols() + if (do_timings) call psb_tic(idx_spspmm) + call psb_par_spspmm(csr_restr,desc_a,acsr3,ac_csr,desc_ac,info) + if (do_timings) call psb_toc(idx_spspmm) + call acsr3%free() + + call psb_cdasb(desc_ac,info) + + call ac_csr%set_nrows(desc_ac%get_local_rows()) + call ac_csr%set_ncols(desc_ac%get_local_cols()) + call ac%mv_from(ac_csr) + call ac%set_asb() + + if (debug) write(0,*) me,' ',trim(name),' After mv_from',psb_get_errstatus() + if (debug) write(0,*) me,' ',trim(name),' ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros(),naggr,ntaggr + ! write(0,*) me,' ',trim(name),' Final AC newstyle ',ac%get_fmt(),ac%get_nrows(),ac%get_ncols(),ac%get_nzeros() + + call coo_prol%set_ncols(desc_ac%get_local_cols()) + !call coo_restr%mv_from_ifmt(csr_restr,info) +!!$ call coo_restr%set_nrows(desc_ac%get_local_rows()) +!!$ call coo_restr%set_ncols(desc_a%get_local_cols()) + if (debug) call check_coo(me,trim(name)//' Check 3 on coo_restr:',coo_restr) end if @@ -386,5 +592,21 @@ subroutine mld_ld_spmm_bld_inner(a_csr,desc_a,nlaggr,parms,ac,& 9999 call psb_error_handler(err_act) return - + +contains + subroutine check_coo(me,string,coo) + implicit none + integer(psb_ipk_) :: me + type(psb_ld_coo_sparse_mat) :: coo + character(len=*) :: string + integer(psb_lpk_) :: nr,nc,nz + nr = coo%get_nrows() + nc = coo%get_ncols() + nz = coo%get_nzeros() + write(0,*) me,string,nr,nc,& + & minval(coo%ia(1:nz)),maxval(coo%ia(1:nz)),& + & minval(coo%ja(1:nz)),maxval(coo%ja(1:nz)) + + end subroutine check_coo + end subroutine mld_ld_spmm_bld_inner diff --git a/mlprec/impl/aggregator/mld_daggrmat_nosmth_bld.f90 b/mlprec/impl/aggregator/mld_daggrmat_nosmth_bld.f90 index 50511fb4..27fd0fb5 100644 --- a/mlprec/impl/aggregator/mld_daggrmat_nosmth_bld.f90 +++ b/mlprec/impl/aggregator/mld_daggrmat_nosmth_bld.f90 @@ -122,11 +122,11 @@ subroutine mld_daggrmat_nosmth_bld(a,desc_a,ilaggr,nlaggr,parms,& type(psb_ldspmat_type) :: la type(psb_ld_coo_sparse_mat) :: ac_coo, tmpcoo, coo_prol, coo_restr type(psb_ld_csr_sparse_mat) :: acsr1, acsr2, acsr - type(psb_desc_type) :: tmp_desc integer(psb_ipk_) :: debug_level, debug_unit integer(psb_lpk_) :: nrow, nglob, ncol, ntaggr, nzl, ip, & & naggr, nzt, naggrm1, naggrp1, i, k integer(psb_ipk_) :: inaggr, nzlp + logical, parameter :: debug = .false. name = 'mld_aggrmat_nosmth_bld' info = psb_success_ @@ -150,14 +150,28 @@ subroutine mld_daggrmat_nosmth_bld(a,desc_a,ilaggr,nlaggr,parms,& call a%cp_to(acsr) call op_prol%mv_to(coo_prol) inaggr = naggr - call psb_cdall(ictxt,tmp_desc,info,nl=inaggr) + call psb_cdall(ictxt,desc_ac,info,nl=inaggr) nzlp = coo_prol%get_nzeros() - call tmp_desc%indxmap%g2lip_ins(coo_prol%ja(1:nzlp),info) - call coo_prol%set_ncols(tmp_desc%get_local_cols()) + call desc_ac%indxmap%g2lip_ins(coo_prol%ja(1:nzlp),info) + call coo_prol%set_ncols(desc_ac%get_local_cols()) + + if (debug) call check_coo(me,trim(name)//' Check 1 on coo_prol:',coo_prol) + + call psb_cdasb(desc_ac,info) + call psb_cd_reinit(desc_ac,info) + call op_prol%cp_from(coo_prol) + call mld_spmm_bld_inner(acsr,desc_a,nlaggr,parms,ac,& - & coo_prol,tmp_desc,coo_restr,info) - - call op_prol%mv_from(coo_prol) + & coo_prol,desc_ac,coo_restr,info) + + !call psb_cdasb(desc_ac,info) + call coo_restr%set_nrows(desc_ac%get_local_rows()) + call coo_restr%set_ncols(desc_a%get_local_cols()) + call coo_prol%set_nrows(desc_a%get_local_rows()) + call coo_prol%set_ncols(desc_ac%get_local_cols()) + + if (debug) call check_coo(me,trim(name)//' Check 1 on coo_restr:',coo_restr) + call op_restr%mv_from(coo_restr) call psb_erractionrestore(err_act) @@ -166,5 +180,19 @@ subroutine mld_daggrmat_nosmth_bld(a,desc_a,ilaggr,nlaggr,parms,& 9999 call psb_error_handler(err_act) return +contains + subroutine check_coo(me,string,coo) + implicit none + integer(psb_ipk_) :: me + type(psb_ld_coo_sparse_mat) :: coo + character(len=*) :: string + integer(psb_lpk_) :: nr,nc,nz + nr = coo%get_nrows() + nc = coo%get_ncols() + nz = coo%get_nzeros() + write(0,*) me,string,nr,nc,& + & minval(coo%ia(1:nz)),maxval(coo%ia(1:nz)),& + & minval(coo%ja(1:nz)),maxval(coo%ja(1:nz)) + end subroutine check_coo end subroutine mld_daggrmat_nosmth_bld diff --git a/mlprec/impl/aggregator/mld_daggrmat_smth_bld.f90 b/mlprec/impl/aggregator/mld_daggrmat_smth_bld.f90 index 1efad4ff..6f52d4eb 100644 --- a/mlprec/impl/aggregator/mld_daggrmat_smth_bld.f90 +++ b/mlprec/impl/aggregator/mld_daggrmat_smth_bld.f90 @@ -127,7 +127,6 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,& integer(psb_ipk_) :: inaggr, nzlp integer(psb_ipk_) :: ictxt, np, me character(len=20) :: name - type(psb_desc_type) :: tmp_desc type(psb_ld_coo_sparse_mat) :: coo_prol, coo_restr, tmpcoo type(psb_ld_csr_sparse_mat) :: acsr1, acsrf, csr_prol, acsr real(psb_dpk_), allocatable :: adiag(:) @@ -258,11 +257,13 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,& if (info /= psb_success_) goto 9999 inaggr = naggr - call psb_cdall(ictxt,tmp_desc,info,nl=inaggr) + call psb_cdall(ictxt,desc_ac,info,nl=inaggr) nzlp = coo_prol%get_nzeros() - call tmp_desc%indxmap%g2lip_ins(coo_prol%ja(1:nzlp),info) - call coo_prol%set_ncols(tmp_desc%get_local_cols()) + call desc_ac%indxmap%g2lip_ins(coo_prol%ja(1:nzlp),info) + call coo_prol%set_ncols(desc_ac%get_local_cols()) call coo_prol%mv_to_fmt(csr_prol,info) + call psb_cdasb(desc_ac,info) + call psb_cd_reinit(desc_ac,info) ! ! Build the smoothed prolongator using either A or Af ! acsr1 = (I-w*D*A) Prol acsr1 = (I-w*D*Af) Prol @@ -270,7 +271,7 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,& ! is a bit less readable, butsaves space and one extra matrix copy ! call omega_smooth(omega,acsrf) - call psb_par_spspmm(acsrf,desc_a,csr_prol,acsr1,tmp_desc,info) + call psb_par_spspmm(acsrf,desc_a,csr_prol,acsr1,desc_ac,info) if(info /= psb_success_) then call psb_errpush(psb_err_from_subroutine_,name,a_err='spspmm 1') goto 9999 @@ -284,8 +285,8 @@ subroutine mld_daggrmat_smth_bld(a,desc_a,ilaggr,nlaggr,parms,& call acsr1%mv_to_coo(coo_prol,info) call mld_spmm_bld_inner(acsr,desc_a,nlaggr,parms,ac,& - & coo_prol,tmp_desc,coo_restr,info) - + & coo_prol,desc_ac,coo_restr,info) + call op_prol%mv_from(coo_prol) call op_restr%mv_from(coo_restr)