From 02586998a37305f8cdf89244f5af6ffde1fbdbcf Mon Sep 17 00:00:00 2001 From: Soren Rasmussen Date: Tue, 30 Apr 2019 15:45:56 +0100 Subject: [PATCH] almost all ready for testing --- test/halo/Makefile | 60 +++++++ test/halo/getp.f90 | 281 +++++++++++++++++++++++++++++++++ test/halo/mpi.sub | 69 ++++++++ test/halo/psb_halo_df_test.f90 | 170 ++++++++++++++------ test/halo/run_tests.sh | 57 +++++++ 5 files changed, 591 insertions(+), 46 deletions(-) create mode 100644 test/halo/Makefile create mode 100644 test/halo/getp.f90 create mode 100644 test/halo/mpi.sub create mode 100755 test/halo/run_tests.sh diff --git a/test/halo/Makefile b/test/halo/Makefile new file mode 100644 index 00000000..4cc54839 --- /dev/null +++ b/test/halo/Makefile @@ -0,0 +1,60 @@ +INSTALLDIR=../.. +INCDIR=$(INSTALLDIR)/include/ +MODDIR=$(INSTALLDIR)/modules/ +include $(INCDIR)/Make.inc.psblas +P=16 +debug= +FFLAGS += -cpp -DITERATIONS=$(ITERATIONS) +FFLAGS += $(debug) +FCOPT += $(FFLAGS) +scorep_vars=SCOREP_WRAPPER_COMPILER_FLAGS="-O2" \ + SCOREP_MPI_ENABLE_GROUP=all SCOREP_TIMER=gettimeofday +scorep_dir=test-results/scorep-2019-test-$(ITERATIONS) +scorep_opt=SCOREP_EXPERIMENT_DIRECTORY=$(scorep_dir) $(scorep_vars) +# +# Libraries used +# +MPFC=scorep-mpif90 +# MPFC=mpif90 +LIBDIR=$(INSTALLDIR)/lib/ +PSBLAS_LIB= -L$(LIBDIR) -lpsb_util -lpsb_krylov -lpsb_prec -lpsb_base +LDLIBS=$(PSBLDLIBS) +FINCLUDES=$(FMFLAG)$(MODDIR) $(FMFLAG). +HALOOBJS=getp.o +EXEDIR=./runs + +all: psb_halo_df_test + +psb_halo_df_test: getp.o psb_halo_df_test.o + $(scorep_opt) $(FLINK) $(LOPT) $^ -o halo_persistant_$(ITERATIONS)_iter $(PSBLAS_LIB) $(LDLIBS) + +psb_halo_df_test1: $(HALOOBJS) psb_halo_df_test1.o + $(FLINK) $(debug) $(LOPT) $(HALOOBJS) -o psb_halo_df_test1 $(PSBLAS_LIB) $(LDLIBS) + +clean: + rm -f *~ *.o psb_halo_df_test1 psb_halo_df_test + /bin/rm -f $(DFOBJS) $(ZFOBJS) $(SFOBJS) $(CFOBJS)\ + *$(.mod) $(EXEDIR)/psb_*f_sample +cleanall: clean + rm -rf scorep-2019* + rm -rf scorep-failed-* +score: + scorep-score -r $(scorep_dir)/profile.cubex | grep -e MPI -e 'type max_buf' + +run: psb_halo_df_test + $(scorep_opt) mpirun -np $(P) ./psb_halo_df_test_$(ITERATIONS) matrices/ck104/ck104.inp + +subrun: + $(scorep_opt) mpirun -machinefile hostfile -ppn 1 -np $(cpus) ./psb_halo_df_test_$(ITERATIONS) matrices/$(MATRIX)/$(MATRIX).inp + +debug: getp.o psb_halo_df_test.o + $(FLINK) $(debug) $(LOPT) $^ -o psb_halo_df_test $(PSBLAS_LIB) $(LDLIBS) + mpirun -np $(P) xterm -hold -e gdb -ex run --args ./psb_halo_df_test matrices/ck104/ck104.inp + +test: getp.o psb_halo_df_test1.o + $(FLINK) $(debug) $(LOPT) $^ -o psb_halo_df_test1 $(PSBLAS_LIB) $(LDLIBS) + mpirun -np $(P) ./psb_halo_df_test1 matrices/pde225/pde225.inp +lib: + (cd ../../; make library) +verycleanlib: + (cd ../../; make veryclean) diff --git a/test/halo/getp.f90 b/test/halo/getp.f90 new file mode 100644 index 00000000..e9ddf7d2 --- /dev/null +++ b/test/halo/getp.f90 @@ -0,0 +1,281 @@ +! +! Parallel Sparse BLAS version 3.5 +! (C) Copyright 2006-2018 +! Salvatore Filippone +! Alfredo Buttari +! +! Redistribution and use in source and binary forms, with or without +! modification, are permitted provided that the following conditions +! are met: +! 1. Redistributions of source code must retain the above copyright +! notice, this list of conditions and the following disclaimer. +! 2. Redistributions in binary form must reproduce the above copyright +! notice, this list of conditions, and the following disclaimer in the +! documentation and/or other materials provided with the distribution. +! 3. The name of the PSBLAS group or the names of its contributors may +! not be used to endorse or promote products derived from this +! software without specific written permission. +! +! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +! ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +! TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +! PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE PSBLAS GROUP OR ITS CONTRIBUTORS +! BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +! CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +! SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +! INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +! CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +! ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +! POSSIBILITY OF SUCH DAMAGE. +! +! +Module getp + interface get_parms + module procedure get_dparms, get_sparms + end interface + +contains + ! + ! Get iteration parameters from the command line + ! + subroutine get_dparms(ictxt,mtrx_file,rhs_file,filefmt,kmethd,ptype,part,& + & afmt,istopc,itmax,itrace,irst,eps) + use psb_base_mod + integer(psb_ipk_) :: ictxt + character(len=2) :: filefmt + character(len=40) :: kmethd, mtrx_file, rhs_file, ptype + character(len=20) :: part + integer(psb_ipk_) :: iret, istopc,itmax,itrace,irst + character(len=40) :: charbuf + real(psb_dpk_) :: eps + character :: afmt*5 + integer(psb_ipk_) :: np, iam + integer(psb_ipk_) :: inparms(40), ip, inp_unit + character(len=1024) :: filename + + call psb_info(ictxt,iam,np) + if (iam == 0) then + if (command_argument_count()>0) then + call get_command_argument(1,filename) + inp_unit = 30 + open(inp_unit,file=filename,action='read',iostat=info) + if (info /= 0) then + write(psb_err_unit,*) 'Could not open file ',filename,' for input' + call psb_abort(ictxt) + stop + else + write(psb_err_unit,*) 'Opened file ',trim(filename),' for input' + end if + else + inp_unit=psb_inp_unit + end if + ! Read Input Parameters + read(inp_unit,*) ip + if (ip >= 5) then + read(inp_unit,*) mtrx_file + read(inp_unit,*) rhs_file + read(inp_unit,*) filefmt + read(inp_unit,*) kmethd + read(inp_unit,*) ptype + read(inp_unit,*) afmt + read(inp_unit,*) part + + + call psb_bcast(ictxt,mtrx_file) + call psb_bcast(ictxt,rhs_file) + call psb_bcast(ictxt,filefmt) + call psb_bcast(ictxt,kmethd) + call psb_bcast(ictxt,ptype) + call psb_bcast(ictxt,afmt) + call psb_bcast(ictxt,part) + + if (ip >= 7) then + read(inp_unit,*) istopc + else + istopc=1 + endif + if (ip >= 8) then + read(inp_unit,*) itmax + else + itmax=500 + endif + if (ip >= 9) then + read(inp_unit,*) itrace + else + itrace=-1 + endif + if (ip >= 10) then + read(inp_unit,*) irst + else + irst = 1 + endif + if (ip >= 11) then + read(inp_unit,*) eps + else + eps=1.d-6 + endif + inparms(1) = istopc + inparms(2) = itmax + inparms(3) = itrace + inparms(4) = irst + call psb_bcast(ictxt,inparms(1:4)) + call psb_bcast(ictxt,eps) + + write(psb_out_unit,'("Solving matrix : ",a)') mtrx_file + write(psb_out_unit,'("Number of processors : ",i3)') np + write(psb_out_unit,'("Data distribution : ",a)') part + write(psb_out_unit,'("Iterative method : ",a)') kmethd + write(psb_out_unit,'("Preconditioner : ",a)') ptype + write(psb_out_unit,'("Restart parameter : ",i2)') irst + write(psb_out_unit,'("Storage format : ",a)') afmt(1:3) + write(psb_out_unit,'(" ")') + else + write(psb_err_unit,*) 'Wrong format for input file' + call psb_abort(ictxt) + stop 1 + end if + if (inp_unit /= psb_inp_unit) then + close(inp_unit) + end if + else + ! Receive Parameters + call psb_bcast(ictxt,mtrx_file) + call psb_bcast(ictxt,rhs_file) + call psb_bcast(ictxt,filefmt) + call psb_bcast(ictxt,kmethd) + call psb_bcast(ictxt,ptype) + call psb_bcast(ictxt,afmt) + call psb_bcast(ictxt,part) + + call psb_bcast(ictxt,inparms(1:4)) + istopc = inparms(1) + itmax = inparms(2) + itrace = inparms(3) + irst = inparms(4) + call psb_bcast(ictxt,eps) + + end if + + end subroutine get_dparms + + subroutine get_sparms(ictxt,mtrx_file,rhs_file,filefmt,kmethd,ptype,part,& + & afmt,istopc,itmax,itrace,irst,eps) + use psb_base_mod + integer(psb_ipk_) :: ictxt + character(len=2) :: filefmt + character(len=40) :: kmethd, mtrx_file, rhs_file, ptype + character(len=20) :: part + integer(psb_ipk_) :: iret, istopc,itmax,itrace,irst + character(len=40) :: charbuf + real(psb_spk_) :: eps + character :: afmt*5 + integer(psb_ipk_) :: np, iam + integer(psb_ipk_) :: inparms(40), ip, inp_unit + character(len=1024) :: filename + + call psb_info(ictxt,iam,np) + if (iam == 0) then + if (command_argument_count()>0) then + call get_command_argument(1,filename) + inp_unit = 30 + open(inp_unit,file=filename,action='read',iostat=info) + if (info /= 0) then + write(psb_err_unit,*) 'Could not open file ',filename,' for input' + call psb_abort(ictxt) + stop + else + write(psb_err_unit,*) 'Opened file ',trim(filename),' for input' + end if + else + inp_unit=inp_unit + end if + ! Read Input Parameters + read(inp_unit,*) ip + if (ip >= 5) then + read(inp_unit,*) mtrx_file + read(inp_unit,*) rhs_file + read(inp_unit,*) filefmt + read(inp_unit,*) kmethd + read(inp_unit,*) ptype + read(inp_unit,*) afmt + read(inp_unit,*) ipart + + + call psb_bcast(ictxt,mtrx_file) + call psb_bcast(ictxt,rhs_file) + call psb_bcast(ictxt,filefmt) + call psb_bcast(ictxt,kmethd) + call psb_bcast(ictxt,ptype) + call psb_bcast(ictxt,afmt) + call psb_bcast(ictxt,part) + + if (ip >= 7) then + read(inp_unit,*) istopc + else + istopc=1 + endif + if (ip >= 8) then + read(inp_unit,*) itmax + else + itmax=500 + endif + if (ip >= 9) then + read(inp_unit,*) itrace + else + itrace=-1 + endif + if (ip >= 10) then + read(inp_unit,*) irst + else + irst = 1 + endif + if (ip >= 11) then + read(inp_unit,*) eps + else + eps=1.d-6 + endif + inparms(1) = istopc + inparms(2) = itmax + inparms(3) = itrace + inparms(4) = irst + call psb_bcast(ictxt,inparms(1:4)) + call psb_bcast(ictxt,eps) + + write(psb_out_unit,'("Solving matrix : ",a)') mtrx_file + write(psb_out_unit,'("Number of processors : ",i3)') np + write(psb_out_unit,'("Data distribution : ",a)') part + write(psb_out_unit,'("Iterative method : ",a)') kmethd + write(psb_out_unit,'("Preconditioner : ",a)') ptype + write(psb_out_unit,'("Restart parameter : ",i2)') irst + write(psb_out_unit,'("Storage format : ",a)') afmt(1:3) + write(psb_out_unit,'(" ")') + else + write(psb_err_unit,*) 'Wrong format for input file' + call psb_abort(ictxt) + stop 1 + end if + if (inp_unit /= psb_inp_unit) then + close(inp_unit) + end if + else + ! Receive Parameters + call psb_bcast(ictxt,mtrx_file) + call psb_bcast(ictxt,rhs_file) + call psb_bcast(ictxt,filefmt) + call psb_bcast(ictxt,kmethd) + call psb_bcast(ictxt,ptype) + call psb_bcast(ictxt,afmt) + call psb_bcast(ictxt,part) + + call psb_bcast(ictxt,inparms(1:4)) + istopc = inparms(1) + itmax = inparms(2) + itrace = inparms(3) + irst = inparms(4) + call psb_bcast(ictxt,eps) + + end if + + end subroutine get_sparms + +end module getp diff --git a/test/halo/mpi.sub b/test/halo/mpi.sub new file mode 100644 index 00000000..f23e48f8 --- /dev/null +++ b/test/halo/mpi.sub @@ -0,0 +1,69 @@ +#!/bin/bash +## +## MPI submission script for PBS on DELTA +## -------------------------------------- +#PBS -N umbrella_corp +#PBS -l select=1:ncpus=16:mpiprocs=16 +## +## half_hour - 30 minutes +## one_hour - 1 hour +## half_day - 12 hours +## one_day - 24 hours +## two_day - 48 hours +## five_day - 120 hours +## ten_day - 240 hours (by special arrangement) +## +#PBS -q one_hour +#PBS -m n +#PBS -M s.rasmussen@cranfield.ac.uk +## +## ==================================== +## DO NOT CHANGE THE LINES BETWEEN HERE +## ==================================== +#PBS -j oe +#PBS -W sandbox=PRIVATE +#PBS -k n +ln -s $PWD $PBS_O_WORKDIR/$PBS_JOBID +## Change to working directory +cd $PBS_O_WORKDIR +rm -f hostfile +touch hostfile +cat $PBS_NODEFILE | sort | uniq >> hostfile +echo "========= PBS_NODEFILE =========" +# cat $PBS_NODEFILE +echo "========= hostfile =========" +cat hostfile +echo "========= end =========" +export NPROCS=`wc -l hostfile |gawk '//{print $1}'` +echo "NPROCS = ${NPROCS}" + +## Calculate number of CPUs +cpus=`cat hostfile | wc -l` +echo "cpus = ${cpus}" +## set some MPI tuning parameters to use the correct transport +export I_MPI_FABRICS=shm:dapl +export I_MPI_DAPL_UD=enable +export I_MPI_PLATFORM=bdw +export I_MPI_ADJUST_ALLREDUCE=5 +## ======== +## AND HERE +## ======== +## +module load score-p-5.0-gcc-8.2 + +ml +echo $PATH +which mpirun +echo "========= START =========" +timeout 5m cpuburn +./run_tests.sh +# echo "mpirun -machinefile hostfile -ppn 1 -np ${cpus} ${PROG} -v" +# export OMP_NUM_THREADS=16 +# mpirun -machinefile hostfile -ppn 1 -np ${cpus} ${PROG} -v +echo "========== FIN ==========" + +## Tidy up the log directory +## DO NOT CHANGE THE LINE BELOW +## ============================ +rm $PBS_O_WORKDIR/$PBS_JOBID +# diff --git a/test/halo/psb_halo_df_test.f90 b/test/halo/psb_halo_df_test.f90 index 8cec8a99..5fe8fa44 100644 --- a/test/halo/psb_halo_df_test.f90 +++ b/test/halo/psb_halo_df_test.f90 @@ -1,3 +1,4 @@ + ! ! Parallel Sparse BLAS version 3.5 ! (C) Copyright 2006-2018 @@ -37,6 +38,7 @@ program psb_df_sample use psb_krylov_mod use psb_util_mod use getp + use mpi implicit none ! input parameters @@ -57,7 +59,7 @@ program psb_df_sample ! communications data structure type(psb_desc_type):: desc_a - integer(psb_ipk_) :: ictxt, iam, np + integer(psb_ipk_) :: ictxt, me, np ! solver paramters integer(psb_ipk_) :: iter, itmax, ierr, itrace, ircode,& @@ -74,7 +76,7 @@ program psb_df_sample ! other variables integer(psb_ipk_) :: i,info,j,m_problem, err_act integer(psb_ipk_) :: internal, m,ii,nnzero - real(psb_dpk_) :: t1, t2, tprec + real(psb_dpk_) :: t1, t2, tprec, t_low, t_high, t_mean, t_sum real(psb_dpk_) :: r_amax, b_amax, scale,resmx,resmxp integer(psb_ipk_) :: nrhs, nrow, n_row, dim, ne, nv, ncol integer(psb_ipk_), allocatable :: ivg(:), perm(:) @@ -84,11 +86,12 @@ program psb_df_sample ! persistant communication integer(psb_lpk_), allocatable :: iv(:) real(psb_dpk_), allocatable :: xa(:) + integer(psb_lpk_) :: num_iterations call psb_init(ictxt) - call psb_info(ictxt,iam,np) + call psb_info(ictxt,me,np) - if (iam < 0) then + if (me < 0) then ! This should not happen, but just in case call psb_exit(ictxt) stop @@ -102,7 +105,7 @@ program psb_df_sample ! ! Hello world ! - if (iam == psb_root_) then + if (me == psb_root_) then write(*,*) 'Welcome to PSBLAS version: ',psb_version_string_ write(*,*) 'This is the ',trim(name),' sample program' end if @@ -117,7 +120,7 @@ program psb_df_sample ! read the input matrix to be processed and (possibly) the rhs nrhs = 1 - if (iam == psb_root_) then + if (me == psb_root_) then select case(psb_toupper(filefmt)) case('MM') ! For Matrix Market we have an input file for the matrix @@ -177,7 +180,7 @@ program psb_df_sample ! switch over different partition types select case(psb_toupper(part)) case('BLOCK') - if (iam == psb_root_) write(psb_out_unit,'("Partition type: block")') + if (me == psb_root_) write(psb_out_unit,'("Partition type: block")') call psb_matdist(aux_a, a, ictxt,desc_a,info,fmt=afmt,parts=part_block) ! call desc_a%cd_get_list ! call desc_a%cd_v_get_list @@ -185,7 +188,7 @@ program psb_df_sample case('GRAPH') - if (iam == psb_root_) then + if (me == psb_root_) then write(psb_out_unit,'("Partition type: graph vector")') write(psb_out_unit,'(" ")') ! write(psb_err_unit,'("Build type: graph")') @@ -198,7 +201,7 @@ program psb_df_sample call psb_matdist(aux_a, a, ictxt,desc_a,info,fmt=afmt,v=ivg) case default - if (iam == psb_root_) write(psb_out_unit,'("Partition type: block")') + if (me == psb_root_) write(psb_out_unit,'("Partition type: block")') call psb_matdist(aux_a, a, ictxt,desc_a,info,fmt=afmt,parts=part_block) end select @@ -214,60 +217,135 @@ program psb_df_sample ! call psb_loc_to_glob(iv,desc_a,info) ! call desc_a%l2gv1(iv,info) + xa = iv ! call desc_a%locgv1(iv,info) - ! ---- test where halo regions are set to -1, sending 10+iam - xa = iv - xa(nrow+1:ncol) = -1 - call x_col%set_vect(xa) - call psb_halo(x_col,desc_a,info,mode=psb_swap_persistent_) - - xa(nrow+1:ncol) = -1 - call x_col%set_vect(xa) - call psb_halo(x_col,desc_a,info,mode=psb_swap_persistent_) - xa = x_col%get_vect() - - do i=nrow+1,ncol - ! print *, iam, ": ", xa(i), "::", iv(i) - ! write(0,*) 'Mismatch ',i,xa(i),iv(i) - if (xa(i) /= iv(i)) then ! ha - write(0,*) iam, ': MISMATCH ',i,xa(i),iv(i) - end if + ! ---- test where halo regions are set to -1, sending 10+me + + ! xa(nrow+1:ncol) = -1 ! (10 + me) * -1 + ! ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + ! ! do i=0,np + ! ! if (me .eq. i) then + ! ! print *, "=======================" + ! ! print *, "me = ", me, "iv =" + ! ! print '(225 I4)', iv + ! ! print *, "-----------------------" + ! ! end if + ! ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + ! ! end do + ! ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + + ! ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + ! ! do i=0,np + ! ! if (me .eq. i) then + ! ! print *, "=======================" + ! ! print *, "me = ", me, "xa =" + ! ! print '(225 F4.0)', xa + ! ! print *, "-----------------------" + ! ! end if + ! ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + ! ! end do + ! ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + + ! call x_col%set_vect(xa) + ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + ! call psb_halo(x_col,desc_a,info,mode=psb_swap_persistent_) + + ! xa = x_col%get_vect() + ! do i=nrow+1,ncol + ! if (xa(i) /= iv(i)) then ! ha + ! write(0,*) me, ': MISMATCH i=',i,"xa(i)=",xa(i),"iv(i)",iv(i) + ! ! goto 9999 + ! end if + ! end do + num_iterations = ITERATIONS + do iter = 1, num_iterations + xa = iv + xa(nrow+1:ncol) = -1 + call x_col%set_vect(xa) + call psb_halo(x_col,desc_a,info,mode=psb_swap_persistent_) + xa = x_col%get_vect() + + do i=nrow+1,ncol + if (xa(i) /= iv(i)) then ! ha + write(0,*) me, ': MISMATCH i=',i,"xa(i)=",xa(i),"iv(i)",iv(i) + goto 9999 + end if + end do end do - call psb_geasb(x_col,desc_a,info) - call psb_geall(r_col,desc_a,info) - call r_col%zero() - call psb_geasb(r_col,desc_a,info) - t2 = psb_wtime() - t1 + ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + ! do i=0,np + ! if (me .eq. i) then + ! print *, "=======================" + ! print *, "me = ", me, " xa after halo" + ! print '(225 F4.0)', xa + ! print *, "me = ", me, " iv after halo" + ! print '(225 I4)', iv + ! print *, "-----------------------" + ! end if + ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + ! end do + ! call MPI_Barrier(MPI_COMM_WORLD, ierr) + ! goto 9999 + + + + + call MPI_Barrier(MPI_COMM_WORLD, ierr) + call flush(output_unit) + call MPI_Barrier(MPI_COMM_WORLD, ierr) + if (allocated(x_col%v)) then + print *, me, "desc_a%p%comm_create_time = ", x_col%v%p%comm_create_time + call MPI_Reduce(x_col%v%p%comm_create_time, t_sum, 1, MPI_DOUBLE_PRECISION, MPI_SUM, & + psb_root_, MPI_COMM_WORLD, ierr) + if (me == psb_root_) then + open(1,file='halo_persistant_output.txt', status='new') + + write(1,*) "flt type max_buf[B] visits time[s] time[%] time/visit[us] region" + t_mean = t_sum / np + write(1,*) "MPI", t_sum, " ", t_mean + end if + end if + call flush(output_unit) + call MPI_Barrier(MPI_COMM_WORLD, ierr) + if (me == psb_root_) then + print *, "did halo communication", num_iterations, "times" + end if + ! artless: commenting these because geasb is breaking it for now + ! call psb_geasb(x_col,desc_a,info) + ! call psb_geall(r_col,desc_a,info) + ! call r_col%zero() + ! call psb_geasb(r_col,desc_a,info) + ! t2 = psb_wtime() - t1 - call psb_amx(ictxt, t2) + ! call psb_amx(ictxt, t2) - if (iam == psb_root_) then - write(psb_out_unit,'(" ")') - write(psb_out_unit,'("Time to read and partition matrix : ",es12.5)')t2 - write(psb_out_unit,'(" ")') - end if + ! if (me == psb_root_) then + ! write(psb_out_unit,'(" ")') + ! write(psb_out_unit,'("Time to read and partition matrix : ",es12.5)')t2 + ! write(psb_out_unit,'(" ")') + ! end if ! ! call perc%init(ictxt,ptype,info) ! building the preconditioner - t1 = psb_wtime() + ! t1 = psb_wtime() ! call prec%build(a,desc_a,info) ! tprec = psb_wtime()-t1 - if (info /= psb_success_) then - call psb_errpush(psb_err_from_subroutine_,name,a_err='psb_precbld') - goto 9999 - end if + ! if (info /= psb_success_) then + ! call psb_errpush(psb_err_from_subroutine_,name,a_err='psb_precbld') + ! goto 9999 + ! end if ! call psb_amx(ictxt,tprec) - ! if(iam == psb_root_) then + ! if(me == psb_root_) then ! write(psb_out_unit,'("Preconditioner time: ",es12.5)')tprec ! write(psb_out_unit,'(" ")') ! end if @@ -294,7 +372,7 @@ program psb_df_sample ! call psb_sum(ictxt,descsize) ! call psb_sum(ictxt,precsize) - ! if (iam == psb_root_) then + ! if (me == psb_root_) then ! call prec%descr() ! write(psb_out_unit,'("Matrix: ",a)')mtrx_file ! write(psb_out_unit,'("Computed solution on ",i8," processors")')np @@ -321,7 +399,7 @@ program psb_df_sample ! & call psb_gather(r_col_glob,r_col,desc_a,info,root=psb_root_) ! if (info /= psb_success_) goto 9999 - ! if (iam == psb_root_) then + ! if (me == psb_root_) then ! write(psb_err_unit,'(" ")') ! write(psb_err_unit,'("Saving x on file")') ! write(20,*) 'matrix: ',mtrx_file @@ -346,7 +424,7 @@ program psb_df_sample call prec%free(info) call psb_cdfree(desc_a,info) call psb_exit(ictxt) - print *, "* FIN for", iam + print *, "* FIN for", me stop 9999 call psb_error(ictxt) diff --git a/test/halo/run_tests.sh b/test/halo/run_tests.sh new file mode 100755 index 00000000..cb2d41a9 --- /dev/null +++ b/test/halo/run_tests.sh @@ -0,0 +1,57 @@ +#!/bin/bash +matrix_dir=../matrices +bin_dir=../bin + # "cz148" + # "cz308" +declare -a matrices=( + "cz628" + "cz1268" + "cz2548" + "cz5108" + "cz10228" + "cz20468" + "cz40948" +) + +declare -a iterations=( + "10" "300" "600" "900" "1200" "1500" "1800" "2100" "2400" "2700" "3000" +) +# declare -a iterations=( +# "10" "100" +# ) + + +# ----------- BUILD ----------- +# for iter in "${iterations[@]}"; do +# make clean +# make ITERATIONS=${iter} +# mv halo_persistant_${iter}_iter bin/ +# done + + +# ------------ RUN ------------ + + + +# go to directory to run +for matrix in "${matrices[@]}"; do +for iter in "${iterations[@]}"; do + # echo $matrix $iter + rm run/* + cd run + bin=halo_persistant_${iter}_iter + # matrix=cz308 + cp ${matrix_dir}/${matrix}/* ./ + cp ${bin_dir}/${bin} ./ + + mkdir ../results/halo-persistant/${matrix}_matrix + write_to=../results/halo-persistant/${matrix}_matrix/${iter}_iter_${matrix}_matrix + SCOREP_TIMER=gettimeofday \ + SCOREP_EXPERIMENT_DIRECTORY=${write_to} \ + SCOREP_WRAPPER_COMPILER_FLAGS="-O2" \ + SCOREP_MPI_ENABLE_GROUP=all \ + mpirun --map-by ppr:16:node -machinefile $PBS_NODEFILE ./${bin} ${matrix}.inp + mv halo_persistant_output.txt ${write_to} + cd .. +done +done