From 8311c65d2f8646564c8bb89db22bd500f7fe19c2 Mon Sep 17 00:00:00 2001 From: Soren Rasmussen Date: Fri, 3 May 2019 12:22:21 +0100 Subject: [PATCH] saveing --- test/halo/Makefile | 10 ++--- test/halo/mpi.sub | 4 +- test/halo/psb_halo_df_test.f90 | 46 ++++++++++++++------ test/halo/run_tests.sh | 76 +++++++++++++++++++++++----------- 4 files changed, 92 insertions(+), 44 deletions(-) diff --git a/test/halo/Makefile b/test/halo/Makefile index 3568301f..ac840696 100644 --- a/test/halo/Makefile +++ b/test/halo/Makefile @@ -2,9 +2,9 @@ INSTALLDIR=../.. INCDIR=$(INSTALLDIR)/include/ MODDIR=$(INSTALLDIR)/modules/ include $(INCDIR)/Make.inc.psblas -P=4 +P=8 debug= -FFLAGS += -cpp -DITERATIONS=$(ITERATIONS) +FFLAGS += -cpp -DITERATIONS=$(ITERATIONS) -DSWAPCHOICE=$(SWAPCHOICE) FFLAGS += $(debug) FCOPT += $(FFLAGS) scorep_vars=SCOREP_WRAPPER_COMPILER_FLAGS="-O2" \ @@ -26,25 +26,25 @@ EXEDIR=./runs all: psb_halo_df_test psb_halo_df_test: getp.o psb_halo_df_test.o - $(scorep_opt) $(FLINK) $(LOPT) $^ -o halo_persistant_$(ITERATIONS)_iter $(PSBLAS_LIB) $(LDLIBS) + $(scorep_opt) $(FLINK) $(LOPT) $^ -o halo_$(SWAPCHOICE)$(ITERATIONS)_iter $(PSBLAS_LIB) $(LDLIBS) halo_test: getp.o psb_halo_df_test.o $(scorep_opt) $(FLINK) $(LOPT) $^ -o halo_test $(PSBLAS_LIB) $(LDLIBS) clean: rm -f *~ *.o psb_halo_df_test1 psb_halo_df_test halo_test \ - halo_persistant_output.txt \ rm -rf test-results/* /bin/rm -f $(DFOBJS) $(ZFOBJS) $(SFOBJS) $(CFOBJS)\ *$(.mod) $(EXEDIR)/psb_*f_sample cleanall: clean + rm -f halo_output.txt rm -rf scorep-2019* rm -rf scorep-failed-* score: scorep-score -r $(scorep_dir)/profile.cubex | grep -e MPI -e 'type max_buf' run: psb_halo_df_test - $(scorep_opt) mpirun -np $(P) ./psb_halo_df_test_$(ITERATIONS) matrices/ck104/ck104.inp + $(scorep_opt) mpirun -np $(P) ./halo_$(SWAPCHOICE)_$(ITERATIONS)_iter matrices/ck104/ck104.inp r: halo_test $(scorep_opt) mpirun -np $(P) ./halo_test matrices/ck104/ck104.inp diff --git a/test/halo/mpi.sub b/test/halo/mpi.sub index 4caed121..129ac6ef 100644 --- a/test/halo/mpi.sub +++ b/test/halo/mpi.sub @@ -3,7 +3,7 @@ ## MPI submission script for PBS on DELTA ## -------------------------------------- #PBS -N umbrella_corp -#PBS -l select=1:ncpus=16:mpiprocs=16 +#PBS -l select=2:ncpus=16:mpiprocs=16 ## ## half_hour - 30 minutes ## one_hour - 1 hour @@ -55,7 +55,7 @@ ml echo $PATH which mpirun echo "========= START =========" -timeout 5m cpuburn +# timeout 5m cpuburn ./run_tests.sh # echo "mpirun -machinefile hostfile -ppn 1 -np ${cpus} ${PROG} -v" # export OMP_NUM_THREADS=16 diff --git a/test/halo/psb_halo_df_test.f90 b/test/halo/psb_halo_df_test.f90 index 164c01ad..f286c3df 100644 --- a/test/halo/psb_halo_df_test.f90 +++ b/test/halo/psb_halo_df_test.f90 @@ -92,14 +92,16 @@ program psb_df_sample integer(psb_ipk_) :: sum_snd, min_snd, max_snd, num_snd, tot_snd, sum_tot_snd integer(psb_ipk_) :: sum_rcv, min_rcv, max_rcv, num_rcv, tot_rcv, sum_tot_rcv real(psb_spk_) :: ave_neighbors, ave_snd_buf, ave_rcv_buf, ave_tot_snd, ave_tot_rcv - real(psb_dpk_) :: alltoall_comm_t, ave_alltoall_comm_t, total_time, ave_time + real(psb_dpk_) :: alltoall_comm_t, ave_alltoall_comm_t, total_time, & + ave_time_pp, ave_time_pi real(psb_dpk_) :: median_comm_t, ave_alltoall_median_comm_t, min_comm_t, max_comm_t, & ave_min_comm_t, ave_max_comm_t, sum_median_comm_t, sum_min_comm_t, sum_max_comm_t, & ave_alltoall_max_comm_t, ave_alltoall_min_comm_t + real(psb_dpk_) :: halo_t_start, halo_t_end, ave_halo_t_pi, & + sum_halo_t, halo_time real(psb_spk_) :: ave_snd, ave_rcv integer(psb_ipk_) :: swap_mode - logical :: swap_persistent, swap_nonpersistent - logical, parameter :: openfile=OPENFILE + logical :: swap_persistent, swap_nonpersistent, file_exists call psb_init(ictxt) call psb_info(ictxt,me,np) @@ -275,12 +277,17 @@ program psb_df_sample swap_mode = SWAPCHOICE swap_persistent = iand(swap_mode,psb_swap_persistent_) /= 0 swap_nonpersistent = iand(swap_mode,psb_swap_nonpersistent_) /= 0 - + halo_time = 0 do iter = 1, num_iterations xa = iv xa(nrow+1:ncol) = -1 call x_col%set_vect(xa) + call MPI_Barrier(MPI_COMM_WORLD,ierr) + halo_t_start = MPI_Wtime() call psb_halo(x_col,desc_a,info,mode=swap_mode) + halo_t_end = MPI_Wtime() - halo_t_start + call MPI_Barrier(MPI_COMM_WORLD,ierr) + halo_time = halo_time + halo_t_end xa = x_col%get_vect() @@ -313,6 +320,10 @@ program psb_df_sample if (allocated(x_col%v)) then if (allocated(x_col%v%p)) then ! -- collect times --- + ! halo time + call MPI_Reduce(halo_time, sum_halo_t, 1, & + MPI_DOUBLE_PRECISION, MPI_SUM, psb_root_, MPI_COMM_WORLD, ierr) + ! total time call MPI_Reduce(x_col%v%p%total_time, total_time, 1, & MPI_DOUBLE_PRECISION, MPI_SUM, psb_root_, MPI_COMM_WORLD, ierr) @@ -346,6 +357,7 @@ program psb_df_sample call MPI_Reduce(ave_snd, ave_snd_buf, 1, MPI_REAL, & MPI_SUM, psb_root_, MPI_COMM_WORLD, ierr) ave_snd_buf = ave_snd_buf / np + print *, "snd_counts = ", x_col%v%p%snd_counts call MPI_Reduce(minval(x_col%v%p%snd_counts), min_snd, 1, MPI_INTEGER, MPI_MIN, & psb_root_, MPI_COMM_WORLD, ierr) call MPI_Reduce(maxval(x_col%v%p%snd_counts), max_snd, 1, MPI_INTEGER, MPI_MAX, & @@ -375,14 +387,17 @@ program psb_df_sample psb_root_, MPI_COMM_WORLD, ierr) if (me == psb_root_) then - if (openfile .and. (np .eq. 2)) then + inquire(file='halo_output.txt', exist=file_exists) + if (.not. file_exists) then open(1,file='halo_output.txt') write(1,'(A)',advance='no') "partition;" write(1,'(A)',advance='no') "np;" write(1,'(A)',advance='no') "num_iterations;" write(1,'(A)',advance='no') "swap_mode;" - write(1,'(A)',advance='no') "total_time;" - write(1,'(A)',advance='no') "ave_time;" + write(1,'(A)',advance='no') "ave_halo_t_pi;" + ! write(1,'(A)',advance='no') "total_time;" + ! write(1,'(A)',advance='no') "ave_time_pp;" + write(1,'(A)',advance='no') "ave_time_pi;" write(1,'(A)',advance='no') "alltoall_comm_t;" write(1,'(A)',advance='no') "ave_alltoall_comm_t;" write(1,'(A)',advance='no') "ave_alltoall_median_comm_t;" @@ -411,7 +426,10 @@ program psb_df_sample ! write(1,*) "" ! converting microseconds total_time = total_time * 1000000 - ave_time = total_time / np + ave_time_pp = total_time / np + ave_time_pi = total_time / (np * num_iterations) + sum_halo_t = sum_halo_t * 1000000 + ave_halo_t_pi = sum_halo_t / (np * num_iterations) alltoall_comm_t = alltoall_comm_t * 1000000 ave_alltoall_comm_t = alltoall_comm_t / (np * num_iterations) sum_median_comm_t = sum_median_comm_t * 1000000 @@ -436,15 +454,17 @@ program psb_df_sample write(1,'(I5 A1)',advance='no') np, ';' write(1,'(I6 A1)',advance='no') num_iterations, ';' write(1,'(I5 A1)' ,advance='no') swap_mode, ';' - write(1,'(F20.4 A1)',advance='no') total_time, ';' - write(1,'(F20.4 A1)',advance='no') ave_time, ';' + write(1,'(I5 A1)' ,advance='no') ave_halo_t_pi, ';' + ! write(1,'(F20.4 A1)',advance='no') total_time, ';' + ! write(1,'(F20.4 A1)',advance='no') ave_time_pp, ';' + write(1,'(F20.4 A1)',advance='no') ave_time_pi, ';' write(1,'(F20.4 A1)',advance='no') alltoall_comm_t, ';' write(1,'(F20.4 A1)',advance='no') ave_alltoall_comm_t, ';' write(1,'(F20.4 A1)',advance='no') ave_alltoall_median_comm_t, ';' write(1,'(F20.4 A1)',advance='no') ave_alltoall_max_comm_t, ';' write(1,'(F20.4 A1)',advance='no') ave_alltoall_min_comm_t, ';' - write(1,'(F10.2 A1)',advance='no') request_create_t, ';' - write(1,'(F10.2 A1)',advance='no') ave_request_create_t, ';' + write(1,'(F10.2 A1)',advance='no') request_create_t, ';' + write(1,'(F10.2 A1)',advance='no') ave_request_create_t, ';' write(1,'(F10.2 A1)',advance='no') ave_neighbors, ';' write(1,'(I5 A1)' ,advance='no') min_neighbors, ';' @@ -456,7 +476,7 @@ program psb_df_sample write(1,'(I7 A1)',advance='no') max_snd, ';' write(1,'(I7 A1)',advance='no') max_rcv, ';' write(1,'(I7 A1)',advance='no') min_snd, ';' - write(1,'(I7 A1)',advance='no') min_rcv, ';' + write(1,'(I7 A1)',advance='no') min_rcv ! write(1,'(F9.2 A1)',advance='no') , ';' ! write(1,'(I5 A1)',advance='no') , ';' diff --git a/test/halo/run_tests.sh b/test/halo/run_tests.sh index cb2d41a9..bebc15cc 100755 --- a/test/halo/run_tests.sh +++ b/test/halo/run_tests.sh @@ -1,9 +1,9 @@ #!/bin/bash matrix_dir=../matrices bin_dir=../bin - # "cz148" - # "cz308" declare -a matrices=( + "cz148" + "cz308" "cz628" "cz1268" "cz2548" @@ -14,44 +14,72 @@ declare -a matrices=( ) declare -a iterations=( - "10" "300" "600" "900" "1200" "1500" "1800" "2100" "2400" "2700" "3000" + "1" "10" "25" "50" "100" "150" "200" "300" "600" "1000" "1500" "2000" "3000" ) -# declare -a iterations=( -# "10" "100" -# ) +declare -a numprocs=( + "2" "3" "4" "5" "6" "7" "8" "9" "10" "11" "12" "13" "14" "15" "16" +) -# ----------- BUILD ----------- -# for iter in "${iterations[@]}"; do -# make clean -# make ITERATIONS=${iter} -# mv halo_persistant_${iter}_iter bin/ -# done +declare -a swapchoices=( + "psb_swap_persistent_" "psb_swap_nonpersistent_" +) -# ------------ RUN ------------ +# -- for testing -- +# declare -a iterations=( +# "1" "10" +# ) +declare -a numprocs=( + "16" +) +declare -a matrices=( + "cz20468" + "cz40948" +) +# ----------------- +# ----------- BUILD ----------- +build=true +if [ "$build" = true ]; then +for swap in "${swapchoices[@]}"; do +for iter in "${iterations[@]}"; do + make clean + make ITERATIONS=${iter} SWAPCHOICE=${swap} + mv halo_${swap}${iter}_iter bin/ +done +done +fi -# go to directory to run +# ------------ RUN ------------ +run=false +if [ "$run" = true ]; then for matrix in "${matrices[@]}"; do -for iter in "${iterations[@]}"; do - # echo $matrix $iter rm run/* cd run - bin=halo_persistant_${iter}_iter - # matrix=cz308 +for swap in "${swapchoices[@]}"; do +for iter in "${iterations[@]}"; do + # echo $matrix $iter + bin=halo_${swap}${iter}_iter + cp ${matrix_dir}/${matrix}/* ./ cp ${bin_dir}/${bin} ./ - mkdir ../results/halo-persistant/${matrix}_matrix - write_to=../results/halo-persistant/${matrix}_matrix/${iter}_iter_${matrix}_matrix + for np in "${numprocs[@]}"; do SCOREP_TIMER=gettimeofday \ - SCOREP_EXPERIMENT_DIRECTORY=${write_to} \ + SCOREP_EXPERIMENT_DIRECTORY=../results/scorep/${matrix}${swap}${iter} \ SCOREP_WRAPPER_COMPILER_FLAGS="-O2" \ SCOREP_MPI_ENABLE_GROUP=all \ - mpirun --map-by ppr:16:node -machinefile $PBS_NODEFILE ./${bin} ${matrix}.inp - mv halo_persistant_output.txt ${write_to} - cd .. + mpirun --map-by ppr:${np}:node -machinefile $PBS_NODEFILE ./${bin} ${matrix}.inp + + # mpirun -np ${np} ./${bin} ${matrix}.inp + + done +done done + write_to=../results/${matrix}.output + mv halo_output.txt ${write_to} + cd .. done +fi