almost all ready for testing
Soren Rasmussen
6 years ago
parent
4d8f7a20c4
commit
02586998a3
@ -0,0 +1,60 @@
|
||||
INSTALLDIR=../..
|
||||
INCDIR=$(INSTALLDIR)/include/
|
||||
MODDIR=$(INSTALLDIR)/modules/
|
||||
include $(INCDIR)/Make.inc.psblas
|
||||
P=16
|
||||
debug=
|
||||
FFLAGS += -cpp -DITERATIONS=$(ITERATIONS)
|
||||
FFLAGS += $(debug)
|
||||
FCOPT += $(FFLAGS)
|
||||
scorep_vars=SCOREP_WRAPPER_COMPILER_FLAGS="-O2" \
|
||||
SCOREP_MPI_ENABLE_GROUP=all SCOREP_TIMER=gettimeofday
|
||||
scorep_dir=test-results/scorep-2019-test-$(ITERATIONS)
|
||||
scorep_opt=SCOREP_EXPERIMENT_DIRECTORY=$(scorep_dir) $(scorep_vars)
|
||||
#
|
||||
# Libraries used
|
||||
#
|
||||
MPFC=scorep-mpif90
|
||||
# MPFC=mpif90
|
||||
LIBDIR=$(INSTALLDIR)/lib/
|
||||
PSBLAS_LIB= -L$(LIBDIR) -lpsb_util -lpsb_krylov -lpsb_prec -lpsb_base
|
||||
LDLIBS=$(PSBLDLIBS)
|
||||
FINCLUDES=$(FMFLAG)$(MODDIR) $(FMFLAG).
|
||||
HALOOBJS=getp.o
|
||||
EXEDIR=./runs
|
||||
|
||||
all: psb_halo_df_test
|
||||
|
||||
psb_halo_df_test: getp.o psb_halo_df_test.o
|
||||
$(scorep_opt) $(FLINK) $(LOPT) $^ -o halo_persistant_$(ITERATIONS)_iter $(PSBLAS_LIB) $(LDLIBS)
|
||||
|
||||
psb_halo_df_test1: $(HALOOBJS) psb_halo_df_test1.o
|
||||
$(FLINK) $(debug) $(LOPT) $(HALOOBJS) -o psb_halo_df_test1 $(PSBLAS_LIB) $(LDLIBS)
|
||||
|
||||
clean:
|
||||
rm -f *~ *.o psb_halo_df_test1 psb_halo_df_test
|
||||
/bin/rm -f $(DFOBJS) $(ZFOBJS) $(SFOBJS) $(CFOBJS)\
|
||||
*$(.mod) $(EXEDIR)/psb_*f_sample
|
||||
cleanall: clean
|
||||
rm -rf scorep-2019*
|
||||
rm -rf scorep-failed-*
|
||||
score:
|
||||
scorep-score -r $(scorep_dir)/profile.cubex | grep -e MPI -e 'type max_buf'
|
||||
|
||||
run: psb_halo_df_test
|
||||
$(scorep_opt) mpirun -np $(P) ./psb_halo_df_test_$(ITERATIONS) matrices/ck104/ck104.inp
|
||||
|
||||
subrun:
|
||||
$(scorep_opt) mpirun -machinefile hostfile -ppn 1 -np $(cpus) ./psb_halo_df_test_$(ITERATIONS) matrices/$(MATRIX)/$(MATRIX).inp
|
||||
|
||||
debug: getp.o psb_halo_df_test.o
|
||||
$(FLINK) $(debug) $(LOPT) $^ -o psb_halo_df_test $(PSBLAS_LIB) $(LDLIBS)
|
||||
mpirun -np $(P) xterm -hold -e gdb -ex run --args ./psb_halo_df_test matrices/ck104/ck104.inp
|
||||
|
||||
test: getp.o psb_halo_df_test1.o
|
||||
$(FLINK) $(debug) $(LOPT) $^ -o psb_halo_df_test1 $(PSBLAS_LIB) $(LDLIBS)
|
||||
mpirun -np $(P) ./psb_halo_df_test1 matrices/pde225/pde225.inp
|
||||
lib:
|
||||
(cd ../../; make library)
|
||||
verycleanlib:
|
||||
(cd ../../; make veryclean)
|
||||
@ -0,0 +1,281 @@
|
||||
!
|
||||
! Parallel Sparse BLAS version 3.5
|
||||
! (C) Copyright 2006-2018
|
||||
! Salvatore Filippone
|
||||
! Alfredo Buttari
|
||||
!
|
||||
! Redistribution and use in source and binary forms, with or without
|
||||
! modification, are permitted provided that the following conditions
|
||||
! are met:
|
||||
! 1. Redistributions of source code must retain the above copyright
|
||||
! notice, this list of conditions and the following disclaimer.
|
||||
! 2. Redistributions in binary form must reproduce the above copyright
|
||||
! notice, this list of conditions, and the following disclaimer in the
|
||||
! documentation and/or other materials provided with the distribution.
|
||||
! 3. The name of the PSBLAS group or the names of its contributors may
|
||||
! not be used to endorse or promote products derived from this
|
||||
! software without specific written permission.
|
||||
!
|
||||
! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
! ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
! TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
! PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE PSBLAS GROUP OR ITS CONTRIBUTORS
|
||||
! BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
! CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
! SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
! INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
! CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
! ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
! POSSIBILITY OF SUCH DAMAGE.
|
||||
!
|
||||
!
|
||||
Module getp
|
||||
interface get_parms
|
||||
module procedure get_dparms, get_sparms
|
||||
end interface
|
||||
|
||||
contains
|
||||
!
|
||||
! Get iteration parameters from the command line
|
||||
!
|
||||
subroutine get_dparms(ictxt,mtrx_file,rhs_file,filefmt,kmethd,ptype,part,&
|
||||
& afmt,istopc,itmax,itrace,irst,eps)
|
||||
use psb_base_mod
|
||||
integer(psb_ipk_) :: ictxt
|
||||
character(len=2) :: filefmt
|
||||
character(len=40) :: kmethd, mtrx_file, rhs_file, ptype
|
||||
character(len=20) :: part
|
||||
integer(psb_ipk_) :: iret, istopc,itmax,itrace,irst
|
||||
character(len=40) :: charbuf
|
||||
real(psb_dpk_) :: eps
|
||||
character :: afmt*5
|
||||
integer(psb_ipk_) :: np, iam
|
||||
integer(psb_ipk_) :: inparms(40), ip, inp_unit
|
||||
character(len=1024) :: filename
|
||||
|
||||
call psb_info(ictxt,iam,np)
|
||||
if (iam == 0) then
|
||||
if (command_argument_count()>0) then
|
||||
call get_command_argument(1,filename)
|
||||
inp_unit = 30
|
||||
open(inp_unit,file=filename,action='read',iostat=info)
|
||||
if (info /= 0) then
|
||||
write(psb_err_unit,*) 'Could not open file ',filename,' for input'
|
||||
call psb_abort(ictxt)
|
||||
stop
|
||||
else
|
||||
write(psb_err_unit,*) 'Opened file ',trim(filename),' for input'
|
||||
end if
|
||||
else
|
||||
inp_unit=psb_inp_unit
|
||||
end if
|
||||
! Read Input Parameters
|
||||
read(inp_unit,*) ip
|
||||
if (ip >= 5) then
|
||||
read(inp_unit,*) mtrx_file
|
||||
read(inp_unit,*) rhs_file
|
||||
read(inp_unit,*) filefmt
|
||||
read(inp_unit,*) kmethd
|
||||
read(inp_unit,*) ptype
|
||||
read(inp_unit,*) afmt
|
||||
read(inp_unit,*) part
|
||||
|
||||
|
||||
call psb_bcast(ictxt,mtrx_file)
|
||||
call psb_bcast(ictxt,rhs_file)
|
||||
call psb_bcast(ictxt,filefmt)
|
||||
call psb_bcast(ictxt,kmethd)
|
||||
call psb_bcast(ictxt,ptype)
|
||||
call psb_bcast(ictxt,afmt)
|
||||
call psb_bcast(ictxt,part)
|
||||
|
||||
if (ip >= 7) then
|
||||
read(inp_unit,*) istopc
|
||||
else
|
||||
istopc=1
|
||||
endif
|
||||
if (ip >= 8) then
|
||||
read(inp_unit,*) itmax
|
||||
else
|
||||
itmax=500
|
||||
endif
|
||||
if (ip >= 9) then
|
||||
read(inp_unit,*) itrace
|
||||
else
|
||||
itrace=-1
|
||||
endif
|
||||
if (ip >= 10) then
|
||||
read(inp_unit,*) irst
|
||||
else
|
||||
irst = 1
|
||||
endif
|
||||
if (ip >= 11) then
|
||||
read(inp_unit,*) eps
|
||||
else
|
||||
eps=1.d-6
|
||||
endif
|
||||
inparms(1) = istopc
|
||||
inparms(2) = itmax
|
||||
inparms(3) = itrace
|
||||
inparms(4) = irst
|
||||
call psb_bcast(ictxt,inparms(1:4))
|
||||
call psb_bcast(ictxt,eps)
|
||||
|
||||
write(psb_out_unit,'("Solving matrix : ",a)') mtrx_file
|
||||
write(psb_out_unit,'("Number of processors : ",i3)') np
|
||||
write(psb_out_unit,'("Data distribution : ",a)') part
|
||||
write(psb_out_unit,'("Iterative method : ",a)') kmethd
|
||||
write(psb_out_unit,'("Preconditioner : ",a)') ptype
|
||||
write(psb_out_unit,'("Restart parameter : ",i2)') irst
|
||||
write(psb_out_unit,'("Storage format : ",a)') afmt(1:3)
|
||||
write(psb_out_unit,'(" ")')
|
||||
else
|
||||
write(psb_err_unit,*) 'Wrong format for input file'
|
||||
call psb_abort(ictxt)
|
||||
stop 1
|
||||
end if
|
||||
if (inp_unit /= psb_inp_unit) then
|
||||
close(inp_unit)
|
||||
end if
|
||||
else
|
||||
! Receive Parameters
|
||||
call psb_bcast(ictxt,mtrx_file)
|
||||
call psb_bcast(ictxt,rhs_file)
|
||||
call psb_bcast(ictxt,filefmt)
|
||||
call psb_bcast(ictxt,kmethd)
|
||||
call psb_bcast(ictxt,ptype)
|
||||
call psb_bcast(ictxt,afmt)
|
||||
call psb_bcast(ictxt,part)
|
||||
|
||||
call psb_bcast(ictxt,inparms(1:4))
|
||||
istopc = inparms(1)
|
||||
itmax = inparms(2)
|
||||
itrace = inparms(3)
|
||||
irst = inparms(4)
|
||||
call psb_bcast(ictxt,eps)
|
||||
|
||||
end if
|
||||
|
||||
end subroutine get_dparms
|
||||
|
||||
subroutine get_sparms(ictxt,mtrx_file,rhs_file,filefmt,kmethd,ptype,part,&
|
||||
& afmt,istopc,itmax,itrace,irst,eps)
|
||||
use psb_base_mod
|
||||
integer(psb_ipk_) :: ictxt
|
||||
character(len=2) :: filefmt
|
||||
character(len=40) :: kmethd, mtrx_file, rhs_file, ptype
|
||||
character(len=20) :: part
|
||||
integer(psb_ipk_) :: iret, istopc,itmax,itrace,irst
|
||||
character(len=40) :: charbuf
|
||||
real(psb_spk_) :: eps
|
||||
character :: afmt*5
|
||||
integer(psb_ipk_) :: np, iam
|
||||
integer(psb_ipk_) :: inparms(40), ip, inp_unit
|
||||
character(len=1024) :: filename
|
||||
|
||||
call psb_info(ictxt,iam,np)
|
||||
if (iam == 0) then
|
||||
if (command_argument_count()>0) then
|
||||
call get_command_argument(1,filename)
|
||||
inp_unit = 30
|
||||
open(inp_unit,file=filename,action='read',iostat=info)
|
||||
if (info /= 0) then
|
||||
write(psb_err_unit,*) 'Could not open file ',filename,' for input'
|
||||
call psb_abort(ictxt)
|
||||
stop
|
||||
else
|
||||
write(psb_err_unit,*) 'Opened file ',trim(filename),' for input'
|
||||
end if
|
||||
else
|
||||
inp_unit=inp_unit
|
||||
end if
|
||||
! Read Input Parameters
|
||||
read(inp_unit,*) ip
|
||||
if (ip >= 5) then
|
||||
read(inp_unit,*) mtrx_file
|
||||
read(inp_unit,*) rhs_file
|
||||
read(inp_unit,*) filefmt
|
||||
read(inp_unit,*) kmethd
|
||||
read(inp_unit,*) ptype
|
||||
read(inp_unit,*) afmt
|
||||
read(inp_unit,*) ipart
|
||||
|
||||
|
||||
call psb_bcast(ictxt,mtrx_file)
|
||||
call psb_bcast(ictxt,rhs_file)
|
||||
call psb_bcast(ictxt,filefmt)
|
||||
call psb_bcast(ictxt,kmethd)
|
||||
call psb_bcast(ictxt,ptype)
|
||||
call psb_bcast(ictxt,afmt)
|
||||
call psb_bcast(ictxt,part)
|
||||
|
||||
if (ip >= 7) then
|
||||
read(inp_unit,*) istopc
|
||||
else
|
||||
istopc=1
|
||||
endif
|
||||
if (ip >= 8) then
|
||||
read(inp_unit,*) itmax
|
||||
else
|
||||
itmax=500
|
||||
endif
|
||||
if (ip >= 9) then
|
||||
read(inp_unit,*) itrace
|
||||
else
|
||||
itrace=-1
|
||||
endif
|
||||
if (ip >= 10) then
|
||||
read(inp_unit,*) irst
|
||||
else
|
||||
irst = 1
|
||||
endif
|
||||
if (ip >= 11) then
|
||||
read(inp_unit,*) eps
|
||||
else
|
||||
eps=1.d-6
|
||||
endif
|
||||
inparms(1) = istopc
|
||||
inparms(2) = itmax
|
||||
inparms(3) = itrace
|
||||
inparms(4) = irst
|
||||
call psb_bcast(ictxt,inparms(1:4))
|
||||
call psb_bcast(ictxt,eps)
|
||||
|
||||
write(psb_out_unit,'("Solving matrix : ",a)') mtrx_file
|
||||
write(psb_out_unit,'("Number of processors : ",i3)') np
|
||||
write(psb_out_unit,'("Data distribution : ",a)') part
|
||||
write(psb_out_unit,'("Iterative method : ",a)') kmethd
|
||||
write(psb_out_unit,'("Preconditioner : ",a)') ptype
|
||||
write(psb_out_unit,'("Restart parameter : ",i2)') irst
|
||||
write(psb_out_unit,'("Storage format : ",a)') afmt(1:3)
|
||||
write(psb_out_unit,'(" ")')
|
||||
else
|
||||
write(psb_err_unit,*) 'Wrong format for input file'
|
||||
call psb_abort(ictxt)
|
||||
stop 1
|
||||
end if
|
||||
if (inp_unit /= psb_inp_unit) then
|
||||
close(inp_unit)
|
||||
end if
|
||||
else
|
||||
! Receive Parameters
|
||||
call psb_bcast(ictxt,mtrx_file)
|
||||
call psb_bcast(ictxt,rhs_file)
|
||||
call psb_bcast(ictxt,filefmt)
|
||||
call psb_bcast(ictxt,kmethd)
|
||||
call psb_bcast(ictxt,ptype)
|
||||
call psb_bcast(ictxt,afmt)
|
||||
call psb_bcast(ictxt,part)
|
||||
|
||||
call psb_bcast(ictxt,inparms(1:4))
|
||||
istopc = inparms(1)
|
||||
itmax = inparms(2)
|
||||
itrace = inparms(3)
|
||||
irst = inparms(4)
|
||||
call psb_bcast(ictxt,eps)
|
||||
|
||||
end if
|
||||
|
||||
end subroutine get_sparms
|
||||
|
||||
end module getp
|
||||
@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
##
|
||||
## MPI submission script for PBS on DELTA
|
||||
## --------------------------------------
|
||||
#PBS -N umbrella_corp
|
||||
#PBS -l select=1:ncpus=16:mpiprocs=16
|
||||
##
|
||||
## half_hour - 30 minutes
|
||||
## one_hour - 1 hour
|
||||
## half_day - 12 hours
|
||||
## one_day - 24 hours
|
||||
## two_day - 48 hours
|
||||
## five_day - 120 hours
|
||||
## ten_day - 240 hours (by special arrangement)
|
||||
##
|
||||
#PBS -q one_hour
|
||||
#PBS -m n
|
||||
#PBS -M s.rasmussen@cranfield.ac.uk
|
||||
##
|
||||
## ====================================
|
||||
## DO NOT CHANGE THE LINES BETWEEN HERE
|
||||
## ====================================
|
||||
#PBS -j oe
|
||||
#PBS -W sandbox=PRIVATE
|
||||
#PBS -k n
|
||||
ln -s $PWD $PBS_O_WORKDIR/$PBS_JOBID
|
||||
## Change to working directory
|
||||
cd $PBS_O_WORKDIR
|
||||
rm -f hostfile
|
||||
touch hostfile
|
||||
cat $PBS_NODEFILE | sort | uniq >> hostfile
|
||||
echo "========= PBS_NODEFILE ========="
|
||||
# cat $PBS_NODEFILE
|
||||
echo "========= hostfile ========="
|
||||
cat hostfile
|
||||
echo "========= end ========="
|
||||
export NPROCS=`wc -l hostfile |gawk '//{print $1}'`
|
||||
echo "NPROCS = ${NPROCS}"
|
||||
|
||||
## Calculate number of CPUs
|
||||
cpus=`cat hostfile | wc -l`
|
||||
echo "cpus = ${cpus}"
|
||||
## set some MPI tuning parameters to use the correct transport
|
||||
export I_MPI_FABRICS=shm:dapl
|
||||
export I_MPI_DAPL_UD=enable
|
||||
export I_MPI_PLATFORM=bdw
|
||||
export I_MPI_ADJUST_ALLREDUCE=5
|
||||
## ========
|
||||
## AND HERE
|
||||
## ========
|
||||
##
|
||||
module load score-p-5.0-gcc-8.2
|
||||
|
||||
ml
|
||||
echo $PATH
|
||||
which mpirun
|
||||
echo "========= START ========="
|
||||
timeout 5m cpuburn
|
||||
./run_tests.sh
|
||||
# echo "mpirun -machinefile hostfile -ppn 1 -np ${cpus} ${PROG} -v"
|
||||
# export OMP_NUM_THREADS=16
|
||||
# mpirun -machinefile hostfile -ppn 1 -np ${cpus} ${PROG} -v
|
||||
echo "========== FIN =========="
|
||||
|
||||
## Tidy up the log directory
|
||||
## DO NOT CHANGE THE LINE BELOW
|
||||
## ============================
|
||||
rm $PBS_O_WORKDIR/$PBS_JOBID
|
||||
#
|
||||
@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
matrix_dir=../matrices
|
||||
bin_dir=../bin
|
||||
# "cz148"
|
||||
# "cz308"
|
||||
declare -a matrices=(
|
||||
"cz628"
|
||||
"cz1268"
|
||||
"cz2548"
|
||||
"cz5108"
|
||||
"cz10228"
|
||||
"cz20468"
|
||||
"cz40948"
|
||||
)
|
||||
|
||||
declare -a iterations=(
|
||||
"10" "300" "600" "900" "1200" "1500" "1800" "2100" "2400" "2700" "3000"
|
||||
)
|
||||
# declare -a iterations=(
|
||||
# "10" "100"
|
||||
# )
|
||||
|
||||
|
||||
# ----------- BUILD -----------
|
||||
# for iter in "${iterations[@]}"; do
|
||||
# make clean
|
||||
# make ITERATIONS=${iter}
|
||||
# mv halo_persistant_${iter}_iter bin/
|
||||
# done
|
||||
|
||||
|
||||
# ------------ RUN ------------
|
||||
|
||||
|
||||
|
||||
# go to directory to run
|
||||
for matrix in "${matrices[@]}"; do
|
||||
for iter in "${iterations[@]}"; do
|
||||
# echo $matrix $iter
|
||||
rm run/*
|
||||
cd run
|
||||
bin=halo_persistant_${iter}_iter
|
||||
# matrix=cz308
|
||||
cp ${matrix_dir}/${matrix}/* ./
|
||||
cp ${bin_dir}/${bin} ./
|
||||
|
||||
mkdir ../results/halo-persistant/${matrix}_matrix
|
||||
write_to=../results/halo-persistant/${matrix}_matrix/${iter}_iter_${matrix}_matrix
|
||||
SCOREP_TIMER=gettimeofday \
|
||||
SCOREP_EXPERIMENT_DIRECTORY=${write_to} \
|
||||
SCOREP_WRAPPER_COMPILER_FLAGS="-O2" \
|
||||
SCOREP_MPI_ENABLE_GROUP=all \
|
||||
mpirun --map-by ppr:16:node -machinefile $PBS_NODEFILE ./${bin} ${matrix}.inp
|
||||
mv halo_persistant_output.txt ${write_to}
|
||||
cd ..
|
||||
done
|
||||
done
|
||||
Loading…
Reference in New Issue