From 021fec40b4d7d863f3001ae6dd79067589b8d755 Mon Sep 17 00:00:00 2001 From: mthevenet Date: Wed, 31 Jan 2018 08:16:05 -0800 Subject: Set of automated performance tests, meant to run every week or so --- Tools/performance_tests/functions_perftest.py | 186 ++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 Tools/performance_tests/functions_perftest.py (limited to 'Tools/performance_tests/functions_perftest.py') diff --git a/Tools/performance_tests/functions_perftest.py b/Tools/performance_tests/functions_perftest.py new file mode 100644 index 000000000..33769b959 --- /dev/null +++ b/Tools/performance_tests/functions_perftest.py @@ -0,0 +1,186 @@ +import os, shutil, re + +def run_batch_nnode(test_list, res_dir, bin_name, config_command, architecture='knl', Cname='knl', n_node=1): + # Clean res_dir + if os.path.exists(res_dir): + shutil.rmtree(res_dir) + os.makedirs(res_dir) + # Copy files to res_dir + cwd = os.environ['WARPX'] + '/Tools/performance_tests/' + bin_dir = cwd + 'Bin/' + shutil.copy(bin_dir + bin_name, res_dir) + os.chdir(res_dir) + # Calculate simulation time. Take 10 min + 10 min / simulation + job_time_min = 5. + len(test_list)*5. + job_time_str = str(int(job_time_min/60)) + ':' + str(int(job_time_min%60)) + ':00' + batch_string = '' + batch_string += '#!/bin/bash\n' + batch_string += '#SBATCH --job-name=' + test_list[0][0] + '\n' + batch_string += '#SBATCH --time=' + job_time_str + '\n' + batch_string += '#SBATCH -C ' + Cname + '\n' + batch_string += '#SBATCH -N ' + str(n_node) + '\n' + batch_string += '#SBATCH -q regular\n' + batch_string += '#SBATCH -e error.txt\n' + batch_string += '#SBATCH --account=m2852\n' + + for count, test_item in enumerate(test_list): + # test_item reads [input_file, int n_mpi, int n_omp] + input_file = test_item[0]; + shutil.copy(cwd + input_file, res_dir) + # test_item[1] is not read since it contain the number of node, which is an + # global parameter. However, we keep it for compatibility with run_alltests.py + n_mpi = test_item[2] + n_omp = test_item[3] + srun_string = '' + srun_string += 'export OMP_NUM_THREADS=' + str(n_omp) + '\n' + # number of logical cores per MPI process + if architecture == 'cpu': + cflag_value = max(1, int(32/n_mpi) * 2) # Follow NERSC directives + elif architecture == 'knl': + cflag_value = max(1, int(64/n_mpi) * 4) # Follow NERSC directives + output_filename = 'out_' + '_'.join([input_file, str(n_node), str(n_mpi), str(n_omp), str(count)]) + '.txt' + srun_string += 'srun --cpu_bind=cores '+ \ + ' -n ' + str(n_node*n_mpi) + \ + ' -c ' + str(cflag_value) + \ + ' ./' + bin_name + \ + ' ' + input_file + \ + ' > ' + output_filename + '\n' + batch_string += srun_string + batch_file = 'slurm' + f_exe = open(batch_file,'w') + f_exe.write(batch_string) + f_exe.close() + os.system('chmod 700 ' + bin_name) + os.system(config_command + 'sbatch ' + batch_file + ' >> ' + cwd + 'log_jobids_tmp.txt') + return 0 + +def run_batch(run_name, res_dir, bin_name, config_command, architecture='knl',\ + Cname='knl', n_node=1, n_mpi=1, n_omp=1): + # Clean res_dir + if os.path.exists(res_dir): + shutil.rmtree(res_dir) + os.makedirs(res_dir) + # Copy files to res_dir + # Copy files to res_dir + cwd = os.environ['WARPX'] + '/Tools/performance_tests/' + bin_dir = cwd + 'Bin/' + shutil.copy(bin_dir + bin_name, res_dir) + shutil.copyfile(cwd + run_name, res_dir + 'inputs') + os.chdir(res_dir) + batch_string = '' + batch_string += '#!/bin/bash\n' + batch_string += '#SBATCH --job-name=' + run_name + str(n_node) + str(n_mpi) + str(n_omp) + '\n' + batch_string += '#SBATCH --time=00:20:00\n' + batch_string += '#SBATCH -C ' + Cname + '\n' + batch_string += '#SBATCH -N ' + str(n_node) + '\n' + batch_string += '#SBATCH -q regular\n' + batch_string += '#SBATCH -e error.txt\n' + batch_string += '#SBATCH --account=m2852\n' + batch_string += 'export OMP_NUM_THREADS=' + str(n_omp) + '\n' + if architecture == 'cpu': + cflag_value = max(1, int(32/n_mpi) * 2) # Follow NERSC directives + batch_string += 'srun --cpu_bind=cores '+ \ + ' -n ' + str(n_node*n_mpi) + \ + ' -c ' + str(cflag_value) + \ + ' ./' + bin_name + ' inputs > perf_output.txt' + elif architecture == 'knl': + # number of logical cores per MPI process + cflag_value = max(1, int(64/n_mpi) * 4) # Follow NERSC directives + batch_string += 'srun --cpu_bind=cores ' + \ + ' -n ' + str(n_node*n_mpi) + \ + ' -c ' + str(cflag_value) + \ + ' ./' + bin_name + ' inputs > perf_output.txt\n' + batch_file = 'slurm' + f_exe = open(batch_file,'w') + f_exe.write(batch_string) + f_exe.close() + os.system('chmod 700 ' + bin_name) + os.system(config_command + 'sbatch ' + batch_file + ' >> ' + cwd + 'log_jobids_tmp.txt') + return 0 + +# Read output file and return init time and 1-step time +def read_run_perf(filename, n_steps): + timing_list = [] + # Search inclusive time to get simulation step time + partition_limit = 'NCalls Incl. Min Incl. Avg Incl. Max Max %' + with open(filename) as file_handler: + output_text = file_handler.read() + # Get total simulation time + line_match_totaltime = re.search('TinyProfiler total time across processes.*', output_text) + total_time = float(line_match_totaltime.group(0).split()[8]) + search_area = output_text.partition(partition_limit)[2] + line_match_looptime = re.search('\nWarpX::Evolve().*', search_area) + time_wo_initialization = float(line_match_looptime.group(0).split()[3]) + timing_list += [str(total_time - time_wo_initialization)] + timing_list += [str(time_wo_initialization/n_steps)] + partition_limit1 = 'NCalls Excl. Min Excl. Avg Excl. Max Max %' + partition_limit2 = 'NCalls Incl. Min Incl. Avg Incl. Max Max %' + file_handler.close() + with open(filename) as file_handler: + output_text = file_handler.read() + # Search EXCLISUSIVE routine timings + search_area = output_text.partition(partition_limit1)[2].partition(partition_limit2)[0] + pattern_list = ['\nParticleContainer::Redistribute().*',\ + '\nFabArray::FillBoundary().*',\ + '\nFabArray::ParallelCopy().*',\ + '\nPICSAR::CurrentDeposition.*',\ + '\nPICSAR::FieldGather.*',\ + '\nPICSAR::ParticlePush.*',\ + '\nPPC::Evolve::Copy.*',\ + '\nWarpX::EvolveEM().*',\ + 'NArrayInt>::Checkpoint().*',\ + 'NArrayInt>::WriteParticles().*',\ + '\nVisMF::Write_FabArray.*',\ + '\nWriteMultiLevelPlotfile().*',\ + '\nParticleContainer::RedistributeMPI().*'] + for pattern in pattern_list: + timing = '0' + line_match = re.search(pattern, search_area) + if line_match is not None: + timing = [str(float(line_match.group(0).split()[3])/n_steps)] + timing_list += timing + return timing_list + +# Write time into logfile +def write_perf_logfile(log_file, log_line): + f_log = open(log_file, 'a') + f_log.write(log_line) + f_log.close() + return 0 + +def get_nsteps(run_name): + with open(run_name) as file_handler: + run_name_text = file_handler.read() + line_match_nsteps = re.search('\nmax_step.*', run_name_text) + nsteps = float(line_match_nsteps.group(0).split()[2]) + return nsteps + + +# Run a performance test in an interactive allocation +# def run_interactive(run_name, res_dir, n_node=1, n_mpi=1, n_omp=1): +# # Clean res_dir # +# if os.path.exists(res_dir): +# shutil.rmtree(res_dir) +# os.makedirs(res_dir) +# # Copy files to res_dir # +# shutil.copyfile(bin_dir + bin_name, res_dir + bin_name) +# shutil.copyfile(cwd + run_name, res_dir + 'inputs') +# os.chdir(res_dir) +# if args.architecture == 'cpu': +# cflag_value = max(1, int(32/n_mpi) * 2) # Follow NERSC directives # +# exec_command = 'export OMP_NUM_THREADS=' + str(n_omp) + ';' +\ +# 'srun --cpu_bind=cores ' + \ +# ' -n ' + str(n_node*n_mpi) + \ +# ' -c ' + str(cflag_value) + \ +# ' ./' + bin_name + ' inputs > perf_output.txt' +# elif args.architecture == 'knl': +# # number of logical cores per MPI process # +# cflag_value = max(1,int(68/n_mpi) * 4) # Follow NERSC directives # +# exec_command = 'export OMP_NUM_THREADS=' + str(n_omp) + ';' +\ +# 'srun --cpu_bind=cores ' + \ +# ' -n ' + str(n_node*n_mpi) + \ +# ' -c ' + str(cflag_value) + \ +# ' ./' + bin_name + ' inputs > perf_output.txt' +# os.system('chmod 700 ' + bin_name) +# os.system(config_command + exec_command) +# return 0 -- cgit v1.2.3 From 9d0a0ef5ae397601470600c6528b85c8b312fffc Mon Sep 17 00:00:00 2001 From: mthevenet Date: Wed, 31 Jan 2018 08:31:51 -0800 Subject: minor change in read_run_perf --- Tools/performance_tests/functions_perftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'Tools/performance_tests/functions_perftest.py') diff --git a/Tools/performance_tests/functions_perftest.py b/Tools/performance_tests/functions_perftest.py index 33769b959..b12f7d0b9 100644 --- a/Tools/performance_tests/functions_perftest.py +++ b/Tools/performance_tests/functions_perftest.py @@ -46,6 +46,7 @@ def run_batch_nnode(test_list, res_dir, bin_name, config_command, architecture=' ' ' + input_file + \ ' > ' + output_filename + '\n' batch_string += srun_string + batch_string += 'rm -r plt* chk* lab_frame_data' batch_file = 'slurm' f_exe = open(batch_file,'w') f_exe.write(batch_string) @@ -128,9 +129,9 @@ def read_run_perf(filename, n_steps): '\nPICSAR::ParticlePush.*',\ '\nPPC::Evolve::Copy.*',\ '\nWarpX::EvolveEM().*',\ - 'NArrayInt>::Checkpoint().*',\ - 'NArrayInt>::WriteParticles().*',\ - '\nVisMF::Write_FabArray.*',\ + 'Checkpoint().*',\ + 'WriteParticles().*',\ + '\nVisMF::Write(FabArray).*',\ '\nWriteMultiLevelPlotfile().*',\ '\nParticleContainer::RedistributeMPI().*'] for pattern in pattern_list: -- cgit v1.2.3 From c15d75e1a55a83d4ecb37ecabaa82a6a2783969b Mon Sep 17 00:00:00 2001 From: mthevenet Date: Thu, 1 Feb 2018 13:18:43 -0800 Subject: Final version for automated performance tests. Last thing to do: run them every week. --- .../performance_tests/automated_test_6_output_2ppc | 6 +- Tools/performance_tests/functions_perftest.py | 5 +- Tools/performance_tests/performance_log.txt | 48 ------------- Tools/performance_tests/run_alltests_1node.py | 84 ++++++++++++++++++---- 4 files changed, 79 insertions(+), 64 deletions(-) (limited to 'Tools/performance_tests/functions_perftest.py') diff --git a/Tools/performance_tests/automated_test_6_output_2ppc b/Tools/performance_tests/automated_test_6_output_2ppc index 0696c8bb1..a1c4172fe 100644 --- a/Tools/performance_tests/automated_test_6_output_2ppc +++ b/Tools/performance_tests/automated_test_6_output_2ppc @@ -1,10 +1,10 @@ # Maximum number of time steps -max_step = 100 +max_step = 10 # number of grid points -amr.n_cell = 256 256 256 +amr.n_cell = 128 128 128 -amr.plot_int = 5 # How often to write plotfiles. +amr.plot_int = 2 # How often to write plotfiles. # Maximum allowable size of each subdomain in the problem domain; # this is used to decompose the domain for parallel calculations. diff --git a/Tools/performance_tests/functions_perftest.py b/Tools/performance_tests/functions_perftest.py index b12f7d0b9..f60da15f0 100644 --- a/Tools/performance_tests/functions_perftest.py +++ b/Tools/performance_tests/functions_perftest.py @@ -20,6 +20,7 @@ def run_batch_nnode(test_list, res_dir, bin_name, config_command, architecture=' batch_string += '#SBATCH -C ' + Cname + '\n' batch_string += '#SBATCH -N ' + str(n_node) + '\n' batch_string += '#SBATCH -q regular\n' + batch_string += '#SBATCH --mail-type=end\n' batch_string += '#SBATCH -e error.txt\n' batch_string += '#SBATCH --account=m2852\n' @@ -46,7 +47,9 @@ def run_batch_nnode(test_list, res_dir, bin_name, config_command, architecture=' ' ' + input_file + \ ' > ' + output_filename + '\n' batch_string += srun_string - batch_string += 'rm -r plt* chk* lab_frame_data' + batch_string += 'rm -rf plt*\n' + batch_string += 'rm -rf chk*\n' + batch_string += 'rm -rf lab_frame_data\n' batch_file = 'slurm' f_exe = open(batch_file,'w') f_exe.write(batch_string) diff --git a/Tools/performance_tests/performance_log.txt b/Tools/performance_tests/performance_log.txt index 330de7756..543d257a0 100644 --- a/Tools/performance_tests/performance_log.txt +++ b/Tools/performance_tests/performance_log.txt @@ -1,52 +1,4 @@ ## year month day run_name compiler architecture n_node n_mpi n_omp time_initialization time_one_iteration Redistribute FillBoundary ParallelCopy CurrentDeposition FieldGather ParthiclePush Copy EvolveEM Checkpoint WriteParticles Write_FabArray WriteMultiLevelPlotfile(unit: second) RedistributeMPI -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.89 0.3941 0.1708 0.01722 0.01598 0.06932 0.03612 0.01871 0.01916 0.003316 0 0 0 0 0.006788 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.88 0.3956 0.1706 0.01787 0.01806 0.06907 0.03605 0.01869 0.01909 0.003086 0 0 0 0 0.006877 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.15 0.3979 0.1708 0.01795 0.0188 0.0695 0.03616 0.01869 0.01916 0.003206 0 0 0 0 0.007259 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.32 0.3996 0.1711 0.01796 0.01894 0.06977 0.03665 0.01881 0.01919 0.003161 0 0 0 0 0.00756 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 0.88 0.4752 0.04437 0.1133 0.0906 0.1073 0.01252 0.003654 0.004123 0.01252 0 0 0 0 0.002832 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 0.97 0.4777 0.04432 0.1148 0.0903 0.1073 0.01278 0.003706 0.00415 0.01251 0 0 0 0 0.003051 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.22 0.4828 0.04471 0.117 0.09294 0.1075 0.01277 0.003697 0.004156 0.01261 0 0 0 0 0.002947 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.18 0.4717 0.0445 0.1109 0.08908 0.1072 0.01262 0.003681 0.004161 0.01253 0 0 0 0 0.003035 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.68 0.1279 0.0308 0.01708 0.01739 0.02001 0.005004 0.003534 0.002822 0.003061 0 0 0 0 0.007921 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.78 0.1272 0.03082 0.01688 0.01721 0.0199 0.004966 0.003509 0.002834 0.003165 0 0 0 0 0.007421 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.65 0.1288 0.03081 0.01707 0.0174 0.02037 0.005359 0.003607 0.002865 0.003093 0 0 0 0 0.007584 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.47 0.1285 0.03057 0.01704 0.01737 0.01986 0.004963 0.003541 0.002774 0.00298 0 0 0 0 0.009253 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.19 0.3035 0.03852 0.06449 0.07505 0.003524 0.003004 0.0001529 0.0001788 0.008249 0 0 0 0.0005316 0.03852 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.93 0.2903 0.03855 0.0644 0.07497 0.003491 0.003002 0.0001525 0.0001778 0.008146 0 0 0 0.001185 0.03855 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.46 0.2926 0.03901 0.06493 0.07521 0.003504 0.003029 0.0001543 0.0001773 0.008053 0 0 0 0.0005591 0.03901 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.52 0.2942 0.03873 0.06463 0.07506 0.003474 0.003009 0.0001519 0.0001748 0.008223 0 0 0 0.000749 0.03873 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.1 0.379 0.08335 0.1011 0.1317 0 0 0 0 0.01243 0 0 0 0 0.08335 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 8.71 0.4016 0.08558 0.1132 0.1394 0 0 0 0 0.0123 0 0 0 0 0.08558 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 8.72 0.4026 0.08542 0.1145 0.1394 0 0 0 0 0.0122 0 0 0 0 0.08542 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.45 0.4045 0.08679 0.1139 0.1398 0 0 0 0 0.01235 0 0 0 0 0.08679 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 3.7 1.083 0.09037 0.1341 0.09628 0.1359 0.02727 0.009255 0.01001 0.02158 0 0 0 0.005835 0.005577 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.1 1.158 0.09151 0.1383 0.09691 0.1385 0.02939 0.009634 0.01047 0.02449 0 0 0 0.03136 0.008183 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.6 1.125 0.09089 0.1447 0.1015 0.136 0.02796 0.009181 0.009982 0.02134 0 0 0 0.03513 0.00643 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.3 2.161 0.09089 0.139 0.09462 0.1397 0.02942 0.009833 0.01048 0.02534 0 0 0 0.002898 0.007857 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.14 0.3986 0.1713 0.01719 0.01615 0.06987 0.03636 0.01901 0.01999 0.003602 0 0 0 0 0.007262 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.39 0.4009 0.1712 0.01676 0.01583 0.07061 0.03684 0.01926 0.02011 0.003687 0 0 0 0 0.007841 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.91 0.4024 0.1716 0.01826 0.01918 0.0703 0.0363 0.01912 0.01989 0.003017 0 0 0 0 0.007256 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.21 0.3997 0.1717 0.01706 0.0162 0.07026 0.03655 0.01928 0.01999 0.003687 0 0 0 0 0.006799 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 0.89 0.4779 0.04441 0.1143 0.09117 0.1072 0.01254 0.003702 0.004217 0.01247 0 0 0 0 0.003441 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.33 0.4813 0.04476 0.1143 0.09043 0.108 0.01365 0.003931 0.00426 0.01378 0 0 0 0 0.002913 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.26 0.5096 0.04494 0.133 0.1033 0.1074 0.01247 0.003781 0.004333 0.0128 0 0 0 0 0.002968 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.32 0.4645 0.04441 0.1053 0.08565 0.1079 0.01307 0.003789 0.004237 0.01269 0 0 0 0 0.002675 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.72 0.1223 0.0309 0.01493 0.01332 0.02019 0.005077 0.003806 0.002812 0.003526 0 0 0 0 0.007072 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.5 0.1241 0.03063 0.0153 0.01366 0.02064 0.005438 0.003966 0.002745 0.003411 0 0 0 0 0.007488 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.6 0.1257 0.0306 0.01596 0.01432 0.02056 0.005598 0.004134 0.002743 0.003329 0 0 0 0 0.00732 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.53 0.128 0.03062 0.01695 0.0174 0.02008 0.005402 0.003621 0.002808 0.00296 0 0 0 0 0.007562 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.06 0.2996 0.03996 0.06474 0.07516 0.003495 0.002986 0.0001529 0.0001777 0.008056 0.03764 0.001542 0 0.006604 0.03996 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.27 0.2921 0.03979 0.06439 0.07494 0.003489 0.003004 0.0001512 0.0001767 0.0083 0.03747 0.001567 0 0.0005017 0.03979 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.38 0.2845 0.03763 0.0628 0.07183 0.003505 0.002981 0.0001538 0.0001797 0.008738 0.03636 0.001559 0 0.0003998 0.03763 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.91 0.2897 0.03867 0.0644 0.07499 0.003516 0.002995 0.0001527 0.00018 0.008797 0.03593 0.001542 0 0.0004542 0.03867 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.39 0.404 0.08675 0.1135 0.1394 0 0 0 0 0.01241 0 0 0 0 0.08675 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.15 0.3993 0.08439 0.1125 0.1387 0 0 0 0 0.01188 0 0 0 0 0.08439 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.13 0.3998 0.08468 0.1116 0.1382 0 0 0 0 0.01239 0 0 0 0 0.08468 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.28 0.405 0.08601 0.1142 0.14 0 0 0 0 0.0125 0 0 0 0 0.08601 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.0 1.084 0.09053 0.13 0.09324 0.1378 0.02747 0.009309 0.01008 0.02333 0.06676 0.04985 0 0.001683 0.007208 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 3.6 1.064 0.08987 0.1301 0.09356 0.1356 0.02736 0.009306 0.01015 0.0225 0.05925 0.05244 0 0.0008205 0.005574 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.2 1.116 0.09026 0.1345 0.09613 0.1364 0.02789 0.009283 0.01019 0.02335 0.09327 0.0508 0 0.0009409 0.005977 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.7 1.091 0.09019 0.1365 0.09802 0.1354 0.02804 0.009118 0.009933 0.02029 0.08396 0.04825 0 0.0009181 0.00505 2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.14 0.3986 0.1713 0.01719 0.01615 0.06987 0.03636 0.01901 0.01999 0.003602 0 0 0 0 0.007262 2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.39 0.4009 0.1712 0.01676 0.01583 0.07061 0.03684 0.01926 0.02011 0.003687 0 0 0 0 0.007841 2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.91 0.4024 0.1716 0.01826 0.01918 0.0703 0.0363 0.01912 0.01989 0.003017 0 0 0 0 0.007256 diff --git a/Tools/performance_tests/run_alltests_1node.py b/Tools/performance_tests/run_alltests_1node.py index 55635a9dc..4c6849c3b 100644 --- a/Tools/performance_tests/run_alltests_1node.py +++ b/Tools/performance_tests/run_alltests_1node.py @@ -9,15 +9,21 @@ from functions_perftest import * # ---- User's manual ---- # Before running performance tests, make sure you have the latest version # of performance_log.txt -# A typical execution reads: -# python run_alltests_1node.py --no-recompile --compiler=intel --architecture=knl --mode=run --input_file=uniform_plasma --n_node=1 --log_file='my_performance_log.txt' -# These are default values, and will give the same result as -# > python run_alltests.py + +# ---- Running a custom set of performance tests ---- +# > python run_alltests_1node.py --no-recompile --compiler=intel +# > --architecture=knl --mode=run --input_file=uniform_plasma +# > --n_node=1 --log_file='my_performance_log.txt' + +# ---- Running the pre-drefined automated tests ---- +# Compile and run: +# > python run_alltests_1node.py --automated --recompile +# Just run: +# > python run_alltests_1node.py --automated + # To add a new test item, extent the test_list with a line like -# test_list.extend([['my_input_file', n_node, n_mpi, n_omp]]*3) +# test_list.extend([['my_input_file', n_node, n_mpi, n_omp]]*n_repeat) # - my_input_file must be in warpx/performance_tests -# - the test will run 3 times, to have some statistics -# - the test must take <1h or it will timeout # ---- Developer's manual ---- # This script can run in two modes: @@ -86,12 +92,11 @@ if args.automated == True: n_repeat = 4 test_list.extend([['automated_test_1_uniform_rest_32ppc', 1, 16, 8]]*n_repeat) test_list.extend([['automated_test_2_uniform_rest_1ppc', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_2_uniform_rest_1ppc', 1, 16, 8]]*n_repeat) test_list.extend([['automated_test_3_uniform_drift_4ppc', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_4_labdiags_2ppc', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_5_loadimbalance', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_6_output_2ppc', 1, 16, 8]]*n_repeat) - do_commit = True + test_list.extend([['automated_test_4_labdiags_2ppc', 1, 16, 8]]*n_repeat) + test_list.extend([['automated_test_5_loadimbalance', 1, 16, 8]]*n_repeat) + test_list.extend([['automated_test_6_output_2ppc', 1, 16, 8]]*n_repeat) + do_commit = False run_name = 'automated_tests' n_tests = len(test_list) @@ -271,3 +276,58 @@ if args.mode == 'read': 'git commit -m "performance tests";'\ 'git push -u origin master') + # Plot file + import numpy as np + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + filename0 = 'performance_log' + filename = filename0 + '.txt' + fontsize = 14 + matplotlib.rcParams.update({'font.size': fontsize}) + nsteps = 100. + nrepeat = 4 + legends = [ 'n_node', 'n_mpi', 'n_omp', 'time_initialization', 'time_one_iteration', \ + 'Redistribute', 'FillBoundary', 'ParallelCopy', 'CurrentDeposition', \ + 'FieldGather', 'ParthiclePush', 'Copy', 'EvolveEM', 'Checkpoint', \ + 'WriteParticles', 'Write_FabArray', 'WriteMultiLevelPlotfile', \ + 'RedistributeMPI'] + date = np.loadtxt( filename, usecols = np.arange(0, 3 )) + data = np.loadtxt( filename, usecols = np.arange(6, 6+len(legends)) ) + # Read run name + with open(filename) as f: + namelist_tmp = zip(*[line.split() for line in f])[3] + # Remove first line = comments + namelist = list(namelist_tmp[1:]) + selector_list = ['automated_test_1_uniform_rest_32ppc',\ + 'automated_test_2_uniform_rest_1ppc',\ + 'automated_test_3_uniform_drift_4ppc',\ + 'automated_test_4_labdiags_2ppc',\ + 'automated_test_5_loadimbalance',\ + 'automated_test_6_output_2ppc'] + selector_string = selector_list[0] + selector = [idx for idx in range(len(namelist)) if selector_string in namelist[idx]] + lin_date = date[:,0]+date[:,1]/12.+date[:,2]/366. + unique_lin_date = np.unique(lin_date) + my_xticks = unique_lin_date +# cmap = plt.get_cmap("tab20") + cycle = plt.rcParams['axes.prop_cycle'].by_key()['color'] + for selector_string in selector_list: + selector = [idx for idx in range(len(namelist)) if selector_string in namelist[idx]] + plt.figure(num=0, figsize=(8,4)) + plt.clf() + plt.title('warpx ' + selector_string) + for i in np.arange(data.shape[1]): + icolors = i-3 + if i>3 and (data[selector,i] > 5./100*data[selector,4]).any(): + plt.plot(lin_date[selector], data[selector,i],'+', ms=6, \ + mew=2, label=legends[i] ) + # plt.plot(lin_date[selector], data[selector,i],'+', ms=6, \ + # mew=2, label=legends[i], color=cmap(i) ) + plt.xlabel('date') + plt.ylabel('time/step (s)') + plt.grid() + plt.legend(loc='best') + plt.legend(bbox_to_anchor=(1.1, 1.05)) + plt.savefig( selector_string + '.pdf', bbox_inches='tight') + plt.savefig( selector_string + '.png', bbox_inches='tight') -- cgit v1.2.3 From 29974a4f715beb6a0e1536e9f9f7e962844df37d Mon Sep 17 00:00:00 2001 From: mthevenet Date: Thu, 1 Feb 2018 13:21:22 -0800 Subject: minor --- Tools/performance_tests/functions_perftest.py | 1 - 1 file changed, 1 deletion(-) (limited to 'Tools/performance_tests/functions_perftest.py') diff --git a/Tools/performance_tests/functions_perftest.py b/Tools/performance_tests/functions_perftest.py index f60da15f0..2085367c7 100644 --- a/Tools/performance_tests/functions_perftest.py +++ b/Tools/performance_tests/functions_perftest.py @@ -20,7 +20,6 @@ def run_batch_nnode(test_list, res_dir, bin_name, config_command, architecture=' batch_string += '#SBATCH -C ' + Cname + '\n' batch_string += '#SBATCH -N ' + str(n_node) + '\n' batch_string += '#SBATCH -q regular\n' - batch_string += '#SBATCH --mail-type=end\n' batch_string += '#SBATCH -e error.txt\n' batch_string += '#SBATCH --account=m2852\n' -- cgit v1.2.3