From ecc099ef7d6f3656484b83eb829377d6bed67044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Gajdu=C5=A1ek?= <gajdusek.pavel@gmail.com> Date: Wed, 20 Sep 2017 15:07:46 +0200 Subject: [PATCH] DELETED ALL DUPLICATES --- docs.it4i/anselm/software/ansys/ansys-cfx.md | 55 - .../anselm/software/ansys/ansys-fluent.md | 162 -- .../anselm/software/ansys/ansys-ls-dyna.md | 55 - .../software/ansys/ansys-mechanical-apdl.md | 57 - docs.it4i/anselm/software/ansys/ansys.md | 15 - docs.it4i/anselm/software/ansys/ls-dyna.md | 35 - docs.it4i/anselm/software/chemistry/molpro.md | 63 - docs.it4i/anselm/software/chemistry/nwchem.md | 42 - docs.it4i/anselm/software/compilers.md | 156 -- .../anselm/software/comsol-multiphysics.md | 120 - .../anselm/software/debuggers/allinea-ddt.md | 94 - .../debuggers/allinea-performance-reports.md | 59 - docs.it4i/anselm/software/debuggers/cube.md | 36 - .../anselm/software/debuggers/debuggers.md | 60 - .../intel-performance-counter-monitor.md | 281 --- .../debuggers/intel-vtune-amplifier.md | 73 - docs.it4i/anselm/software/debuggers/papi.md | 237 -- .../anselm/software/debuggers/scalasca.md | 70 - .../anselm/software/debuggers/score-p.md | 117 - .../anselm/software/debuggers/total-view.md | 158 -- .../anselm/software/debuggers/valgrind.md | 262 -- docs.it4i/anselm/software/debuggers/vampir.md | 22 - .../software/intel-suite/intel-compilers.md | 36 - .../software/intel-suite/intel-debugger.md | 73 - ...intel-integrated-performance-primitives.md | 81 - .../anselm/software/intel-suite/intel-mkl.md | 114 - .../anselm/software/intel-suite/intel-tbb.md | 42 - .../software/intel-suite/introduction.md | 62 - docs.it4i/anselm/software/isv_licenses.md | 105 - docs.it4i/anselm/software/java.md | 27 - .../software/machine-learning/introduction.md | 19 - .../software/machine-learning/tensorflow.md | 66 - .../anselm/software/mpi/Running_OpenMPI.md | 207 -- docs.it4i/anselm/software/mpi/mpi.md | 146 -- .../software/mpi/mpi4py-mpi-for-python.md | 171 -- .../anselm/software/mpi/running-mpich2.md | 155 -- .../numerical-languages/introduction.md | 40 - .../software/numerical-languages/matlab.md | 280 --- .../numerical-languages/matlab_1314.md | 206 -- .../software/numerical-languages/octave.md | 110 - .../anselm/software/numerical-languages/r.md | 399 --- .../software/numerical-libraries/fftw.md | 73 - .../software/numerical-libraries/gsl.md | 145 -- .../software/numerical-libraries/hdf5.md | 89 - .../intel-numerical-libraries.md | 33 - .../magma-for-intel-xeon-phi.md | 76 - .../software/numerical-libraries/petsc.md | 60 - .../software/numerical-libraries/trilinos.md | 49 - docs.it4i/anselm/software/openfoam.md | 228 -- docs.it4i/anselm/software/operating-system.md | 3 - docs.it4i/anselm/software/paraview.md | 85 - docs.it4i/salomon/software/ansys/ansys-cfx.md | 55 - .../salomon/software/ansys/ansys-fluent.md | 162 -- .../salomon/software/ansys/ansys-ls-dyna.md | 55 - .../software/ansys/ansys-mechanical-apdl.md | 56 - docs.it4i/salomon/software/ansys/ansys.md | 15 - docs.it4i/salomon/software/ansys/licensing.md | 25 - .../ansys/setting-license-preferences.md | 23 - docs.it4i/salomon/software/ansys/workbench.md | 62 - docs.it4i/salomon/software/chemistry/INCAR | 13 - docs.it4i/salomon/software/chemistry/KPOINTS | 6 - docs.it4i/salomon/software/chemistry/POSCAR | 16 - docs.it4i/salomon/software/chemistry/POTCAR | 2208 ----------------- .../software/chemistry/gofree-cond1.sh | 23 - .../salomon/software/chemistry/prepare.sh | 44 - docs.it4i/salomon/software/chemistry/run.sh | 31 - .../salomon/software/chemistry/submit.sh | 35 - docs.it4i/salomon/software/compilers.md | 193 -- .../software/comsol/comsol-multiphysics.md | 118 - .../licensing-and-available-versions.md | 19 - .../software/debuggers/Introduction.md | 62 - .../salomon/software/debuggers/aislinn.md | 102 - .../salomon/software/debuggers/allinea-ddt.md | 94 - .../debuggers/allinea-performance-reports.md | 59 - .../debuggers/intel-vtune-amplifier.md | 94 - .../mympiprog_32p_2014-10-15_16-56.html | 610 ----- .../mympiprog_32p_2014-10-15_16-56.txt | 50 - .../salomon/software/debuggers/total-view.md | 150 -- .../salomon/software/debuggers/valgrind.md | 265 -- .../salomon/software/debuggers/vampir.md | 24 - .../software/intel-suite/intel-advisor.md | 31 - .../software/intel-suite/intel-compilers.md | 36 - .../software/intel-suite/intel-debugger.md | 73 - .../software/intel-suite/intel-inspector.md | 39 - ...intel-integrated-performance-primitives.md | 78 - .../salomon/software/intel-suite/intel-mkl.md | 119 - .../intel-parallel-studio-introduction.md | 69 - .../salomon/software/intel-suite/intel-tbb.md | 40 - .../intel-trace-analyzer-and-collector.md | 40 - docs.it4i/salomon/software/java.md | 25 - .../software/machine-learning/introduction.md | 19 - .../software/machine-learning/tensorflow.md | 68 - .../salomon/software/mpi/Running_OpenMPI.md | 203 -- docs.it4i/salomon/software/mpi/mpi.md | 142 -- .../software/mpi/mpi4py-mpi-for-python.md | 110 - .../numerical-languages/introduction.md | 40 - .../software/numerical-languages/matlab.md | 278 --- .../software/numerical-languages/octave.md | 56 - .../numerical-languages/opencoarrays.md | 127 - .../salomon/software/numerical-languages/r.md | 405 --- docs.it4i/salomon/software/paraview.md | 81 - docs.it4i/software/debuggers/Introduction.md | 2 +- docs.it4i/software/debuggers/score-p.md | 4 +- .../numerical-languages/opencoarrays.md | 2 +- docs.it4i/software/numerical-languages/r.md | 4 +- .../omics-master/diagnostic-component-team.md | 2 +- docs.it4i/software/omics-master/overview.md | 22 +- .../priorization-component-bierapp.md | 2 +- 108 files changed, 19 insertions(+), 12176 deletions(-) delete mode 100644 docs.it4i/anselm/software/ansys/ansys-cfx.md delete mode 100644 docs.it4i/anselm/software/ansys/ansys-fluent.md delete mode 100644 docs.it4i/anselm/software/ansys/ansys-ls-dyna.md delete mode 100644 docs.it4i/anselm/software/ansys/ansys-mechanical-apdl.md delete mode 100644 docs.it4i/anselm/software/ansys/ansys.md delete mode 100644 docs.it4i/anselm/software/ansys/ls-dyna.md delete mode 100644 docs.it4i/anselm/software/chemistry/molpro.md delete mode 100644 docs.it4i/anselm/software/chemistry/nwchem.md delete mode 100644 docs.it4i/anselm/software/compilers.md delete mode 100644 docs.it4i/anselm/software/comsol-multiphysics.md delete mode 100644 docs.it4i/anselm/software/debuggers/allinea-ddt.md delete mode 100644 docs.it4i/anselm/software/debuggers/allinea-performance-reports.md delete mode 100644 docs.it4i/anselm/software/debuggers/cube.md delete mode 100644 docs.it4i/anselm/software/debuggers/debuggers.md delete mode 100644 docs.it4i/anselm/software/debuggers/intel-performance-counter-monitor.md delete mode 100644 docs.it4i/anselm/software/debuggers/intel-vtune-amplifier.md delete mode 100644 docs.it4i/anselm/software/debuggers/papi.md delete mode 100644 docs.it4i/anselm/software/debuggers/scalasca.md delete mode 100644 docs.it4i/anselm/software/debuggers/score-p.md delete mode 100644 docs.it4i/anselm/software/debuggers/total-view.md delete mode 100644 docs.it4i/anselm/software/debuggers/valgrind.md delete mode 100644 docs.it4i/anselm/software/debuggers/vampir.md delete mode 100644 docs.it4i/anselm/software/intel-suite/intel-compilers.md delete mode 100644 docs.it4i/anselm/software/intel-suite/intel-debugger.md delete mode 100644 docs.it4i/anselm/software/intel-suite/intel-integrated-performance-primitives.md delete mode 100644 docs.it4i/anselm/software/intel-suite/intel-mkl.md delete mode 100644 docs.it4i/anselm/software/intel-suite/intel-tbb.md delete mode 100644 docs.it4i/anselm/software/intel-suite/introduction.md delete mode 100644 docs.it4i/anselm/software/isv_licenses.md delete mode 100644 docs.it4i/anselm/software/java.md delete mode 100644 docs.it4i/anselm/software/machine-learning/introduction.md delete mode 100644 docs.it4i/anselm/software/machine-learning/tensorflow.md delete mode 100644 docs.it4i/anselm/software/mpi/Running_OpenMPI.md delete mode 100644 docs.it4i/anselm/software/mpi/mpi.md delete mode 100644 docs.it4i/anselm/software/mpi/mpi4py-mpi-for-python.md delete mode 100644 docs.it4i/anselm/software/mpi/running-mpich2.md delete mode 100644 docs.it4i/anselm/software/numerical-languages/introduction.md delete mode 100644 docs.it4i/anselm/software/numerical-languages/matlab.md delete mode 100644 docs.it4i/anselm/software/numerical-languages/matlab_1314.md delete mode 100644 docs.it4i/anselm/software/numerical-languages/octave.md delete mode 100644 docs.it4i/anselm/software/numerical-languages/r.md delete mode 100644 docs.it4i/anselm/software/numerical-libraries/fftw.md delete mode 100644 docs.it4i/anselm/software/numerical-libraries/gsl.md delete mode 100644 docs.it4i/anselm/software/numerical-libraries/hdf5.md delete mode 100644 docs.it4i/anselm/software/numerical-libraries/intel-numerical-libraries.md delete mode 100644 docs.it4i/anselm/software/numerical-libraries/magma-for-intel-xeon-phi.md delete mode 100644 docs.it4i/anselm/software/numerical-libraries/petsc.md delete mode 100644 docs.it4i/anselm/software/numerical-libraries/trilinos.md delete mode 100644 docs.it4i/anselm/software/openfoam.md delete mode 100644 docs.it4i/anselm/software/operating-system.md delete mode 100644 docs.it4i/anselm/software/paraview.md delete mode 100644 docs.it4i/salomon/software/ansys/ansys-cfx.md delete mode 100644 docs.it4i/salomon/software/ansys/ansys-fluent.md delete mode 100644 docs.it4i/salomon/software/ansys/ansys-ls-dyna.md delete mode 100644 docs.it4i/salomon/software/ansys/ansys-mechanical-apdl.md delete mode 100644 docs.it4i/salomon/software/ansys/ansys.md delete mode 100644 docs.it4i/salomon/software/ansys/licensing.md delete mode 100644 docs.it4i/salomon/software/ansys/setting-license-preferences.md delete mode 100644 docs.it4i/salomon/software/ansys/workbench.md delete mode 100644 docs.it4i/salomon/software/chemistry/INCAR delete mode 100644 docs.it4i/salomon/software/chemistry/KPOINTS delete mode 100644 docs.it4i/salomon/software/chemistry/POSCAR delete mode 100644 docs.it4i/salomon/software/chemistry/POTCAR delete mode 100644 docs.it4i/salomon/software/chemistry/gofree-cond1.sh delete mode 100644 docs.it4i/salomon/software/chemistry/prepare.sh delete mode 100644 docs.it4i/salomon/software/chemistry/run.sh delete mode 100644 docs.it4i/salomon/software/chemistry/submit.sh delete mode 100644 docs.it4i/salomon/software/compilers.md delete mode 100644 docs.it4i/salomon/software/comsol/comsol-multiphysics.md delete mode 100644 docs.it4i/salomon/software/comsol/licensing-and-available-versions.md delete mode 100644 docs.it4i/salomon/software/debuggers/Introduction.md delete mode 100644 docs.it4i/salomon/software/debuggers/aislinn.md delete mode 100644 docs.it4i/salomon/software/debuggers/allinea-ddt.md delete mode 100644 docs.it4i/salomon/software/debuggers/allinea-performance-reports.md delete mode 100644 docs.it4i/salomon/software/debuggers/intel-vtune-amplifier.md delete mode 100644 docs.it4i/salomon/software/debuggers/mympiprog_32p_2014-10-15_16-56.html delete mode 100644 docs.it4i/salomon/software/debuggers/mympiprog_32p_2014-10-15_16-56.txt delete mode 100644 docs.it4i/salomon/software/debuggers/total-view.md delete mode 100644 docs.it4i/salomon/software/debuggers/valgrind.md delete mode 100644 docs.it4i/salomon/software/debuggers/vampir.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-advisor.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-compilers.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-debugger.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-inspector.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-integrated-performance-primitives.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-mkl.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-parallel-studio-introduction.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-tbb.md delete mode 100644 docs.it4i/salomon/software/intel-suite/intel-trace-analyzer-and-collector.md delete mode 100644 docs.it4i/salomon/software/java.md delete mode 100644 docs.it4i/salomon/software/machine-learning/introduction.md delete mode 100644 docs.it4i/salomon/software/machine-learning/tensorflow.md delete mode 100644 docs.it4i/salomon/software/mpi/Running_OpenMPI.md delete mode 100644 docs.it4i/salomon/software/mpi/mpi.md delete mode 100644 docs.it4i/salomon/software/mpi/mpi4py-mpi-for-python.md delete mode 100644 docs.it4i/salomon/software/numerical-languages/introduction.md delete mode 100644 docs.it4i/salomon/software/numerical-languages/matlab.md delete mode 100644 docs.it4i/salomon/software/numerical-languages/octave.md delete mode 100644 docs.it4i/salomon/software/numerical-languages/opencoarrays.md delete mode 100644 docs.it4i/salomon/software/numerical-languages/r.md delete mode 100644 docs.it4i/salomon/software/paraview.md diff --git a/docs.it4i/anselm/software/ansys/ansys-cfx.md b/docs.it4i/anselm/software/ansys/ansys-cfx.md deleted file mode 100644 index b816f0264..000000000 --- a/docs.it4i/anselm/software/ansys/ansys-cfx.md +++ /dev/null @@ -1,55 +0,0 @@ -# ANSYS CFX - -[ANSYS CFX](http://www.ansys.com/products/fluids/ansys-cfx) software is a high-performance, general purpose fluid dynamics program that has been applied to solve wide-ranging fluid flow problems for over 20 years. At the heart of ANSYS CFX is its advanced solver technology, the key to achieving reliable and accurate solutions quickly and robustly. The modern, highly parallelized solver is the foundation for an abundant choice of physical models to capture virtually any type of phenomena related to fluid flow. The solver and its many physical models are wrapped in a modern, intuitive, and flexible GUI and user environment, with extensive capabilities for customization and automation using session files, scripting and a powerful expression language. - -To run ANSYS CFX in batch mode you can utilize/modify the default cfx.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l nodes=2:ppn=16 -#PBS -q qprod -#PBS -N $USER-CFX-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> (working directory must exists) -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` -echo This jobs runs on the following processors: -echo `cat $PBS_NODEFILE` - -module load ansys - -#### Set number of processors per host listing -#### (set to 1 as $PBS_NODEFILE lists each node twice if :ppn=2) -procs_per_host=1 -#### Create host list -hl="" -for host in `cat $PBS_NODEFILE` -do - if ["$hl" = "" ] - then hl="$host:$procs_per_host" - else hl="${hl}:$host:$procs_per_host" - fi -done - -echo Machines: $hl - -#-dev input.def includes the input of CFX analysis in DEF format -#-P the name of prefered license feature (aa_r=ANSYS Academic Research, ane3fl=Multiphysics(commercial)) -/ansys_inc/v145/CFX/bin/cfx5solve -def input.def -size 4 -size-ni 4x -part-large -start-method "Platform MPI Distributed Parallel" -par-dist $hl -P aa_r -``` - -Header of the PBS file (above) is common and description can be find on [this site](../../job-submission-and-execution/). SVS FEM recommends to utilize sources by keywords: nodes, ppn. These keywords allows to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending PBS job into the queue. Input file should be in working directory or full path to input file has to be specified. >Input file has to be defined by common CFX def file which is attached to the cfx solver via parameter -def - -**License** should be selected by parameter -P (Big letter **P**). Licensed products are the following: aa_r (ANSYS **Academic** Research), ane3fl (ANSYS Multiphysics)-**Commercial**. -[More about licensing here](licensing/) diff --git a/docs.it4i/anselm/software/ansys/ansys-fluent.md b/docs.it4i/anselm/software/ansys/ansys-fluent.md deleted file mode 100644 index 4521c758e..000000000 --- a/docs.it4i/anselm/software/ansys/ansys-fluent.md +++ /dev/null @@ -1,162 +0,0 @@ -# ANSYS Fluent - -[ANSYS Fluent](http://www.ansys.com/products/fluids/ansys-fluent) -software contains the broad physical modeling capabilities needed to model flow, turbulence, heat transfer, and reactions for industrial applications ranging from air flow over an aircraft wing to combustion in a furnace, from bubble columns to oil platforms, from blood flow to semiconductor manufacturing, and from clean room design to wastewater treatment plants. Special models that give the software the ability to model in-cylinder combustion, aeroacoustics, turbomachinery, and multiphase systems have served to broaden its reach. - -## Common Way to Run Fluent Over PBS File - -To run ANSYS Fluent in batch mode you can utilize/modify the default fluent.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -S /bin/bash -#PBS -l nodes=2:ppn=16 -#PBS -q qprod -#PBS -N $USER-Fluent-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> (working directory must exists) -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` -echo This jobs runs on the following processors: -echo `cat $PBS_NODEFILE` - -#### Load ansys module so that we find the cfx5solve command -module load ansys - -# Use following line to specify MPI for message-passing instead -NCORES=`wc -l $PBS_NODEFILE |awk '{print $1}'` - -/ansys_inc/v145/fluent/bin/fluent 3d -t$NCORES -cnf=$PBS_NODEFILE -g -i fluent.jou -``` - -Header of the pbs file (above) is common and description can be find on [this site](../../resources-allocation-policy/). [SVS FEM](http://www.svsfem.cz) recommends to utilize sources by keywords: nodes, ppn. These keywords allows to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending pbs job into the queue. Input file should be in working directory or full path to input file has to be specified. Input file has to be defined by common Fluent journal file which is attached to the Fluent solver via parameter -i fluent.jou - -Journal file with definition of the input geometry and boundary conditions and defined process of solution has e.g. the following structure: - -```console - /file/read-case aircraft_2m.cas.gz - /solve/init - init - /solve/iterate - 10 - /file/write-case-dat aircraft_2m-solution - /exit yes -``` - -The appropriate dimension of the problem has to be set by parameter (2d/3d). - -## Fast Way to Run Fluent From Command Line - -```console -fluent solver_version [FLUENT_options] -i journal_file -pbs -``` - -This syntax will start the ANSYS FLUENT job under PBS Professional using the qsub command in a batch manner. When resources are available, PBS Professional will start the job and return a job ID, usually in the form of _job_ID.hostname_. This job ID can then be used to query, control, or stop the job using standard PBS Professional commands, such as qstat or qdel. The job will be run out of the current working directory, and all output will be written to the file fluent.o _job_ID_. - -## Running Fluent via User's Config File - -The sample script uses a configuration file called pbs_fluent.conf if no command line arguments are present. This configuration file should be present in the directory from which the jobs are submitted (which is also the directory in which the jobs are executed). The following is an example of what the content of pbs_fluent.conf can be: - -```console -input="example_small.flin" -case="Small-1.65m.cas" -fluent_args="3d -pmyrinet" -outfile="fluent_test.out" -mpp="true" -``` - -The following is an explanation of the parameters: - -input is the name of the input file. - -case is the name of the .cas file that the input file will utilize. - -fluent_args are extra ANSYS FLUENT arguments. As shown in the previous example, you can specify the interconnect by using the -p interconnect command. The available interconnects include ethernet (the default), myrinet, infiniband, vendor, altix, and crayx. The MPI is selected automatically, based on the specified interconnect. - -outfile is the name of the file to which the standard output will be sent. - - mpp="true" will tell the job script to execute the job across multiple processors. - -To run ANSYS Fluent in batch mode with user's config file you can utilize/modify the following script and execute it via the qsub command. - -```bash -#!/bin/sh -#PBS -l nodes=2:ppn=4 -#PBS -1 qprod -#PBS -N $USE-Fluent-Project -#PBS -A XX-YY-ZZ - - cd $PBS_O_WORKDIR - - #We assume that if they didn’t specify arguments then they should use the - #config file if ["xx${input}${case}${mpp}${fluent_args}zz" = "xxzz" ]; then - if [ -f pbs_fluent.conf ]; then - . pbs_fluent.conf - else - printf "No command line arguments specified, " - printf "and no configuration file found. Exiting n" - fi - fi - - - #Augment the ANSYS FLUENT command line arguments case "$mpp" in - true) - #MPI job execution scenario - num_nodes=â€cat $PBS_NODEFILE | sort -u | wc -l†- cpus=â€expr $num_nodes * $NCPUS†- #Default arguments for mpp jobs, these should be changed to suit your - #needs. - fluent_args="-t${cpus} $fluent_args -cnf=$PBS_NODEFILE" - ;; - *) - #SMP case - #Default arguments for smp jobs, should be adjusted to suit your - #needs. - fluent_args="-t$NCPUS $fluent_args" - ;; - esac - #Default arguments for all jobs - fluent_args="-ssh -g -i $input $fluent_args" - - echo "---------- Going to start a fluent job with the following settings: - Input: $input - Case: $case - Output: $outfile - Fluent arguments: $fluent_args" - - #run the solver - /ansys_inc/v145/fluent/bin/fluent $fluent_args > $outfile -``` - -It runs the jobs out of the directory from which they are submitted (PBS_O_WORKDIR). - -## Running Fluent in Parralel - -Fluent could be run in parallel only under Academic Research license. To do so this ANSYS Academic Research license must be placed before ANSYS CFD license in user preferences. To make this change anslic_admin utility should be run - -```console -/ansys_inc/shared_les/licensing/lic_admin/anslic_admin -``` - -ANSLIC_ADMIN Utility will be run - - - - - - - -ANSYS Academic Research license should be moved up to the top of the list. - - diff --git a/docs.it4i/anselm/software/ansys/ansys-ls-dyna.md b/docs.it4i/anselm/software/ansys/ansys-ls-dyna.md deleted file mode 100644 index af46af93a..000000000 --- a/docs.it4i/anselm/software/ansys/ansys-ls-dyna.md +++ /dev/null @@ -1,55 +0,0 @@ -# ANSYS LS-DYNA - -**[ANSYSLS-DYNA](http://www.ansys.com/products/structures/ansys-ls-dyna)** software provides convenient and easy-to-use access to the technology-rich, time-tested explicit solver without the need to contend with the complex input requirements of this sophisticated program. Introduced in 1996, ANSYS LS-DYNA capabilities have helped customers in numerous industries to resolve highly intricate design issues. ANSYS Mechanical users have been able take advantage of complex explicit solutions for a long time utilizing the traditional ANSYS Parametric Design Language (APDL) environment. These explicit capabilities are available to ANSYS Workbench users as well. The Workbench platform is a powerful, comprehensive, easy-to-use environment for engineering simulation. CAD import from all sources, geometry cleanup, automatic meshing, solution, parametric optimization, result visualization and comprehensive report generation are all available within a single fully interactive modern graphical user environment. - -To run ANSYS LS-DYNA in batch mode you can utilize/modify the default ansysdyna.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l nodes=2:ppn=16 -#PBS -q qprod -#PBS -N $USER-DYNA-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` -echo This jobs runs on the following processors: -echo `cat $PBS_NODEFILE` - -#! Counts the number of processors -NPROCS=`wc -l < $PBS_NODEFILE` - -echo This job has allocated $NPROCS nodes - -module load ansys - -#### Set number of processors per host listing -#### (set to 1 as $PBS_NODEFILE lists each node twice if :ppn=2) -procs_per_host=1 -#### Create host list -hl="" -for host in `cat $PBS_NODEFILE` -do - if ["$hl" = "" ] - then hl="$host:$procs_per_host" - else hl="${hl}:$host:$procs_per_host" - fi -done - -echo Machines: $hl - -/ansys_inc/v145/ansys/bin/ansys145 -dis -lsdynampp i=input.k -machines $hl -``` - -Header of the PBS file (above) is common and description can be find on [this site](../../job-submission-and-execution/). [SVS FEM](http://www.svsfem.cz) recommends to utilize sources by keywords: nodes, ppn. These keywords allows to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending PBS job into the queue. Input file should be in working directory or full path to input file has to be specified. Input file has to be defined by common LS-DYNA .**k** file which is attached to the ANSYS solver via parameter i= diff --git a/docs.it4i/anselm/software/ansys/ansys-mechanical-apdl.md b/docs.it4i/anselm/software/ansys/ansys-mechanical-apdl.md deleted file mode 100644 index cdaac19ff..000000000 --- a/docs.it4i/anselm/software/ansys/ansys-mechanical-apdl.md +++ /dev/null @@ -1,57 +0,0 @@ -# ANSYS MAPDL - -**[ANSYS Multiphysics](http://www.ansys.com/products/multiphysics)** -software offers a comprehensive product solution for both multiphysics and single-physics analysis. The product includes structural, thermal, fluid and both high- and low-frequency electromagnetic analysis. The product also contains solutions for both direct and sequentially coupled physics problems including direct coupled-field elements and the ANSYS multi-field solver. - -To run ANSYS MAPDL in batch mode you can utilize/modify the default mapdl.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l nodes=2:ppn=16 -#PBS -q qprod -#PBS -N $USER-ANSYS-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> (working directory must exists) -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` -echo This jobs runs on the following processors: -echo `cat $PBS_NODEFILE` - -module load ansys - -#### Set number of processors per host listing -#### (set to 1 as $PBS_NODEFILE lists each node twice if :ppn=2) -procs_per_host=1 -#### Create host list -hl="" -for host in `cat $PBS_NODEFILE` -do - if ["$hl" = "" ] - then hl="$host:$procs_per_host" - else hl="${hl}:$host:$procs_per_host" - fi -done - -echo Machines: $hl - -#-i input.dat includes the input of analysis in APDL format -#-o file.out is output file from ansys where all text outputs will be redirected -#-p the name of license feature (aa_r=ANSYS Academic Research, ane3fl=Multiphysics(commercial), aa_r_dy=Academic AUTODYN) -/ansys_inc/v145/ansys/bin/ansys145 -b -dis -p aa_r -i input.dat -o file.out -machines $hl -dir $WORK_DIR -``` - -Header of the PBS file (above) is common and description can be found on [this site](../../resource-allocation-policy/). [SVS FEM](http://www.svsfem.cz) recommends to utilize sources by keywords: nodes, ppn. These keywords allow to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending PBS job into the queue. Input file should be in working directory or full path to input file has to be specified. Input file has to be defined by common APDL file which is attached to the ANSYS solver via parameter -i - -**License** should be selected by parameter -p. Licensed products are the following: aa_r (ANSYS **Academic** Research), ane3fl (ANSYS Multiphysics)-**Commercial**, aa_r_dy (ANSYS **Academic** AUTODYN) -[More about licensing here](licensing/) diff --git a/docs.it4i/anselm/software/ansys/ansys.md b/docs.it4i/anselm/software/ansys/ansys.md deleted file mode 100644 index 24b8b1c09..000000000 --- a/docs.it4i/anselm/software/ansys/ansys.md +++ /dev/null @@ -1,15 +0,0 @@ -# Overview of ANSYS Products - -**[SVS FEM](http://www.svsfem.cz/)** as **[ANSYS Channel partner](http://www.ansys.com/)** for Czech Republic provided all ANSYS licenses for ANSELM cluster and supports of all ANSYS Products (Multiphysics, Mechanical, MAPDL, CFX, Fluent, Maxwell, LS-DYNA...) to IT staff and ANSYS users. If you are challenging to problem of ANSYS functionality contact please [hotline@svsfem.cz](mailto:hotline@svsfem.cz?subject=Ostrava%20-%20ANSELM) - -Anselm provides commercial as well as academic variants. Academic variants are distinguished by "**Academic...**" word in the name of license or by two letter preposition "**aa\_**" in the license feature name. Change of license is realized on command line respectively directly in user's PBS file (see individual products). [More about licensing here](ansys/licensing/) - -To load the latest version of any ANSYS product (Mechanical, Fluent, CFX, MAPDL,...) load the module: - -```console -$ ml ansys -``` - -ANSYS supports interactive regime, but due to assumed solution of extremely difficult tasks it is not recommended. - -If user needs to work in interactive regime we recommend to configure the RSM service on the client machine which allows to forward the solution to the Anselm directly from the client's Workbench project (see ANSYS RSM service). diff --git a/docs.it4i/anselm/software/ansys/ls-dyna.md b/docs.it4i/anselm/software/ansys/ls-dyna.md deleted file mode 100644 index 063bcf245..000000000 --- a/docs.it4i/anselm/software/ansys/ls-dyna.md +++ /dev/null @@ -1,35 +0,0 @@ -# LS-DYNA - -[LS-DYNA](http://www.lstc.com/) is a multi-purpose, explicit and implicit finite element program used to analyze the nonlinear dynamic response of structures. Its fully automated contact analysis capability, a wide range of constitutive models to simulate a whole range of engineering materials (steels, composites, foams, concrete, etc.), error-checking features and the high scalability have enabled users worldwide to solve successfully many complex problems. Additionally LS-DYNA is extensively used to simulate impacts on structures from drop tests, underwater shock, explosions or high-velocity impacts. Explosive forming, process engineering, accident reconstruction, vehicle dynamics, thermal brake disc analysis or nuclear safety are further areas in the broad range of possible applications. In leading-edge research LS-DYNA is used to investigate the behavior of materials like composites, ceramics, concrete, or wood. Moreover, it is used in biomechanics, human modeling, molecular structures, casting, forging, or virtual testing. - -Anselm provides **1 commercial license of LS-DYNA without HPC** support now. - -To run LS-DYNA in batch mode you can utilize/modify the default lsdyna.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l nodes=1:ppn=16 -#PBS -q qprod -#PBS -N $USER-LSDYNA-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> (working directory must exists) -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` - -module load lsdyna - -/apps/engineering/lsdyna/lsdyna700s i=input.k -``` - -Header of the PBS file (above) is common and description can be find on [this site](../../job-submission-and-execution/). [SVS FEM](http://www.svsfem.cz) recommends to utilize sources by keywords: nodes, ppn. These keywords allows to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending PBS job into the queue. Input file should be in working directory or full path to input file has to be specified. Input file has to be defined by common LS-DYNA **.k** file which is attached to the LS-DYNA solver via parameter i= diff --git a/docs.it4i/anselm/software/chemistry/molpro.md b/docs.it4i/anselm/software/chemistry/molpro.md deleted file mode 100644 index 9b08cb6ec..000000000 --- a/docs.it4i/anselm/software/chemistry/molpro.md +++ /dev/null @@ -1,63 +0,0 @@ -# Molpro - -Molpro is a complete system of ab initio programs for molecular electronic structure calculations. - -## About Molpro - -Molpro is a software package used for accurate ab-initio quantum chemistry calculations. More information can be found at the [official webpage](http://www.molpro.net/). - -## License - -Molpro software package is available only to users that have a valid license. Please contact support to enable access to Molpro if you have a valid license appropriate for running on our cluster (eg. academic research group licence, parallel execution). - -To run Molpro, you need to have a valid license token present in " $HOME/.molpro/token". You can download the token from [Molpro website](https://www.molpro.net/licensee/?portal=licensee). - -## Installed Version - -Currently on Anselm is installed version 2010.1, patch level 45, parallel version compiled with Intel compilers and Intel MPI. - -Compilation parameters are default: - -| Parameter | Value | -| ---------------------------------- | ------------ | -| max number of atoms | 200 | -| max number of valence orbitals | 300 | -| max number of basis functions | 4095 | -| max number of states per symmmetry | 20 | -| max number of state symmetries | 16 | -| max number of records | 200 | -| max number of primitives | maxbfn x [2] | - -## Running - -Molpro is compiled for parallel execution using MPI and OpenMP. By default, Molpro reads the number of allocated nodes from PBS and launches a data server on one node. On the remaining allocated nodes, compute processes are launched, one process per node, each with 16 threads. You can modify this behavior by using -n, -t and helper-server options. Please refer to the [Molpro documentation](http://www.molpro.net/info/2010.1/doc/manual/node9.html) for more details. - -!!! note - The OpenMP parallelization in Molpro is limited and has been observed to produce limited scaling. We therefore recommend to use MPI parallelization only. This can be achieved by passing option mpiprocs=16:ompthreads=1 to PBS. - -You are advised to use the -d option to point to a directory in [SCRATCH file system](../../storage/storage/). Molpro can produce a large amount of temporary data during its run, and it is important that these are placed in the fast scratch file system. - -### Example jobscript - -```bash - #PBS -A IT4I-0-0 - #PBS -q qprod - #PBS -l select=1:ncpus=16:mpiprocs=16:ompthreads=1 - - cd $PBS_O_WORKDIR - - # load Molpro module - module add molpro - - # create a directory in the SCRATCH filesystem - mkdir -p /scratch/$USER/$PBS_JOBID - - # copy an example input - cp /apps/chem/molpro/2010.1/molprop_2010_1_Linux_x86_64_i8/examples/caffeine_opt_diis.com . - - # run Molpro with default options - molpro -d /scratch/$USER/$PBS_JOBID caffeine_opt_diis.com - - # delete scratch directory - rm -rf /scratch/$USER/$PBS_JOBID -``` diff --git a/docs.it4i/anselm/software/chemistry/nwchem.md b/docs.it4i/anselm/software/chemistry/nwchem.md deleted file mode 100644 index e4f84d49f..000000000 --- a/docs.it4i/anselm/software/chemistry/nwchem.md +++ /dev/null @@ -1,42 +0,0 @@ -# NWChem - -## Introduction - -NWChem aims to provide its users with computational chemistry tools that are scalable both in their ability to treat large scientific computational chemistry problems efficiently, and in their use of available parallel computing resources from high-performance parallel supercomputers to conventional workstation clusters. - -[Homepage](http://www.nwchem-sw.org/index.php/Main_Page) - -## Installed Versions - -The following versions are currently installed: - -* 6.1.1, not recommended, problems have been observed with this version -* 6.3-rev2-patch1, current release with QMD patch applied. Compiled with Intel compilers, MKL and Intel MPI -* 6.3-rev2-patch1-openmpi, same as above, but compiled with OpenMPI and NWChem provided BLAS instead of MKL. This version is expected to be slower -* 6.3-rev2-patch1-venus, this version contains only libraries for VENUS interface linking. Does not provide standalone NWChem executable - -For a current list of installed versions, execute: - -```console -$ ml av nwchem -``` - -## Running - -NWChem is compiled for parallel MPI execution. Normal procedure for MPI jobs applies. Sample jobscript: - -```bash - #PBS -A IT4I-0-0 - #PBS -q qprod - #PBS -l select=1:ncpus=16 - - module add nwchem/6.3-rev2-patch1 - mpirun -np 16 nwchem h2o.nw -``` - -## Options - -Please refer to [the documentation](http://www.nwchem-sw.org/index.php/Release62:Top-level) and in the input file set the following directives : - -* MEMORY : controls the amount of memory NWChem will use -* SCRATCH_DIR : set this to a directory in [SCRATCH file system](../../storage/storage/#scratch) (or run the calculation completely in a scratch directory). For certain calculations, it might be advisable to reduce I/O by forcing "direct" mode, e.g.. "scf direct" diff --git a/docs.it4i/anselm/software/compilers.md b/docs.it4i/anselm/software/compilers.md deleted file mode 100644 index 63fa258ae..000000000 --- a/docs.it4i/anselm/software/compilers.md +++ /dev/null @@ -1,156 +0,0 @@ -# Compilers - -## Available Compilers, Including GNU, INTEL, and UPC Compilers - -Currently there are several compilers for different programming languages available on the Anselm cluster: - -* C/C++ -* Fortran 77/90/95 -* Unified Parallel C -* Java -* NVIDIA CUDA - -The C/C++ and Fortran compilers are divided into two main groups GNU and Intel. - -## Intel Compilers - -For information about the usage of Intel Compilers and other Intel products, please read the [Intel Parallel studio](intel-suite/) page. - -## GNU C/C++ and Fortran Compilers - -For compatibility reasons there are still available the original (old 4.4.6-4) versions of GNU compilers as part of the OS. These are accessible in the search path by default. - -It is strongly recommended to use the up to date version (4.8.1) which comes with the module gcc: - -```console -$ ml gcc -$ gcc -v -$ g++ -v -$ gfortran -v -``` - -With the module loaded two environment variables are predefined. One for maximum optimizations on the Anselm cluster architecture, and the other for debugging purposes: - -```console -$ echo $OPTFLAGS - -O3 -march=corei7-avx -$ echo $DEBUGFLAGS - -O0 -g -``` - -For more information about the possibilities of the compilers, please see the man pages. - -## Unified Parallel C - - UPC is supported by two compiler/runtime implementations: - -* GNU - SMP/multi-threading support only -* Berkley - multi-node support as well as SMP/multi-threading support - -### GNU UPC Compiler - -To use the GNU UPC compiler and run the compiled binaries use the module gupc - -```console -$ module add gupc -$ gupc -v -$ g++ -v -``` - -Simple program to test the compiler - -```cpp -$ cat count.upc - -/* hello.upc - a simple UPC example */ -#include <upc.h> -#include <stdio.h> - -int main() { - if (MYTHREAD == 0) { - printf("Welcome to GNU UPC!!!n"); - } - upc_barrier; - printf(" - Hello from thread %in", MYTHREAD); - return 0; -} -``` - -To compile the example use - -```console -$ gupc -o count.upc.x count.upc -``` - -To run the example with 5 threads issue - -```console -$ ./count.upc.x -fupc-threads-5 -``` - -For more information see the man pages. - -### Berkley UPC Compiler - -To use the Berkley UPC compiler and runtime environment to run the binaries use the module bupc - -```console -$ module add bupc -$ upcc -version -``` - -As default UPC network the "smp" is used. This is very quick and easy way for testing/debugging, but limited to one node only. - -For production runs, it is recommended to use the native Infiband implementation of UPC network "ibv". For testing/debugging using multiple nodes, the "mpi" UPC network is recommended. - -!!! warning - Selection of the network is done at the compile time and not at runtime (as expected)! - -Example UPC code: - -```cpp -$ cat hello.upc - -/* hello.upc - a simple UPC example */ -#include <upc.h> -#include <stdio.h> - -int main() { - if (MYTHREAD == 0) { - printf("Welcome to Berkeley UPC!!!n"); - } - upc_barrier; - printf(" - Hello from thread %in", MYTHREAD); - return 0; -} -``` - -To compile the example with the "ibv" UPC network use - -```console -$ upcc -network=ibv -o hello.upc.x hello.upc -``` - -To run the example with 5 threads issue - -```console -$ upcrun -n 5 ./hello.upc.x -``` - -To run the example on two compute nodes using all 32 cores, with 32 threads, issue - -```console -$ qsub -I -q qprod -A PROJECT_ID -l select=2:ncpus=16 -$ module add bupc -$ upcrun -n 32 ./hello.upc.x -``` - -For more information see the man pages. - -## Java - -For information how to use Java (runtime and/or compiler), please read the [Java page](java/). - -## NVIDIA CUDA - -For information on how to work with NVIDIA CUDA, please read the [NVIDIA CUDA page](nvidia-cuda/). diff --git a/docs.it4i/anselm/software/comsol-multiphysics.md b/docs.it4i/anselm/software/comsol-multiphysics.md deleted file mode 100644 index 746724285..000000000 --- a/docs.it4i/anselm/software/comsol-multiphysics.md +++ /dev/null @@ -1,120 +0,0 @@ -# COMSOL Multiphysics - -## Introduction - -[COMSOL](http://www.comsol.com) is a powerful environment for modelling and solving various engineering and scientific problems based on partial differential equations. COMSOL is designed to solve coupled or multiphysics phenomena. For many -standard engineering problems COMSOL provides add-on products such as electrical, mechanical, fluid flow, and chemical -applications. - -* [Structural Mechanics Module](http://www.comsol.com/structural-mechanics-module), -* [Heat Transfer Module](http://www.comsol.com/heat-transfer-module), -* [CFD Module](http://www.comsol.com/cfd-module), -* [Acoustics Module](http://www.comsol.com/acoustics-module), -* and [many others](http://www.comsol.com/products) - -COMSOL also allows an interface support for equation-based modelling of partial differential equations. - -## Execution - -On the Anselm cluster COMSOL is available in the latest stable version. There are two variants of the release: - -* **Non commercial** or so called **EDU variant**, which can be used for research and educational purposes. -* **Commercial** or so called **COM variant**, which can used also for commercial activities. **COM variant** has only subset of features compared to the **EDU variant** available. More about licensing will be posted here soon. - -To load the of COMSOL load the module - -```console -$ ml comsol -``` - -By default the **EDU variant** will be loaded. If user needs other version or variant, load the particular version. To obtain the list of available versions use - -```console -$ ml av comsol -``` - -If user needs to prepare COMSOL jobs in the interactive mode it is recommend to use COMSOL on the compute nodes via PBS Pro scheduler. In order run the COMSOL Desktop GUI on Windows is recommended to use the Virtual Network Computing (VNC). - -```console -$ xhost + -$ qsub -I -X -A PROJECT_ID -q qprod -l select=1:ncpus=16 -$ ml comsol -$ comsol -``` - -To run COMSOL in batch mode, without the COMSOL Desktop GUI environment, user can utilized the default (comsol.pbs) job script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l select=3:ncpus=16 -#PBS -q qprod -#PBS -N JOB_NAME -#PBS -A PROJECT_ID - -cd /scratch/$USER/ || exit - -echo Time is `date` -echo Directory is `pwd` -echo '**PBS_NODEFILE***START*******' -cat $PBS_NODEFILE -echo '**PBS_NODEFILE***END*********' - -text_nodes < cat $PBS_NODEFILE - -module load comsol -# module load comsol/43b-COM - -ntask=$(wc -l $PBS_NODEFILE) - -comsol -nn ${ntask} batch -configuration /tmp –mpiarg –rmk –mpiarg pbs -tmpdir /scratch/$USER/ -inputfile name_input_f.mph -outputfile name_output_f.mph -batchlog name_log_f.log -``` - -Working directory has to be created before sending the (comsol.pbs) job script into the queue. Input file (name_input_f.mph) has to be in working directory or full path to input file has to be specified. The appropriate path to the temp directory of the job has to be set by command option (-tmpdir). - -## LiveLink for MATLAB - -COMSOL is the software package for the numerical solution of the partial differential equations. LiveLink for MATLAB allows connection to the COMSOL API (Application Programming Interface) with the benefits of the programming language and computing environment of the MATLAB. - -LiveLink for MATLAB is available in both **EDU** and **COM** **variant** of the COMSOL release. On Anselm 1 commercial (**COM**) license and the 5 educational (**EDU**) licenses of LiveLink for MATLAB (please see the [ISV Licenses](isv_licenses/)) are available. -Following example shows how to start COMSOL model from MATLAB via LiveLink in the interactive mode. - -```console -$ xhost + -$ qsub -I -X -A PROJECT_ID -q qexp -l select=1:ncpus=16 -$ ml matlab -$ ml comsol -$ comsol server matlab -``` - -At the first time to launch the LiveLink for MATLAB (client-MATLAB/server-COMSOL connection) the login and password is requested and this information is not requested again. - -To run LiveLink for MATLAB in batch mode with (comsol_matlab.pbs) job script you can utilize/modify the following script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l select=3:ncpus=16 -#PBS -q qprod -#PBS -N JOB_NAME -#PBS -A PROJECT_ID - -cd /scratch/$USER || exit - -echo Time is `date` -echo Directory is `pwd` -echo '**PBS_NODEFILE***START*******' -cat $PBS_NODEFILE -echo '**PBS_NODEFILE***END*********' - -text_nodes < cat $PBS_NODEFILE - -module load matlab -module load comsol/43b-EDU - -ntask=$(wc -l $PBS_NODEFILE) - -comsol -nn ${ntask} server -configuration /tmp -mpiarg -rmk -mpiarg pbs -tmpdir /scratch/$USER & -cd /apps/engineering/comsol/comsol43b/mli -matlab -nodesktop -nosplash -r "mphstart; addpath /scratch/$USER; test_job" -``` - -This example shows, how to run LiveLink for MATLAB with following configuration: 3 nodes and 16 cores per node. Working directory has to be created before submitting (comsol_matlab.pbs) job script into the queue. Input file (test_job.m) has to be in working directory or full path to input file has to be specified. The MATLAB command option (-r ”mphstart”) created a connection with a COMSOL server using the default port number. diff --git a/docs.it4i/anselm/software/debuggers/allinea-ddt.md b/docs.it4i/anselm/software/debuggers/allinea-ddt.md deleted file mode 100644 index f85848417..000000000 --- a/docs.it4i/anselm/software/debuggers/allinea-ddt.md +++ /dev/null @@ -1,94 +0,0 @@ -# Allinea Forge (DDT,MAP) - -Allinea Forge consist of two tools - debugger DDT and profiler MAP. - -Allinea DDT, is a commercial debugger primarily for debugging parallel MPI or OpenMP programs. It also has a support for GPU (CUDA) and Intel Xeon Phi accelerators. DDT provides all the standard debugging features (stack trace, breakpoints, watches, view variables, threads etc.) for every thread running as part of your program, or for every process - even if these processes are distributed across a cluster using an MPI implementation. - -Allinea MAP is a profiler for C/C++/Fortran HPC codes. It is designed for profiling parallel code, which uses pthreads, OpenMP or MPI. - -## License and Limitations for Anselm Users - -On Anselm users can debug OpenMP or MPI code that runs up to 64 parallel processes. In case of debugging GPU or Xeon Phi accelerated codes the limit is 8 accelerators. These limitation means that: - -* 1 user can debug up 64 processes, or -* 32 users can debug 2 processes, etc. - -In case of debugging on accelerators: - -* 1 user can debug on up to 8 accelerators, or -* 8 users can debug on single accelerator. - -## Compiling Code to Run With DDT - -### Modules - -Load all necessary modules to compile the code. For example: - -```console -$ ml intel -$ ml impi ... or ... module load openmpi/X.X.X-icc -``` - -Load the Allinea DDT module: - -```console -$ ml Forge -``` - -Compile the code: - -```console -$ mpicc -g -O0 -o test_debug test.c - -$ mpif90 -g -O0 -o test_debug test.f -``` - -### Compiler Flags - -Before debugging, you need to compile your code with theses flags: - -!!! note - \* **g** : Generates extra debugging information usable by GDB. -g3 includes even more debugging information. This option is available for GNU and INTEL C/C++ and Fortran compilers. - \* **O0** : Suppress all optimizations. - -## Starting a Job With DDT - -Be sure to log in with an X window forwarding enabled. This could mean using the -X in the ssh: - -```console -$ ssh -X username@anselm.it4i.cz -``` - -Other options is to access login node using VNC. Please see the detailed information on how to [use graphic user interface on Anselm](/general/accessing-the-clusters/graphical-user-interface/x-window-system/) - -From the login node an interactive session **with X windows forwarding** (-X option) can be started by following command: - -```console -$ qsub -I -X -A NONE-0-0 -q qexp -lselect=1:ncpus=16:mpiprocs=16,walltime=01:00:00 -``` - -Then launch the debugger with the ddt command followed by the name of the executable to debug: - -```console -$ ddt test_debug -``` - -A submission window that appears have a prefilled path to the executable to debug. You can select the number of MPI processors and/or OpenMP threads on which to run and press run. Command line arguments to a program can be entered to the "Arguments " box. - - - -To start the debugging directly without the submission window, user can specify the debugging and execution parameters from the command line. For example the number of MPI processes is set by option "-np 4". Skipping the dialog is done by "-start" option. To see the list of the "ddt" command line parameters, run "ddt --help". - -```console -ddt -start -np 4 ./hello_debug_impi -``` - -## Documentation - -Users can find original User Guide after loading the DDT module: - -```console -$DDTPATH/doc/userguide.pdf -``` - -[1] Discipline, Magic, Inspiration and Science: Best Practice Debugging with Allinea DDT, Workshop conducted at LLNL by Allinea on May 10, 2013, [link](https://computing.llnl.gov/tutorials/allineaDDT/index.html) diff --git a/docs.it4i/anselm/software/debuggers/allinea-performance-reports.md b/docs.it4i/anselm/software/debuggers/allinea-performance-reports.md deleted file mode 100644 index a5399a61e..000000000 --- a/docs.it4i/anselm/software/debuggers/allinea-performance-reports.md +++ /dev/null @@ -1,59 +0,0 @@ -# Allinea Performance Reports - -## Introduction - -Allinea Performance Reports characterize the performance of HPC application runs. After executing your application through the tool, a synthetic HTML report is generated automatically, containing information about several metrics along with clear behavior statements and hints to help you improve the efficiency of your runs. - -The Allinea Performance Reports is most useful in profiling MPI programs. - -Our license is limited to 64 MPI processes. - -## Modules - -Allinea Performance Reports version 6.0 is available - -```console -$ ml PerformanceReports/6.0 -``` - -The module sets up environment variables, required for using the Allinea Performance Reports. This particular command loads the default module, which is performance reports version 4.2. - -## Usage - -!!! note - Use the the perf-report wrapper on your (MPI) program. - -Instead of [running your MPI program the usual way](../mpi/), use the the perf report wrapper: - -```console -$ perf-report mpirun ./mympiprog.x -``` - -The mpi program will run as usual. The perf-report creates two additional files, in \*.txt and \*.html format, containing the performance report. Note that [demanding MPI codes should be run within the queue system](../../job-submission-and-execution/). - -## Example - -In this example, we will be profiling the mympiprog.x MPI program, using Allinea performance reports. Assume that the code is compiled with Intel compilers and linked against Intel MPI library: - -First, we allocate some nodes via the express queue: - -```console -$ qsub -q qexp -l select=2:ncpus=16:mpiprocs=16:ompthreads=1 -I - qsub: waiting for job 262197.dm2 to start - qsub: job 262197.dm2 ready -``` - -Then we load the modules and run the program the usual way: - -```console -$ ml intel impi allinea-perf-report/4.2 -$ mpirun ./mympiprog.x -``` - -Now lets profile the code: - -```console -$ perf-report mpirun ./mympiprog.x -``` - -Performance report files [mympiprog_32p\*.txt](../../../src/mympiprog_32p_2014-10-15_16-56.txt) and [mympiprog_32p\*.html](../../../src/mympiprog_32p_2014-10-15_16-56.html) were created. We can see that the code is very efficient on MPI and is CPU bounded. diff --git a/docs.it4i/anselm/software/debuggers/cube.md b/docs.it4i/anselm/software/debuggers/cube.md deleted file mode 100644 index a7f88955e..000000000 --- a/docs.it4i/anselm/software/debuggers/cube.md +++ /dev/null @@ -1,36 +0,0 @@ -# CUBE - -## Introduction - -CUBE is a graphical performance report explorer for displaying data from Score-P and Scalasca (and other compatible tools). The name comes from the fact that it displays performance data in a three-dimensions : - -* **performance metric**, where a number of metrics are available, such as communication time or cache misses, -* **call path**, which contains the call tree of your program -* **system resource**, which contains system's nodes, processes and threads, depending on the parallel programming model. - -Each dimension is organized in a tree, for example the time performance metric is divided into Execution time and Overhead time, call path dimension is organized by files and routines in your source code etc. - - - -\*Figure 1. Screenshot of CUBE displaying data from Scalasca.\* - -Each node in the tree is colored by severity (the color scheme is displayed at the bottom of the window, ranging from the least severe blue to the most severe being red). For example in Figure 1, we can see that most of the point-to-point MPI communication happens in routine exch_qbc, colored red. - -## Installed Versions - -Currently, there are two versions of CUBE 4.2.3 available as [modules](../../environment-and-modules/): - -* cube/4.2.3-gcc, compiled with GCC -* cube/4.2.3-icc, compiled with Intel compiler - -## Usage - -CUBE is a graphical application. Refer to Graphical User Interface documentation for a list of methods to launch graphical applications on Anselm. - -!!! note - Analyzing large data sets can consume large amount of CPU and RAM. Do not perform large analysis on login nodes. - -After loading the appropriate module, simply launch cube command, or alternatively you can use scalasca -examine command to launch the GUI. Note that for Scalasca datasets, if you do not analyze the data with scalasca -examine before to opening them with CUBE, not all performance data will be available. - -References -1\. <http://www.scalasca.org/software/cube-4.x/download.html> diff --git a/docs.it4i/anselm/software/debuggers/debuggers.md b/docs.it4i/anselm/software/debuggers/debuggers.md deleted file mode 100644 index 3d38fd6a5..000000000 --- a/docs.it4i/anselm/software/debuggers/debuggers.md +++ /dev/null @@ -1,60 +0,0 @@ -# Debuggers and profilers summary - -## Introduction - -We provide state of the art programms and tools to develop, profile and debug HPC codes at IT4Innovations. On these pages, we provide an overview of the profiling and debugging tools available on Anslem at IT4I. - -## Intel Debugger - -The intel debugger version 13.0 is available, via module intel. The debugger works for applications compiled with C and C++ compiler and the ifort fortran 77/90/95 compiler. The debugger provides java GUI environment. Use X display for running the GUI. - -```console -$ ml intel -$ idb -``` - -Read more at the [Intel Debugger](intel-suite/intel-debugger/) page. - -## Allinea Forge (DDT/MAP) - -Allinea DDT, is a commercial debugger primarily for debugging parallel MPI or OpenMP programs. It also has a support for GPU (CUDA) and Intel Xeon Phi accelerators. DDT provides all the standard debugging features (stack trace, breakpoints, watches, view variables, threads etc.) for every thread running as part of your program, or for every process even if these processes are distributed across a cluster using an MPI implementation. - -```console -$ ml Forge -$ forge -``` - -Read more at the [Allinea DDT](debuggers/allinea-ddt/) page. - -## Allinea Performance Reports - -Allinea Performance Reports characterize the performance of HPC application runs. After executing your application through the tool, a synthetic HTML report is generated automatically, containing information about several metrics along with clear behavior statements and hints to help you improve the efficiency of your runs. Our license is limited to 64 MPI processes. - -```console -$ ml PerformanceReports/6.0 -$ perf-report mpirun -n 64 ./my_application argument01 argument02 -``` - -Read more at the [Allinea Performance Reports](debuggers/allinea-performance-reports/) page. - -## RougeWave Totalview - -TotalView is a source- and machine-level debugger for multi-process, multi-threaded programs. Its wide range of tools provides ways to analyze, organize, and test programs, making it easy to isolate and identify problems in individual threads and processes in programs of great complexity. - -```console -$ ml totalview -$ totalview -``` - -Read more at the [Totalview](debuggers/total-view/) page. - -## Vampir Trace Analyzer - -Vampir is a GUI trace analyzer for traces in OTF format. - -```console -$ ml Vampir/8.5.0 -$ vampir -``` - -Read more at the [Vampir](vampir/) page. diff --git a/docs.it4i/anselm/software/debuggers/intel-performance-counter-monitor.md b/docs.it4i/anselm/software/debuggers/intel-performance-counter-monitor.md deleted file mode 100644 index b46b472b6..000000000 --- a/docs.it4i/anselm/software/debuggers/intel-performance-counter-monitor.md +++ /dev/null @@ -1,281 +0,0 @@ -# Intel Performance Counter Monitor - -## Introduction - -Intel PCM (Performance Counter Monitor) is a tool to monitor performance hardware counters on Intel>® processors, similar to [PAPI](papi/). The difference between PCM and PAPI is that PCM supports only Intel hardware, but PCM can monitor also uncore metrics, like memory controllers and >QuickPath Interconnect links. - -## Installed Version - -Currently installed version 2.6. To load the [module](../../environment-and-modules/), issue: - -```console -$ ml intelpcm -``` - -## Command Line Tools - -PCM provides a set of tools to monitor system/or application. - -### Pcm-Memory - - Measures memory bandwidth of your application or the whole system. Usage: - -```console -$ pcm-memory.x <delay>|[external_program parameters] -``` - -Specify either a delay of updates in seconds or an external program to monitor. If you get an error about PMU in use, respond "y" and relaunch the program. - -Sample output: - -```console - ---------------------------------------||--------------------------------------- - -- Socket 0 --||-- Socket 1 -- - ---------------------------------------||--------------------------------------- - ---------------------------------------||--------------------------------------- - ---------------------------------------||--------------------------------------- - -- Memory Performance Monitoring --||-- Memory Performance Monitoring -- - ---------------------------------------||--------------------------------------- - -- Mem Ch 0: Reads (MB/s): 2.44 --||-- Mem Ch 0: Reads (MB/s): 0.26 -- - -- Writes(MB/s): 2.16 --||-- Writes(MB/s): 0.08 -- - -- Mem Ch 1: Reads (MB/s): 0.35 --||-- Mem Ch 1: Reads (MB/s): 0.78 -- - -- Writes(MB/s): 0.13 --||-- Writes(MB/s): 0.65 -- - -- Mem Ch 2: Reads (MB/s): 0.32 --||-- Mem Ch 2: Reads (MB/s): 0.21 -- - -- Writes(MB/s): 0.12 --||-- Writes(MB/s): 0.07 -- - -- Mem Ch 3: Reads (MB/s): 0.36 --||-- Mem Ch 3: Reads (MB/s): 0.20 -- - -- Writes(MB/s): 0.13 --||-- Writes(MB/s): 0.07 -- - -- NODE0 Mem Read (MB/s): 3.47 --||-- NODE1 Mem Read (MB/s): 1.45 -- - -- NODE0 Mem Write (MB/s): 2.55 --||-- NODE1 Mem Write (MB/s): 0.88 -- - -- NODE0 P. Write (T/s) : 31506 --||-- NODE1 P. Write (T/s): 9099 -- - -- NODE0 Memory (MB/s): 6.02 --||-- NODE1 Memory (MB/s): 2.33 -- - ---------------------------------------||--------------------------------------- - -- System Read Throughput(MB/s): 4.93 -- - -- System Write Throughput(MB/s): 3.43 -- - -- System Memory Throughput(MB/s): 8.35 -- - ---------------------------------------||--------------------------------------- -``` - -### Pcm-Msr - -Command pcm-msr.x can be used to read/write model specific registers of the CPU. - -### Pcm-Numa - -NUMA monitoring utility does not work on Anselm. - -### Pcm-Pcie - -Can be used to monitor PCI Express bandwith. Usage: pcm-pcie.x <delay> - -### Pcm-Power - -Displays energy usage and thermal headroom for CPU and DRAM sockets. Usage: `pcm-power.x <delay> | <external program>` - -### Pcm - -This command provides an overview of performance counters and memory usage. Usage: `pcm.x <delay> | <external program>` - -Sample output : - -```console - $ pcm.x ./matrix - - Intel(r) Performance Counter Monitor V2.6 (2013-11-04 13:43:31 +0100 ID=db05e43) - - Copyright (c) 2009-2013 Intel Corporation - - Number of physical cores: 16 - Number of logical cores: 16 - Threads (logical cores) per physical core: 1 - Num sockets: 2 - Core PMU (perfmon) version: 3 - Number of core PMU generic (programmable) counters: 8 - Width of generic (programmable) counters: 48 bits - Number of core PMU fixed counters: 3 - Width of fixed counters: 48 bits - Nominal core frequency: 2400000000 Hz - Package thermal spec power: 115 Watt; Package minimum power: 51 Watt; Package maximum power: 180 Watt; - Socket 0: 1 memory controllers detected with total number of 4 channels. 2 QPI ports detected. - Socket 1: 1 memory controllers detected with total number of 4 channels. 2 QPI ports detected. - Number of PCM instances: 2 - Max QPI link speed: 16.0 GBytes/second (8.0 GT/second) - - Detected Intel(R) Xeon(R) CPU E5-2665 0 @ 2.40GHz "Intel(r) microarchitecture codename Sandy Bridge-EP/Jaketown" - - Executing "./matrix" command: - - Exit code: 0 - - EXEC : instructions per nominal CPU cycle - IPC : instructions per CPU cycle - FREQ : relation to nominal CPU frequency='unhalted clock ticks'/'invariant timer ticks' (includes Intel Turbo Boost) - AFREQ : relation to nominal CPU frequency while in active state (not in power-saving C state)='unhalted clock ticks'/'invariant timer ticks while in C0-state' (includes Intel Turbo Boost) - L3MISS: L3 cache misses - L2MISS: L2 cache misses (including other core's L2 cache *hits*) - L3HIT : L3 cache hit ratio (0.00-1.00) - L2HIT : L2 cache hit ratio (0.00-1.00) - L3CLK : ratio of CPU cycles lost due to L3 cache misses (0.00-1.00), in some cases could be >1.0 due to a higher memory latency - L2CLK : ratio of CPU cycles lost due to missing L2 cache but still hitting L3 cache (0.00-1.00) - READ : bytes read from memory controller (in GBytes) - WRITE : bytes written to memory controller (in GBytes) - TEMP : Temperature reading in 1 degree Celsius relative to the TjMax temperature (thermal headroom): 0 corresponds to the max temperature - - Core (SKT) | EXEC | IPC | FREQ | AFREQ | L3MISS | L2MISS | L3HIT | L2HIT | L3CLK | L2CLK | READ | WRITE | TEMP - - 0 0 0.00 0.64 0.01 0.80 5592 11 K 0.49 0.13 0.32 0.06 N/A N/A 67 - 1 0 0.00 0.18 0.00 0.69 3086 5552 0.44 0.07 0.48 0.08 N/A N/A 68 - 2 0 0.00 0.23 0.00 0.81 300 562 0.47 0.06 0.43 0.08 N/A N/A 67 - 3 0 0.00 0.21 0.00 0.99 437 862 0.49 0.06 0.44 0.09 N/A N/A 73 - 4 0 0.00 0.23 0.00 0.93 293 559 0.48 0.07 0.42 0.09 N/A N/A 73 - 5 0 0.00 0.21 0.00 1.00 423 849 0.50 0.06 0.43 0.10 N/A N/A 69 - 6 0 0.00 0.23 0.00 0.94 285 558 0.49 0.06 0.41 0.09 N/A N/A 71 - 7 0 0.00 0.18 0.00 0.81 674 1130 0.40 0.05 0.53 0.08 N/A N/A 65 - 8 1 0.00 0.47 0.01 1.26 6371 13 K 0.51 0.35 0.31 0.07 N/A N/A 64 - 9 1 2.30 1.80 1.28 1.29 179 K 15 M 0.99 0.59 0.04 0.71 N/A N/A 60 - 10 1 0.00 0.22 0.00 1.26 315 570 0.45 0.06 0.43 0.08 N/A N/A 67 - 11 1 0.00 0.23 0.00 0.74 321 579 0.45 0.05 0.45 0.07 N/A N/A 66 - 12 1 0.00 0.22 0.00 1.25 305 570 0.46 0.05 0.42 0.07 N/A N/A 68 - 13 1 0.00 0.22 0.00 1.26 336 581 0.42 0.04 0.44 0.06 N/A N/A 69 - 14 1 0.00 0.22 0.00 1.25 314 565 0.44 0.06 0.43 0.07 N/A N/A 69 - 15 1 0.00 0.29 0.00 1.19 2815 6926 0.59 0.39 0.29 0.08 N/A N/A 69 - ------------------------------------------------------------------------------------------------------------------- - SKT 0 0.00 0.46 0.00 0.79 11 K 21 K 0.47 0.10 0.38 0.07 0.00 0.00 65 - SKT 1 0.29 1.79 0.16 1.29 190 K 15 M 0.99 0.59 0.05 0.70 0.01 0.01 61 - ------------------------------------------------------------------------------------------------------------------- - TOTAL * 0.14 1.78 0.08 1.28 201 K 15 M 0.99 0.59 0.05 0.70 0.01 0.01 N/A - - Instructions retired: 1345 M ; Active cycles: 755 M ; Time (TSC): 582 Mticks ; C0 (active,non-halted) core residency: 6.30 % - - C1 core residency: 0.14 %; C3 core residency: 0.20 %; C6 core residency: 0.00 %; C7 core residency: 93.36 %; - C2 package residency: 48.81 %; C3 package residency: 0.00 %; C6 package residency: 0.00 %; C7 package residency: 0.00 %; - - PHYSICAL CORE IPC : 1.78 => corresponds to 44.50 % utilization for cores in active state - Instructions per nominal CPU cycle: 0.14 => corresponds to 3.60 % core utilization over time interval - - Intel(r) QPI data traffic estimation in bytes (data traffic coming to CPU/socket through QPI links): - - QPI0 QPI1 | QPI0 QPI1 - ---------------------------------------------------------------------------------------------- - SKT 0 0 0 | 0% 0% - SKT 1 0 0 | 0% 0% - ---------------------------------------------------------------------------------------------- - Total QPI incoming data traffic: 0 QPI data traffic/Memory controller traffic: 0.00 - - Intel(r) QPI traffic estimation in bytes (data and non-data traffic outgoing from CPU/socket through QPI links): - - QPI0 QPI1 | QPI0 QPI1 - ---------------------------------------------------------------------------------------------- - SKT 0 0 0 | 0% 0% - SKT 1 0 0 | 0% 0% - ---------------------------------------------------------------------------------------------- - Total QPI outgoing data and non-data traffic: 0 - - ---------------------------------------------------------------------------------------------- - SKT 0 package consumed 4.06 Joules - SKT 1 package consumed 9.40 Joules - ---------------------------------------------------------------------------------------------- - TOTAL: 13.46 Joules - - ---------------------------------------------------------------------------------------------- - SKT 0 DIMMs consumed 4.18 Joules - SKT 1 DIMMs consumed 4.28 Joules - ---------------------------------------------------------------------------------------------- - TOTAL: 8.47 Joules - Cleaning up -``` - -### Pcm-Sensor - -Can be used as a sensor for ksysguard GUI, which is currently not installed on Anselm. - -## API - -In a similar fashion to PAPI, PCM provides a C++ API to access the performance counter from within your application. Refer to the [Doxygen documentation](http://intel-pcm-api-documentation.github.io/classPCM.html) for details of the API. - -!!! note - Due to security limitations, using PCM API to monitor your applications is currently not possible on Anselm. (The application must be run as root user) - -Sample program using the API : - -```cpp - #include <stdlib.h> - #include <stdio.h> - #include "cpucounters.h" - - #define SIZE 1000 - - using namespace std; - - int main(int argc, char **argv) { - float matrixa[SIZE][SIZE], matrixb[SIZE][SIZE], mresult[SIZE][SIZE]; - float real_time, proc_time, mflops; - long long flpins; - int retval; - int i,j,k; - - PCM * m = PCM::getInstance(); - - if (m->program() != PCM::Success) return 1; - - SystemCounterState before_sstate = getSystemCounterState(); - - /* Initialize the Matrix arrays */ - for ( i=0; i<SIZE*SIZE; i++ ){ - mresult[0][i] = 0.0; - matrixa[0][i] = matrixb[0][i] = rand()*(float)1.1; } - - /* A naive Matrix-Matrix multiplication */ - for (i=0;i<SIZE;i++) - for(j=0;j<SIZE;j++) - for(k=0;k<SIZE;k++) - mresult[i][j]=mresult[i][j] + matrixa[i][k]*matrixb[k][j]; - - SystemCounterState after_sstate = getSystemCounterState(); - - cout << "Instructions per clock:" << getIPC(before_sstate,after_sstate) - << "L3 cache hit ratio:" << getL3CacheHitRatio(before_sstate,after_sstate) - << "Bytes read:" << getBytesReadFromMC(before_sstate,after_sstate); - - for (i=0; i<SIZE;i++) - for (j=0; j<SIZE; j++) - if (mresult[i][j] == -1) printf("x"); - - return 0; - } -``` - -Compile it with : - -```console -$ icc matrix.cpp -o matrix -lpthread -lpcm -``` - -Sample output: - -```console -$ ./matrix - Number of physical cores: 16 - Number of logical cores: 16 - Threads (logical cores) per physical core: 1 - Num sockets: 2 - Core PMU (perfmon) version: 3 - Number of core PMU generic (programmable) counters: 8 - Width of generic (programmable) counters: 48 bits - Number of core PMU fixed counters: 3 - Width of fixed counters: 48 bits - Nominal core frequency: 2400000000 Hz - Package thermal spec power: 115 Watt; Package minimum power: 51 Watt; Package maximum power: 180 Watt; - Socket 0: 1 memory controllers detected with total number of 4 channels. 2 QPI ports detected. - Socket 1: 1 memory controllers detected with total number of 4 channels. 2 QPI ports detected. - Number of PCM instances: 2 - Max QPI link speed: 16.0 GBytes/second (8.0 GT/second) - Instructions per clock:1.7 - L3 cache hit ratio:1.0 - Bytes read:12513408 -``` - -## References - -1. <https://software.intel.com/en-us/articles/intel-performance-counter-monitor-a-better-way-to-measure-cpu-utilization> -1. <https://software.intel.com/sites/default/files/m/3/2/2/xeon-e5-2600-uncore-guide.pdf> Intel® Xeon® Processor E5-2600 Product Family Uncore Performance Monitoring Guide. -1. <http://intel-pcm-api-documentation.github.io/classPCM.html> API Documentation diff --git a/docs.it4i/anselm/software/debuggers/intel-vtune-amplifier.md b/docs.it4i/anselm/software/debuggers/intel-vtune-amplifier.md deleted file mode 100644 index 1d90aacfe..000000000 --- a/docs.it4i/anselm/software/debuggers/intel-vtune-amplifier.md +++ /dev/null @@ -1,73 +0,0 @@ -# Intel VTune Amplifier - -## Introduction - -Intel VTune Amplifier, part of Intel Parallel studio, is a GUI profiling tool designed for Intel processors. It offers a graphical performance analysis of single core and multithreaded applications. A highlight of the features: - -* Hotspot analysis -* Locks and waits analysis -* Low level specific counters, such as branch analysis and memory - bandwidth -* Power usage analysis - frequency and sleep states. - - - -## Usage - -To launch the GUI, first load the module: - -```console -$ module add VTune/2016_update1 -``` - -and launch the GUI : - -```console -$ amplxe-gui -``` - -!!! note - To profile an application with VTune Amplifier, special kernel modules need to be loaded. The modules are not loaded on Anselm login nodes, thus direct profiling on login nodes is not possible. Use VTune on compute nodes and refer to the documentation on using GUI applications. - -The GUI will open in new window. Click on "_New Project..._" to create a new project. After clicking _OK_, a new window with project properties will appear. At "_Application:_", select the bath to your binary you want to profile (the binary should be compiled with -g flag). Some additional options such as command line arguments can be selected. At "_Managed code profiling mode:_" select "_Native_" (unless you want to profile managed mode .NET/Mono applications). After clicking _OK_, your project is created. - -To run a new analysis, click "_New analysis..._". You will see a list of possible analysis. Some of them will not be possible on the current CPU (e.g. Intel Atom analysis is not possible on Sandy Bridge CPU), the GUI will show an error box if you select the wrong analysis. For example, select "_Advanced Hotspots_". Clicking on _Start _will start profiling of the application. - -## Remote Analysis - -VTune Amplifier also allows a form of remote analysis. In this mode, data for analysis is collected from the command line without GUI, and the results are then loaded to GUI on another machine. This allows profiling without interactive graphical jobs. To perform a remote analysis, launch a GUI somewhere, open the new analysis window and then click the button "_Command line_" in bottom right corner. It will show the command line needed to perform the selected analysis. - -The command line will look like this: - -```console -$ /apps/all/VTune/2016_update1/vtune_amplifier_xe_2016.1.1.434111/bin64/amplxe-cl -collect advanced-hotspots -knob collection-detail=stack-and-callcount -mrte-mode=native -target-duration-type=veryshort -app-working-dir /home/sta545/test -- /home/sta545/test_pgsesv -``` - -Copy the line to clipboard and then you can paste it in your jobscript or in command line. After the collection is run, open the GUI once again, click the menu button in the upper right corner, and select "_Open > Result..._". The GUI will load the results from the run. - -## Xeon Phi - -!!! note - This section is outdated. It will be updated with new information soon. - -It is possible to analyze both native and offload Xeon Phi applications. For offload mode, just specify the path to the binary. For native mode, you need to specify in project properties: - -Application: ssh - -Application parameters: mic0 source ~/.profile && /path/to/your/bin - -Note that we include source ~/.profile in the command to setup environment paths [as described here](../intel-xeon-phi/). - -!!! note - If the analysis is interrupted or aborted, further analysis on the card might be impossible and you will get errors like "ERROR connecting to MIC card". In this case please contact our support to reboot the MIC card. - -You may also use remote analysis to collect data from the MIC and then analyze it in the GUI later : - -```console -$ amplxe-cl -collect knc-hotspots -no-auto-finalize -- ssh mic0 - "export LD_LIBRARY_PATH=/apps/intel/composer_xe_2015.2.164/compiler/lib/mic/:/apps/intel/composer_xe_2015.2.164/mkl/lib/mic/; export KMP_AFFINITY=compact; /tmp/app.mic" -``` - -## References - -1. <https://www.rcac.purdue.edu/tutorials/phi/PerformanceTuningXeonPhi-Tullos.pdf> Performance Tuning for Intel® Xeon Phi™ Coprocessors diff --git a/docs.it4i/anselm/software/debuggers/papi.md b/docs.it4i/anselm/software/debuggers/papi.md deleted file mode 100644 index d03dd8354..000000000 --- a/docs.it4i/anselm/software/debuggers/papi.md +++ /dev/null @@ -1,237 +0,0 @@ -# PAPI - -## Introduction - -Performance Application Programming Interface (PAPI) is a portable interface to access hardware performance counters (such as instruction counts and cache misses) found in most modern architectures. With the new component framework, PAPI is not limited only to CPU counters, but offers also components for CUDA, network, Infiniband etc. - -PAPI provides two levels of interface - a simpler, high level interface and more detailed low level interface. - -PAPI can be used with parallel as well as serial programs. - -## Usage - -To use PAPI, load [module](../../environment-and-modules/) papi: - -```console -$ ml papi -``` - -This will load the default version. Execute module avail papi for a list of installed versions. - -## Utilities - -The bin directory of PAPI (which is automatically added to $PATH upon loading the module) contains various utilites. - -### Papi_avail - -Prints which preset events are available on the current CPU. The third column indicated whether the preset event is available on the current CPU. - -```console -$ papi_avail - Available events and hardware information. - -------------------------------------------------------------------------------- - PAPI Version : 5.3.2.0 - Vendor string and code : GenuineIntel (1) - Model string and code : Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz (45) - CPU Revision : 7.000000 - CPUID Info : Family: 6 Model: 45 Stepping: 7 - CPU Max Megahertz : 2601 - CPU Min Megahertz : 1200 - Hdw Threads per core : 1 - Cores per Socket : 8 - Sockets : 2 - NUMA Nodes : 2 - CPUs per Node : 8 - Total CPUs : 16 - Running in a VM : no - Number Hardware Counters : 11 - Max Multiplex Counters : 32 - -------------------------------------------------------------------------------- - Name Code Avail Deriv Description (Note) - PAPI_L1_DCM 0x80000000 Yes No Level 1 data cache misses - PAPI_L1_ICM 0x80000001 Yes No Level 1 instruction cache misses - PAPI_L2_DCM 0x80000002 Yes Yes Level 2 data cache misses - PAPI_L2_ICM 0x80000003 Yes No Level 2 instruction cache misses - PAPI_L3_DCM 0x80000004 No No Level 3 data cache misses - PAPI_L3_ICM 0x80000005 No No Level 3 instruction cache misses - PAPI_L1_TCM 0x80000006 Yes Yes Level 1 cache misses - PAPI_L2_TCM 0x80000007 Yes No Level 2 cache misses - PAPI_L3_TCM 0x80000008 Yes No Level 3 cache misses - .... -``` - -### Papi_native_avail - -Prints which native events are available on the current CPU. - -### Papi_cost - -Measures the cost (in cycles) of basic PAPI operations. - -### Papi_mem_info - -Prints information about the memory architecture of the current CPU. - -## PAPI API - -PAPI provides two kinds of events: - -* **Preset events** is a set of predefined common CPU events, standardized across platforms. -* **Native events **is a set of all events supported by the current hardware. This is a larger set of features than preset. For other components than CPU, only native events are usually available. - -To use PAPI in your application, you need to link the appropriate include file. - -* papi.h for C -* f77papi.h for Fortran 77 -* f90papi.h for Fortran 90 -* fpapi.h for Fortran with preprocessor - -The include path is automatically added by papi module to $INCLUDE. - -### High Level API - -Please refer to [this description of the High level API](http://icl.cs.utk.edu/projects/papi/wiki/PAPIC:High_Level). - -### Low Level API - -Please refer to [this description of the Low level API](http://icl.cs.utk.edu/projects/papi/wiki/PAPIC:Low_Level). - -### Timers - -PAPI provides the most accurate timers the platform can support. [See](http://icl.cs.utk.edu/projects/papi/wiki/PAPIC:Timers). - -### System Information - -PAPI can be used to query some system infromation, such as CPU name and MHz. [See](http://icl.cs.utk.edu/projects/papi/wiki/PAPIC:System_Information). - -## Example - -The following example prints MFLOPS rate of a naive matrix-matrix multiplication: - -```cpp - #include <stdlib.h> - #include <stdio.h> - #include "papi.h" - #define SIZE 1000 - - int main(int argc, char **argv) { - float matrixa[SIZE][SIZE], matrixb[SIZE][SIZE], mresult[SIZE][SIZE]; - float real_time, proc_time, mflops; - long long flpins; - int retval; - int i,j,k; - - /* Initialize the Matrix arrays */ - for ( i=0; i<SIZE*SIZE; i++ ){ - mresult[0][i] = 0.0; - matrixa[0][i] = matrixb[0][i] = rand()*(float)1.1; - } - - /* Setup PAPI library and begin collecting data from the counters */ - if((retval=PAPI_flops( &real_time, &proc_time, &flpins, &mflops))<PAPI_OK) - printf("Error!"); - - /* A naive Matrix-Matrix multiplication */ - for (i=0;i<SIZE;i++) - for(j=0;j<SIZE;j++) - for(k=0;k<SIZE;k++) - mresult[i][j]=mresult[i][j] + matrixa[i][k]*matrixb[k][j]; - - /* Collect the data into the variables passed in */ - if((retval=PAPI_flops( &real_time, &proc_time, &flpins, &mflops))<PAPI_OK) - printf("Error!"); - - printf("Real_time:t%fnProc_time:t%fnTotal flpins:t%lldnMFLOPS:tt%fn", real_time, proc_time, flpins, mflops); - PAPI_shutdown(); - return 0; - } -``` - -Now compile and run the example : - -```console -$ gcc matrix.c -o matrix -lpapi -$ ./matrix - Real_time: 8.852785 - Proc_time: 8.850000 - Total flpins: 6012390908 - MFLOPS: 679.366211 -``` - -Let's try with optimizations enabled : - -```console -$ gcc -O3 matrix.c -o matrix -lpapi -$ ./matrix - Real_time: 0.000020 - Proc_time: 0.000000 - Total flpins: 6 - MFLOPS: inf -``` - -Now we see a seemingly strange result - the multiplication took no time and only 6 floating point instructions were issued. This is because the compiler optimizations have completely removed the multiplication loop, as the result is actually not used anywhere in the program. We can fix this by adding some "dummy" code at the end of the Matrix-Matrix multiplication routine : - -```cpp - for (i=0; i<SIZE;i++) - for (j=0; j<SIZE; j++) - if (mresult[i][j] == -1.0) printf("x"); -``` - -Now the compiler won't remove the multiplication loop. (However it is still not that smart to see that the result won't ever be negative). Now run the code again: - -```console -$ gcc -O3 matrix.c -o matrix -lpapi -$ ./matrix - Real_time: 8.795956 - Proc_time: 8.790000 - Total flpins: 18700983160 - MFLOPS: 2127.529297 -``` - -### Intel Xeon Phi - -!!! note - PAPI currently supports only a subset of counters on the Intel Xeon Phi processor compared to Intel Xeon, for example the floating point operations counter is missing. - -To use PAPI in [Intel Xeon Phi](../intel-xeon-phi/) native applications, you need to load module with " -mic" suffix, for example " papi/5.3.2-mic" : - -```console -$ ml papi/5.3.2-mic -``` - -Then, compile your application in the following way: - -```console -$ ml intel -$ icc -mmic -Wl,-rpath,/apps/intel/composer_xe_2013.5.192/compiler/lib/mic matrix-mic.c -o matrix-mic -lpapi -lpfm -``` - -To execute the application on MIC, you need to manually set LD_LIBRARY_PATH: - -```console -$ qsub -q qmic -A NONE-0-0 -I -$ ssh mic0 -$ export LD_LIBRARY_PATH="/apps/tools/papi/5.4.0-mic/lib/" -$ ./matrix-mic -``` - -Alternatively, you can link PAPI statically (-static flag), then LD_LIBRARY_PATH does not need to be set. - -You can also execute the PAPI tools on MIC : - -```console -$ /apps/tools/papi/5.4.0-mic/bin/papi_native_avail -``` - -To use PAPI in offload mode, you need to provide both host and MIC versions of PAPI: - -```console -$ ml papi/5.4.0 -$ icc matrix-offload.c -o matrix-offload -offload-option,mic,compiler,"-L$PAPI_HOME-mic/lib -lpapi" -lpapi -``` - -## References - -1. [Main project page](http://icl.cs.utk.edu/papi/) -1. [Wiki](http://icl.cs.utk.edu/projects/papi/wiki/Main_Page) -1. [API Documentation](http://icl.cs.utk.edu/papi/docs/) diff --git a/docs.it4i/anselm/software/debuggers/scalasca.md b/docs.it4i/anselm/software/debuggers/scalasca.md deleted file mode 100644 index a7cd44b1d..000000000 --- a/docs.it4i/anselm/software/debuggers/scalasca.md +++ /dev/null @@ -1,70 +0,0 @@ -# Scalasca - -## Introduction - -[Scalasca](http://www.scalasca.org/) is a software tool that supports the performance optimization of parallel programs by measuring and analyzing their runtime behavior. The analysis identifies potential performance bottlenecks – in particular those concerning communication and synchronization – and offers guidance in exploring their causes. - -Scalasca supports profiling of MPI, OpenMP and hybrid MPI+OpenMP applications. - -## Installed Versions - -There are currently two versions of Scalasca 2.0 [modules](../../environment-and-modules/) installed on Anselm: - -* scalasca2/2.0-gcc-openmpi, for usage with [GNU Compiler](../compilers/) and [OpenMPI](../mpi/Running_OpenMPI/), -* scalasca2/2.0-icc-impi, for usage with [Intel Compiler](../compilers/) and [Intel MPI](../mpi/running-mpich2/). - -## Usage - -Profiling a parallel application with Scalasca consists of three steps: - -1. Instrumentation, compiling the application such way, that the profiling data can be generated. -1. Runtime measurement, running the application with the Scalasca profiler to collect performance data. -1. Analysis of reports - -### Instrumentation - -Instrumentation via " scalasca -instrument" is discouraged. Use [Score-P instrumentation](score-p/). - -### Runtime Measurement - -After the application is instrumented, runtime measurement can be performed with the `scalasca -analyze` command. The syntax is: - -`scalasca -analyze [scalasca options][launcher] [launcher options][program] [program options]` - -An example : - -```console - $ scalasca -analyze mpirun -np 4 ./mympiprogram -``` - -Some notable Scalasca options are: - -* **-t Enable trace data collection. By default, only summary data are collected.** -* **-e <directory> Specify a directory to save the collected data to. By default, Scalasca saves the data to a directory with prefix scorep\_, followed by name of the executable and launch configuration.** - -!!! note - Scalasca can generate a huge amount of data, especially if tracing is enabled. Please consider saving the data to a [scratch directory](../../storage/storage/). - -### Analysis of Reports - -For the analysis, you must have [Score-P](score-p/) and [CUBE](cube/) modules loaded. The analysis is done in two steps, first, the data is preprocessed and then CUBE GUI tool is launched. - -To launch the analysis, run : - -```console -scalasca -examine [options] <experiment_directory> -``` - -If you do not wish to launch the GUI tool, use the "-s" option : - -```console -scalasca -examine -s <experiment_directory> -``` - -Alternatively you can open CUBE and load the data directly from here. Keep in mind that in that case the preprocessing is not done and not all metrics will be shown in the viewer. - -Refer to [CUBE documentation](cube/) on usage of the GUI viewer. - -## References - -1. <http://www.scalasca.org/> diff --git a/docs.it4i/anselm/software/debuggers/score-p.md b/docs.it4i/anselm/software/debuggers/score-p.md deleted file mode 100644 index 3295933c4..000000000 --- a/docs.it4i/anselm/software/debuggers/score-p.md +++ /dev/null @@ -1,117 +0,0 @@ -# Score-P - -## Introduction - -The [Score-P measurement infrastructure](http://www.vi-hps.org/projects/score-p/) is a highly scalable and easy-to-use tool suite for profiling, event tracing, and online analysis of HPC applications. - -Score-P can be used as an instrumentation tool for [Scalasca](scalasca/). - -## Installed Versions - -There are currently two versions of Score-P version 1.2.6 [modules](../../environment-and-modules/) installed on Anselm : - -* scorep/1.2.3-gcc-openmpi, for usage with [GNU Compiler](../compilers/) and [OpenMPI](../mpi/Running_OpenMPI/) -* scorep/1.2.3-icc-impi, for usage with [Intel Compiler](../compilers/)> and [Intel MPI](../mpi/running-mpich2/)>. - -## Instrumentation - -There are three ways to instrument your parallel applications in order to enable performance data collection: - -1. Automated instrumentation using compiler -1. Manual instrumentation using API calls -1. Manual instrumentation using directives - -### Automated Instrumentation - -is the easiest method. Score-P will automatically add instrumentation to every routine entry and exit using compiler hooks, and will intercept MPI calls and OpenMP regions. This method might, however, produce a large number of data. If you want to focus on profiler a specific regions of your code, consider using the manual instrumentation methods. To use automated instrumentation, simply prepend scorep to your compilation command. For example, replace: - -```console -$ mpif90 -c foo.f90 -$ mpif90 -c bar.f90 -$ mpif90 -o myapp foo.o bar.o -``` - -with: - -```console -$ scorep mpif90 -c foo.f90 -$ scorep mpif90 -c bar.f90 -$ scorep mpif90 -o myapp foo.o bar.o -``` - -Usually your program is compiled using a Makefile or similar script, so it advisable to add the scorep command to your definition of variables CC, CXX, FCC etc. - -It is important that scorep is prepended also to the linking command, in order to link with Score-P instrumentation libraries. - -### Manual Instrumentation Using API Calls - -To use this kind of instrumentation, use scorep with switch --user. You will then mark regions to be instrumented by inserting API calls. - -An example in C/C++ : - -```cpp - #include <scorep/SCOREP_User.h> - void foo() - { - SCOREP_USER_REGION_DEFINE( my_region_handle ) - // more declarations - SCOREP_USER_REGION_BEGIN( my_region_handle, "foo", SCOREP_USER_REGION_TYPE_COMMON ) - // do something - SCOREP_USER_REGION_END( my_region_handle ) - } -``` - - and Fortran : - -```cpp - #include "scorep/SCOREP_User.inc" - subroutine foo - SCOREP_USER_REGION_DEFINE( my_region_handle ) - ! more declarations - SCOREP_USER_REGION_BEGIN( my_region_handle, "foo", SCOREP_USER_REGION_TYPE_COMMON ) - ! do something - SCOREP_USER_REGION_END( my_region_handle ) - end subroutine foo -``` - -Please refer to the [documentation for description of the API](https://silc.zih.tu-dresden.de/scorep-current/pdf/scorep.pdf). - -### Manual Instrumentation Using Directives - -This method uses POMP2 directives to mark regions to be instrumented. To use this method, use command scorep --pomp. - -Example directives in C/C++ : - -```cpp - void foo(...) - { - /* declarations */ - #pragma pomp inst begin(foo) - ... - if (<condition>) - { - #pragma pomp inst altend(foo) - return; - } - ... - #pragma pomp inst end(foo) - } -``` - -and in Fortran : - -```cpp - subroutine foo(...) - !declarations - !POMP$ INST BEGIN(foo) - ... - if (<condition>) then - !POMP$ INST ALTEND(foo) - return - end if - ... - !POMP$ INST END(foo) - end subroutine foo -``` - -The directives are ignored if the program is compiled without Score-P. Again, please refer to the [documentation](https://silc.zih.tu-dresden.de/scorep-current/pdf/scorep.pdf) for a more elaborate description. diff --git a/docs.it4i/anselm/software/debuggers/total-view.md b/docs.it4i/anselm/software/debuggers/total-view.md deleted file mode 100644 index de618ace5..000000000 --- a/docs.it4i/anselm/software/debuggers/total-view.md +++ /dev/null @@ -1,158 +0,0 @@ -# Total View - -TotalView is a GUI-based source code multi-process, multi-thread debugger. - -## License and Limitations for Anselm Users - -On Anselm users can debug OpenMP or MPI code that runs up to 64 parallel processes. These limitation means that: - -```console - 1 user can debug up 64 processes, or - 32 users can debug 2 processes, etc. -``` - -Debugging of GPU accelerated codes is also supported. - -You can check the status of the licenses here: - -```console -$ cat /apps/user/licenses/totalview_features_state.txt - - # totalview - # ------------------------------------------------- - # FEATURE TOTAL USED AVAIL - # ------------------------------------------------- - TotalView_Team 64 0 64 - Replay 64 0 64 - CUDA 64 0 64 -``` - -## Compiling Code to Run With TotalView - -### Modules - -Load all necessary modules to compile the code. For example: - -```console -$ ml intel **or** ml foss -``` - -Load the TotalView module: - -```console -$ ml totalview/8.12 -``` - -Compile the code: - -```console -$ mpicc -g -O0 -o test_debug test.c -$ mpif90 -g -O0 -o test_debug test.f -``` - -### Compiler Flags - -Before debugging, you need to compile your code with theses flags: - -!!! note - \* **-g** : Generates extra debugging information usable by GDB. **-g3** includes even more debugging information. This option is available for GNU and INTEL C/C++ and Fortran compilers. - \* **-O0** : Suppress all optimizations. - -## Starting a Job With TotalView - -Be sure to log in with an X window forwarding enabled. This could mean using the -X in the ssh: - -```console -local $ ssh -X username@anselm.it4i.cz -``` - -Other options is to access login node using VNC. Please see the detailed information on how to use graphic user interface on Anselm. - -From the login node an interactive session with X windows forwarding (-X option) can be started by following command: - -```console -$ qsub -I -X -A NONE-0-0 -q qexp -lselect=1:ncpus=16:mpiprocs=16,walltime=01:00:00 -``` - -Then launch the debugger with the totalview command followed by the name of the executable to debug. - -### Debugging a Serial Code - -To debug a serial code use: - -```console -$ totalview test_debug -``` - -### Debugging a Parallel Code - Option 1 - -To debug a parallel code compiled with **OpenMPI** you need to setup your TotalView environment: - -!!! hint - To be able to run parallel debugging procedure from the command line without stopping the debugger in the mpiexec source code you have to add the following function to your `~/.tvdrc` file: - -```console - proc mpi_auto_run_starter {loaded_id} { - set starter_programs {mpirun mpiexec orterun} - set executable_name [TV::symbol get $loaded_id full_pathname] - set file_component [file tail $executable_name] - - if {[lsearch -exact $starter_programs $file_component] != -1} { - puts "*************************************" - puts "Automatically starting $file_component" - puts "*************************************" - dgo - } - } - - # Append this function to TotalView's image load callbacks so that - # TotalView run this program automatically. - - dlappend TV::image_load_callbacks mpi_auto_run_starter -``` - -The source code of this function can be also found in - -```console -$ /apps/mpi/openmpi/intel/1.6.5/etc/openmpi-totalview.tcl -``` - -!!! note - You can also add only following line to you ~/.tvdrc file instead of the entire function: - **source /apps/mpi/openmpi/intel/1.6.5/etc/openmpi-totalview.tcl** - -You need to do this step only once. - -Now you can run the parallel debugger using: - -```console -$ mpirun -tv -n 5 ./test_debug -``` - -When following dialog appears click on "Yes" - - - -At this point the main TotalView GUI window will appear and you can insert the breakpoints and start debugging: - - - -### Debugging a Parallel Code - Option 2 - -Other option to start new parallel debugging session from a command line is to let TotalView to execute mpirun by itself. In this case user has to specify a MPI implementation used to compile the source code. - -The following example shows how to start debugging session with Intel MPI: - -```console -$ ml intel -$ ml totalview -$ totalview -mpi "Intel MPI-Hydra" -np 8 ./hello_debug_impi -``` - -After running previous command you will see the same window as shown in the screenshot above. - -More information regarding the command line parameters of the TotalView can be found TotalView Reference Guide, Chapter 7: TotalView Command Syntax. - -## Documentation - -[1] The [TotalView documentation](http://www.roguewave.com/support/product-documentation/totalview-family.aspx#totalview) web page is a good resource for learning more about some of the advanced TotalView features. diff --git a/docs.it4i/anselm/software/debuggers/valgrind.md b/docs.it4i/anselm/software/debuggers/valgrind.md deleted file mode 100644 index f6c3f6aaf..000000000 --- a/docs.it4i/anselm/software/debuggers/valgrind.md +++ /dev/null @@ -1,262 +0,0 @@ -# Valgrind - -Valgrind is a tool for memory debugging and profiling. - -## About Valgrind - -Valgrind is an open-source tool, used mainly for debuggig memory-related problems, such as memory leaks, use of uninitalized memory etc. in C/C++ applications. The toolchain was however extended over time with more functionality, such as debugging of threaded applications, cache profiling, not limited only to C/C++. - -Valgind is an extremely useful tool for debugging memory errors such as [off-by-one](http://en.wikipedia.org/wiki/Off-by-one_error). Valgrind uses a virtual machine and dynamic recompilation of binary code, because of that, you can expect that programs being debugged by Valgrind run 5-100 times slower. - -The main tools available in Valgrind are : - -* **Memcheck**, the original, must used and default tool. Verifies memory access in you program and can detect use of unitialized memory, out of bounds memory access, memory leaks, double free, etc. -* **Massif**, a heap profiler. -* **Hellgrind** and **DRD** can detect race conditions in multi-threaded applications. -* **Cachegrind**, a cache profiler. -* **Callgrind**, a callgraph analyzer. -* For a full list and detailed documentation, please refer to the [official Valgrind documentation](http://valgrind.org/docs/). - -## Installed Versions - -There are two versions of Valgrind available on Anselm. - -* Version 3.6.0, installed by operating system vendor in /usr/bin/valgrind. This version is available by default, without the need to load any module. This version however does not provide additional MPI support. -* Version 3.9.0 with support for Intel MPI, available in [module](../../environment-and-modules/) valgrind/3.9.0-impi. After loading the module, this version replaces the default valgrind. - -## Usage - -Compile the application which you want to debug as usual. It is advisable to add compilation flags -g (to add debugging information to the binary so that you will see original source code lines in the output) and -O0 (to disable compiler optimizations). - -For example, lets look at this C code, which has two problems : - -```cpp -#include <stdlib.h> - -void f(void) -{ - int* x = malloc(10 * sizeof(int)); - x[10] = 0; // problem 1: heap block overrun -} // problem 2: memory leak -- x not freed - -int main(void) -{ - f(); - return 0; -} -``` - -Now, compile it with Intel compiler : - -```console -$ module add intel -$ icc -g valgrind-example.c -o valgrind-example -``` - -Now, lets run it with Valgrind. The syntax is : - -`valgrind [valgrind options] <your program binary> [your program options]` - -If no Valgrind options are specified, Valgrind defaults to running Memcheck tool. Please refer to the Valgrind documentation for a full description of command line options. - -```console -$ valgrind ./valgrind-example - ==12652== Memcheck, a memory error detector - ==12652== Copyright (C) 2002-2013, and GNU GPL'd, by Julian Seward et al. - ==12652== Using Valgrind-3.9.0 and LibVEX; rerun with -h for copyright info - ==12652== Command: ./valgrind-example - ==12652== - ==12652== Invalid write of size 4 - ==12652== at 0x40053E: f (valgrind-example.c:6) - ==12652== by 0x40054E: main (valgrind-example.c:11) - ==12652== Address 0x5861068 is 0 bytes after a block of size 40 alloc'd - ==12652== at 0x4C27AAA: malloc (vg_replace_malloc.c:291) - ==12652== by 0x400528: f (valgrind-example.c:5) - ==12652== by 0x40054E: main (valgrind-example.c:11) - ==12652== - ==12652== - ==12652== HEAP SUMMARY: - ==12652== in use at exit: 40 bytes in 1 blocks - ==12652== total heap usage: 1 allocs, 0 frees, 40 bytes allocated - ==12652== - ==12652== LEAK SUMMARY: - ==12652== definitely lost: 40 bytes in 1 blocks - ==12652== indirectly lost: 0 bytes in 0 blocks - ==12652== possibly lost: 0 bytes in 0 blocks - ==12652== still reachable: 0 bytes in 0 blocks - ==12652== suppressed: 0 bytes in 0 blocks - ==12652== Rerun with --leak-check=full to see details of leaked memory - ==12652== - ==12652== For counts of detected and suppressed errors, rerun with: -v - ==12652== ERROR SUMMARY: 1 errors from 1 contexts (suppressed: 6 from 6) -``` - -In the output we can see that Valgrind has detected both errors - the off-by-one memory access at line 5 and a memory leak of 40 bytes. If we want a detailed analysis of the memory leak, we need to run Valgrind with --leak-check=full option : - -```console -$ valgrind --leak-check=full ./valgrind-example - ==23856== Memcheck, a memory error detector - ==23856== Copyright (C) 2002-2010, and GNU GPL'd, by Julian Seward et al. - ==23856== Using Valgrind-3.6.0 and LibVEX; rerun with -h for copyright info - ==23856== Command: ./valgrind-example - ==23856== - ==23856== Invalid write of size 4 - ==23856== at 0x40067E: f (valgrind-example.c:6) - ==23856== by 0x40068E: main (valgrind-example.c:11) - ==23856== Address 0x66e7068 is 0 bytes after a block of size 40 alloc'd - ==23856== at 0x4C26FDE: malloc (vg_replace_malloc.c:236) - ==23856== by 0x400668: f (valgrind-example.c:5) - ==23856== by 0x40068E: main (valgrind-example.c:11) - ==23856== - ==23856== - ==23856== HEAP SUMMARY: - ==23856== in use at exit: 40 bytes in 1 blocks - ==23856== total heap usage: 1 allocs, 0 frees, 40 bytes allocated - ==23856== - ==23856== 40 bytes in 1 blocks are definitely lost in loss record 1 of 1 - ==23856== at 0x4C26FDE: malloc (vg_replace_malloc.c:236) - ==23856== by 0x400668: f (valgrind-example.c:5) - ==23856== by 0x40068E: main (valgrind-example.c:11) - ==23856== - ==23856== LEAK SUMMARY: - ==23856== definitely lost: 40 bytes in 1 blocks - ==23856== indirectly lost: 0 bytes in 0 blocks - ==23856== possibly lost: 0 bytes in 0 blocks - ==23856== still reachable: 0 bytes in 0 blocks - ==23856== suppressed: 0 bytes in 0 blocks - ==23856== - ==23856== For counts of detected and suppressed errors, rerun with: -v - ==23856== ERROR SUMMARY: 2 errors from 2 contexts (suppressed: 6 from 6) -``` - -Now we can see that the memory leak is due to the malloc() at line 6. - -## Usage With MPI - -Although Valgrind is not primarily a parallel debugger, it can be used to debug parallel applications as well. When launching your parallel applications, prepend the valgrind command. For example : - -```console -$ mpirun -np 4 valgrind myapplication -``` - -The default version without MPI support will however report a large number of false errors in the MPI library, such as : - -```console - ==30166== Conditional jump or move depends on uninitialised value(s) - ==30166== at 0x4C287E8: strlen (mc_replace_strmem.c:282) - ==30166== by 0x55443BD: I_MPI_Processor_model_number (init_interface.c:427) - ==30166== by 0x55439E0: I_MPI_Processor_arch_code (init_interface.c:171) - ==30166== by 0x558D5AE: MPID_nem_impi_init_shm_configuration (mpid_nem_impi_extensions.c:1091) - ==30166== by 0x5598F4C: MPID_nem_init_ckpt (mpid_nem_init.c:566) - ==30166== by 0x5598B65: MPID_nem_init (mpid_nem_init.c:489) - ==30166== by 0x539BD75: MPIDI_CH3_Init (ch3_init.c:64) - ==30166== by 0x5578743: MPID_Init (mpid_init.c:193) - ==30166== by 0x554650A: MPIR_Init_thread (initthread.c:539) - ==30166== by 0x553369F: PMPI_Init (init.c:195) - ==30166== by 0x4008BD: main (valgrind-example-mpi.c:18) -``` - -so it is better to use the MPI-enabled valgrind from module. The MPI version requires library /apps/tools/valgrind/3.9.0/impi/lib/valgrind/libmpiwrap-amd64-linux.so, which must be included in the LD_PRELOAD environment variable. - -Lets look at this MPI example : - -```cpp -#include <stdlib.h> -#include <mpi.h> - -int main(int argc, char *argv[]) -{ - int *data = malloc(sizeof(int)*99); - - MPI_Init(&argc, &argv); - MPI_Bcast(data, 100, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Finalize(); - - return 0; -} -``` - -There are two errors - use of uninitialized memory and invalid length of the buffer. Lets debug it with valgrind : - -```console -$ module add intel impi -$ mpicc -g valgrind-example-mpi.c -o valgrind-example-mpi -$ module add valgrind/3.9.0-impi -$ mpirun -np 2 -env LD_PRELOAD /apps/tools/valgrind/3.9.0/impi/lib/valgrind/libmpiwrap-amd64-linux.so valgrind ./valgrind-example-mpi -``` - -Prints this output : (note that there is output printed for every launched MPI process) - -```console - ==31318== Memcheck, a memory error detector - ==31318== Copyright (C) 2002-2013, and GNU GPL'd, by Julian Seward et al. - ==31318== Using Valgrind-3.9.0 and LibVEX; rerun with -h for copyright info - ==31318== Command: ./valgrind-example-mpi - ==31318== - ==31319== Memcheck, a memory error detector - ==31319== Copyright (C) 2002-2013, and GNU GPL'd, by Julian Seward et al. - ==31319== Using Valgrind-3.9.0 and LibVEX; rerun with -h for copyright info - ==31319== Command: ./valgrind-example-mpi - ==31319== - valgrind MPI wrappers 31319: Active for pid 31319 - valgrind MPI wrappers 31319: Try MPIWRAP_DEBUG=help for possible options - valgrind MPI wrappers 31318: Active for pid 31318 - valgrind MPI wrappers 31318: Try MPIWRAP_DEBUG=help for possible options - ==31319== Unaddressable byte(s) found during client check request - ==31319== at 0x4E35974: check_mem_is_addressable_untyped (libmpiwrap.c:960) - ==31319== by 0x4E5D0FE: PMPI_Bcast (libmpiwrap.c:908) - ==31319== by 0x400911: main (valgrind-example-mpi.c:20) - ==31319== Address 0x69291cc is 0 bytes after a block of size 396 alloc'd - ==31319== at 0x4C27AAA: malloc (vg_replace_malloc.c:291) - ==31319== by 0x4007BC: main (valgrind-example-mpi.c:8) - ==31319== - ==31318== Uninitialised byte(s) found during client check request - ==31318== at 0x4E3591D: check_mem_is_defined_untyped (libmpiwrap.c:952) - ==31318== by 0x4E5D06D: PMPI_Bcast (libmpiwrap.c:908) - ==31318== by 0x400911: main (valgrind-example-mpi.c:20) - ==31318== Address 0x6929040 is 0 bytes inside a block of size 396 alloc'd - ==31318== at 0x4C27AAA: malloc (vg_replace_malloc.c:291) - ==31318== by 0x4007BC: main (valgrind-example-mpi.c:8) - ==31318== - ==31318== Unaddressable byte(s) found during client check request - ==31318== at 0x4E3591D: check_mem_is_defined_untyped (libmpiwrap.c:952) - ==31318== by 0x4E5D06D: PMPI_Bcast (libmpiwrap.c:908) - ==31318== by 0x400911: main (valgrind-example-mpi.c:20) - ==31318== Address 0x69291cc is 0 bytes after a block of size 396 alloc'd - ==31318== at 0x4C27AAA: malloc (vg_replace_malloc.c:291) - ==31318== by 0x4007BC: main (valgrind-example-mpi.c:8) - ==31318== - ==31318== - ==31318== HEAP SUMMARY: - ==31318== in use at exit: 3,172 bytes in 67 blocks - ==31318== total heap usage: 191 allocs, 124 frees, 81,203 bytes allocated - ==31318== - ==31319== - ==31319== HEAP SUMMARY: - ==31319== in use at exit: 3,172 bytes in 67 blocks - ==31319== total heap usage: 175 allocs, 108 frees, 48,435 bytes allocated - ==31319== - ==31318== LEAK SUMMARY: - ==31318== definitely lost: 408 bytes in 3 blocks - ==31318== indirectly lost: 256 bytes in 1 blocks - ==31318== possibly lost: 0 bytes in 0 blocks - ==31318== still reachable: 2,508 bytes in 63 blocks - ==31318== suppressed: 0 bytes in 0 blocks - ==31318== Rerun with --leak-check=full to see details of leaked memory - ==31318== - ==31318== For counts of detected and suppressed errors, rerun with: -v - ==31318== Use --track-origins=yes to see where uninitialised values come from - ==31318== ERROR SUMMARY: 2 errors from 2 contexts (suppressed: 4 from 4) - ==31319== LEAK SUMMARY: - ==31319== definitely lost: 408 bytes in 3 blocks - ==31319== indirectly lost: 256 bytes in 1 blocks - ==31319== possibly lost: 0 bytes in 0 blocks - ==31319== still reachable: 2,508 bytes in 63 blocks - ==31319== suppressed: 0 bytes in 0 blocks - ==31319== Rerun with --leak-check=full to see details of leaked memory - ==31319== - ==31319== For counts of detected and suppressed errors, rerun with: -v - ==31319== ERROR SUMMARY: 1 errors from 1 contexts (suppressed: 4 from 4) -``` - -We can see that Valgrind has reported use of unitialised memory on the master process (which reads the array to be broadcast) and use of unaddresable memory on both processes. diff --git a/docs.it4i/anselm/software/debuggers/vampir.md b/docs.it4i/anselm/software/debuggers/vampir.md deleted file mode 100644 index 1dfa23e7b..000000000 --- a/docs.it4i/anselm/software/debuggers/vampir.md +++ /dev/null @@ -1,22 +0,0 @@ -# Vampir - -Vampir is a commercial trace analysis and visualization tool. It can work with traces in OTF and OTF2 formats. It does not have the functionality to collect traces, you need to use a trace collection tool (such as [Score-P](score-p/)) first to collect the traces. - - - -## Installed Versions - -Version 8.5.0 is currently installed as module Vampir/8.5.0 : - -```console -$ ml Vampir/8.5.0 -$ vampir & -``` - -## User Manual - -You can find the detailed user manual in PDF format in $EBROOTVAMPIR/doc/vampir-manual.pdf - -## References - -[1]. <https://www.vampir.eu> diff --git a/docs.it4i/anselm/software/intel-suite/intel-compilers.md b/docs.it4i/anselm/software/intel-suite/intel-compilers.md deleted file mode 100644 index d446655d9..000000000 --- a/docs.it4i/anselm/software/intel-suite/intel-compilers.md +++ /dev/null @@ -1,36 +0,0 @@ -# Intel Compilers - -The Intel compilers version 13.1.1 are available, via module intel. The compilers include the icc C and C++ compiler and the ifort fortran 77/90/95 compiler. - -```console -$ ml intel -$ icc -v -$ ifort -v -``` - -The intel compilers provide for vectorization of the code, via the AVX instructions and support threading parallelization via OpenMP - -For maximum performance on the Anselm cluster, compile your programs using the AVX instructions, with reporting where the vectorization was used. We recommend following compilation options for high performance - -```console -$ icc -ipo -O3 -vec -xAVX -vec-report1 myprog.c mysubroutines.c -o myprog.x -$ ifort -ipo -O3 -vec -xAVX -vec-report1 myprog.f mysubroutines.f -o myprog.x -``` - -In this example, we compile the program enabling interprocedural optimizations between source files (-ipo), aggressive loop optimizations (-O3) and vectorization (-vec -xAVX) - -The compiler recognizes the omp, simd, vector and ivdep pragmas for OpenMP parallelization and AVX vectorization. Enable the OpenMP parallelization by the **-openmp** compiler switch. - -```console -$ icc -ipo -O3 -vec -xAVX -vec-report1 -openmp myprog.c mysubroutines.c -o myprog.x -$ ifort -ipo -O3 -vec -xAVX -vec-report1 -openmp myprog.f mysubroutines.f -o myprog.x -``` - -Read more at <http://software.intel.com/sites/products/documentation/doclib/stdxe/2013/composerxe/compiler/cpp-lin/index.htm> - -## Sandy Bridge/Haswell Binary Compatibility - -Anselm nodes are currently equipped with Sandy Bridge CPUs, while Salomon will use Haswell architecture. >The new processors are backward compatible with the Sandy Bridge nodes, so all programs that ran on the Sandy Bridge processors, should also run on the new Haswell nodes. >To get optimal performance out of the Haswell processors a program should make use of the special AVX2 instructions for this processor. One can do this by recompiling codes with the compiler flags >designated to invoke these instructions. For the Intel compiler suite, there are two ways of doing this: - -* Using compiler flag (both for Fortran and C): -xCORE-AVX2. This will create a binary with AVX2 instructions, specifically for the Haswell processors. Note that the executable will not run on Sandy Bridge nodes. -* Using compiler flags (both for Fortran and C): -xAVX -axCORE-AVX2. This will generate multiple, feature specific auto-dispatch code paths for Intel® processors, if there is a performance benefit. So this binary will run both on Sandy Bridge and Haswell processors. During runtime it will be decided which path to follow, dependent on which processor you are running on. In general this will result in larger binaries. diff --git a/docs.it4i/anselm/software/intel-suite/intel-debugger.md b/docs.it4i/anselm/software/intel-suite/intel-debugger.md deleted file mode 100644 index d3a5807fc..000000000 --- a/docs.it4i/anselm/software/intel-suite/intel-debugger.md +++ /dev/null @@ -1,73 +0,0 @@ -# Intel Debugger - -## Debugging Serial Applications - -The intel debugger version 13.0 is available, via module intel. The debugger works for applications compiled with C and C++ compiler and the ifort fortran 77/90/95 compiler. The debugger provides java GUI environment. Use X display for running the GUI. - -```baconsolesh -$ ml intel -$ idb -``` - -The debugger may run in text mode. To debug in text mode, use - -```console -$ idbc -``` - -To debug on the compute nodes, module intel must be loaded. The GUI on compute nodes may be accessed using the same way as in the GUI section - -Example: - -```console -$ qsub -q qexp -l select=1:ncpus=16 -X -I - qsub: waiting for job 19654.srv11 to start - qsub: job 19654.srv11 ready - -$ ml intel -$ ml java -$ icc -O0 -g myprog.c -o myprog.x -$ idb ./myprog.x -``` - -In this example, we allocate 1 full compute node, compile program myprog.c with debugging options -O0 -g and run the idb debugger interactively on the myprog.x executable. The GUI access is via X11 port forwarding provided by the PBS workload manager. - -## Debugging Parallel Applications - -Intel debugger is capable of debugging multithreaded and MPI parallel programs as well. - -### Small Number of MPI Ranks - -For debugging small number of MPI ranks, you may execute and debug each rank in separate xterm terminal (do not forget the X display. Using Intel MPI, this may be done in following way: - -```console -$ qsub -q qexp -l select=2:ncpus=16 -X -I - qsub: waiting for job 19654.srv11 to start - qsub: job 19655.srv11 ready - -$ ml intel -$ mpirun -ppn 1 -hostfile $PBS_NODEFILE --enable-x xterm -e idbc ./mympiprog.x -``` - -In this example, we allocate 2 full compute node, run xterm on each node and start idb debugger in command line mode, debugging two ranks of mympiprog.x application. The xterm will pop up for each rank, with idb prompt ready. The example is not limited to use of Intel MPI - -### Large Number of MPI Ranks - -Run the idb debugger from within the MPI debug option. This will cause the debugger to bind to all ranks and provide aggregated outputs across the ranks, pausing execution automatically just after startup. You may then set break points and step the execution manually. Using Intel MPI: - -```console - $ qsub -q qexp -l select=2:ncpus=16 -X -I - qsub: waiting for job 19654.srv11 to start - qsub: job 19655.srv11 ready - -$ ml intel -$ mpirun -n 32 -idb ./mympiprog.x -``` - -### Debugging Multithreaded Application - -Run the idb debugger in GUI mode. The menu Parallel contains number of tools for debugging multiple threads. One of the most useful tools is the **Serialize Execution** tool, which serializes execution of concurrent threads for easy orientation and identification of concurrency related bugs. - -## Further Information - -Exhaustive manual on idb features and usage is published at [Intel website](http://software.intel.com/sites/products/documentation/doclib/stdxe/2013/composerxe/debugger/user_guide/index.htm) diff --git a/docs.it4i/anselm/software/intel-suite/intel-integrated-performance-primitives.md b/docs.it4i/anselm/software/intel-suite/intel-integrated-performance-primitives.md deleted file mode 100644 index 8e0451c69..000000000 --- a/docs.it4i/anselm/software/intel-suite/intel-integrated-performance-primitives.md +++ /dev/null @@ -1,81 +0,0 @@ -# Intel IPP - -## Intel Integrated Performance Primitives - -Intel Integrated Performance Primitives, version 7.1.1, compiled for AVX vector instructions is available, via module ipp. The IPP is a very rich library of highly optimized algorithmic building blocks for media and data applications. This includes signal, image and frame processing algorithms, such as FFT, FIR, Convolution, Optical Flow, Hough transform, Sum, MinMax, as well as cryptographic functions, linear algebra functions and many more. - -!!! note - Check out IPP before implementing own math functions for data processing, it is likely already there. - -```console -$ ml ipp -``` - -The module sets up environment variables, required for linking and running ipp enabled applications. - -## IPP Example - -```cpp - #include "ipp.h" - #include <stdio.h> - int main(int argc, char* argv[]) - { - const IppLibraryVersion *lib; - Ipp64u fm; - IppStatus status; - - status= ippInit(); //IPP initialization with the best optimization layer - if( status != ippStsNoErr ) { - printf("IppInit() Error:n"); - printf("%sn", ippGetStatusString(status) ); - return -1; - } - - //Get version info - lib = ippiGetLibVersion(); - printf("%s %sn", lib->Name, lib->Version); - - //Get CPU features enabled with selected library level - fm=ippGetEnabledCpuFeatures(); - printf("SSE :%cn",(fm>1)&1?'Y':'N'); - printf("SSE2 :%cn",(fm>2)&1?'Y':'N'); - printf("SSE3 :%cn",(fm>3)&1?'Y':'N'); - printf("SSSE3 :%cn",(fm>4)&1?'Y':'N'); - printf("SSE41 :%cn",(fm>6)&1?'Y':'N'); - printf("SSE42 :%cn",(fm>7)&1?'Y':'N'); - printf("AVX :%cn",(fm>8)&1 ?'Y':'N'); - printf("AVX2 :%cn", (fm>15)&1 ?'Y':'N' ); - printf("----------n"); - printf("OS Enabled AVX :%cn", (fm>9)&1 ?'Y':'N'); - printf("AES :%cn", (fm>10)&1?'Y':'N'); - printf("CLMUL :%cn", (fm>11)&1?'Y':'N'); - printf("RDRAND :%cn", (fm>13)&1?'Y':'N'); - printf("F16C :%cn", (fm>14)&1?'Y':'N'); - - return 0; - } -``` - -Compile above example, using any compiler and the ipp module. - -```console -$ ml intel -$ ml ipp - -$ icc testipp.c -o testipp.x -lippi -lipps -lippcore -``` - -You will need the ipp module loaded to run the ipp enabled executable. This may be avoided, by compiling library search paths into the executable - -```console -$ ml intel -$ ml ipp - -$ icc testipp.c -o testipp.x -Wl,-rpath=$LIBRARY_PATH -lippi -lipps -lippcore -``` - -## Code Samples and Documentation - -Intel provides number of [Code Samples for IPP](https://software.intel.com/en-us/articles/code-samples-for-intel-integrated-performance-primitives-library), illustrating use of IPP. - -Read full documentation on IPP [on Intel website,](http://software.intel.com/sites/products/search/search.php?q=&x=15&y=6&product=ipp&version=7.1&docos=lin) in particular the [IPP Reference manual.](http://software.intel.com/sites/products/documentation/doclib/ipp_sa/71/ipp_manual/index.htm) diff --git a/docs.it4i/anselm/software/intel-suite/intel-mkl.md b/docs.it4i/anselm/software/intel-suite/intel-mkl.md deleted file mode 100644 index 6594f8193..000000000 --- a/docs.it4i/anselm/software/intel-suite/intel-mkl.md +++ /dev/null @@ -1,114 +0,0 @@ -# Intel MKL - -## Intel Math Kernel Library - -Intel Math Kernel Library (Intel MKL) is a library of math kernel subroutines, extensively threaded and optimized for maximum performance. Intel MKL provides these basic math kernels: - -* BLAS (level 1, 2, and 3) and LAPACK linear algebra routines, offering vector, vector-matrix, and matrix-matrix operations. -* The PARDISO direct sparse solver, an iterative sparse solver, and supporting sparse BLAS (level 1, 2, and 3) routines for solving sparse systems of equations. -* ScaLAPACK distributed processing linear algebra routines for Linux and Windows operating systems, as well as the Basic Linear Algebra Communications Subprograms (BLACS) and the Parallel Basic Linear Algebra Subprograms (PBLAS). -* Fast Fourier transform (FFT) functions in one, two, or three dimensions with support for mixed radices (not limited to sizes that are powers of 2), as well as distributed versions of these functions. -* Vector Math Library (VML) routines for optimized mathematical operations on vectors. -* Vector Statistical Library (VSL) routines, which offer high-performance vectorized random number generators (RNG) for several probability distributions, convolution and correlation routines, and summary statistics functions. -* Data Fitting Library, which provides capabilities for spline-based approximation of functions, derivatives and integrals of functions, and search. -* Extended Eigensolver, a shared memory version of an eigensolver based on the Feast Eigenvalue Solver. - -For details see the [Intel MKL Reference Manual](http://software.intel.com/sites/products/documentation/doclib/mkl_sa/11/mklman/index.htm). - -Intel MKL is available on Anselm - -```console -$ ml imkl -``` - -The module sets up environment variables, required for linking and running mkl enabled applications. The most important variables are the $MKLROOT, $MKL_INC_DIR, $MKL_LIB_DIR and $MKL_EXAMPLES - -!!! note - The MKL library may be linked using any compiler. With intel compiler use -mkl option to link default threaded MKL. - -### Interfaces - -The MKL library provides number of interfaces. The fundamental once are the LP64 and ILP64. The Intel MKL ILP64 libraries use the 64-bit integer type (necessary for indexing large arrays, with more than 231^-1 elements), whereas the LP64 libraries index arrays with the 32-bit integer type. - -| Interface | Integer type | -| --------- | -------------------------------------------- | -| LP64 | 32-bit, int, integer(kind=4), MPI_INT | -| ILP64 | 64-bit, long int, integer(kind=8), MPI_INT64 | - -### Linking - -Linking MKL libraries may be complex. Intel [mkl link line advisor](http://software.intel.com/en-us/articles/intel-mkl-link-line-advisor) helps. See also [examples](intel-mkl/#examples) below. - -You will need the mkl module loaded to run the mkl enabled executable. This may be avoided, by compiling library search paths into the executable. Include rpath on the compile line: - -```console -$ icc .... -Wl,-rpath=$LIBRARY_PATH ... -``` - -### Threading - -!!! note - Advantage in using the MKL library is that it brings threaded parallelization to applications that are otherwise not parallel. - -For this to work, the application must link the threaded MKL library (default). Number and behaviour of MKL threads may be controlled via the OpenMP environment variables, such as OMP_NUM_THREADS and KMP_AFFINITY. MKL_NUM_THREADS takes precedence over OMP_NUM_THREADS - -```console -$ export OMP_NUM_THREADS=16 -$ export KMP_AFFINITY=granularity=fine,compact,1,0 -``` - -The application will run with 16 threads with affinity optimized for fine grain parallelization. - -## Examples - -Number of examples, demonstrating use of the MKL library and its linking is available on Anselm, in the $MKL_EXAMPLES directory. In the examples below, we demonstrate linking MKL to Intel and GNU compiled program for multi-threaded matrix multiplication. - -### Working With Examples - -```console -$ ml intel -$ cp -a $MKL_EXAMPLES/cblas /tmp/ -$ cd /tmp/cblas -$ make sointel64 function=cblas_dgemm -``` - -In this example, we compile, link and run the cblas_dgemm example, demonstrating use of MKL example suite installed on Anselm. - -### Example: MKL and Intel Compiler - -```console -$ ml intel -$ cp -a $MKL_EXAMPLES/cblas /tmp/ -$ cd /tmp/cblas -$ icc -w source/cblas_dgemmx.c source/common_func.c -mkl -o cblas_dgemmx.x -$ ./cblas_dgemmx.x data/cblas_dgemmx.d -``` - -In this example, we compile, link and run the cblas_dgemm example, demonstrating use of MKL with icc -mkl option. Using the -mkl option is equivalent to: - -```console -$ icc -w source/cblas_dgemmx.c source/common_func.c -o cblas_dgemmx.x -I$MKL_INC_DIR -L$MKL_LIB_DIR -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -liomp5 -``` - -In this example, we compile and link the cblas_dgemm example, using LP64 interface to threaded MKL and Intel OMP threads implementation. - -### Example: MKL and GNU Compiler - -```console -$ ml gcc -$ ml imkl -$ cp -a $MKL_EXAMPLES/cblas /tmp/ -$ cd /tmp/cblas -$ gcc -w source/cblas_dgemmx.c source/common_func.c -o cblas_dgemmx.x -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lm -$ ./cblas_dgemmx.x data/cblas_dgemmx.d -``` - -In this example, we compile, link and run the cblas_dgemm example, using LP64 interface to threaded MKL and gnu OMP threads implementation. - -## MKL and MIC Accelerators - -The MKL is capable to automatically offload the computations o the MIC accelerator. See section [Intel XeonPhi](../intel-xeon-phi/) for details. - -## Further Reading - -Read more on [Intel website](http://software.intel.com/en-us/intel-mkl), in particular the [MKL users guide](https://software.intel.com/en-us/intel-mkl/documentation/linux). diff --git a/docs.it4i/anselm/software/intel-suite/intel-tbb.md b/docs.it4i/anselm/software/intel-suite/intel-tbb.md deleted file mode 100644 index 497b26f5e..000000000 --- a/docs.it4i/anselm/software/intel-suite/intel-tbb.md +++ /dev/null @@ -1,42 +0,0 @@ -# Intel TBB - -## Intel Threading Building Blocks - -Intel Threading Building Blocks (Intel TBB) is a library that supports scalable parallel programming using standard ISO C++ code. It does not require special languages or compilers. To use the library, you specify tasks, not threads, and let the library map tasks onto threads in an efficient manner. The tasks are executed by a runtime scheduler and may -be offloaded to [MIC accelerator](../intel-xeon-phi/). - -Intel TBB version 4.1 is available on Anselm - -```console -$ ml tbb -``` - -The module sets up environment variables, required for linking and running tbb enabled applications. - -!!! note - Link the tbb library, using -ltbb - -## Examples - -Number of examples, demonstrating use of TBB and its built-in scheduler is available on Anselm, in the $TBB_EXAMPLES directory. - -```console -$ ml intel -$ ml tbb -$ cp -a $TBB_EXAMPLES/common $TBB_EXAMPLES/parallel_reduce /tmp/ -$ cd /tmp/parallel_reduce/primes -$ icc -O2 -DNDEBUG -o primes.x main.cpp primes.cpp -ltbb -$ ./primes.x -``` - -In this example, we compile, link and run the primes example, demonstrating use of parallel task-based reduce in computation of prime numbers. - -You will need the tbb module loaded to run the tbb enabled executable. This may be avoided, by compiling library search paths into the executable. - -```console -$ icc -O2 -o primes.x main.cpp primes.cpp -Wl,-rpath=$LIBRARY_PATH -ltbb -``` - -## Further Reading - -Read more on Intel website, <http://software.intel.com/sites/products/documentation/doclib/tbb_sa/help/index.htm> diff --git a/docs.it4i/anselm/software/intel-suite/introduction.md b/docs.it4i/anselm/software/intel-suite/introduction.md deleted file mode 100644 index 879389f3f..000000000 --- a/docs.it4i/anselm/software/intel-suite/introduction.md +++ /dev/null @@ -1,62 +0,0 @@ -# Intel Parallel Studio - -The Anselm cluster provides following elements of the Intel Parallel Studio XE - -* Intel Compilers -* Intel Debugger -* Intel MKL Library -* Intel Integrated Performance Primitives Library -* Intel Threading Building Blocks Library - -## Intel Compilers - -The Intel compilers version 13.1.3 are available, via module intel. The compilers include the icc C and C++ compiler and the ifort fortran 77/90/95 compiler. - -```console -$ ml intel -$ icc -v -$ ifort -v -``` - -Read more at the [Intel Compilers](intel-compilers/) page. - -## Intel Debugger - -The intel debugger version 13.0 is available, via module intel. The debugger works for applications compiled with C and C++ compiler and the ifort fortran 77/90/95 compiler. The debugger provides java GUI environment. Use X display for running the GUI. - -```console -$ ml intel -$ idb -``` - -Read more at the [Intel Debugger](intel-debugger/) page. - -## Intel Math Kernel Library - -Intel Math Kernel Library (Intel MKL) is a library of math kernel subroutines, extensively threaded and optimized for maximum performance. Intel MKL unites and provides these basic components: BLAS, LAPACK, ScaLapack, PARDISO, FFT, VML, VSL, Data fitting, Feast Eigensolver and many more. - -```console -$ ml imkl -``` - -Read more at the [Intel MKL](intel-mkl/) page. - -## Intel Integrated Performance Primitives - -Intel Integrated Performance Primitives, version 7.1.1, compiled for AVX is available, via module ipp. The IPP is a library of highly optimized algorithmic building blocks for media and data applications. This includes signal, image and frame processing algorithms, such as FFT, FIR, Convolution, Optical Flow, Hough transform, Sum, MinMax and many more. - -```console -$ ml ipp -``` - -Read more at the [Intel IPP](intel-integrated-performance-primitives/) page. - -## Intel Threading Building Blocks - -Intel Threading Building Blocks (Intel TBB) is a library that supports scalable parallel programming using standard ISO C++ code. It does not require special languages or compilers. It is designed to promote scalable data parallel programming. Additionally, it fully supports nested parallelism, so you can build larger parallel components from smaller parallel components. To use the library, you specify tasks, not threads, and let the library map tasks onto threads in an efficient manner. - -```console -$ ml tbb -``` - -Read more at the [Intel TBB](intel-tbb/) page. diff --git a/docs.it4i/anselm/software/isv_licenses.md b/docs.it4i/anselm/software/isv_licenses.md deleted file mode 100644 index f26319ec1..000000000 --- a/docs.it4i/anselm/software/isv_licenses.md +++ /dev/null @@ -1,105 +0,0 @@ -# ISV Licenses - -## Guide to Managing Independent Software Vendor Licenses - -On Anselm cluster there are also installed commercial software applications, also known as ISV (Independent Software Vendor), which are subjects to licensing. The licenses are limited and their usage may be restricted only to some users or user groups. - -Currently Flex License Manager based licensing is supported on the cluster for products ANSYS, Comsol and MATLAB. More information about the applications can be found in the general software section. - -If an ISV application was purchased for educational (research) purposes and also for commercial purposes, then there are always two separate versions maintained and suffix "edu" is used in the name of the non-commercial version. - -## Overview of the Licenses Usage - -!!! note - The overview is generated every minute and is accessible from web or command line interface. - -### Web Interface - -For each license there is a table, which provides the information about the name, number of available (purchased/licensed), number of used and number of free license features <https://extranet.it4i.cz/anselm/licenses> - -### Text Interface - -For each license there is a unique text file, which provides the information about the name, number of available (purchased/licensed), number of used and number of free license features. The text files are accessible from the Anselm command prompt. - -| Product | File with license state | Note | -| ---------- | ------------------------------------------------- | ------------------- | -| ansys | /apps/user/licenses/ansys_features_state.txt | Commercial | -| comsol | /apps/user/licenses/comsol_features_state.txt | Commercial | -| comsol-edu | /apps/user/licenses/comsol-edu_features_state.txt | Non-commercial only | -| matlab | /apps/user/licenses/matlab_features_state.txt | Commercial | -| matlab-edu | /apps/user/licenses/matlab-edu_features_state.txt | Non-commercial only | - -The file has a header which serves as a legend. All the info in the legend starts with a hash (#) so it can be easily filtered when parsing the file via a script. - -Example of the Commercial Matlab license state: - -```console -$ cat /apps/user/licenses/matlab_features_state.txt - # matlab - # ------------------------------------------------- - # FEATURE TOTAL USED AVAIL - # ------------------------------------------------- - MATLAB 1 1 0 - SIMULINK 1 0 1 - Curve_Fitting_Toolbox 1 0 1 - Signal_Blocks 1 0 1 - GADS_Toolbox 1 0 1 - Image_Toolbox 1 0 1 - Compiler 1 0 1 - Neural_Network_Toolbox 1 0 1 - Optimization_Toolbox 1 0 1 - Signal_Toolbox 1 0 1 - Statistics_Toolbox 1 0 1 -``` - -## License Tracking in PBS Pro Scheduler and Users Usage - -Each feature of each license is accounted and checked by the scheduler of PBS Pro. If you ask for certain licenses, the scheduler won't start the job until the asked licenses are free (available). This prevents to crash batch jobs, just because of unavailability of the needed licenses. - -The general format of the name is `feature__APP__FEATURE`. - -Names of applications (APP): - -* ansys -* comsol -* comsol-edu -* matlab -* matlab-edu - -To get the FEATUREs of a license take a look into the corresponding state file ([see above](isv_licenses/#Licence)), or use: - -### Application and List of Provided Features - -* **ansys** $ grep -v "#" /apps/user/licenses/ansys_features_state.txt | cut -f1 -d' ' -* **comsol** $ grep -v "#" /apps/user/licenses/comsol_features_state.txt | cut -f1 -d' ' -* **comsol-ed** $ grep -v "#" /apps/user/licenses/comsol-edu_features_state.txt | cut -f1 -d' ' -* **matlab** $ grep -v "#" /apps/user/licenses/matlab_features_state.txt | cut -f1 -d' ' -* **matlab-edu** $ grep -v "#" /apps/user/licenses/matlab-edu_features_state.txt | cut -f1 -d' ' - -Example of PBS Pro resource name, based on APP and FEATURE name: - -| Application | Feature | PBS Pro resource name | -| ----------- | -------------------------- | ----------------------------------------------- | -| ansys | acfd | feature_ansys_acfd | -| ansys | aa_r | feature_ansys_aa_r | -| comsol | COMSOL | feature_comsol_COMSOL | -| comsol | HEATTRANSFER | feature_comsol_HEATTRANSFER | -| comsol-edu | COMSOLBATCH | feature_comsol-edu_COMSOLBATCH | -| comsol-edu | STRUCTURALMECHANICS | feature_comsol-edu_STRUCTURALMECHANICS | -| matlab | MATLAB | feature_matlab_MATLAB | -| matlab | Image_Toolbox | feature_matlab_Image_Toolbox | -| matlab-edu | MATLAB_Distrib_Comp_Engine | feature_matlab-edu_MATLAB_Distrib_Comp_Engine | -| matlab-edu | Image_Acquisition_Toolbox | feature_matlab-edu_Image_Acquisition_Toolbox\\ | - -!!! Warnig -Resource names in PBS Pro are case sensitive. - -### Example of qsub Statement - -Run an interactive PBS job with 1 Matlab EDU license, 1 Distributed Computing Toolbox and 32 Distributed Computing Engines (running on 32 cores): - -```console -$ qsub -I -q qprod -A PROJECT_ID -l select=2:ncpus=16 -l feature__matlab-edu__MATLAB=1 -l feature__matlab-edu__Distrib_Computing_Toolbox=1 -l feature__matlab-edu__MATLAB_Distrib_Comp_Engine=32 -``` - -The license is used and accounted only with the real usage of the product. So in this example, the general Matlab is used after Matlab is run by the user and not at the time, when the shell of the interactive job is started. Also the Distributed Computing licenses are used at the time, when the user uses the distributed parallel computation in Matlab (e. g. issues pmode start, matlabpool, etc.). diff --git a/docs.it4i/anselm/software/java.md b/docs.it4i/anselm/software/java.md deleted file mode 100644 index a9de12676..000000000 --- a/docs.it4i/anselm/software/java.md +++ /dev/null @@ -1,27 +0,0 @@ -# Java - -## Java on ANSELM - -Java is available on Anselm cluster. Activate java by loading the java module - -```console -$ ml Java -``` - -Note that the java module must be loaded on the compute nodes as well, in order to run java on compute nodes. - -Check for java version and path - -```console -$ java -version -$ which java -``` - -With the module loaded, not only the runtime environment (JRE), but also the development environment (JDK) with the compiler is available. - -```console -$ javac -version -$ which javac -``` - -Java applications may use MPI for inter-process communication, in conjunction with OpenMPI. Read more on <http://www.open-mpi.org/faq/?category=java>. This functionality is currently not supported on Anselm cluster. In case you require the java interface to MPI, please contact [Anselm support](https://support.it4i.cz/rt/). diff --git a/docs.it4i/anselm/software/machine-learning/introduction.md b/docs.it4i/anselm/software/machine-learning/introduction.md deleted file mode 100644 index 49a1e4f05..000000000 --- a/docs.it4i/anselm/software/machine-learning/introduction.md +++ /dev/null @@ -1,19 +0,0 @@ -# Machine Learning - -This section overviews machine learning frameworks and libraries available on the Anselm cluster. - -## TensorFlow - -Load TensorFlow module: - -```console -$ ml Tensorflow -``` - -Test module: - -```console -$ ml Tensorflow -``` - -Read more about available versions at the [TensorFlow page](tensorflow). diff --git a/docs.it4i/anselm/software/machine-learning/tensorflow.md b/docs.it4i/anselm/software/machine-learning/tensorflow.md deleted file mode 100644 index d35b92fec..000000000 --- a/docs.it4i/anselm/software/machine-learning/tensorflow.md +++ /dev/null @@ -1,66 +0,0 @@ -# TensorFlow - -TensorFlow is an open-source software library for machine intelligence. - -## TensorFlow modules - -Anselm provides three different TensorFlow modules: - -* Tensorflow/1.1.0 -* Tensorflow/1.1.0-CUDA-7.5.18-Python-3.6.1 -* Tensorflow/1.1.0-CUDA-8.0.44-Python-3.6.1 - -### Tensorflow/1.1.0 (CPU only) - -TensorFlow 1.1 build. - -```console -$ ml Tensorflow/1.1.0 -``` - -This module was built with: - -* GCC/4.9.3 -* Python/3.6.1 - -### Tensorflow/1.1.0-CUDA-7.5.18-Python-3.6.1 (GPU enabled) - -TensorFlow 1.1 with GPU support. - -```console -$ ml Tensorflow/1.1.0-CUDA-7.5.18-Python-3.6.1 -``` - -This module was built with: - -* GCC/4.9.3 -* Python/3.6.1 -* CUDA/7.5.18 -* cuDNN/5.1-CUDA-7.5.18 - -### Tensorflow/1.1.0-CUDA-8.0.44-Python-3.6.1 (GPU enabled) - -TensorFlow 1.1 with GPU support. - -```console -$ ml Tensorflow/1.1.0-CUDA-8.0.44-Python-3.6.1 -``` - -This module was built with: - -* GCC/4.9.3 -* Python/3.6.1 -* CUDA/8.0.44 -* cuDNN/5.1-CUDA-8.0.44 - -## TensorFlow application example - -After loading one of the available TensorFlow modules, you can check the functionality running the following python script. - -```python -import tensorflow as tf - -c = tf.constant('Hello World!') -sess = tf.Session() -print(sess.run(c)) -``` diff --git a/docs.it4i/anselm/software/mpi/Running_OpenMPI.md b/docs.it4i/anselm/software/mpi/Running_OpenMPI.md deleted file mode 100644 index 4974eb5b1..000000000 --- a/docs.it4i/anselm/software/mpi/Running_OpenMPI.md +++ /dev/null @@ -1,207 +0,0 @@ -# Running OpenMPI - -## OpenMPI Program Execution - -The OpenMPI programs may be executed only via the PBS Workload manager, by entering an appropriate queue. On Anselm, the **bullxmpi-1.2.4.1** and **OpenMPI 1.6.5** are OpenMPI based MPI implementations. - -### Basic Usage - -!!! note - Use the mpiexec to run the OpenMPI code. - -Example: - -```console -$ qsub -q qexp -l select=4:ncpus=16 -I - qsub: waiting for job 15210.srv11 to start - qsub: job 15210.srv11 ready -$ pwd - /home/username -$ ml OpenMPI -$ mpiexec -pernode ./helloworld_mpi.x - Hello world! from rank 0 of 4 on host cn17 - Hello world! from rank 1 of 4 on host cn108 - Hello world! from rank 2 of 4 on host cn109 - Hello world! from rank 3 of 4 on host cn110 -``` - -!!! note - Please be aware, that in this example, the directive **-pernode** is used to run only **one task per node**, which is normally an unwanted behaviour (unless you want to run hybrid code with just one MPI and 16 OpenMP tasks per node). In normal MPI programs **omit the -pernode directive** to run up to 16 MPI tasks per each node. - -In this example, we allocate 4 nodes via the express queue interactively. We set up the openmpi environment and interactively run the helloworld_mpi.x program. Note that the executable helloworld_mpi.x must be available within the -same path on all nodes. This is automatically fulfilled on the /home and /scratch filesystem. - -You need to preload the executable, if running on the local scratch /lscratch filesystem - -```console -$ pwd - /lscratch/15210.srv11 -$ mpiexec -pernode --preload-binary ./helloworld_mpi.x - Hello world! from rank 0 of 4 on host cn17 - Hello world! from rank 1 of 4 on host cn108 - Hello world! from rank 2 of 4 on host cn109 - Hello world! from rank 3 of 4 on host cn110 -``` - -In this example, we assume the executable helloworld_mpi.x is present on compute node cn17 on local scratch. We call the mpiexec whith the **--preload-binary** argument (valid for openmpi). The mpiexec will copy the executable from cn17 to the /lscratch/15210.srv11 directory on cn108, cn109 and cn110 and execute the program. - -!!! note - MPI process mapping may be controlled by PBS parameters. - -The mpiprocs and ompthreads parameters allow for selection of number of running MPI processes per node as well as number of OpenMP threads per MPI process. - -### One MPI Process Per Node - -Follow this example to run one MPI process per node, 16 threads per process. - -```console -$ qsub -q qexp -l select=4:ncpus=16:mpiprocs=1:ompthreads=16 -I -$ ml OpenMPI -$ mpiexec --bind-to-none ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 1 MPI processes per node and 16 threads per socket, on 4 nodes. - -### Two MPI Processes Per Node - -Follow this example to run two MPI processes per node, 8 threads per process. Note the options to mpiexec. - -```console -$ qsub -q qexp -l select=4:ncpus=16:mpiprocs=2:ompthreads=8 -I -$ ml openmpi -$ mpiexec -bysocket -bind-to-socket ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 2 MPI processes per node and 8 threads per socket, each process and its threads bound to a separate processor socket of the node, on 4 nodes - -### 16 MPI Processes Per Node - -Follow this example to run 16 MPI processes per node, 1 thread per process. Note the options to mpiexec. - -```console -$ qsub -q qexp -l select=4:ncpus=16:mpiprocs=16:ompthreads=1 -I -$ ml OpenMPI -$ mpiexec -bycore -bind-to-core ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 16 MPI processes per node, single threaded. Each process is bound to separate processor core, on 4 nodes. - -### OpenMP Thread Affinity - -!!! note - Important! Bind every OpenMP thread to a core! - -In the previous two examples with one or two MPI processes per node, the operating system might still migrate OpenMP threads between cores. You might want to avoid this by setting these environment variable for GCC OpenMP: - -```console -$ export GOMP_CPU_AFFINITY="0-15" -``` - -or this one for Intel OpenMP: - -```console -$ export KMP_AFFINITY=granularity=fine,compact,1,0 -``` - -As of OpenMP 4.0 (supported by GCC 4.9 and later and Intel 14.0 and later) the following variables may be used for Intel or GCC: - -```console -$ export OMP_PROC_BIND=true -$ export OMP_PLACES=cores -``` - -## OpenMPI Process Mapping and Binding - -The mpiexec allows for precise selection of how the MPI processes will be mapped to the computational nodes and how these processes will bind to particular processor sockets and cores. - -MPI process mapping may be specified by a hostfile or rankfile input to the mpiexec program. Altough all implementations of MPI provide means for process mapping and binding, following examples are valid for the openmpi only. - -### Hostfile - -Example hostfile - -```console - cn110.bullx - cn109.bullx - cn108.bullx - cn17.bullx -``` - -Use the hostfile to control process placement - -```console -$ mpiexec -hostfile hostfile ./helloworld_mpi.x - Hello world! from rank 0 of 4 on host cn110 - Hello world! from rank 1 of 4 on host cn109 - Hello world! from rank 2 of 4 on host cn108 - Hello world! from rank 3 of 4 on host cn17 -``` - -In this example, we see that ranks have been mapped on nodes according to the order in which nodes show in the hostfile - -### Rankfile - -Exact control of MPI process placement and resource binding is provided by specifying a rankfile - -!!! note - Appropriate binding may boost performance of your application. - -Example rankfile - -```console - rank 0=cn110.bullx slot=1:0,1 - rank 1=cn109.bullx slot=0:* - rank 2=cn108.bullx slot=1:1-2 - rank 3=cn17.bullx slot=0:1,1:0-2 - rank 4=cn109.bullx slot=0:*,1:* -``` - -This rankfile assumes 5 ranks will be running on 4 nodes and provides exact mapping and binding of the processes to the processor sockets and cores - -Explanation: -rank 0 will be bounded to cn110, socket1 core0 and core1 -rank 1 will be bounded to cn109, socket0, all cores -rank 2 will be bounded to cn108, socket1, core1 and core2 -rank 3 will be bounded to cn17, socket0 core1, socket1 core0, core1, core2 -rank 4 will be bounded to cn109, all cores on both sockets - -```console -$ mpiexec -n 5 -rf rankfile --report-bindings ./helloworld_mpi.x - [cn17:11180] MCW rank 3 bound to socket 0[core 1] socket 1[core 0-2]: [. B . . . . . .][B B B . . . . .] (slot list 0:1,1:0-2) - [cn110:09928] MCW rank 0 bound to socket 1[core 0-1]: [. . . . . . . .][B B . . . . . .] (slot list 1:0,1) - [cn109:10395] MCW rank 1 bound to socket 0[core 0-7]: [B B B B B B B B][. . . . . . . .] (slot list 0:*) - [cn108:10406] MCW rank 2 bound to socket 1[core 1-2]: [. . . . . . . .][. B B . . . . .] (slot list 1:1-2) - [cn109:10406] MCW rank 4 bound to socket 0[core 0-7] socket 1[core 0-7]: [B B B B B B B B][B B B B B B B B] (slot list 0:*,1:*) - Hello world! from rank 3 of 5 on host cn17 - Hello world! from rank 1 of 5 on host cn109 - Hello world! from rank 0 of 5 on host cn110 - Hello world! from rank 4 of 5 on host cn109 - Hello world! from rank 2 of 5 on host cn108 -``` - -In this example we run 5 MPI processes (5 ranks) on four nodes. The rankfile defines how the processes will be mapped on the nodes, sockets and cores. The **--report-bindings** option was used to print out the actual process location and bindings. Note that ranks 1 and 4 run on the same node and their core binding overlaps. - -It is users responsibility to provide correct number of ranks, sockets and cores. - -### Bindings Verification - -In all cases, binding and threading may be verified by executing for example: - -```console -$ mpiexec -bysocket -bind-to-socket --report-bindings echo -$ mpiexec -bysocket -bind-to-socket numactl --show -$ mpiexec -bysocket -bind-to-socket echo $OMP_NUM_THREADS -``` - -## Changes in OpenMPI 1.8 - -Some options have changed in OpenMPI version 1.8. - -| version 1.6.5 | version 1.8.1 | -| ---------------- | ------------------- | -| --bind-to-none | --bind-to none | -| --bind-to-core | --bind-to core | -| --bind-to-socket | --bind-to socket | -| -bysocket | --map-by socket | -| -bycore | --map-by core | -| -pernode | --map-by ppr:1:node | diff --git a/docs.it4i/anselm/software/mpi/mpi.md b/docs.it4i/anselm/software/mpi/mpi.md deleted file mode 100644 index c8e9c7d6d..000000000 --- a/docs.it4i/anselm/software/mpi/mpi.md +++ /dev/null @@ -1,146 +0,0 @@ -# MPI - -## Setting Up MPI Environment - -The Anselm cluster provides several implementations of the MPI library: - -| MPI Library | Thread support | -| ---------------------------------------------------- | --------------------------------------------------------------- | -| The highly optimized and stable **bullxmpi 1.2.4.1** | Partial thread support up to MPI_THREAD_SERIALIZED | -| The **Intel MPI 4.1** | Full thread support up to MPI_THREAD_MULTIPLE | -| The [OpenMPI 1.6.5](href="http://www.open-mpi.org) | Full thread support up to MPI_THREAD_MULTIPLE, BLCR c/r support | -| The OpenMPI 1.8.1 | Full thread support up to MPI_THREAD_MULTIPLE, MPI-3.0 support | -| The **mpich2 1.9** | Full thread support up to MPI_THREAD_MULTIPLE, BLCR c/r support | - -MPI libraries are activated via the environment modules. - -```console -$ ml av mpi/ - ------------------------- /opt/modules/modulefiles/mpi ------------------------- - bullxmpi/bullxmpi-1.2.4.1 mvapich2/1.9-icc - impi/4.0.3.008 openmpi/1.6.5-gcc(default) - impi/4.1.0.024 openmpi/1.6.5-gcc46 - impi/4.1.0.030 openmpi/1.6.5-icc - impi/4.1.1.036(default) openmpi/1.8.1-gcc - openmpi/1.8.1-gcc46 - mvapich2/1.9-gcc(default) openmpi/1.8.1-gcc49 - mvapich2/1.9-gcc46 openmpi/1.8.1-icc -``` - -There are default compilers associated with any particular MPI implementation. The defaults may be changed, the MPI libraries may be used in conjunction with any compiler. The defaults are selected via the modules in following way - -| Module | MPI | Compiler suite | -| ------------ | ---------------- | ------------------------------------------------------------------------------ | -| PrgEnv-gnu | bullxmpi-1.2.4.1 | bullx GNU 4.4.6 | -| PrgEnv-intel | Intel MPI 4.1.1 | Intel 13.1.1 | -| bullxmpi | bullxmpi-1.2.4.1 | none, select via module | -| impi | Intel MPI 4.1.1 | none, select via module | -| openmpi | OpenMPI 1.6.5 | GNU compilers 4.8.1, GNU compilers 4.4.6, Intel Compilers | -| openmpi | OpenMPI 1.8.1 | GNU compilers 4.8.1, GNU compilers 4.4.6, GNU compilers 4.9.0, Intel Compilers | -| mvapich2 | MPICH2 1.9 | GNU compilers 4.8.1, GNU compilers 4.4.6, Intel Compilers | - -Examples: - -```console -$ ml OpenMPI **or** ml openmpi **for older versions** -``` - -In this example, we activate the latest openmpi with latest GNU compilers - -To use openmpi with the intel compiler suite, use - -```console -$ ml intel -$ ml openmpi/1.6.5-icc -``` - -In this example, the openmpi 1.6.5 using intel compilers is activated - -## Compiling MPI Programs - -!!! note - After setting up your MPI environment, compile your program using one of the mpi wrappers - -```console -$ mpicc -v -$ mpif77 -v -$ mpif90 -v -``` - -Example program: - -```cpp -// helloworld_mpi.c -#include <stdio.h> - -#include<mpi.h> - -int main(int argc, char **argv) { - -int len; -int rank, size; -char node[MPI_MAX_PROCESSOR_NAME]; - -// Initiate MPI -MPI_Init(&argc, &argv); -MPI_Comm_rank(MPI_COMM_WORLD,&rank); -MPI_Comm_size(MPI_COMM_WORLD,&size); - -// Get hostame and print -MPI_Get_processor_name(node,&len); -printf("Hello world! from rank %d of %d on host %sn",rank,size,node); - -// Finalize and exit -MPI_Finalize(); - -return 0; -} -``` - -Compile the above example with - -```console -$ mpicc helloworld_mpi.c -o helloworld_mpi.x -``` - -## Running MPI Programs - -!!! note - The MPI program executable must be compatible with the loaded MPI module. - Always compile and execute using the very same MPI module. - -It is strongly discouraged to mix mpi implementations. Linking an application with one MPI implementation and running mpirun/mpiexec form other implementation may result in unexpected errors. - -The MPI program executable must be available within the same path on all nodes. This is automatically fulfilled on the /home and /scratch file system. You need to preload the executable, if running on the local scratch /lscratch file system. - -### Ways to Run MPI Programs - -Optimal way to run an MPI program depends on its memory requirements, memory access pattern and communication pattern. - -!!! note - Consider these ways to run an MPI program: - - 1. One MPI process per node, 16 threads per process - 2. Two MPI processes per node, 8 threads per process - 3. 16 MPI processes per node, 1 thread per process. - -**One MPI** process per node, using 16 threads, is most useful for memory demanding applications, that make good use of processor cache memory and are not memory bound. This is also a preferred way for communication intensive applications as one process per node enjoys full bandwidth access to the network interface. - -**Two MPI** processes per node, using 8 threads each, bound to processor socket is most useful for memory bandwidth bound applications such as BLAS1 or FFT, with scalable memory demand. However, note that the two processes will share access to the network interface. The 8 threads and socket binding should ensure maximum memory access bandwidth and minimize communication, migration and NUMA effect overheads. - -!!! note - Important! Bind every OpenMP thread to a core! - -In the previous two cases with one or two MPI processes per node, the operating system might still migrate OpenMP threads between cores. You want to avoid this by setting the KMP_AFFINITY or GOMP_CPU_AFFINITY environment variables. - -**16 MPI** processes per node, using 1 thread each bound to processor core is most suitable for highly scalable applications with low communication demand. - -### Running OpenMPI - -The **bullxmpi-1.2.4.1** and [**OpenMPI 1.6.5**](http://www.open-mpi.org/) are both based on OpenMPI. Read more on [how to run OpenMPI](Running_OpenMPI/) based MPI. - -### Running MPICH2 - -The **Intel MPI** and **mpich2 1.9** are MPICH2 based implementations. Read more on [how to run MPICH2](running-mpich2/) based MPI. - -The Intel MPI may run on the Intel Xeon Phi accelerators as well. Read more on [how to run Intel MPI on accelerators](../intel-xeon-phi/). diff --git a/docs.it4i/anselm/software/mpi/mpi4py-mpi-for-python.md b/docs.it4i/anselm/software/mpi/mpi4py-mpi-for-python.md deleted file mode 100644 index 591e0cd47..000000000 --- a/docs.it4i/anselm/software/mpi/mpi4py-mpi-for-python.md +++ /dev/null @@ -1,171 +0,0 @@ -# MPI4Py (MPI for Python) - -## Introduction - -MPI for Python provides bindings of the Message Passing Interface (MPI) standard for the Python programming language, allowing any Python program to exploit multiple processors. - -This package is constructed on top of the MPI-1/2 specifications and provides an object oriented interface which closely follows MPI-2 C++ bindings. It supports point-to-point (sends, receives) and collective (broadcasts, scatters, gathers) communications of any picklable Python object, as well as optimized communications of Python object exposing the single-segment buffer interface (NumPy arrays, builtin bytes/string/array objects). - -On Anselm MPI4Py is available in standard Python modules. - -## Modules - -MPI4Py is build for OpenMPI. Before you start with MPI4Py you need to load Python and OpenMPI modules. - -```console -$ ml av Python/ ---------------------------------------- /apps/modules/lang ------------------------- - Python/2.7.8-intel-2015b Python/2.7.11-intel-2016a Python/3.5.1-intel-2017.00 - Python/2.7.11-intel-2017a Python/2.7.9-foss-2015b Python/2.7.9-intel-2015b - Python/2.7.11-foss-2016a Python/3.5.2-foss-2016a Python/3.5.1 - Python/2.7.9-foss-2015g Python/3.4.3-intel-2015b Python/2.7.9 - Python/2.7.11-intel-2015b Python/3.5.2 - -$ ml av OpenMPI/ ---------------------------------------- /apps/modules/mpi -------------------------- -OpenMPI/1.8.6-GCC-4.4.7-system OpenMPI/1.8.8-GNU-4.9.3-2.25 OpenMPI/1.10.1-GCC-4.9.3-2.25 -OpenMPI/1.8.6-GNU-5.1.0-2.25 OpenMPI/1.8.8-GNU-5.1.0-2.25 OpenMPI/1.10.1-GNU-4.9.3-2.25 - OpenMPI/1.8.8-iccifort-2015.3.187-GNU-4.9.3-2.25 OpenMPI/2.0.2-GCC-6.3.0-2.27 -``` - -!!! Warning "Flavours" - - * modules Python/x.x.x-intel... - intel MPI - * modules Python/x.x.x-foss... - OpenMPI - * modules Python/x.x.x - without MPI - -## Execution - -You need to import MPI to your python program. Include the following line to the python script: - -```python -from mpi4py import MPI -``` - -The MPI4Py enabled python programs [execute as any other OpenMPI](Running_OpenMPI/) code.The simpliest way is to run - -```console -$ mpiexec python <script>.py -``` - -For example - -```console -$ mpiexec python hello_world.py -``` - -## Examples - -### Hello World! - -```python -from mpi4py import MPI - -comm = MPI.COMM_WORLD - -print "Hello! I'm rank %d from %d running in total..." % (comm.rank, comm.size) - -comm.Barrier() # wait for everybody to synchronize -``` - -### Collective Communication With NumPy Arrays - -```python -from mpi4py import MPI -from __future__ import division -import numpy as np - -comm = MPI.COMM_WORLD - -print("-"*78) -print(" Running on %d cores" % comm.size) -print("-"*78) - -comm.Barrier() - -# Prepare a vector of N=5 elements to be broadcasted... -N = 5 -if comm.rank == 0: - A = np.arange(N, dtype=np.float64) # rank 0 has proper data -else: - A = np.empty(N, dtype=np.float64) # all other just an empty array - -# Broadcast A from rank 0 to everybody -comm.Bcast( [A, MPI.DOUBLE] ) - -# Everybody should now have the same... -print "[%02d] %s" % (comm.rank, A) -``` - -Execute the above code as: - -```console -$ qsub -q qexp -l select=4:ncpus=16:mpiprocs=16:ompthreads=1 -I -$ ml Python -$ ml OpenMPI -$ mpiexec -bycore -bind-to-core python hello_world.py -``` - -In this example, we run MPI4Py enabled code on 4 nodes, 16 cores per node (total of 64 processes), each python process is bound to a different core. More examples and documentation can be found on [MPI for Python webpage](https://pypi.python.org/pypi/mpi4py). - -### Adding numbers - -Task: count sum of numbers from 1 to 1 000 000. (There is an easy formula to count the sum of arithmetic sequence, but we are showing the MPI solution with adding numbers one by one). - -```python -#!/usr/bin/python - -import numpy -from mpi4py import MPI -import time - -comm = MPI.COMM_WORLD -rank = comm.Get_rank() -size = comm.Get_size() - -a = 1 -b = 1000000 - -perrank = b//size -summ = numpy.zeros(1) - -comm.Barrier() -start_time = time.time() - -temp = 0 -for i in range(a + rank*perrank, a + (rank+1)*perrank): - temp = temp + i - -summ[0] = temp - -if rank == 0: - total = numpy.zeros(1) -else: - total = None - -comm.Barrier() -#collect the partial results and add to the total sum -comm.Reduce(summ, total, op=MPI.SUM, root=0) - -stop_time = time.time() - -if rank == 0: - #add the rest numbers to 1 000 000 - for i in range(a + (size)*perrank, b+1): - total[0] = total[0] + i - print ("The sum of numbers from 1 to 1 000 000: ", int(total[0])) - print ("time spent with ", size, " threads in milliseconds") - print ("-----", int((time.time()-start_time)*1000), "-----") -``` - -Execute the code above as: - -```console -$ qsub -I -q qexp -l select=4:ncpus=16,walltime=00:30:00 - -$ ml Python/3.5.2-intel-2017.00 - -$ mpirun -n 2 python myprogram.py -``` - -You can increase n and watch time lowering. diff --git a/docs.it4i/anselm/software/mpi/running-mpich2.md b/docs.it4i/anselm/software/mpi/running-mpich2.md deleted file mode 100644 index 7b37a8118..000000000 --- a/docs.it4i/anselm/software/mpi/running-mpich2.md +++ /dev/null @@ -1,155 +0,0 @@ -# Running MPICH2 - -## MPICH2 Program Execution - -The MPICH2 programs use mpd daemon or ssh connection to spawn processes, no PBS support is needed. However the PBS allocation is required to access compute nodes. On Anselm, the **Intel MPI** and **mpich2 1.9** are MPICH2 based MPI implementations. - -### Basic Usage - -!!! note - Use the mpirun to execute the MPICH2 code. - -Example: - -```console -$ qsub -q qexp -l select=4:ncpus=16 -I - qsub: waiting for job 15210.srv11 to start - qsub: job 15210.srv11 ready -$ ml impi -$ mpirun -ppn 1 -hostfile $PBS_NODEFILE ./helloworld_mpi.x - Hello world! from rank 0 of 4 on host cn17 - Hello world! from rank 1 of 4 on host cn108 - Hello world! from rank 2 of 4 on host cn109 - Hello world! from rank 3 of 4 on host cn110 -``` - -In this example, we allocate 4 nodes via the express queue interactively. We set up the intel MPI environment and interactively run the helloworld_mpi.x program. We request MPI to spawn 1 process per node. -Note that the executable helloworld_mpi.x must be available within the same path on all nodes. This is automatically fulfilled on the /home and /scratch filesystem. - -You need to preload the executable, if running on the local scratch /lscratch filesystem - -```console -$ pwd - /lscratch/15210.srv11 -$ mpirun -ppn 1 -hostfile $PBS_NODEFILE cp /home/username/helloworld_mpi.x . -$ mpirun -ppn 1 -hostfile $PBS_NODEFILE ./helloworld_mpi.x - Hello world! from rank 0 of 4 on host cn17 - Hello world! from rank 1 of 4 on host cn108 - Hello world! from rank 2 of 4 on host cn109 - Hello world! from rank 3 of 4 on host cn110 -``` - -In this example, we assume the executable helloworld_mpi.x is present on shared home directory. We run the cp command via mpirun, copying the executable from shared home to local scratch . Second mpirun will execute the binary in the /lscratch/15210.srv11 directory on nodes cn17, cn108, cn109 and cn110, one process per node. - -!!! note - MPI process mapping may be controlled by PBS parameters. - -The mpiprocs and ompthreads parameters allow for selection of number of running MPI processes per node as well as number of OpenMP threads per MPI process. - -### One MPI Process Per Node - -Follow this example to run one MPI process per node, 16 threads per process. Note that no options to mpirun are needed - -```console -$ qsub -q qexp -l select=4:ncpus=16:mpiprocs=1:ompthreads=16 -I -$ ml mvapich2 -$ mpirun ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 1 MPI processes per node and 16 threads per socket, on 4 nodes. - -### Two MPI Processes Per Node - -Follow this example to run two MPI processes per node, 8 threads per process. Note the options to mpirun for mvapich2. No options are needed for impi. - -```console -$ qsub -q qexp -l select=4:ncpus=16:mpiprocs=2:ompthreads=8 -I -$ ml mvapich2 -$ mpirun -bind-to numa ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 2 MPI processes per node and 8 threads per socket, each process and its threads bound to a separate processor socket of the node, on 4 nodes - -### 16 MPI Processes Per Node - -Follow this example to run 16 MPI processes per node, 1 thread per process. Note the options to mpirun for mvapich2. No options are needed for impi. - -```console -$ qsub -q qexp -l select=4:ncpus=16:mpiprocs=16:ompthreads=1 -I -$ ml mvapich2 -$ mpirun -bind-to core ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 16 MPI processes per node, single threaded. Each process is bound to separate processor core, on 4 nodes. - -### OpenMP Thread Affinity - -!!! note - Important! Bind every OpenMP thread to a core! - -In the previous two examples with one or two MPI processes per node, the operating system might still migrate OpenMP threads between cores. You might want to avoid this by setting these environment variable for GCC OpenMP: - -```console -$ export GOMP_CPU_AFFINITY="0-15" -``` - -or this one for Intel OpenMP: - -```console -$ export KMP_AFFINITY=granularity=fine,compact,1,0 -``` - -As of OpenMP 4.0 (supported by GCC 4.9 and later and Intel 14.0 and later) the following variables may be used for Intel or GCC: - -```console -$ export OMP_PROC_BIND=true -$ export OMP_PLACES=cores -``` - -## MPICH2 Process Mapping and Binding - -The mpirun allows for precise selection of how the MPI processes will be mapped to the computational nodes and how these processes will bind to particular processor sockets and cores. - -### Machinefile - -Process mapping may be controlled by specifying a machinefile input to the mpirun program. Altough all implementations of MPI provide means for process mapping and binding, following examples are valid for the impi and mvapich2 only. - -Example machinefile - -```console - cn110.bullx - cn109.bullx - cn108.bullx - cn17.bullx - cn108.bullx -``` - -Use the machinefile to control process placement - -```console -$ mpirun -machinefile machinefile helloworld_mpi.x - Hello world! from rank 0 of 5 on host cn110 - Hello world! from rank 1 of 5 on host cn109 - Hello world! from rank 2 of 5 on host cn108 - Hello world! from rank 3 of 5 on host cn17 - Hello world! from rank 4 of 5 on host cn108 -``` - -In this example, we see that ranks have been mapped on nodes according to the order in which nodes show in the machinefile - -### Process Binding - -The Intel MPI automatically binds each process and its threads to the corresponding portion of cores on the processor socket of the node, no options needed. The binding is primarily controlled by environment variables. Read more about mpi process binding on [Intel website](https://software.intel.com/sites/products/documentation/hpc/ics/impi/41/lin/Reference_Manual/Environment_Variables_Process_Pinning.htm). The MPICH2 uses the -bind-to option Use -bind-to numa or -bind-to core to bind the process on single core or entire socket. - -### Bindings Verification - -In all cases, binding and threading may be verified by executing - -```console -$ mpirun -bindto numa numactl --show -$ mpirun -bindto numa echo $OMP_NUM_THREADS -``` - -## Intel MPI on Xeon Phi - -The[MPI section of Intel Xeon Phi chapter](../intel-xeon-phi/) provides details on how to run Intel MPI code on Xeon Phi architecture. diff --git a/docs.it4i/anselm/software/numerical-languages/introduction.md b/docs.it4i/anselm/software/numerical-languages/introduction.md deleted file mode 100644 index 8646fe6fe..000000000 --- a/docs.it4i/anselm/software/numerical-languages/introduction.md +++ /dev/null @@ -1,40 +0,0 @@ -# Numerical languages - -Interpreted languages for numerical computations and analysis - -## Introduction - -This section contains a collection of high-level interpreted languages, primarily intended for numerical computations. - -## Matlab - -MATLAB® is a high-level language and interactive environment for numerical computation, visualization, and programming. - -```console -$ ml MATLAB/2015b-EDU -$ matlab -``` - -Read more at the [Matlab page](matlab/). - -## Octave - -GNU Octave is a high-level interpreted language, primarily intended for numerical computations. The Octave language is quite similar to Matlab so that most programs are easily portable. - -```console -$ ml Octave -$ octave -``` - -Read more at the [Octave page](octave/). - -## R - -The R is an interpreted language and environment for statistical computing and graphics. - -```console -$ ml R -$ R -``` - -Read more at the [R page](r/). diff --git a/docs.it4i/anselm/software/numerical-languages/matlab.md b/docs.it4i/anselm/software/numerical-languages/matlab.md deleted file mode 100644 index f39ca4d3c..000000000 --- a/docs.it4i/anselm/software/numerical-languages/matlab.md +++ /dev/null @@ -1,280 +0,0 @@ -# Matlab - -## Introduction - -Matlab is available in versions R2015a and R2015b. There are always two variants of the release: - -* Non commercial or so called EDU variant, which can be used for common research and educational purposes. -* Commercial or so called COM variant, which can used also for commercial activities. The licenses for commercial variant are much more expensive, so usually the commercial variant has only subset of features compared to the EDU available. - -To load the latest version of Matlab load the module - -```console -$ ml MATLAB -``` - -By default the EDU variant is marked as default. If you need other version or variant, load the particular version. To obtain the list of available versions use - -```console -$ ml av MATLAB -``` - -If you need to use the Matlab GUI to prepare your Matlab programs, you can use Matlab directly on the login nodes. But for all computations use Matlab on the compute nodes via PBS Pro scheduler. - -If you require the Matlab GUI, please follow the general information about [running graphical applications](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/). - -Matlab GUI is quite slow using the X forwarding built in the PBS (qsub -X), so using X11 display redirection either via SSH or directly by xauth (please see the "GUI Applications on Compute Nodes over VNC" part [here](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/x-window-system/)) is recommended. - -To run Matlab with GUI, use - -```console -$ matlab -``` - -To run Matlab in text mode, without the Matlab Desktop GUI environment, use - -```console -$ matlab -nodesktop -nosplash -``` - -plots, images, etc... will be still available. - -## Running Parallel Matlab Using Distributed Computing Toolbox / Engine - -!!! note - Distributed toolbox is available only for the EDU variant - -The MPIEXEC mode available in previous versions is no longer available in MATLAB 2015. Also, the programming interface has changed. Refer to [Release Notes](http://www.mathworks.com/help/distcomp/release-notes.html#buanp9e-1). - -Delete previously used file mpiLibConf.m, we have observed crashes when using Intel MPI. - -To use Distributed Computing, you first need to setup a parallel profile. We have provided the profile for you, you can either import it in MATLAB command line: - -```console - >> parallel.importProfile('/apps/all/MATLAB/2015a-EDU/SalomonPBSPro.settings') - - ans = - - SalomonPBSPro -``` - -Or in the GUI, go to tab HOME -> Parallel -> Manage Cluster Profiles..., click Import and navigate to: - -/apps/all/MATLAB/2015a-EDU/SalomonPBSPro.settings - -With the new mode, MATLAB itself launches the workers via PBS, so you can either use interactive mode or a batch mode on one node, but the actual parallel processing will be done in a separate job started by MATLAB itself. Alternatively, you can use "local" mode to run parallel code on just a single node. - -!!! note - The profile is confusingly named Salomon, but you can use it also on Anselm. - -### Parallel Matlab Interactive Session - -Following example shows how to start interactive session with support for Matlab GUI. For more information about GUI based applications on Anselm see [this page](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/x-window-system/). - -```console -$ xhost + -$ qsub -I -v DISPLAY=$(uname -n):$(echo $DISPLAY | cut -d ':' -f 2) -A NONE-0-0 -q qexp -l select=1 -l walltime=00:30:00 -l feature__matlab__MATLAB=1 -``` - -This qsub command example shows how to run Matlab on a single node. - -The second part of the command shows how to request all necessary licenses. In this case 1 Matlab-EDU license and 48 Distributed Computing Engines licenses. - -Once the access to compute nodes is granted by PBS, user can load following modules and start Matlab: - -```console -r1i0n17$ ml MATLAB/2015b-EDU -r1i0n17$ matlab & -``` - -### Parallel Matlab Batch Job in Local Mode - -To run matlab in batch mode, write an matlab script, then write a bash jobscript and execute via the qsub command. By default, matlab will execute one matlab worker instance per allocated core. - -```bash -#!/bin/bash -#PBS -A PROJECT ID -#PBS -q qprod -#PBS -l select=1:ncpus=16:mpiprocs=16:ompthreads=1 - -# change to shared scratch directory -SCR=/scratch/work/user/$USER/$PBS_JOBID -mkdir -p $SCR ; cd $SCR || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/matlabcode.m . - -# load modules -module load MATLAB/2015a-EDU - -# execute the calculation -matlab -nodisplay -r matlabcode > output.out - -# copy output file to home -cp output.out $PBS_O_WORKDIR/. -``` - -This script may be submitted directly to the PBS workload manager via the qsub command. The inputs and matlab script are in matlabcode.m file, outputs in output.out file. Note the missing .m extension in the matlab -r matlabcodefile call, **the .m must not be included**. Note that the **shared /scratch must be used**. Further, it is **important to include quit** statement at the end of the matlabcode.m script. - -Submit the jobscript using qsub - -```console -$ qsub ./jobscript -``` - -### Parallel Matlab Local Mode Program Example - -The last part of the configuration is done directly in the user Matlab script before Distributed Computing Toolbox is started. - -```console - cluster = parcluster('local') -``` - -This script creates scheduler object "cluster" of type "local" that starts workers locally. - -!!! note - Every Matlab script that needs to initialize/use matlabpool has to contain these three lines prior to calling parpool(sched, ...) function. - -The last step is to start matlabpool with "cluster" object and correct number of workers. We have 24 cores per node, so we start 24 workers. - -```console -parpool(cluster,16); - - -... parallel code ... - - -parpool close -``` - -The complete example showing how to use Distributed Computing Toolbox in local mode is shown here. - -```console -cluster = parcluster('local'); -cluster - -parpool(cluster,24); - -n=2000; - -W = rand(n,n); -W = distributed(W); -x = (1:n)'; -x = distributed(x); -spmd -[~, name] = system('hostname') - - T = W*x; % Calculation performed on labs, in parallel. - % T and W are both codistributed arrays here. -end -T; -whos % T and W are both distributed arrays here. - -parpool close -quit -``` - -You can copy and paste the example in a .m file and execute. Note that the parpool size should correspond to **total number of cores** available on allocated nodes. - -### Parallel Matlab Batch Job Using PBS Mode (Workers Spawned in a Separate Job) - -This mode uses PBS scheduler to launch the parallel pool. It uses the SalomonPBSPro profile that needs to be imported to Cluster Manager, as mentioned before. This methodod uses MATLAB's PBS Scheduler interface - it spawns the workers in a separate job submitted by MATLAB using qsub. - -This is an example of m-script using PBS mode: - -```console -cluster = parcluster('SalomonPBSPro'); -set(cluster, 'SubmitArguments', '-A OPEN-0-0'); -set(cluster, 'ResourceTemplate', '-q qprod -l select=10:ncpus=16'); -set(cluster, 'NumWorkers', 160); - -pool = parpool(cluster, 160); - -n=2000; - -W = rand(n,n); -W = distributed(W); -x = (1:n)'; -x = distributed(x); -spmd -[~, name] = system('hostname') - - T = W*x; % Calculation performed on labs, in parallel. - % T and W are both codistributed arrays here. -end -whos % T and W are both distributed arrays here. - -% shut down parallel pool -delete(pool) -``` - -Note that we first construct a cluster object using the imported profile, then set some important options, namely: SubmitArguments, where you need to specify accounting id, and ResourceTemplate, where you need to specify number of nodes to run the job. - -You can start this script using batch mode the same way as in Local mode example. - -### Parallel Matlab Batch With Direct Launch (Workers Spawned Within the Existing Job) - -This method is a "hack" invented by us to emulate the mpiexec functionality found in previous MATLAB versions. We leverage the MATLAB Generic Scheduler interface, but instead of submitting the workers to PBS, we launch the workers directly within the running job, thus we avoid the issues with master script and workers running in separate jobs (issues with license not available, waiting for the worker's job to spawn etc.) - -!!! warning - This method is experimental. - -For this method, you need to use SalomonDirect profile, import it using [the same way as SalomonPBSPro](matlab/#running-parallel-matlab-using-distributed-computing-toolbox---engine) - -This is an example of m-script using direct mode: - -```console -parallel.importProfile('/apps/all/MATLAB/2015a-EDU/SalomonDirect.settings') -cluster = parcluster('SalomonDirect'); -set(cluster, 'NumWorkers', 48); - -pool = parpool(cluster, 48); - -n=2000; - -W = rand(n,n); -W = distributed(W); -x = (1:n)'; -x = distributed(x); -spmd -[~, name] = system('hostname') - - T = W*x; % Calculation performed on labs, in parallel. - % T and W are both codistributed arrays here. -end -whos % T and W are both distributed arrays here. - -% shut down parallel pool -delete(pool) -``` - -### Non-Interactive Session and Licenses - -If you want to run batch jobs with Matlab, be sure to request appropriate license features with the PBS Pro scheduler, at least the `-l _feature_matlab_MATLAB=1` for EDU variant of Matlab. More information about how to check the license features states and how to request them with PBS Pro, please [look here](../isv_licenses/). - -In case of non-interactive session please read the [following information](../isv_licenses/) on how to modify the qsub command to test for available licenses prior getting the resource allocation. - -### Matlab Distributed Computing Engines Start Up Time - -Starting Matlab workers is an expensive process that requires certain amount of time. For your information please see the following table: - -| compute nodes | number of workers | start-up time[s] | -| ------------- | ----------------- | ---------------- | -| 16 | 384 | 831 | -| 8 | 192 | 807 | -| 4 | 96 | 483 | -| 2 | 48 | 16 | - -## MATLAB on UV2000 - -UV2000 machine available in queue "qfat" can be used for MATLAB computations. This is a SMP NUMA machine with large amount of RAM, which can be beneficial for certain types of MATLAB jobs. CPU cores are allocated in chunks of 8 for this machine. - -You can use MATLAB on UV2000 in two parallel modes: - -### Threaded Mode - -Since this is a SMP machine, you can completely avoid using Parallel Toolbox and use only MATLAB's threading. MATLAB will automatically detect the number of cores you have allocated and will set maxNumCompThreads accordingly and certain operations, such as fft, , eig, svd, etc. will be automatically run in threads. The advantage of this mode is that you don't need to modify your existing sequential codes. - -### Local Cluster Mode - -You can also use Parallel Toolbox on UV2000. Use l[ocal cluster mode](matlab/#parallel-matlab-batch-job-in-local-mode), "SalomonPBSPro" profile will not work. diff --git a/docs.it4i/anselm/software/numerical-languages/matlab_1314.md b/docs.it4i/anselm/software/numerical-languages/matlab_1314.md deleted file mode 100644 index 41dca0561..000000000 --- a/docs.it4i/anselm/software/numerical-languages/matlab_1314.md +++ /dev/null @@ -1,206 +0,0 @@ -# Matlab 2013-2014 - -## Introduction - -!!! note - This document relates to the old versions R2013 and R2014. For MATLAB 2015, please use [this documentation instead](matlab/). - -Matlab is available in the latest stable version. There are always two variants of the release: - -* Non commercial or so called EDU variant, which can be used for common research and educational purposes. -* Commercial or so called COM variant, which can used also for commercial activities. The licenses for commercial variant are much more expensive, so usually the commercial variant has only subset of features compared to the EDU available. - -To load the latest version of Matlab load the module - -```console -$ ml matlab -``` - -By default the EDU variant is marked as default. If you need other version or variant, load the particular version. To obtain the list of available versions use - -```console -$ ml matlab -``` - -If you need to use the Matlab GUI to prepare your Matlab programs, you can use Matlab directly on the login nodes. But for all computations use Matlab on the compute nodes via PBS Pro scheduler. - -If you require the Matlab GUI, please follow the general information about running graphical applications - -Matlab GUI is quite slow using the X forwarding built in the PBS (qsub -X), so using X11 display redirection either via SSH or directly by xauth (please see the "GUI Applications on Compute Nodes over VNC" part) is recommended. - -To run Matlab with GUI, use - -```console -$ matlab -``` - -To run Matlab in text mode, without the Matlab Desktop GUI environment, use - -```console -$ matlab -nodesktop -nosplash -``` - -Plots, images, etc... will be still available. - -## Running Parallel Matlab Using Distributed Computing Toolbox / Engine - -Recommended parallel mode for running parallel Matlab on Anselm is MPIEXEC mode. In this mode user allocates resources through PBS prior to starting Matlab. Once resources are granted the main Matlab instance is started on the first compute node assigned to job by PBS and workers are started on all remaining nodes. User can use both interactive and non-interactive PBS sessions. This mode guarantees that the data processing is not performed on login nodes, but all processing is on compute nodes. - - - -For the performance reasons Matlab should use system MPI. On Anselm the supported MPI implementation for Matlab is Intel MPI. To switch to system MPI user has to override default Matlab setting by creating new configuration file in its home directory. The path and file name has to be exactly the same as in the following listing: - -```console -$ vim ~/matlab/mpiLibConf.m - -function [lib, extras] = mpiLibConf -%MATLAB MPI Library overloading for Infiniband Networks - -mpich = '/opt/intel/impi/4.1.1.036/lib64/'; - -disp('Using Intel MPI 4.1.1.036 over Infiniband') - -lib = strcat(mpich, 'libmpich.so'); -mpl = strcat(mpich, 'libmpl.so'); -opa = strcat(mpich, 'libopa.so'); - -extras = {}; -``` - -System MPI library allows Matlab to communicate through 40 Gbit/s InfiniBand QDR interconnect instead of slower 1 Gbit Ethernet network. - -!!! note - The path to MPI library in "mpiLibConf.m" has to match with version of loaded Intel MPI module. In this example the version 4.1.1.036 of Intel MPI is used by Matlab and therefore module impi/4.1.1.036 has to be loaded prior to starting Matlab. - -### Parallel Matlab Interactive Session - -Once this file is in place, user can request resources from PBS. Following example shows how to start interactive session with support for Matlab GUI. For more information about GUI based applications on Anselm see. - -```console -$ xhost + -$ qsub -I -v DISPLAY=$(uname -n):$(echo $DISPLAY | cut -d ':' -f 2) -A NONE-0-0 -q qexp -l select=4:ncpus=16:mpiprocs=16 -l walltime=00:30:00 -l feature__matlab__MATLAB=1 -``` - -This qsub command example shows how to run Matlab with 32 workers in following configuration: 2 nodes (use all 16 cores per node) and 16 workers = mpirocs per node (-l select=2:ncpus=16:mpiprocs=16). If user requires to run smaller number of workers per node then the "mpiprocs" parameter has to be changed. - -The second part of the command shows how to request all necessary licenses. In this case 1 Matlab-EDU license and 32 Distributed Computing Engines licenses. - -Once the access to compute nodes is granted by PBS, user can load following modules and start Matlab: - -```console - cn79$ ml matlab/R2013a-EDU - cn79$ ml impi/4.1.1.036 - cn79$ matlab & -``` - -### Parallel Matlab Batch Job - -To run matlab in batch mode, write an matlab script, then write a bash jobscript and execute via the qsub command. By default, matlab will execute one matlab worker instance per allocated core. - -```bash -#!/bin/bash -#PBS -A PROJECT ID -#PBS -q qprod -#PBS -l select=2:ncpus=16:mpiprocs=16:ompthreads=1 - -# change to shared scratch directory -SCR=/scratch/$USER/$PBS_JOBID -mkdir -p $SCR ; cd $SCR || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/matlabcode.m . - -# load modules -module load matlab/R2013a-EDU -module load impi/4.1.1.036 - -# execute the calculation -matlab -nodisplay -r matlabcode > output.out - -# copy output file to home -cp output.out $PBS_O_WORKDIR/. -``` - -This script may be submitted directly to the PBS workload manager via the qsub command. The inputs and matlab script are in matlabcode.m file, outputs in output.out file. Note the missing .m extension in the matlab -r matlabcodefile call, **the .m must not be included**. Note that the **shared /scratch must be used**. Further, it is **important to include quit** statement at the end of the matlabcode.m script. - -Submit the jobscript using qsub - -```console -$ qsub ./jobscript -``` - -### Parallel Matlab Program Example - -The last part of the configuration is done directly in the user Matlab script before Distributed Computing Toolbox is started. - -```console -sched = findResource('scheduler', 'type', 'mpiexec'); -set(sched, 'MpiexecFileName', '/apps/intel/impi/4.1.1/bin/mpirun'); -set(sched, 'EnvironmentSetMethod', 'setenv'); -``` - -This script creates scheduler object "sched" of type "mpiexec" that starts workers using mpirun tool. To use correct version of mpirun, the second line specifies the path to correct version of system Intel MPI library. - -!!! note - Every Matlab script that needs to initialize/use matlabpool has to contain these three lines prior to calling matlabpool(sched, ...) function. - -The last step is to start matlabpool with "sched" object and correct number of workers. In this case qsub asked for total number of 32 cores, therefore the number of workers is also set to 32. - -```console -matlabpool(sched,32); - - -... parallel code ... - - -matlabpool close -``` - -The complete example showing how to use Distributed Computing Toolbox is show here. - -```console -sched = findResource('scheduler', 'type', 'mpiexec'); -set(sched, 'MpiexecFileName', '/apps/intel/impi/4.1.1/bin/mpirun') -set(sched, 'EnvironmentSetMethod', 'setenv') -set(sched, 'SubmitArguments', '') -sched - -matlabpool(sched,32); - -n=2000; - -W = rand(n,n); -W = distributed(W); -x = (1:n)'; -x = distributed(x); -spmd -[~, name] = system('hostname') - - T = W*x; % Calculation performed on labs, in parallel. - % T and W are both codistributed arrays here. -end -T; -whos % T and W are both distributed arrays here. - -matlabpool close -quit -``` - -You can copy and paste the example in a .m file and execute. Note that the matlabpool size should correspond to **total number of cores** available on allocated nodes. - -### Non-Interactive Session and Licenses - -If you want to run batch jobs with Matlab, be sure to request appropriate license features with the PBS Pro scheduler, at least the ` -l __feature__matlab__MATLAB=1` for EDU variant of Matlab. More information about how to check the license features states and how to request them with PBS Pro, please [look here](../isv_licenses/). - -In case of non-interactive session please read the [following information](../isv_licenses/) on how to modify the qsub command to test for available licenses prior getting the resource allocation. - -### Matlab Distributed Computing Engines Start Up Time - -Starting Matlab workers is an expensive process that requires certain amount of time. For your information please see the following table: - -| compute nodes | number of workers | start-up time[s] | -| ------------- | ----------------- | ---------------- | -| 16 | 256 | 1008 | -| 8 | 128 | 534 | -| 4 | 64 | 333 | -| 2 | 32 | 210 | diff --git a/docs.it4i/anselm/software/numerical-languages/octave.md b/docs.it4i/anselm/software/numerical-languages/octave.md deleted file mode 100644 index 12651d9f0..000000000 --- a/docs.it4i/anselm/software/numerical-languages/octave.md +++ /dev/null @@ -1,110 +0,0 @@ -# Octave - -## Introduction - -GNU Octave is a high-level interpreted language, primarily intended for numerical computations. It provides capabilities for the numerical solution of linear and nonlinear problems, and for performing other numerical experiments. It also provides extensive graphics capabilities for data visualization and manipulation. Octave is normally used through its interactive command line interface, but it can also be used to write non-interactive programs. The Octave language is quite similar to Matlab so that most programs are easily portable. Read more on <http://www.gnu.org/software/octave/> - -Two versions of octave are available on Anselm, via module - -| Version | module | -| ----------------------------------------------------- | ------------------------- | -| Octave 3.8.2, compiled with GCC and Multithreaded MKL | Octave/3.8.2-gimkl-2.11.5 | -| Octave 4.0.1, compiled with GCC and Multithreaded MKL | Octave/4.0.1-gimkl-2.11.5 | -| Octave 4.0.0, compiled with >GCC and OpenBLAS | Octave/4.0.0-foss-2015g | - -## Modules and Execution - -```console -$ ml Octave -``` - -The octave on Anselm is linked to highly optimized MKL mathematical library. This provides threaded parallelization to many octave kernels, notably the linear algebra subroutines. Octave runs these heavy calculation kernels without any penalty. By default, octave would parallelize to 16 threads. You may control the threads by setting the OMP_NUM_THREADS environment variable. - -To run octave interactively, log in with ssh -X parameter for X11 forwarding. Run octave: - -```console -$ octave -``` - -To run octave in batch mode, write an octave script, then write a bash jobscript and execute via the qsub command. By default, octave will use 16 threads when running MKL kernels. - -```bash -#!/bin/bash - -# change to local scratch directory -cd /lscratch/$PBS_JOBID || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/octcode.m . - -# load octave module -module load octave - -# execute the calculation -octave -q --eval octcode > output.out - -# copy output file to home -cp output.out $PBS_O_WORKDIR/. - -#exit -exit -``` - -This script may be submitted directly to the PBS workload manager via the qsub command. The inputs are in octcode.m file, outputs in output.out file. See the single node jobscript example in the [Job execution section](../../job-submission-and-execution/). - -The octave c compiler mkoctfile calls the GNU gcc 4.8.1 for compiling native c code. This is very useful for running native c subroutines in octave environment. - -```console -$ mkoctfile -v -``` - -Octave may use MPI for interprocess communication This functionality is currently not supported on Anselm cluster. In case you require the octave interface to MPI, please contact [Anselm support](https://support.it4i.cz/rt/). - -## Xeon Phi Support - -Octave may take advantage of the Xeon Phi accelerators. This will only work on the [Intel Xeon Phi](../intel-xeon-phi/) [accelerated nodes](../../compute-nodes/). - -### Automatic Offload Support - -Octave can accelerate BLAS type operations (in particular the Matrix Matrix multiplications] on the Xeon Phi accelerator, via [Automatic Offload using the MKL library](../intel-xeon-phi/#section-3) - -Example - -```console -$ export OFFLOAD_REPORT=2 -$ export MKL_MIC_ENABLE=1 -$ ml octave -$ octave -q - octave:1> A=rand(10000); B=rand(10000); - octave:2> tic; C=A*B; toc - [MKL] [MIC --] [AO Function] DGEMM - [MKL] [MIC --] [AO DGEMM Workdivision] 0.32 0.68 - [MKL] [MIC 00] [AO DGEMM CPU Time] 2.896003 seconds - [MKL] [MIC 00] [AO DGEMM MIC Time] 1.967384 seconds - [MKL] [MIC 00] [AO DGEMM CPU->MIC Data] 1347200000 bytes - [MKL] [MIC 00] [AO DGEMM MIC->CPU Data] 2188800000 bytes - Elapsed time is 2.93701 seconds. -``` - -In this example, the calculation was automatically divided among the CPU cores and the Xeon Phi MIC accelerator, reducing the total runtime from 6.3 secs down to 2.9 secs. - -### Native Support - -A version of [native](../intel-xeon-phi/#section-4) Octave is compiled for Xeon Phi accelerators. Some limitations apply for this version: - -* Only command line support. GUI, graph plotting etc. is not supported. -* Command history in interactive mode is not supported. - -Octave is linked with parallel Intel MKL, so it best suited for batch processing of tasks that utilize BLAS, LAPACK and FFT operations. By default, number of threads is set to 120, you can control this with > OMP_NUM_THREADS environment -variable. - -!!! note - Calculations that do not employ parallelism (either by using parallel MKL e.g. via matrix operations, fork() function, [parallel package](http://octave.sourceforge.net/parallel/) or other mechanism) will actually run slower than on host CPU. - -To use Octave on a node with Xeon Phi: - -```console -$ ssh mic0 # login to the MIC card -$ source /apps/tools/octave/3.8.2-mic/bin/octave-env.sh # set up environment variables -$ octave -q /apps/tools/octave/3.8.2-mic/example/test0.m # run an example -``` diff --git a/docs.it4i/anselm/software/numerical-languages/r.md b/docs.it4i/anselm/software/numerical-languages/r.md deleted file mode 100644 index 9d189e85b..000000000 --- a/docs.it4i/anselm/software/numerical-languages/r.md +++ /dev/null @@ -1,399 +0,0 @@ -# R - -## Introduction - -The R is a language and environment for statistical computing and graphics. R provides a wide variety of statistical (linear and nonlinear modelling, classical statistical tests, time-series analysis, classification, clustering, ...) and graphical techniques, and is highly extensible. - -One of R's strengths is the ease with which well-designed publication-quality plots can be produced, including mathematical symbols and formulae where needed. Great care has been taken over the defaults for the minor design choices in graphics, but the user retains full control. - -Another convenience is the ease with which the C code or third party libraries may be integrated within R. - -Extensive support for parallel computing is available within R. - -Read more on <http://www.r-project.org/>, <http://cran.r-project.org/doc/manuals/r-release/R-lang.html> - -## Modules - -The R version 3.0.1 is available on Anselm, along with GUI interface Rstudio - -| Application | Version | module | -| ----------- | ------------ | ------- | -| **R** | R 3.0.1 | R | -| **Rstudio** | Rstudio 0.97 | Rstudio | - -```console -$ ml R -``` - -## Execution - -The R on Anselm is linked to highly optimized MKL mathematical library. This provides threaded parallelization to many R kernels, notably the linear algebra subroutines. The R runs these heavy calculation kernels without any penalty. By default, the R would parallelize to 16 threads. You may control the threads by setting the OMP_NUM_THREADS environment variable. - -### Interactive Execution - -To run R interactively, using Rstudio GUI, log in with ssh -X parameter for X11 forwarding. Run rstudio: - -```console -$ ml Rstudio -$ rstudio -``` - -### Batch Execution - -To run R in batch mode, write an R script, then write a bash jobscript and execute via the qsub command. By default, R will use 16 threads when running MKL kernels. - -Example jobscript: - -```bash -#!/bin/bash - -# change to local scratch directory -cd /lscratch/$PBS_JOBID || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/rscript.R . - -# load R module -module load R - -# execute the calculation -R CMD BATCH rscript.R routput.out - -# copy output file to home -cp routput.out $PBS_O_WORKDIR/. - -#exit -exit -``` - -This script may be submitted directly to the PBS workload manager via the qsub command. The inputs are in rscript.R file, outputs in routput.out file. See the single node jobscript example in the [Job execution section](../../job-submission-and-execution/). - -## Parallel R - -Parallel execution of R may be achieved in many ways. One approach is the implied parallelization due to linked libraries or specially enabled functions, as [described above](r/#interactive-execution). In the following sections, we focus on explicit parallelization, where parallel constructs are directly stated within the R script. - -## Package Parallel - -The package parallel provides support for parallel computation, including by forking (taken from package multicore), by sockets (taken from package snow) and random-number generation. - -The package is activated this way: - -```console -$ R - > library(parallel) -``` - -More information and examples may be obtained directly by reading the documentation available in R - -```console - > ?parallel - > library(help = "parallel") - > vignette("parallel") -``` - -Download the package [parallell](package-parallel-vignette.pdf) vignette. - -The forking is the most simple to use. Forking family of functions provide parallelized, drop in replacement for the serial apply() family of functions. - -!!! note - Forking via package parallel provides functionality similar to OpenMP construct - - omp parallel for - - Only cores of single node can be utilized this way! - -Forking example: - -```r -library(parallel) - -#integrand function -f <- function(i,h) { -x <- h*(i-0.5) -return (4/(1 + x*x)) -} - -#initialize -size <- detectCores() - -while (TRUE) -{ - #read number of intervals - cat("Enter the number of intervals: (0 quits) ") - fp<-file("stdin"); n<-scan(fp,nmax=1); close(fp) - - if(n<=0) break - - #run the calculation - n <- max(n,size) - h <- 1.0/n - - i <- seq(1,n); - pi3 <- h*sum(simplify2array(mclapply(i,f,h,mc.cores=size))); - - #print results - cat(sprintf("Value of PI %16.14f, diff= %16.14fn",pi3,pi3-pi)) -} -``` - -The above example is the classic parallel example for calculating the number Ď€. Note the **detectCores()** and **mclapply()** functions. Execute the example as: - -```console -$ R --slave --no-save --no-restore -f pi3p.R -``` - -Every evaluation of the integrad function runs in parallel on different process. - -## Package Rmpi - -!!! note - package Rmpi provides an interface (wrapper) to MPI APIs. - -It also provides interactive R slave environment. On Anselm, Rmpi provides interface to the [OpenMPI](../mpi-1/Running_OpenMPI/). - -Read more on Rmpi at <http://cran.r-project.org/web/packages/Rmpi/>, reference manual is available at <http://cran.r-project.org/web/packages/Rmpi/Rmpi.pdf> - -When using package Rmpi, both openmpi and R modules must be loaded - -```console -$ ml OpenMPI -$ ml R -``` - -Rmpi may be used in three basic ways. The static approach is identical to executing any other MPI programm. In addition, there is Rslaves dynamic MPI approach and the mpi.apply approach. In the following section, we will use the number Ď€ integration example, to illustrate all these concepts. - -### Static Rmpi - -Static Rmpi programs are executed via mpiexec, as any other MPI programs. Number of processes is static - given at the launch time. - -Static Rmpi example: - -```r -library(Rmpi) - -#integrand function -f <- function(i,h) { -x <- h*(i-0.5) -return (4/(1 + x*x)) -} - -#initialize -invisible(mpi.comm.dup(0,1)) -rank <- mpi.comm.rank() -size <- mpi.comm.size() -n<-0 - -while (TRUE) -{ - #read number of intervals - if (rank==0) { - cat("Enter the number of intervals: (0 quits) ") - fp<-file("stdin"); n<-scan(fp,nmax=1); close(fp) - } - - #broadcat the intervals - n <- mpi.bcast(as.integer(n),type=1) - - if(n<=0) break - - #run the calculation - n <- max(n,size) - h <- 1.0/n - - i <- seq(rank+1,n,size); - mypi <- h*sum(sapply(i,f,h)); - - pi3 <- mpi.reduce(mypi) - - #print results - if (rank==0) cat(sprintf("Value of PI %16.14f, diff= %16.14fn",pi3,pi3-pi)) -} - -mpi.quit() -``` - -The above is the static MPI example for calculating the number Ď€. Note the **library(Rmpi)** and **mpi.comm.dup()** function calls. - -Execute the example as: - -```console -$ mpiexec R --slave --no-save --no-restore -f pi3.R -``` - -### Dynamic Rmpi - -Dynamic Rmpi programs are executed by calling the R directly. openmpi module must be still loaded. The R slave processes will be spawned by a function call within the Rmpi program. - -Dynamic Rmpi example: - -```r -#integrand function -f <- function(i,h) { -x <- h*(i-0.5) -return (4/(1 + x*x)) -} - -#the worker function -workerpi <- function() -{ -#initialize -rank <- mpi.comm.rank() -size <- mpi.comm.size() -n<-0 - -while (TRUE) -{ - #read number of intervals - if (rank==0) { - cat("Enter the number of intervals: (0 quits) ") - fp<-file("stdin"); n<-scan(fp,nmax=1); close(fp) - } - - #broadcat the intervals - n <- mpi.bcast(as.integer(n),type=1) - - if(n<=0) break - - #run the calculation - n <- max(n,size) - h <- 1.0/n - - i <- seq(rank+1,n,size); - mypi <- h*sum(sapply(i,f,h)); - - pi3 <- mpi.reduce(mypi) - - #print results - if (rank==0) cat(sprintf("Value of PI %16.14f, diff= %16.14fn",pi3,pi3-pi)) -} -} - -#main -library(Rmpi) - -cat("Enter the number of slaves: ") -fp<-file("stdin"); ns<-scan(fp,nmax=1); close(fp) - -mpi.spawn.Rslaves(nslaves=ns) -mpi.bcast.Robj2slave(f) -mpi.bcast.Robj2slave(workerpi) - -mpi.bcast.cmd(workerpi()) -workerpi() - -mpi.quit() -``` - -The above example is the dynamic MPI example for calculating the number Ď€. Both master and slave processes carry out the calculation. Note the mpi.spawn.Rslaves(), mpi.bcast.Robj2slave()** and the mpi.bcast.cmd()** function calls. - -Execute the example as: - -```console -$ R --slave --no-save --no-restore -f pi3Rslaves.R -``` - -### mpi.apply Rmpi - -mpi.apply is a specific way of executing Dynamic Rmpi programs. - -!!! note - mpi.apply() family of functions provide MPI parallelized, drop in replacement for the serial apply() family of functions. - -Execution is identical to other dynamic Rmpi programs. - -mpi.apply Rmpi example: - -```r -#integrand function -f <- function(i,h) { -x <- h*(i-0.5) -return (4/(1 + x*x)) -} - -#the worker function -workerpi <- function(rank,size,n) -{ - #run the calculation - n <- max(n,size) - h <- 1.0/n - - i <- seq(rank,n,size); - mypi <- h*sum(sapply(i,f,h)); - - return(mypi) -} - -#main -library(Rmpi) - -cat("Enter the number of slaves: ") -fp<-file("stdin"); ns<-scan(fp,nmax=1); close(fp) - -mpi.spawn.Rslaves(nslaves=ns) -mpi.bcast.Robj2slave(f) -mpi.bcast.Robj2slave(workerpi) - -while (TRUE) -{ - #read number of intervals - cat("Enter the number of intervals: (0 quits) ") - fp<-file("stdin"); n<-scan(fp,nmax=1); close(fp) - if(n<=0) break - - #run workerpi - i=seq(1,2*ns) - pi3=sum(mpi.parSapply(i,workerpi,2*ns,n)) - - #print results - cat(sprintf("Value of PI %16.14f, diff= %16.14fn",pi3,pi3-pi)) -} - -mpi.quit() -``` - -The above is the mpi.apply MPI example for calculating the number Ď€. Only the slave processes carry out the calculation. Note the **mpi.parSapply()**, function call. The package parallel [example](r/#package-parallel)[above](r/#package-parallel) may be trivially adapted (for much better performance) to this structure using the mclapply() in place of mpi.parSapply(). - -Execute the example as: - -```console -$ R --slave --no-save --no-restore -f pi3parSapply.R -``` - -## Combining Parallel and Rmpi - -Currently, the two packages can not be combined for hybrid calculations. - -## Parallel Execution - -The R parallel jobs are executed via the PBS queue system exactly as any other parallel jobs. User must create an appropriate jobscript and submit via the **qsub** - -Example jobscript for [static Rmpi](r/#static-rmpi) parallel R execution, running 1 process per core: - -```bash -#!/bin/bash -#PBS -q qprod -#PBS -N Rjob -#PBS -l select=100:ncpus=16:mpiprocs=16:ompthreads=1 - -# change to scratch directory -SCRDIR=/scratch/$USER/myjob -cd $SCRDIR || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/rscript.R . - -# load R and openmpi module -module load R -module load openmpi - -# execute the calculation -mpiexec -bycore -bind-to-core R --slave --no-save --no-restore -f rscript.R - -# copy output file to home -cp routput.out $PBS_O_WORKDIR/. - -#exit -exit -``` - -For more information about jobscript and MPI execution refer to the [Job submission](../../job-submission-and-execution/) and general [MPI](../mpi/mpi/) sections. diff --git a/docs.it4i/anselm/software/numerical-libraries/fftw.md b/docs.it4i/anselm/software/numerical-libraries/fftw.md deleted file mode 100644 index 7345a8116..000000000 --- a/docs.it4i/anselm/software/numerical-libraries/fftw.md +++ /dev/null @@ -1,73 +0,0 @@ -# FFTW - -The discrete Fourier transform in one or more dimensions, MPI parallel - -FFTW is a C subroutine library for computing the discrete Fourier transform in one or more dimensions, of arbitrary input size, and of both real and complex data (as well as of even/odd data, e.g. the discrete cosine/sine transforms or DCT/DST). The FFTW library allows for MPI parallel, in-place discrete Fourier transform, with data distributed over number of nodes. - -Two versions, **3.3.3** and **2.1.5** of FFTW are available on Anselm, each compiled for **Intel MPI** and **OpenMPI** using **intel** and **gnu** compilers. These are available via modules: - -| Version | Parallelization | module | linker options | -| -------------- | --------------- | ------------------- | ----------------------------------- | -| FFTW3 gcc3.3.3 | pthread, OpenMP | fftw3/3.3.3-gcc | -lfftw3, -lfftw3_threads-lfftw3_omp | -| FFTW3 icc3.3.3 | pthread, OpenMP | fftw3 | -lfftw3, -lfftw3_threads-lfftw3_omp | -| FFTW2 gcc2.1.5 | pthread | fftw2/2.1.5-gcc | -lfftw, -lfftw_threads | -| FFTW2 icc2.1.5 | pthread | fftw2 | -lfftw, -lfftw_threads | -| FFTW3 gcc3.3.3 | OpenMPI | fftw-mpi3/3.3.3-gcc | -lfftw3_mpi | -| FFTW3 icc3.3.3 | Intel MPI | fftw3-mpi | -lfftw3_mpi | -| FFTW2 gcc2.1.5 | OpenMPI | fftw2-mpi/2.1.5-gcc | -lfftw_mpi | -| FFTW2 gcc2.1.5 | IntelMPI | fftw2-mpi/2.1.5-gcc | -lfftw_mpi | - -```console -$ ml fftw3 **or** ml FFTW -``` - -The module sets up environment variables, required for linking and running FFTW enabled applications. Make sure that the choice of FFTW module is consistent with your choice of MPI library. Mixing MPI of different implementations may have unpredictable results. - -## Example - -```cpp - #include <fftw3-mpi.h> - int main(int argc, char **argv) - { - const ptrdiff_t N0 = 100, N1 = 1000; - fftw_plan plan; - fftw_complex *data; - ptrdiff_t alloc_local, local_n0, local_0_start, i, j; - - MPI_Init(&argc, &argv); - fftw_mpi_init(); - - /* get local data size and allocate */ - alloc_local = fftw_mpi_local_size_2d(N0, N1, MPI_COMM_WORLD, - &local_n0, &local_0_start); - data = fftw_alloc_complex(alloc_local); - - /* create plan for in-place forward DFT */ - plan = fftw_mpi_plan_dft_2d(N0, N1, data, data, MPI_COMM_WORLD, - FFTW_FORWARD, FFTW_ESTIMATE); - - /* initialize data */ - for (i = 0; i < local_n0; ++i) for (j = 0; j < N1; ++j) - { data[i*N1 + j][0] = i; - data[i*N1 + j][1] = j; } - - /* compute transforms, in-place, as many times as desired */ - fftw_execute(plan); - - fftw_destroy_plan(plan); - - MPI_Finalize(); - } -``` - -Load modules and compile: - -```console -$ ml intel -$ ml fftw3-mpi -$ mpicc testfftw3mpi.c -o testfftw3mpi.x -Wl,-rpath=$LIBRARY_PATH -lfftw3_mpi -``` - -Run the example as [Intel MPI program](../mpi/running-mpich2/). - -Read more on FFTW usage on the [FFTW website.](http://www.fftw.org/fftw3_doc/) diff --git a/docs.it4i/anselm/software/numerical-libraries/gsl.md b/docs.it4i/anselm/software/numerical-libraries/gsl.md deleted file mode 100644 index 3299492dd..000000000 --- a/docs.it4i/anselm/software/numerical-libraries/gsl.md +++ /dev/null @@ -1,145 +0,0 @@ -# GSL - -The GNU Scientific Library. Provides a wide range of mathematical routines. - -## Introduction - -The GNU Scientific Library (GSL) provides a wide range of mathematical routines such as random number generators, special functions and least-squares fitting. There are over 1000 functions in total. The routines have been written from scratch in C, and present a modern Applications Programming Interface (API) for C programmers, allowing wrappers to be written for very high level languages. - -The library covers a wide range of topics in numerical computing. Routines are available for the following areas: - - Complex Numbers Roots of Polynomials - - Special Functions Vectors and Matrices - - Permutations Combinations - - Sorting BLAS Support - - Linear Algebra CBLAS Library - - Fast Fourier Transforms Eigensystems - - Random Numbers Quadrature - - Random Distributions Quasi-Random Sequences - - Histograms Statistics - - Monte Carlo Integration N-Tuples - - Differential Equations Simulated Annealing - - Numerical Differentiation Interpolation - - Series Acceleration Chebyshev Approximations - - Root-Finding Discrete Hankel Transforms - - Least-Squares Fitting Minimization - - IEEE Floating-Point Physical Constants - - Basis Splines Wavelets - -## Modules - -The GSL 1.16 is available on Anselm, compiled for GNU and Intel compiler. These variants are available via modules: - -| Module | Compiler | -| --------------------- | --------- | -| gsl/1.16-gcc | gcc 4.8.6 | -| gsl/1.16-icc(default) | icc | - -```console -$ ml gsl -``` - -The module sets up environment variables, required for linking and running GSL enabled applications. This particular command loads the default module, which is gsl/1.16-icc - -## Linking - -Load an appropriate gsl module. Link using **-lgsl** switch to link your code against GSL. The GSL depends on cblas API to BLAS library, which must be supplied for linking. The BLAS may be provided, for example from the MKL library, as well as from the BLAS GSL library (-lgslcblas). Using the MKL is recommended. - -### Compiling and Linking With Intel Compilers - -```console -$ ml intel -$ ml gsl -$ icc myprog.c -o myprog.x -Wl,-rpath=$LIBRARY_PATH -mkl -lgsl -``` - -### Compiling and Linking With GNU Compilers - -```console -$ ml gcc -$ ml imkl **or** ml mkl -$ ml gsl/1.16-gcc -$ gcc myprog.c -o myprog.x -Wl,-rpath=$LIBRARY_PATH -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lgsl -``` - -## Example - -Following is an example of discrete wavelet transform implemented by GSL: - -```cpp - #include <stdio.h> - #include <math.h> - #include <gsl/gsl_sort.h> - #include <gsl/gsl_wavelet.h> - - int - main (int argc, char **argv) - { - int i, n = 256, nc = 20; - double *data = malloc (n * sizeof (double)); - double *abscoeff = malloc (n * sizeof (double)); - size_t *p = malloc (n * sizeof (size_t)); - - gsl_wavelet *w; - gsl_wavelet_workspace *work; - - w = gsl_wavelet_alloc (gsl_wavelet_daubechies, 4); - work = gsl_wavelet_workspace_alloc (n); - - for (i=0; i<n; i++) - data[i] = sin (3.141592654*(double)i/256.0); - - gsl_wavelet_transform_forward (w, data, 1, n, work); - - for (i = 0; i < n; i++) - { - abscoeff[i] = fabs (data[i]); - } - - gsl_sort_index (p, abscoeff, 1, n); - - for (i = 0; (i + nc) < n; i++) - data[p[i]] = 0; - - gsl_wavelet_transform_inverse (w, data, 1, n, work); - - for (i = 0; i < n; i++) - { - printf ("%gn", data[i]); - } - - gsl_wavelet_free (w); - gsl_wavelet_workspace_free (work); - - free (data); - free (abscoeff); - free (p); - return 0; - } -``` - -Load modules and compile: - -```console -$ ml intel -$ ml gsl -$ icc dwt.c -o dwt.x -Wl,-rpath=$LIBRARY_PATH -mkl -lgsl -``` - -In this example, we compile the dwt.c code using the Intel compiler and link it to the MKL and GSL library, note the -mkl and -lgsl options. The library search path is compiled in, so that no modules are necessary to run the code. diff --git a/docs.it4i/anselm/software/numerical-libraries/hdf5.md b/docs.it4i/anselm/software/numerical-libraries/hdf5.md deleted file mode 100644 index 13f626264..000000000 --- a/docs.it4i/anselm/software/numerical-libraries/hdf5.md +++ /dev/null @@ -1,89 +0,0 @@ -# HDF5 - -Hierarchical Data Format library. Serial and MPI parallel version. - -[HDF5 (Hierarchical Data Format)](http://www.hdfgroup.org/HDF5/) is a general purpose library and file format for storing scientific data. HDF5 can store two primary objects: datasets and groups. A dataset is essentially a multidimensional array of data elements, and a group is a structure for organizing objects in an HDF5 file. Using these two basic objects, one can create and store almost any kind of scientific data structure, such as images, arrays of vectors, and structured and unstructured grids. You can also mix and match them in HDF5 files according to your needs. - -Versions **1.8.11** and **1.8.13** of HDF5 library are available on Anselm, compiled for **Intel MPI** and **OpenMPI** using **intel** and **gnu** compilers. These are available via modules: - -| Version | Parallelization | module | C linker options | C++ linker options | Fortran linker options | -| --------------------- | --------------------------------- | -------------------------- | --------------------- | ----------------------- | ----------------------- | -| HDF5 icc serial | pthread | hdf5/1.8.11 | $HDF5_INC $HDF5_SHLIB | $HDF5_INC $HDF5_CPP_LIB | $HDF5_INC $HDF5_F90_LIB | -| HDF5 icc parallel MPI | pthread, IntelMPI | hdf5-parallel/1.8.11 | $HDF5_INC $HDF5_SHLIB | Not supported | $HDF5_INC $HDF5_F90_LIB | -| HDF5 icc serial | pthread | hdf5/1.8.13 | $HDF5_INC $HDF5_SHLIB | $HDF5_INC $HDF5_CPP_LIB | $HDF5_INC $HDF5_F90_LIB | -| HDF5 icc parallel MPI | pthread, IntelMPI | hdf5-parallel/1.8.13 | $HDF5_INC $HDF5_SHLIB | Not supported | $HDF5_INC $HDF5_F90_LIB | -| HDF5 gcc parallel MPI | pthread, OpenMPI 1.6.5, gcc 4.8.1 | hdf5-parallel/1.8.11-gcc | $HDF5_INC $HDF5_SHLIB | Not supported | $HDF5_INC $HDF5_F90_LIB | -| HDF5 gcc parallel MPI | pthread, OpenMPI 1.6.5, gcc 4.8.1 | hdf5-parallel/1.8.13-gcc | $HDF5_INC $HDF5_SHLIB | Not supported | $HDF5_INC $HDF5_F90_LIB | -| HDF5 gcc parallel MPI | pthread, OpenMPI 1.8.1, gcc 4.9.0 | hdf5-parallel/1.8.13-gcc49 | $HDF5_INC $HDF5_SHLIB | Not supported | $HDF5_INC $HDF5_F90_LIB | - -```console - -$ ml hdf5-parallel -``` - -The module sets up environment variables, required for linking and running HDF5 enabled applications. Make sure that the choice of HDF5 module is consistent with your choice of MPI library. Mixing MPI of different implementations may have unpredictable results. - -!!! note - Be aware, that GCC version of **HDF5 1.8.11** has serious performance issues, since it's compiled with -O0 optimization flag. This version is provided only for testing of code compiled only by GCC and IS NOT recommended for production computations. For more information, please see: <http://www.hdfgroup.org/ftp/HDF5/prev-releases/ReleaseFiles/release5-1811> - - All GCC versions of **HDF5 1.8.13** are not affected by the bug, are compiled with -O3 optimizations and are recommended for production computations. - -## Example - -```cpp - #include "hdf5.h" - #define FILE "dset.h5" - - int main() { - - hid_t file_id, dataset_id, dataspace_id; /* identifiers */ - hsize_t dims[2]; - herr_t status; - int i, j, dset_data[4][6]; - - /* Create a new file using default properties. */ - file_id = H5Fcreate(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - - /* Create the data space for the dataset. */ - dims[0] = 4; - dims[1] = 6; - dataspace_id = H5Screate_simple(2, dims, NULL); - - /* Initialize the dataset. */ - for (i = 0; i < 4; i++) - for (j = 0; j < 6; j++) - dset_data[i][j] = i * 6 + j + 1; - - /* Create the dataset. */ - dataset_id = H5Dcreate2(file_id, "/dset", H5T_STD_I32BE, dataspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - /* Write the dataset. */ - status = H5Dwrite(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, - dset_data); - - status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, - dset_data); - - /* End access to the dataset and release resources used by it. */ - status = H5Dclose(dataset_id); - - /* Terminate access to the data space. */ - status = H5Sclose(dataspace_id); - - /* Close the file. */ - status = H5Fclose(file_id); - } -``` - -Load modules and compile: - -```console -$ ml intel -$ ml hdf5-parallel -$ mpicc hdf5test.c -o hdf5test.x -Wl,-rpath=$LIBRARY_PATH $HDF5_INC $HDF5_SHLIB -``` - -Run the example as [Intel MPI program](../mpi/running-mpich2/). - -For further information, please see the website: <http://www.hdfgroup.org/HDF5/> diff --git a/docs.it4i/anselm/software/numerical-libraries/intel-numerical-libraries.md b/docs.it4i/anselm/software/numerical-libraries/intel-numerical-libraries.md deleted file mode 100644 index 5f3834ffa..000000000 --- a/docs.it4i/anselm/software/numerical-libraries/intel-numerical-libraries.md +++ /dev/null @@ -1,33 +0,0 @@ -# Intel numerical libraries - -Intel libraries for high performance in numerical computing - -## Intel Math Kernel Library - -Intel Math Kernel Library (Intel MKL) is a library of math kernel subroutines, extensively threaded and optimized for maximum performance. Intel MKL unites and provides these basic components: BLAS, LAPACK, ScaLapack, PARDISO, FFT, VML, VSL, Data fitting, Feast Eigensolver and many more. - -```console -$ ml mkl **or** ml imkl -``` - -Read more at the [Intel MKL](../intel-suite/intel-mkl/) page. - -## Intel Integrated Performance Primitives - -Intel Integrated Performance Primitives, version 7.1.1, compiled for AVX is available, via module ipp. The IPP is a library of highly optimized algorithmic building blocks for media and data applications. This includes signal, image and frame processing algorithms, such as FFT, FIR, Convolution, Optical Flow, Hough transform, Sum, MinMax and many more. - -```console -$ ml ipp -``` - -Read more at the [Intel IPP](../intel-suite/intel-integrated-performance-primitives/) page. - -## Intel Threading Building Blocks - -Intel Threading Building Blocks (Intel TBB) is a library that supports scalable parallel programming using standard ISO C++ code. It does not require special languages or compilers. It is designed to promote scalable data parallel programming. Additionally, it fully supports nested parallelism, so you can build larger parallel components from smaller parallel components. To use the library, you specify tasks, not threads, and let the library map tasks onto threads in an efficient manner. - -```console -$ ml tbb -``` - -Read more at the [Intel TBB](../intel-suite/intel-tbb/) page. diff --git a/docs.it4i/anselm/software/numerical-libraries/magma-for-intel-xeon-phi.md b/docs.it4i/anselm/software/numerical-libraries/magma-for-intel-xeon-phi.md deleted file mode 100644 index 64c443796..000000000 --- a/docs.it4i/anselm/software/numerical-libraries/magma-for-intel-xeon-phi.md +++ /dev/null @@ -1,76 +0,0 @@ -# MAGMA for Intel Xeon Phi - -Next generation dense algebra library for heterogeneous systems with accelerators - -## Compiling and Linking With MAGMA - -To be able to compile and link code with MAGMA library user has to load following module: - -```console -$ ml magma/1.3.0-mic -``` - -To make compilation more user friendly module also sets these two environment variables: - -!!! note - MAGMA_INC - contains paths to the MAGMA header files (to be used for compilation step) - -!!! note - MAGMA_LIBS - contains paths to MAGMA libraries (to be used for linking step). - -Compilation example: - -```console -$ icc -mkl -O3 -DHAVE_MIC -DADD_ -Wall $MAGMA_INC -c testing_dgetrf_mic.cpp -o testing_dgetrf_mic.o -$ icc -mkl -O3 -DHAVE_MIC -DADD_ -Wall -fPIC -Xlinker -zmuldefs -Wall -DNOCHANGE -DHOST testing_dgetrf_mic.o -o testing_dgetrf_mic $MAGMA_LIBS -``` - -### Running MAGMA Code - -MAGMA implementation for Intel MIC requires a MAGMA server running on accelerator prior to executing the user application. The server can be started and stopped using following scripts: - -!!! note - To start MAGMA server use: - **$MAGMAROOT/start_magma_server** - -!!! note - To stop the server use: - **$MAGMAROOT/stop_magma_server** - -!!! note - For deeper understanding how the MAGMA server is started, see the following script: - **$MAGMAROOT/launch_anselm_from_mic.sh** - -To test if the MAGMA server runs properly we can run one of examples that are part of the MAGMA installation: - -```console -[user@cn204 ~]$ $MAGMAROOT/testing/testing_dgetrf_mic -[user@cn204 ~]$ export OMP_NUM_THREADS=16 -[lriha@cn204 ~]$ $MAGMAROOT/testing/testing_dgetrf_mic - Usage: /apps/libs/magma-mic/magmamic-1.3.0/testing/testing_dgetrf_mic [options] [-h|--help] - - M N CPU GFlop/s (sec) MAGMA GFlop/s (sec) ||PA-LU||/(||A||*N) - ========================================================================= - 1088 1088 --- ( --- ) 13.93 ( 0.06) --- - 2112 2112 --- ( --- ) 77.85 ( 0.08) --- - 3136 3136 --- ( --- ) 183.21 ( 0.11) --- - 4160 4160 --- ( --- ) 227.52 ( 0.21) --- - 5184 5184 --- ( --- ) 258.61 ( 0.36) --- - 6208 6208 --- ( --- ) 333.12 ( 0.48) --- - 7232 7232 --- ( --- ) 416.52 ( 0.61) --- - 8256 8256 --- ( --- ) 446.97 ( 0.84) --- - 9280 9280 --- ( --- ) 461.15 ( 1.16) --- - 10304 10304 --- ( --- ) 500.70 ( 1.46) --- -``` - -!!! hint - MAGMA contains several benchmarks and examples in `$MAGMAROOT/testing/` - -!!! note - MAGMA relies on the performance of all CPU cores as well as on the performance of the accelerator. Therefore on Anselm number of CPU OpenMP threads has to be set to 16 with `export OMP_NUM_THREADS=16`. - -See more details at [MAGMA home page](http://icl.cs.utk.edu/magma/). - -## References - -[1] MAGMA MIC: Linear Algebra Library for Intel Xeon Phi Coprocessors, Jack Dongarra et. al, <http://icl.utk.edu/projectsfiles/magma/pubs/24-MAGMA_MIC_03.pdf> diff --git a/docs.it4i/anselm/software/numerical-libraries/petsc.md b/docs.it4i/anselm/software/numerical-libraries/petsc.md deleted file mode 100644 index 214e4074a..000000000 --- a/docs.it4i/anselm/software/numerical-libraries/petsc.md +++ /dev/null @@ -1,60 +0,0 @@ -# PETSc - -PETSc is a suite of building blocks for the scalable solution of scientific and engineering applications modeled by partial differential equations. It supports MPI, shared memory, and GPU through CUDA or OpenCL, as well as hybrid MPI-shared memory or MPI-GPU parallelism. - -## Introduction - -PETSc (Portable, Extensible Toolkit for Scientific Computation) is a suite of building blocks (data structures and routines) for the scalable solution of scientific and engineering applications modelled by partial differential equations. It allows thinking in terms of high-level objects (matrices) instead of low-level objects (raw arrays). Written in C language but can also be called from FORTRAN, C++, Python and Java codes. It supports MPI, shared memory, and GPUs through CUDA or OpenCL, as well as hybrid MPI-shared memory or MPI-GPU parallelism. - -## Resources - -* [project webpage](http://www.mcs.anl.gov/petsc/) -* [documentation](http://www.mcs.anl.gov/petsc/documentation/) - * [PETSc Users Manual (PDF)](http://www.mcs.anl.gov/petsc/petsc-current/docs/manual.pdf) - * [index of all manual pages](http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/singleindex.html) -* PRACE Video Tutorial [part1](http://www.youtube.com/watch?v=asVaFg1NDqY), [part2](http://www.youtube.com/watch?v=ubp_cSibb9I), [part3](http://www.youtube.com/watch?v=vJAAAQv-aaw), [part4](http://www.youtube.com/watch?v=BKVlqWNh8jY), [part5](http://www.youtube.com/watch?v=iXkbLEBFjlM) - -## Modules - -You can start using PETSc on Anselm by loading the PETSc module. Module names obey this pattern: - -```console -$# ml petsc/version-compiler-mpi-blas-variant, e.g. -$ ml petsc/3.4.4-icc-impi-mkl-opt -``` - -where `variant` is replaced by one of `{dbg, opt, threads-dbg, threads-opt}`. The `opt` variant is compiled without debugging information (no `-g` option) and with aggressive compiler optimizations (`-O3 -xAVX`). This variant is suitable for performance measurements and production runs. In all other cases use the debug (`dbg`) variant, because it contains debugging information, performs validations and self-checks, and provides a clear stack trace and message in case of an error. The other two variants `threads-dbg` and `threads-opt` are `dbg` and `opt`, respectively, built with [OpenMP and pthreads threading support](https://www.mcs.anl.gov/petsc/miscellaneous/petscthreads.html). - -## External Libraries - -PETSc needs at least MPI, BLAS and LAPACK. These dependencies are currently satisfied with Intel MPI and Intel MKL in Anselm `petsc` modules. - -PETSc can be linked with a plethora of [external numerical libraries](http://www.mcs.anl.gov/petsc/miscellaneous/external.html), extending PETSc functionality, e.g. direct linear system solvers, preconditioners or partitioners. See below a list of libraries currently included in Anselm `petsc` modules. - -All these libraries can be used also alone, without PETSc. Their static or shared program libraries are available in -`$PETSC_DIR/$PETSC_ARCH/lib` and header files in `$PETSC_DIR/$PETSC_ARCH/include`. `PETSC_DIR` and `PETSC_ARCH` are environment variables pointing to a specific PETSc instance based on the petsc module loaded. - -### Libraries Linked to PETSc on Anselm (As of 11 April 2015) - -* dense linear algebra - * [Elemental](http://libelemental.org/) -* sparse linear system solvers - * [Intel MKL Pardiso](https://software.intel.com/en-us/node/470282) - * [MUMPS](http://mumps.enseeiht.fr/) - * [PaStiX](http://pastix.gforge.inria.fr/) - * [SuiteSparse](http://faculty.cse.tamu.edu/davis/suitesparse.html) - * [SuperLU](http://crd.lbl.gov/~xiaoye/SuperLU/#superlu) - * [SuperLU_Dist](http://crd.lbl.gov/~xiaoye/SuperLU/#superlu_dist) -* input/output - * [ExodusII](http://sourceforge.net/projects/exodusii/) - * [HDF5](http://www.hdfgroup.org/HDF5/) - * [NetCDF](http://www.unidata.ucar.edu/software/netcdf/) -* partitioning - * [Chaco](http://www.cs.sandia.gov/CRF/chac.html) - * [METIS](http://glaros.dtc.umn.edu/gkhome/metis/metis/overview) - * [ParMETIS](http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview) - * [PT-Scotch](http://www.labri.fr/perso/pelegrin/scotch/) -* preconditioners & multigrid - * [Hypre](http://www.nersc.gov/users/software/programming-libraries/math-libraries/petsc/) - * [Trilinos ML](http://trilinos.sandia.gov/packages/ml/) - * [SPAI - Sparse Approximate Inverse](https://bitbucket.org/petsc/pkg-spai) diff --git a/docs.it4i/anselm/software/numerical-libraries/trilinos.md b/docs.it4i/anselm/software/numerical-libraries/trilinos.md deleted file mode 100644 index 36688e989..000000000 --- a/docs.it4i/anselm/software/numerical-libraries/trilinos.md +++ /dev/null @@ -1,49 +0,0 @@ -# Trilinos - -Packages for large scale scientific and engineering problems. Provides MPI and hybrid parallelization. - -## Introduction - -Trilinos is a collection of software packages for the numerical solution of large scale scientific and engineering problems. It is based on C++ and features modern object-oriented design. Both serial as well as parallel computations based on MPI and hybrid parallelization are supported within Trilinos packages. - -## Installed Packages - -Current Trilinos installation on ANSELM contains (among others) the following main packages - -* **Epetra** - core linear algebra package containing classes for manipulation with serial and distributed vectors, matrices, and graphs. Dense linear solvers are supported via interface to BLAS and LAPACK (Intel MKL on ANSELM). Its extension **EpetraExt** contains e.g. methods for matrix-matrix multiplication. -* **Tpetra** - next-generation linear algebra package. Supports 64-bit indexing and arbitrary data type using C++ templates. -* **Belos** - library of various iterative solvers (CG, block CG, GMRES, block GMRES etc.). -* **Amesos** - interface to direct sparse solvers. -* **Anasazi** - framework for large-scale eigenvalue algorithms. -* **IFPACK** - distributed algebraic preconditioner (includes e.g. incomplete LU factorization) -* **Teuchos** - common tools packages. This package contains classes for memory management, output, performance monitoring, BLAS and LAPACK wrappers etc. - -For the full list of Trilinos packages, descriptions of their capabilities, and user manuals see [http://trilinos.sandia.gov.](http://trilinos.sandia.gov) - -## Installed Version - -Currently, Trilinos in version 11.2.3 compiled with Intel Compiler is installed on ANSELM. - -## Compiling Against Trilinos - -First, load the appropriate module: - -```console -$ ml trilinos -``` - -For the compilation of CMake-aware project, Trilinos provides the FIND_PACKAGE( Trilinos ) capability, which makes it easy to build against Trilinos, including linking against the correct list of libraries. For details, see <http://trilinos.sandia.gov/Finding_Trilinos.txt> - -For compiling using simple makefiles, Trilinos provides Makefile.export system, which allows users to include important Trilinos variables directly into their makefiles. This can be done simply by inserting the following line into the makefile: - -```cpp -include Makefile.export.Trilinos -``` - -or - -```cpp -include Makefile.export.<package> -``` - -if you are interested only in a specific Trilinos package. This will give you access to the variables such as Trilinos_CXX_COMPILER, Trilinos_INCLUDE_DIRS, Trilinos_LIBRARY_DIRS etc. For the detailed description and example makefile see <http://trilinos.sandia.gov/Export_Makefile.txt>. diff --git a/docs.it4i/anselm/software/openfoam.md b/docs.it4i/anselm/software/openfoam.md deleted file mode 100644 index 865f054d3..000000000 --- a/docs.it4i/anselm/software/openfoam.md +++ /dev/null @@ -1,228 +0,0 @@ -# OpenFOAM - -a Free, Open Source CFD Software Package - -## Introduction - -OpenFOAM is a free, open source CFD software package developed by [**OpenCFD Ltd**](http://www.openfoam.com/about) at [**ESI Group**](http://www.esi-group.com/) and distributed by the [**OpenFOAM Foundation **](http://www.openfoam.org/). It has a large user base across most areas of engineering and science, from both commercial and academic organisations. - -Homepage: <http://www.openfoam.com/> - -### Installed Version - -Currently, several version compiled by GCC/ICC compilers in single/double precision with several version of openmpi are available on Anselm. - -For example syntax of available OpenFOAM module is: - -\<openfoam\/2.2.1-icc-openmpi1.6.5-DP\> - -this means openfoam version 2.2.1 compiled by ICC compiler with openmpi1.6.5 in double precision. - -Naming convection of the installed versions is following: - -openfoam\<VERSION\>-\<COMPILER\>\<openmpiVERSION\>-\<PRECISION\> - -* \<VERSION\> - version of openfoam -* \<COMPILER\> - version of used compiler -* \<openmpiVERSION\> - version of used openmpi/impi -* \<PRECISION\> - DP/SP – double/single precision - -### Available OpenFOAM Modules - -To check available modules use - -```console -$ ml av -``` - -In /opt/modules/modulefiles/engineering you can see installed engineering softwares: - -```console - ------------------------------------ /opt/modules/modulefiles/engineering ------------------------------------------------------------- - ansys/14.5.x matlab/R2013a-COM openfoam/2.2.1-icc-impi4.1.1.036-DP - comsol/43b-COM matlab/R2013a-EDU openfoam/2.2.1-icc-openmpi1.6.5-DP - comsol/43b-EDU openfoam/2.2.1-gcc481-openmpi1.6.5-DP paraview/4.0.1-gcc481-bullxmpi1.2.4.1-osmesa10.0 - lsdyna/7.x.x openfoam/2.2.1-gcc481-openmpi1.6.5-SP -``` - -For information how to use modules please [look here](../environment-and-modules/). - -## Getting Started - -To create OpenFOAM environment on ANSELM give the commands: - -```console -$ ml openfoam/2.2.1-icc-openmpi1.6.5-DP -$ source $FOAM_BASHRC -``` - -!!! note - Please load correct module with your requirements “compiler - GCC/ICC, precision - DP/SP”. - -Create a project directory within the $HOME/OpenFOAM directory named \<USER\>-\<OFversion\> and create a directory named run within it, e.g. by typing: - -```console -$ mkdir -p $FOAM_RUN -``` - -Project directory is now available by typing: - -```console -$ cd /home/<USER>/OpenFOAM/<USER>-<OFversion>/run -``` - -\<OFversion\> - for example \<2.2.1\> - -or - -```console -$ cd $FOAM_RUN -``` - -Copy the tutorial examples directory in the OpenFOAM distribution to the run directory: - -```console -$ cp -r $FOAM_TUTORIALS $FOAM_RUN -``` - -Now you can run the first case for example incompressible laminar flow in a cavity. - -## Running Serial Applications - -Create a Bash script test.sh - -```bash - #!/bin/bash - module load openfoam/2.2.1-icc-openmpi1.6.5-DP - source $FOAM_BASHRC - - # source to run functions - . $WM_PROJECT_DIR/bin/tools/RunFunctions - - cd $FOAM_RUN/tutorials/incompressible/icoFoam/cavity - - runApplication blockMesh - runApplication icoFoam -``` - -Job submission - -```console -$ qsub -A OPEN-0-0 -q qprod -l select=1:ncpus=16,walltime=03:00:00 test.sh -``` - -For information about job submission please [look here](../job-submission-and-execution/). - -## Running Applications in Parallel - -Run the second case for example external incompressible turbulent flow - case - motorBike. - -First we must run serial application bockMesh and decomposePar for preparation of parallel computation. - -!!! note - Create a Bash scrip test.sh: - -```bash - #!/bin/bash - module load openfoam/2.2.1-icc-openmpi1.6.5-DP - source $FOAM_BASHRC - - # source to run functions - . $WM_PROJECT_DIR/bin/tools/RunFunctions - - cd $FOAM_RUN/tutorials/incompressible/simpleFoam/motorBike - - runApplication blockMesh - runApplication decomposePar -``` - -Job submission - -```console -$ qsub -A OPEN-0-0 -q qprod -l select=1:ncpus=16,walltime=03:00:00 test.sh -``` - -This job create simple block mesh and domain decomposition. Check your decomposition, and submit parallel computation: - -!!! note - Create a PBS script testParallel.pbs: - -```bash - #!/bin/bash - #PBS -N motorBike - #PBS -l select=2:ncpus=16 - #PBS -l walltime=01:00:00 - #PBS -q qprod - #PBS -A OPEN-0-0 - - module load openfoam/2.2.1-icc-openmpi1.6.5-DP - source $FOAM_BASHRC - - cd $FOAM_RUN/tutorials/incompressible/simpleFoam/motorBike - - nproc = 32 - - mpirun -hostfile ${PBS_NODEFILE} -np $nproc snappyHexMesh -overwrite -parallel | tee snappyHexMesh.log - - mpirun -hostfile ${PBS_NODEFILE} -np $nproc potentialFoam -noFunctionObject-writep -parallel | tee potentialFoam.log - - mpirun -hostfile ${PBS_NODEFILE} -np $nproc simpleFoam -parallel | tee simpleFoam.log -``` - -nproc – number of subdomains - -Job submission - -```console -$ qsub testParallel.pbs -``` - -## Compile Your Own Solver - -Initialize OpenFOAM environment before compiling your solver - -```console -$ ml openfoam/2.2.1-icc-openmpi1.6.5-DP -$ source $FOAM_BASHRC -$ cd $FOAM_RUN/ -``` - -Create directory applications/solvers in user directory - -```console -$ mkdir -p applications/solvers -$ cd applications/solvers -``` - -Copy icoFoam solver’s source files - -```console -$ cp -r $FOAM_SOLVERS/incompressible/icoFoam/ My_icoFoam -$ cd My_icoFoam -``` - -Rename icoFoam.C to My_icoFOAM.C - -```console -$ mv icoFoam.C My_icoFoam.C -``` - -Edit _files_ file in _Make_ directory: - -```bash - icoFoam.C - EXE = $(FOAM_APPBIN)/icoFoam -``` - -and change to: - -```bash - My_icoFoam.C - EXE = $(FOAM_USER_APPBIN)/My_icoFoam -``` - -In directory My_icoFoam give the compilation command: - -```console -$ wmake -``` diff --git a/docs.it4i/anselm/software/operating-system.md b/docs.it4i/anselm/software/operating-system.md deleted file mode 100644 index e43800e0d..000000000 --- a/docs.it4i/anselm/software/operating-system.md +++ /dev/null @@ -1,3 +0,0 @@ -# Operating System - -The operating system on Anselm is Linux - [**Red Hat Enterprise Linux release 6.x**](https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux). diff --git a/docs.it4i/anselm/software/paraview.md b/docs.it4i/anselm/software/paraview.md deleted file mode 100644 index 171a79c10..000000000 --- a/docs.it4i/anselm/software/paraview.md +++ /dev/null @@ -1,85 +0,0 @@ -# ParaView - -Open-Source, Multi-Platform Data Analysis and Visualization Application - -## Introduction - -**ParaView** is an open-source, multi-platform data analysis and visualization application. ParaView users can quickly build visualizations to analyze their data using qualitative and quantitative techniques. The data exploration can be done interactively in 3D or programmatically using ParaView's batch processing capabilities. - -ParaView was developed to analyze extremely large datasets using distributed memory computing resources. It can be run on supercomputers to analyze datasets of exascale size as well as on laptops for smaller data. - -Homepage : <http://www.paraview.org/> - -## Installed Version - -Currently, version 5.1.2 compiled with intel/2017a against intel MPI library and OSMesa 12.0.2 is installed on Anselm. - -## Usage - -On Anselm, ParaView is to be used in client-server mode. A parallel ParaView server is launched on compute nodes by the user, and client is launched on your desktop PC to control and view the visualization. Download ParaView client application for your OS here: <http://paraview.org/paraview/resources/software.php>. - -!!!Warning - Your version must match the version number installed on Anselm. - -### Launching Server - -To launch the server, you must first allocate compute nodes, for example - -```console -$ qsub -I -q qprod -A OPEN-0-0 -l select=2 -``` - -to launch an interactive session on 2 nodes. Refer to [Resource Allocation and Job Execution](../job-submission-and-execution/) for details. - -After the interactive session is opened, load the ParaView module : - -```console -$ ml ParaView/5.1.2-intel-2017a-mpi -``` - -Now launch the parallel server, with number of nodes times 16 processes: - -```console -$ mpirun -np 32 pvserver --use-offscreen-rendering - Waiting for client... - Connection URL: cs://cn77:11111 - Accepting connection(s): cn77:11111 -``` - -Note the that the server is listening on compute node cn77 in this case, we shall use this information later. - -### Client Connection - -Because a direct connection is not allowed to compute nodes on Anselm, you must establish a SSH tunnel to connect to the server. Choose a port number on your PC to be forwarded to ParaView server, for example 12345. If your PC is running Linux, use this command to establish a SSH tunnel: - -```console -$ ssh -TN -L 12345:cn77:11111 username@anselm.it4i.cz -``` - -replace username with your login and cn77 with the name of compute node your ParaView server is running on (see previous step). - -If you use PuTTY on Windows, load Anselm connection configuration, then go to *Connection* -> *SSH* -> *Tunnels* to set up the port forwarding. - -Fill the Source port and Destination fields. **Do not forget to click the Add button.** - - - -Now launch ParaView client installed on your desktop PC. Select *File* -> *Connect*... and fill in the following : - - - -The configuration is now saved for later use. Now click Connect to connect to the ParaView server. In your terminal where you have interactive session with ParaView server launched, you should see: - -```console -Client connected. -``` - -You can now use Parallel ParaView. - -### Close Server - -Remember to close the interactive session after you finish working with ParaView server, as it will remain launched even after your client is disconnected and will continue to consume resources. - -## GPU Support - -Currently, GPU acceleration is not supported in the server and ParaView will not take advantage of accelerated nodes on Anselm. Support for GPU acceleration might be added in the future. diff --git a/docs.it4i/salomon/software/ansys/ansys-cfx.md b/docs.it4i/salomon/software/ansys/ansys-cfx.md deleted file mode 100644 index 21ce8f93b..000000000 --- a/docs.it4i/salomon/software/ansys/ansys-cfx.md +++ /dev/null @@ -1,55 +0,0 @@ -# ANSYS CFX - -[ANSYS CFX](http://www.ansys.com/products/fluids/ansys-cfx) software is a high-performance, general purpose fluid dynamics program that has been applied to solve wide-ranging fluid flow problems for over 20 years. At the heart of ANSYS CFX is its advanced solver technology, the key to achieving reliable and accurate solutions quickly and robustly. The modern, highly parallelized solver is the foundation for an abundant choice of physical models to capture virtually any type of phenomena related to fluid flow. The solver and its many physical models are wrapped in a modern, intuitive, and flexible GUI and user environment, with extensive capabilities for customization and automation using session files, scripting and a powerful expression language. - -To run ANSYS CFX in batch mode you can utilize/modify the default cfx.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l nodes=2:ppn=16 -#PBS -q qprod -#PBS -N $USER-CFX-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> (working directory must exists) -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` -echo This jobs runs on the following processors: -echo `cat $PBS_NODEFILE` - -module load ansys - -#### Set number of processors per host listing -#### (set to 1 as $PBS_NODEFILE lists each node twice if :ppn=2) -procs_per_host=1 -#### Create host list -hl="" -for host in `cat $PBS_NODEFILE` -do - if ["$hl" = "" ] - then hl="$host:$procs_per_host" - else hl="${hl}:$host:$procs_per_host" - fi -done - -echo Machines: $hl - -#-dev input.def includes the input of CFX analysis in DEF format -#-P the name of prefered license feature (aa_r=ANSYS Academic Research, ane3fl=Multiphysics(commercial)) -/ansys_inc/v145/CFX/bin/cfx5solve -def input.def -size 4 -size-ni 4x -part-large -start-method "Platform MPI Distributed Parallel" -par-dist $hl -P aa_r -``` - -Header of the pbs file (above) is common and description can be find on [this site](../../job-submission-and-execution/). SVS FEM recommends to utilize sources by keywords: nodes, ppn. These keywords allows to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending pbs job into the queue. Input file should be in working directory or full path to input file has to be specified. >Input file has to be defined by common CFX def file which is attached to the cfx solver via parameter -def - -**License** should be selected by parameter -P (Big letter **P**). Licensed products are the following: aa_r (ANSYS **Academic** Research), ane3fl (ANSYS Multiphysics)-**Commercial**. -[More about licensing here](licensing/) diff --git a/docs.it4i/salomon/software/ansys/ansys-fluent.md b/docs.it4i/salomon/software/ansys/ansys-fluent.md deleted file mode 100644 index 27469a1c5..000000000 --- a/docs.it4i/salomon/software/ansys/ansys-fluent.md +++ /dev/null @@ -1,162 +0,0 @@ -# ANSYS Fluent - -[ANSYS Fluent](http://www.ansys.com/products/fluids/ansys-fluent) -software contains the broad physical modeling capabilities needed to model flow, turbulence, heat transfer, and reactions for industrial applications ranging from air flow over an aircraft wing to combustion in a furnace, from bubble columns to oil platforms, from blood flow to semiconductor manufacturing, and from clean room design to wastewater treatment plants. Special models that give the software the ability to model in-cylinder combustion, aeroacoustics, turbomachinery, and multiphase systems have served to broaden its reach. - -1. Common way to run Fluent over pbs file - -To run ANSYS Fluent in batch mode you can utilize/modify the default fluent.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -S /bin/bash -#PBS -l nodes=2:ppn=16 -#PBS -q qprod -#PBS -N $USER-Fluent-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> (working directory must exists) -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` -echo This jobs runs on the following processors: -echo `cat $PBS_NODEFILE` - -#### Load ansys module so that we find the cfx5solve command -module load ansys - -# Use following line to specify MPI for message-passing instead -NCORES=`wc -l $PBS_NODEFILE |awk '{print $1}'` - -/ansys_inc/v145/fluent/bin/fluent 3d -t$NCORES -cnf=$PBS_NODEFILE -g -i fluent.jou -``` - -Header of the pbs file (above) is common and description can be find on [this site](../../resources-allocation-policy/). [SVS FEM](http://www.svsfem.cz) recommends to utilize sources by keywords: nodes, ppn. These keywords allows to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending pbs job into the queue. Input file should be in working directory or full path to input file has to be specified. Input file has to be defined by common Fluent journal file which is attached to the Fluent solver via parameter -i fluent.jou - -Journal file with definition of the input geometry and boundary conditions and defined process of solution has e.g. the following structure: - -```console - /file/read-case aircraft_2m.cas.gz - /solve/init - init - /solve/iterate - 10 - /file/write-case-dat aircraft_2m-solution - /exit yes -``` - -The appropriate dimension of the problem has to be set by parameter (2d/3d). - -1. Fast way to run Fluent from command line - -```console -fluent solver_version [FLUENT_options] -i journal_file -pbs -``` - -This syntax will start the ANSYS FLUENT job under PBS Professional using the qsub command in a batch manner. When resources are available, PBS Professional will start the job and return a job ID, usually in the form of _job_ID.hostname_. This job ID can then be used to query, control, or stop the job using standard PBS Professional commands, such as qstat or qdel. The job will be run out of the current working directory, and all output will be written to the file fluent.o _job_ID_. - -1. Running Fluent via user's config file - -The sample script uses a configuration file called pbs_fluent.conf if no command line arguments are present. This configuration file should be present in the directory from which the jobs are submitted (which is also the directory in which the jobs are executed). The following is an example of what the content of pbs_fluent.conf can be: - -```console -input="example_small.flin" -case="Small-1.65m.cas" -fluent_args="3d -pmyrinet" -outfile="fluent_test.out" -mpp="true" -``` - -The following is an explanation of the parameters: - -input is the name of the input file. - -case is the name of the .cas file that the input file will utilize. - -fluent_args are extra ANSYS FLUENT arguments. As shown in the previous example, you can specify the interconnect by using the -p interconnect command. The available interconnects include ethernet (the default), myrinet, infiniband, vendor, altix, and crayx. The MPI is selected automatically, based on the specified interconnect. - -outfile is the name of the file to which the standard output will be sent. - -mpp="true" will tell the job script to execute the job across multiple processors. - -To run ANSYS Fluent in batch mode with user's config file you can utilize/modify the following script and execute it via the qsub command. - -```bash -#!/bin/sh -#PBS -l nodes=2:ppn=4 -#PBS -1 qprod -#PBS -N $USE-Fluent-Project -#PBS -A XX-YY-ZZ - - cd $PBS_O_WORKDIR - - #We assume that if they didn’t specify arguments then they should use the - #config file if ["xx${input}${case}${mpp}${fluent_args}zz" = "xxzz" ]; then - if [ -f pbs_fluent.conf ]; then - . pbs_fluent.conf - else - printf "No command line arguments specified, " - printf "and no configuration file found. Exiting n" - fi - fi - - - #Augment the ANSYS FLUENT command line arguments case "$mpp" in - true) - #MPI job execution scenario - num_nodes=â€cat $PBS_NODEFILE | sort -u | wc -l†- cpus=â€expr $num_nodes * $NCPUS†- #Default arguments for mpp jobs, these should be changed to suit your - #needs. - fluent_args="-t${cpus} $fluent_args -cnf=$PBS_NODEFILE" - ;; - *) - #SMP case - #Default arguments for smp jobs, should be adjusted to suit your - #needs. - fluent_args="-t$NCPUS $fluent_args" - ;; - esac - #Default arguments for all jobs - fluent_args="-ssh -g -i $input $fluent_args" - - echo "---------- Going to start a fluent job with the following settings: - Input: $input - Case: $case - Output: $outfile - Fluent arguments: $fluent_args" - - #run the solver - /ansys_inc/v145/fluent/bin/fluent $fluent_args > $outfile -``` - -It runs the jobs out of the directory from which they are submitted (PBS_O_WORKDIR). - -1. Running Fluent in parralel - -Fluent could be run in parallel only under Academic Research license. To do so this ANSYS Academic Research license must be placed before ANSYS CFD license in user preferences. To make this change anslic_admin utility should be run - -```console -/ansys_inc/shared_les/licensing/lic_admin/anslic_admin -``` - -ANSLIC_ADMIN Utility will be run - - - - - - - -ANSYS Academic Research license should be moved up to the top of the list. - - diff --git a/docs.it4i/salomon/software/ansys/ansys-ls-dyna.md b/docs.it4i/salomon/software/ansys/ansys-ls-dyna.md deleted file mode 100644 index 8646c2666..000000000 --- a/docs.it4i/salomon/software/ansys/ansys-ls-dyna.md +++ /dev/null @@ -1,55 +0,0 @@ -# ANSYS LS-DYNA - -**[ANSYSLS-DYNA](http://www.ansys.com/products/structures/ansys-ls-dyna)** software provides convenient and easy-to-use access to the technology-rich, time-tested explicit solver without the need to contend with the complex input requirements of this sophisticated program. Introduced in 1996, ANSYS LS-DYNA capabilities have helped customers in numerous industries to resolve highly intricate design issues. ANSYS Mechanical users have been able take advantage of complex explicit solutions for a long time utilizing the traditional ANSYS Parametric Design Language (APDL) environment. These explicit capabilities are available to ANSYS Workbench users as well. The Workbench platform is a powerful, comprehensive, easy-to-use environment for engineering simulation. CAD import from all sources, geometry cleanup, automatic meshing, solution, parametric optimization, result visualization and comprehensive report generation are all available within a single fully interactive modern graphical user environment. - -To run ANSYS LS-DYNA in batch mode you can utilize/modify the default ansysdyna.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l nodes=2:ppn=16 -#PBS -q qprod -#PBS -N $USER-DYNA-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` -echo This jobs runs on the following processors: -echo `cat $PBS_NODEFILE` - -#! Counts the number of processors -NPROCS=`wc -l < $PBS_NODEFILE` - -echo This job has allocated $NPROCS nodes - -module load ansys - -#### Set number of processors per host listing -#### (set to 1 as $PBS_NODEFILE lists each node twice if :ppn=2) -procs_per_host=1 -#### Create host list -hl="" -for host in `cat $PBS_NODEFILE` -do - if ["$hl" = "" ] - then hl="$host:$procs_per_host" - else hl="${hl}:$host:$procs_per_host" - fi -done - -echo Machines: $hl - -/ansys_inc/v145/ansys/bin/ansys145 -dis -lsdynampp i=input.k -machines $hl -``` - -Header of the pbs file (above) is common and description can be find on [this site](../../job-submission-and-execution/). [SVS FEM](http://www.svsfem.cz) recommends to utilize sources by keywords: nodes, ppn. These keywords allows to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending pbs job into the queue. Input file should be in working directory or full path to input file has to be specified. Input file has to be defined by common LS-DYNA .**k** file which is attached to the ansys solver via parameter i= diff --git a/docs.it4i/salomon/software/ansys/ansys-mechanical-apdl.md b/docs.it4i/salomon/software/ansys/ansys-mechanical-apdl.md deleted file mode 100644 index c1562c1c2..000000000 --- a/docs.it4i/salomon/software/ansys/ansys-mechanical-apdl.md +++ /dev/null @@ -1,56 +0,0 @@ -# ANSYS MAPDL - -**[ANSYS Multiphysics](http://www.ansys.com/products/multiphysics)** -software offers a comprehensive product solution for both multiphysics and single-physics analysis. The product includes structural, thermal, fluid and both high- and low-frequency electromagnetic analysis. The product also contains solutions for both direct and sequentially coupled physics problems including direct coupled-field elements and the ANSYS multi-field solver. - -To run ANSYS MAPDL in batch mode you can utilize/modify the default mapdl.pbs script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l nodes=2:ppn=16 -#PBS -q qprod -#PBS -N $USER-ANSYS-Project -#PBS -A XX-YY-ZZ - -#! Mail to user when job terminate or abort -#PBS -m ae - -#!change the working directory (default is home directory) -#cd <working directory> (working directory must exists) -WORK_DIR="/scratch/$USER/work" -cd $WORK_DIR - -echo Running on host `hostname` -echo Time is `date` -echo Directory is `pwd` -echo This jobs runs on the following processors: -echo `cat $PBS_NODEFILE` - -module load ansys - -#### Set number of processors per host listing -#### (set to 1 as $PBS_NODEFILE lists each node twice if :ppn=2) -procs_per_host=1 -#### Create host list -hl="" -for host in `cat $PBS_NODEFILE` -do - if ["$hl" = "" ] - then hl="$host:$procs_per_host" - else hl="${hl}:$host:$procs_per_host" - fi -done - -echo Machines: $hl - -#-i input.dat includes the input of analysis in APDL format -#-o file.out is output file from ansys where all text outputs will be redirected -#-p the name of license feature (aa_r=ANSYS Academic Research, ane3fl=Multiphysics(commercial), aa_r_dy=Academic AUTODYN) -/ansys_inc/v145/ansys/bin/ansys145 -b -dis -p aa_r -i input.dat -o file.out -machines $hl -dir $WORK_DIR -``` - -Header of the pbs file (above) is common and description can be find on [this site](../../resources-allocation-policy/). [SVS FEM](http://www.svsfem.cz) recommends to utilize sources by keywords: nodes, ppn. These keywords allows to address directly the number of nodes (computers) and cores (ppn) which will be utilized in the job. Also the rest of code assumes such structure of allocated resources. - -Working directory has to be created before sending pbs job into the queue. Input file should be in working directory or full path to input file has to be specified. Input file has to be defined by common APDL file which is attached to the ansys solver via parameter -i - -**License** should be selected by parameter -p. Licensed products are the following: aa_r (ANSYS **Academic** Research), ane3fl (ANSYS Multiphysics)-**Commercial**, aa_r_dy (ANSYS **Academic** AUTODYN) [More about licensing here](licensing/) diff --git a/docs.it4i/salomon/software/ansys/ansys.md b/docs.it4i/salomon/software/ansys/ansys.md deleted file mode 100644 index d7e0f2e14..000000000 --- a/docs.it4i/salomon/software/ansys/ansys.md +++ /dev/null @@ -1,15 +0,0 @@ -# Overview of ANSYS Products - -**[SVS FEM](http://www.svsfem.cz/)** as **[ANSYS Channel partner](http://www.ansys.com/)** for Czech Republic provided all ANSYS licenses for ANSELM cluster and supports of all ANSYS Products (Multiphysics, Mechanical, MAPDL, CFX, Fluent, Maxwell, LS-DYNA...) to IT staff and ANSYS users. If you are challenging to problem of ANSYS functionality contact please [hotline@svsfem.cz](mailto:hotline@svsfem.cz?subject=Ostrava%20-%20ANSELM) - -Anselm provides as commercial as academic variants. Academic variants are distinguished by "**Academic...**" word in the name of license or by two letter preposition "**aa\_**" in the license feature name. Change of license is realized on command line respectively directly in user's pbs file (see individual products). [More about licensing here](licensing/) - -To load the latest version of any ANSYS product (Mechanical, Fluent, CFX, MAPDL,...) load the module: - -```console -$ ml ansys -``` - -ANSYS supports interactive regime, but due to assumed solution of extremely difficult tasks it is not recommended. - -If user needs to work in interactive regime we recommend to configure the RSM service on the client machine which allows to forward the solution to the Anselm directly from the client's Workbench project (see ANSYS RSM service). diff --git a/docs.it4i/salomon/software/ansys/licensing.md b/docs.it4i/salomon/software/ansys/licensing.md deleted file mode 100644 index eac78966d..000000000 --- a/docs.it4i/salomon/software/ansys/licensing.md +++ /dev/null @@ -1,25 +0,0 @@ -# Licensing and Available Versions - -## ANSYS Licence Can Be Used By: - -* all persons in the carrying out of the CE IT4Innovations Project (In addition to the primary licensee, which is VSB - Technical University of Ostrava, users are CE IT4Innovations third parties - CE IT4Innovations project partners, particularly the University of Ostrava, the Brno University of Technology - Faculty of Informatics, the Silesian University in Opava, Institute of Geonics AS CR.) -* all persons who have a valid license -* students of the Technical University - -## ANSYS Academic Research - -The licence intended to be used for science and research, publications, students’ projects (academic licence). - -## ANSYS COM - -The licence intended to be used for science and research, publications, students’ projects, commercial research with no commercial use restrictions. - -## Available Versions - -* 16.1 -* 17.0 -* 18.0 - -## License Preferences - -Please [see this page to set license preferences](setting-license-preferences/). diff --git a/docs.it4i/salomon/software/ansys/setting-license-preferences.md b/docs.it4i/salomon/software/ansys/setting-license-preferences.md deleted file mode 100644 index b3f594d14..000000000 --- a/docs.it4i/salomon/software/ansys/setting-license-preferences.md +++ /dev/null @@ -1,23 +0,0 @@ -# Setting license preferences - -Some ANSYS tools allow you to explicitly specify usage of academic or commercial licenses in the command line (eg. ansys161 -p aa_r to select Academic Research license). However, we have observed that not all tools obey this option and choose commercial license. - -Thus you need to configure preferred license order with ANSLIC_ADMIN. Please follow these steps and move Academic Research license to the top or bottom of the list accordingly. - -Launch the ANSLIC_ADMIN utility in a graphical environment: - -```console -$ANSYSLIC_DIR/lic_admin/anslic_admin -``` - -ANSLIC_ADMIN Utility will be run - - - - - - - -ANSYS Academic Research license should be moved up to the top or down to the bottom of the list. - - diff --git a/docs.it4i/salomon/software/ansys/workbench.md b/docs.it4i/salomon/software/ansys/workbench.md deleted file mode 100644 index 1b138ccd0..000000000 --- a/docs.it4i/salomon/software/ansys/workbench.md +++ /dev/null @@ -1,62 +0,0 @@ -# Workbench - -## Workbench Batch Mode - -It is possible to run Workbench scripts in batch mode. You need to configure solvers of individual components to run in parallel mode. Open your project in Workbench. Then, for example, in Mechanical, go to Tools - Solve Process Settings ... - - - -Enable Distribute Solution checkbox and enter number of cores (eg. 48 to run on two Salomon nodes). If you want the job to run on more then 1 node, you must also provide a so called MPI appfile. In the Additional Command Line Arguments input field, enter: - -```console - -mpifile /path/to/my/job/mpifile.txt -``` - -Where /path/to/my/job is the directory where your project is saved. We will create the file mpifile.txt programatically later in the batch script. For more information, refer to \*ANSYS Mechanical APDL Parallel Processing\* \*Guide\*. - -Now, save the project and close Workbench. We will use this script to launch the job: - -```bash - #!/bin/bash - #PBS -l select=2:ncpus=24 - #PBS -q qprod - #PBS -N test9_mpi_2 - #PBS -A OPEN-0-0 - - # Mail to user when job terminate or abort - #PBS -m a - - # change the working directory - WORK_DIR="$PBS_O_WORKDIR" - cd $WORK_DIR - - echo Running on host `hostname` - echo Time is `date` - echo Directory is `pwd` - echo This jobs runs on the following nodes: - echo `cat $PBS_NODEFILE` - - module load ANSYS - - #### Set number of processors per host listing - procs_per_host=24 - #### Create MPI appfile - echo -n "" > mpifile.txt - for host in `cat $PBS_NODEFILE` - do - echo "-h $host -np $procs_per_host $ANSYS160_DIR/bin/ansysdis161 -dis" > mpifile.txt - done - - #-i input.dat includes the input of analysis in APDL format - #-o file.out is output file from ansys where all text outputs will be redirected - #-p the name of license feature (aa_r=ANSYS Academic Research, ane3fl=Multiphysics(commercial), aa_r_dy=Academic AUTODYN) - - # prevent using scsif0 interface on accelerated nodes - export MPI_IC_ORDER="UDAPL" - # spawn remote process using SSH (default is RSH) - export MPI_REMSH="/usr/bin/ssh" - - runwb2 -R jou6.wbjn -B -F test9.wbpj -``` - -The solver settings are saved in file solvehandlers.xml, which is not located in the project directory. Verify your solved settings when uploading a project from your local computer. diff --git a/docs.it4i/salomon/software/chemistry/INCAR b/docs.it4i/salomon/software/chemistry/INCAR deleted file mode 100644 index 4d2848958..000000000 --- a/docs.it4i/salomon/software/chemistry/INCAR +++ /dev/null @@ -1,13 +0,0 @@ - PREC = Accurate - IBRION = -1 - ENCUT = 500 - EDIFF = 1.0e-08 - ISMEAR = 0 - SIGMA = 0.01 - IALGO = 38 - LREAL = .FALSE. - ADDGRID = .TRUE. - LWAVE = .FALSE. - LCHARG = .FALSE. - NCORE = 8 - KPAR = 8 diff --git a/docs.it4i/salomon/software/chemistry/KPOINTS b/docs.it4i/salomon/software/chemistry/KPOINTS deleted file mode 100644 index cae740039..000000000 --- a/docs.it4i/salomon/software/chemistry/KPOINTS +++ /dev/null @@ -1,6 +0,0 @@ -Automatic mesh -0 -Monkhorst Pack - 3 3 3 -0.5 0.5 0.5 - diff --git a/docs.it4i/salomon/software/chemistry/POSCAR b/docs.it4i/salomon/software/chemistry/POSCAR deleted file mode 100644 index f72a61b99..000000000 --- a/docs.it4i/salomon/software/chemistry/POSCAR +++ /dev/null @@ -1,16 +0,0 @@ - Si - 1.0 - 5.4335600309153529 0.0000000000000000 0.0000000000000000 - 0.0000000000000000 5.4335600309153529 0.0000000000000000 - 0.0000000000000000 0.0000000000000000 5.4335600309153529 - Si - 8 -Direct - 0.8750000000000000 0.8750000000000000 0.8750000000000000 - 0.8750000000000000 0.3750000000000000 0.3750000000000000 - 0.3750000000000000 0.8750000000000000 0.3750000000000000 - 0.3750000000000000 0.3750000000000000 0.8750000000000000 - 0.1250000000000000 0.1250000000000000 0.1250000000000000 - 0.1250000000000000 0.6250000000000000 0.6250000000000000 - 0.6250000000000000 0.1250000000000000 0.6250000000000000 - 0.6250000000000000 0.6250000000000000 0.1250000000000000 diff --git a/docs.it4i/salomon/software/chemistry/POTCAR b/docs.it4i/salomon/software/chemistry/POTCAR deleted file mode 100644 index e69e8a304..000000000 --- a/docs.it4i/salomon/software/chemistry/POTCAR +++ /dev/null @@ -1,2208 +0,0 @@ - PAW_PBE Si 05Jan2001 - 4.00000000000000 - parameters from PSCTR are: - VRHFIN =Si: s2p2 - LEXCH = PE - EATOM = 103.0669 eV, 7.5752 Ry - - TITEL = PAW_PBE Si 05Jan2001 - LULTRA = F use ultrasoft PP ? - IUNSCR = 1 unscreen: 0-lin 1-nonlin 2-no - RPACOR = 1.500 partial core radius - POMASS = 28.085; ZVAL = 4.000 mass and valenz - RCORE = 1.900 outmost cutoff radius - RWIGS = 2.480; RWIGS = 1.312 wigner-seitz radius (au A) - ENMAX = 245.345; ENMIN = 184.009 eV - ICORE = 2 local potential - LCOR = T correct aug charges - LPAW = T paw PP - EAUG = 322.069 - DEXC = 0.000 - RMAX = 1.950 core radius for proj-oper - RAUG = 1.300 factor for augmentation sphere - RDEP = 1.993 radius for radial grids - RDEPT = 1.837 core radius for aug-charge - - Atomic configuration - 6 entries - n l j E occ. - 1 0 0.50 -1785.8828 2.0000 - 2 0 0.50 -139.4969 2.0000 - 2 1 1.50 -95.5546 6.0000 - 3 0 0.50 -10.8127 2.0000 - 3 1 0.50 -4.0811 2.0000 - 3 2 1.50 -4.0817 0.0000 - Description - l E TYP RCUT TYP RCUT - 0 -10.8127223 23 1.900 - 0 -7.6451159 23 1.900 - 1 -4.0811372 23 1.900 - 1 2.4879257 23 1.900 - 2 -4.0817478 7 1.900 - Error from kinetic energy argument (eV) - NDATA = 100 - STEP = 20.000 1.050 - 10.1 9.04 8.56 7.65 7.23 6.44 5.73 5.40 - 4.79 4.25 4.00 3.54 3.13 2.77 2.45 2.16 - 1.91 1.69 1.50 1.24 1.10 0.975 0.812 0.718 - 0.636 0.529 0.440 0.388 0.322 0.266 0.219 0.180 - 0.148 0.121 0.986E-01 0.804E-01 0.614E-01 0.504E-01 0.392E-01 0.328E-01 - 0.265E-01 0.220E-01 0.189E-01 0.166E-01 0.149E-01 0.135E-01 0.123E-01 0.109E-01 - 0.977E-02 0.840E-02 0.707E-02 0.605E-02 0.488E-02 0.387E-02 0.290E-02 0.229E-02 - 0.185E-02 0.152E-02 0.134E-02 0.125E-02 0.121E-02 0.117E-02 0.112E-02 0.102E-02 - 0.915E-03 0.776E-03 0.640E-03 0.524E-03 0.425E-03 0.369E-03 0.331E-03 0.310E-03 - 0.294E-03 0.273E-03 0.242E-03 0.210E-03 0.175E-03 0.146E-03 0.124E-03 0.113E-03 - 0.105E-03 0.973E-04 0.879E-04 0.755E-04 0.633E-04 0.539E-04 0.478E-04 0.438E-04 - 0.404E-04 0.362E-04 0.308E-04 0.264E-04 0.229E-04 0.209E-04 0.192E-04 0.170E-04 - 0.145E-04 0.126E-04 0.112E-04 0.103E-04 -END of PSCTR-controll parameters - local part - 98.2657514061040 - 0.84157827E+01 0.84210738E+01 0.84276982E+01 0.84387529E+01 0.84542581E+01 - 0.84742396E+01 0.84987267E+01 0.85277503E+01 0.85613408E+01 0.85995259E+01 - 0.86423293E+01 0.86897698E+01 0.87418601E+01 0.87986073E+01 0.88600119E+01 - 0.89260684E+01 0.89967646E+01 0.90720817E+01 0.91519931E+01 0.92364635E+01 - 0.93254478E+01 0.94188892E+01 0.95167180E+01 0.96188495E+01 0.97251831E+01 - 0.98356006E+01 0.99499656E+01 0.10068122E+02 0.10189896E+02 0.10315092E+02 - 0.10443494E+02 0.10574869E+02 0.10708960E+02 0.10845494E+02 0.10984175E+02 - 0.11124688E+02 0.11266699E+02 0.11409854E+02 0.11553782E+02 0.11698093E+02 - 0.11842379E+02 0.11986220E+02 0.12129179E+02 0.12270807E+02 0.12410647E+02 - 0.12548229E+02 0.12683078E+02 0.12814715E+02 0.12942655E+02 0.13066413E+02 - 0.13185507E+02 0.13299453E+02 0.13407778E+02 0.13510011E+02 0.13605696E+02 - 0.13694386E+02 0.13775649E+02 0.13849071E+02 0.13914257E+02 0.13970832E+02 - 0.14018447E+02 0.14056776E+02 0.14085521E+02 0.14104413E+02 0.14113214E+02 - 0.14111717E+02 0.14099750E+02 0.14077174E+02 0.14043887E+02 0.13999823E+02 - 0.13944953E+02 0.13879287E+02 0.13802871E+02 0.13715791E+02 0.13618172E+02 - 0.13510174E+02 0.13391995E+02 0.13263871E+02 0.13126072E+02 0.12978902E+02 - 0.12822701E+02 0.12657837E+02 0.12484712E+02 0.12303752E+02 0.12115414E+02 - 0.11920177E+02 0.11718542E+02 0.11511032E+02 0.11298186E+02 0.11080557E+02 - 0.10858712E+02 0.10633228E+02 0.10404687E+02 0.10173679E+02 0.99407916E+01 - 0.97066145E+01 0.94717325E+01 0.92367246E+01 0.90021610E+01 0.87686002E+01 - 0.85365875E+01 0.83066515E+01 0.80793028E+01 0.78550311E+01 0.76343038E+01 - 0.74175639E+01 0.72052283E+01 0.69976864E+01 0.67952987E+01 0.65983955E+01 - 0.64072761E+01 0.62222079E+01 0.60434255E+01 0.58711305E+01 0.57054914E+01 - 0.55466428E+01 0.53946864E+01 0.52496904E+01 0.51116908E+01 0.49806913E+01 - 0.48566647E+01 0.47395535E+01 0.46292714E+01 0.45257040E+01 0.44287107E+01 - 0.43381259E+01 0.42537609E+01 0.41754051E+01 0.41028283E+01 0.40357824E+01 - 0.39740032E+01 0.39172126E+01 0.38651205E+01 0.38174269E+01 0.37738239E+01 - 0.37339980E+01 0.36976316E+01 0.36644057E+01 0.36340010E+01 0.36061008E+01 - 0.35803917E+01 0.35565662E+01 0.35343243E+01 0.35133744E+01 0.34934356E+01 - 0.34742386E+01 0.34555273E+01 0.34370594E+01 0.34186082E+01 0.33999625E+01 - 0.33809283E+01 0.33613289E+01 0.33410051E+01 0.33198163E+01 0.32976400E+01 - 0.32743723E+01 0.32499276E+01 0.32242386E+01 0.31972561E+01 0.31689481E+01 - 0.31393000E+01 0.31083133E+01 0.30760055E+01 0.30424088E+01 0.30075693E+01 - 0.29715464E+01 0.29344112E+01 0.28962459E+01 0.28571426E+01 0.28172018E+01 - 0.27765317E+01 0.27352466E+01 0.26934661E+01 0.26513137E+01 0.26089155E+01 - 0.25663991E+01 0.25238926E+01 0.24815235E+01 0.24394172E+01 0.23976965E+01 - 0.23564804E+01 0.23158831E+01 0.22760134E+01 0.22369739E+01 0.21988601E+01 - 0.21617599E+01 0.21257533E+01 0.20909117E+01 0.20572975E+01 0.20249639E+01 - 0.19939549E+01 0.19643047E+01 0.19360380E+01 0.19091702E+01 0.18837070E+01 - 0.18596451E+01 0.18369721E+01 0.18156671E+01 0.17957010E+01 0.17770369E+01 - 0.17596305E+01 0.17434311E+01 0.17283815E+01 0.17144192E+01 0.17014767E+01 - 0.16894823E+01 0.16783606E+01 0.16680337E+01 0.16584212E+01 0.16494414E+01 - 0.16410120E+01 0.16330503E+01 0.16254746E+01 0.16182040E+01 0.16111597E+01 - 0.16042653E+01 0.15974473E+01 0.15906355E+01 0.15837638E+01 0.15767703E+01 - 0.15695978E+01 0.15621942E+01 0.15545124E+01 0.15465111E+01 0.15381545E+01 - 0.15294125E+01 0.15202610E+01 0.15106817E+01 0.15006618E+01 0.14901946E+01 - 0.14792787E+01 0.14679181E+01 0.14561220E+01 0.14439045E+01 0.14312843E+01 - 0.14182845E+01 0.14049320E+01 0.13912573E+01 0.13772941E+01 0.13630790E+01 - 0.13486508E+01 0.13340503E+01 0.13193195E+01 0.13045019E+01 0.12896413E+01 - 0.12747816E+01 0.12599667E+01 0.12452398E+01 0.12306429E+01 0.12162169E+01 - 0.12020007E+01 0.11880315E+01 0.11743438E+01 0.11609698E+01 0.11479386E+01 - 0.11352766E+01 0.11230067E+01 0.11111486E+01 0.10997185E+01 0.10887293E+01 - 0.10781901E+01 0.10681067E+01 0.10584812E+01 0.10493126E+01 0.10405963E+01 - 0.10323245E+01 0.10244864E+01 0.10170684E+01 0.10100541E+01 0.10034245E+01 - 0.99715842E+00 0.99123278E+00 0.98562260E+00 0.98030147E+00 0.97524176E+00 - 0.97041492E+00 0.96579177E+00 0.96134272E+00 0.95703813E+00 0.95284851E+00 - 0.94874481E+00 0.94469866E+00 0.94068261E+00 0.93667036E+00 0.93263699E+00 - 0.92855911E+00 0.92441506E+00 0.92018508E+00 0.91585139E+00 0.91139837E+00 - 0.90681257E+00 0.90208284E+00 0.89720031E+00 0.89215846E+00 0.88695307E+00 - 0.88158221E+00 0.87604623E+00 0.87034764E+00 0.86449105E+00 0.85848310E+00 - 0.85233228E+00 0.84604885E+00 0.83964467E+00 0.83313304E+00 0.82652853E+00 - 0.81984678E+00 0.81310436E+00 0.80631854E+00 0.79950711E+00 0.79268821E+00 - 0.78588011E+00 0.77910103E+00 0.77236900E+00 0.76570163E+00 0.75911595E+00 - 0.75262828E+00 0.74625406E+00 0.74000769E+00 0.73390245E+00 0.72795034E+00 - 0.72216201E+00 0.71654666E+00 0.71111201E+00 0.70586419E+00 0.70080779E+00 - 0.69594576E+00 0.69127948E+00 0.68680873E+00 0.68253171E+00 0.67844512E+00 - 0.67454419E+00 0.67082277E+00 0.66727336E+00 0.66388728E+00 0.66065470E+00 - 0.65756482E+00 0.65460595E+00 0.65176565E+00 0.64903085E+00 0.64638804E+00 - 0.64382333E+00 0.64132264E+00 0.63887182E+00 0.63645681E+00 0.63406371E+00 - 0.63167898E+00 0.62928953E+00 0.62688282E+00 0.62444703E+00 0.62197109E+00 - 0.61944485E+00 0.61685909E+00 0.61420564E+00 0.61147745E+00 0.60866857E+00 - 0.60577426E+00 0.60279097E+00 0.59971637E+00 0.59654936E+00 0.59329001E+00 - 0.58993961E+00 0.58650059E+00 0.58297648E+00 0.57937187E+00 0.57569237E+00 - 0.57194448E+00 0.56813556E+00 0.56427372E+00 0.56036773E+00 0.55642691E+00 - 0.55246106E+00 0.54848032E+00 0.54449508E+00 0.54051586E+00 0.53655323E+00 - 0.53261770E+00 0.52871958E+00 0.52486894E+00 0.52107546E+00 0.51734836E+00 - 0.51369633E+00 0.51012743E+00 0.50664899E+00 0.50326761E+00 0.49998904E+00 - 0.49681819E+00 0.49375905E+00 0.49081466E+00 0.48798714E+00 0.48527765E+00 - 0.48268635E+00 0.48021249E+00 0.47785436E+00 0.47560937E+00 0.47347400E+00 - 0.47144395E+00 0.46951410E+00 0.46767862E+00 0.46593101E+00 0.46426418E+00 - 0.46267051E+00 0.46114196E+00 0.45967012E+00 0.45824631E+00 0.45686165E+00 - 0.45550718E+00 0.45417389E+00 0.45285284E+00 0.45153526E+00 0.45021257E+00 - 0.44887651E+00 0.44751921E+00 0.44613322E+00 0.44471163E+00 0.44324809E+00 - 0.44173687E+00 0.44017292E+00 0.43855189E+00 0.43687018E+00 0.43512494E+00 - 0.43331410E+00 0.43143638E+00 0.42949128E+00 0.42747908E+00 0.42540083E+00 - 0.42325834E+00 0.42105410E+00 0.41879135E+00 0.41647393E+00 0.41410629E+00 - 0.41169346E+00 0.40924092E+00 0.40675463E+00 0.40424087E+00 0.40170626E+00 - 0.39915764E+00 0.39660202E+00 0.39404651E+00 0.39149826E+00 0.38896435E+00 - 0.38645179E+00 0.38396739E+00 0.38151775E+00 0.37910915E+00 0.37674752E+00 - 0.37443839E+00 0.37218682E+00 0.36999739E+00 0.36787410E+00 0.36582042E+00 - 0.36383920E+00 0.36193268E+00 0.36010247E+00 0.35834954E+00 0.35667423E+00 - 0.35507622E+00 0.35355459E+00 0.35210779E+00 0.35073369E+00 0.34942958E+00 - 0.34819223E+00 0.34701790E+00 0.34590241E+00 0.34484115E+00 0.34382917E+00 - 0.34286120E+00 0.34193171E+00 0.34103498E+00 0.34016513E+00 0.33931621E+00 - 0.33848221E+00 0.33765717E+00 0.33683518E+00 0.33601049E+00 0.33517751E+00 - 0.33433089E+00 0.33346559E+00 0.33257685E+00 0.33166032E+00 0.33071204E+00 - 0.32972848E+00 0.32870659E+00 0.32764378E+00 0.32653800E+00 0.32538767E+00 - 0.32419176E+00 0.32294975E+00 0.32166166E+00 0.32032801E+00 0.31894983E+00 - 0.31752862E+00 0.31606638E+00 0.31456552E+00 0.31302889E+00 0.31145970E+00 - 0.30986149E+00 0.30823814E+00 0.30659375E+00 0.30493266E+00 0.30325937E+00 - 0.30157850E+00 0.29989478E+00 0.29821295E+00 0.29653774E+00 0.29487384E+00 - 0.29322581E+00 0.29159810E+00 0.28999494E+00 0.28842037E+00 0.28687815E+00 - 0.28537173E+00 0.28390425E+00 0.28247850E+00 0.28109688E+00 0.27976142E+00 - 0.27847372E+00 0.27723499E+00 0.27604598E+00 0.27490705E+00 0.27381811E+00 - 0.27277867E+00 0.27178782E+00 0.27084424E+00 0.26994624E+00 0.26909178E+00 - 0.26827846E+00 0.26750359E+00 0.26676420E+00 0.26605705E+00 0.26537872E+00 - 0.26472559E+00 0.26409390E+00 0.26347982E+00 0.26287940E+00 0.26228871E+00 - 0.26170380E+00 0.26112078E+00 0.26053582E+00 0.25994525E+00 0.25934552E+00 - 0.25873328E+00 0.25810539E+00 0.25745897E+00 0.25679138E+00 0.25610030E+00 - 0.25538370E+00 0.25463989E+00 0.25386749E+00 0.25306548E+00 0.25223319E+00 - 0.25137030E+00 0.25047681E+00 0.24955311E+00 0.24859991E+00 0.24761822E+00 - 0.24660941E+00 0.24557510E+00 0.24451722E+00 0.24343793E+00 0.24233963E+00 - 0.24122491E+00 0.24009654E+00 0.23895742E+00 0.23781058E+00 0.23665912E+00 - 0.23550619E+00 0.23435497E+00 0.23320861E+00 0.23207023E+00 0.23094290E+00 - 0.22982954E+00 0.22873298E+00 0.22765588E+00 0.22660071E+00 0.22556975E+00 - 0.22456503E+00 0.22358838E+00 0.22264133E+00 0.22172517E+00 0.22084090E+00 - 0.21998924E+00 0.21917062E+00 0.21838518E+00 0.21763278E+00 0.21691298E+00 - 0.21622507E+00 0.21556808E+00 0.21494076E+00 0.21434166E+00 0.21376905E+00 - 0.21322105E+00 0.21269556E+00 0.21219033E+00 0.21170298E+00 0.21123101E+00 - 0.21077184E+00 0.21032282E+00 0.20988128E+00 0.20944453E+00 0.20900989E+00 - 0.20857473E+00 0.20813650E+00 0.20769272E+00 0.20724104E+00 0.20677924E+00 - 0.20630528E+00 0.20581727E+00 0.20531352E+00 0.20479256E+00 0.20425312E+00 - 0.20369419E+00 0.20311496E+00 0.20251488E+00 0.20189363E+00 0.20125114E+00 - 0.20058759E+00 0.19990339E+00 0.19919917E+00 0.19847580E+00 0.19773436E+00 - 0.19697612E+00 0.19620254E+00 0.19541525E+00 0.19461602E+00 0.19380675E+00 - 0.19298946E+00 0.19216623E+00 0.19133925E+00 0.19051071E+00 0.18968285E+00 - 0.18885789E+00 0.18803806E+00 0.18722550E+00 0.18642234E+00 0.18563058E+00 - 0.18485213E+00 0.18408879E+00 0.18334221E+00 0.18261386E+00 0.18190508E+00 - 0.18121702E+00 0.18055061E+00 0.17990663E+00 0.17928561E+00 0.17868791E+00 - 0.17811368E+00 0.17756283E+00 0.17703511E+00 0.17653003E+00 0.17604693E+00 - 0.17558494E+00 0.17514305E+00 0.17472004E+00 0.17431456E+00 0.17392512E+00 - 0.17355012E+00 0.17318784E+00 0.17283649E+00 0.17249419E+00 0.17215904E+00 - 0.17182910E+00 0.17150240E+00 0.17117700E+00 0.17085099E+00 0.17052249E+00 - 0.17018968E+00 0.16985083E+00 0.16950432E+00 0.16914863E+00 0.16878235E+00 - 0.16840425E+00 0.16801322E+00 0.16760831E+00 0.16718876E+00 0.16675395E+00 - 0.16630348E+00 0.16583708E+00 0.16535470E+00 0.16485644E+00 0.16434260E+00 - 0.16381364E+00 0.16327019E+00 0.16271304E+00 0.16214313E+00 0.16156154E+00 - 0.16096948E+00 0.16036827E+00 0.15975933E+00 0.15914418E+00 0.15852437E+00 - 0.15790155E+00 0.15727738E+00 0.15665353E+00 0.15603169E+00 0.15541354E+00 - 0.15480073E+00 0.15419485E+00 0.15359745E+00 0.15300998E+00 0.15243381E+00 - 0.15187022E+00 0.15132035E+00 0.15078523E+00 0.15026574E+00 0.14976264E+00 - 0.14927652E+00 0.14880782E+00 0.14835684E+00 0.14792369E+00 0.14750835E+00 - 0.14711063E+00 0.14673020E+00 0.14636656E+00 0.14601907E+00 0.14568697E+00 - 0.14536935E+00 0.14506518E+00 0.14477334E+00 0.14449260E+00 0.14422166E+00 - 0.14395913E+00 0.14370359E+00 0.14345355E+00 0.14320753E+00 0.14296401E+00 - 0.14272149E+00 0.14247846E+00 0.14223347E+00 0.14198509E+00 0.14173199E+00 - 0.14147286E+00 0.14120650E+00 0.14093181E+00 0.14064779E+00 0.14035355E+00 - 0.14004833E+00 0.13973149E+00 0.13940254E+00 0.13906109E+00 0.13870694E+00 - 0.13833999E+00 0.13796029E+00 0.13756804E+00 0.13716357E+00 0.13674733E+00 - 0.13631991E+00 0.13588203E+00 0.13543449E+00 0.13497823E+00 0.13451425E+00 - 0.13404367E+00 0.13356764E+00 0.13308739E+00 0.13260420E+00 0.13211936E+00 - 0.13163422E+00 0.13115008E+00 0.13066829E+00 0.13019015E+00 0.12971693E+00 - 0.12924988E+00 0.12879016E+00 0.12833890E+00 0.12789711E+00 0.12746576E+00 - 0.12704568E+00 0.12663761E+00 0.12624221E+00 0.12585997E+00 0.12549131E+00 - 0.12513650E+00 0.12479568E+00 0.12446890E+00 0.12415605E+00 0.12385692E+00 - 0.12357117E+00 0.12329835E+00 0.12303789E+00 0.12278912E+00 0.12255128E+00 - 0.12232351E+00 0.12210488E+00 0.12189437E+00 0.12169094E+00 0.12149345E+00 - 0.12130077E+00 0.12111171E+00 0.12092509E+00 0.12073970E+00 0.12055436E+00 - 0.12036791E+00 0.12017920E+00 0.11998713E+00 0.11979067E+00 0.11958882E+00 - 0.11938067E+00 0.11916539E+00 0.11894223E+00 0.11871054E+00 0.11846977E+00 - 0.11821946E+00 0.11795928E+00 0.11768898E+00 0.11740846E+00 0.11711769E+00 - 0.11681676E+00 0.11650590E+00 0.11618540E+00 0.11585569E+00 0.11551727E+00 - 0.11517075E+00 0.11481682E+00 0.11445624E+00 0.11408986E+00 0.11371856E+00 - 0.11334329E+00 0.11296505E+00 0.11258484E+00 0.11220371E+00 0.11182270E+00 - 0.11144287E+00 0.11106526E+00 0.11069088E+00 0.11032073E+00 0.10995575E+00 - 0.10959686E+00 0.10924489E+00 0.10890064E+00 0.10856481E+00 0.10823802E+00 - 0.10792083E+00 0.10761369E+00 0.10731697E+00 0.10703093E+00 0.10675575E+00 - 0.10649150E+00 0.10623816E+00 0.10599560E+00 0.10576362E+00 0.10554191E+00 - 0.10533006E+00 0.10512760E+00 0.10493397E+00 0.10474853E+00 0.10457057E+00 - 0.10439934E+00 0.10423403E+00 0.10407377E+00 0.10391769E+00 0.10376486E+00 - 0.10361436E+00 0.10346525E+00 0.10331660E+00 0.10316749E+00 0.10301700E+00 - 0.10286427E+00 0.10270845E+00 0.10254875E+00 0.10238443E+00 0.10221481E+00 - 0.10203926E+00 0.10185725E+00 0.10166830E+00 0.10147204E+00 0.10126815E+00 - 0.10105642E+00 0.10083671E+00 0.10060898E+00 0.10037327E+00 0.10012971E+00 - 0.99878499E-01 0.99619941E-01 0.99354403E-01 0.99082329E-01 0.98804234E-01 - 0.98520694E-01 0.98232343E-01 0.97939865E-01 0.97643987E-01 0.97345468E-01 - 0.97045098E-01 0.96743682E-01 0.96442039E-01 0.96140988E-01 0.95841346E-01 - 0.95543917E-01 0.95249483E-01 0.94958801E-01 0.94672589E-01 0.94391526E-01 - 0.94116241E-01 0.93847307E-01 0.93585238E-01 0.93330481E-01 0.93083416E-01 - 0.92844349E-01 0.92613514E-01 0.92391067E-01 0.92177088E-01 0.91971582E-01 - 0.91774473E-01 0.91585615E-01 0.91404784E-01 0.91231686E-01 0.91065960E-01 - 0.90907179E-01 0.90754858E-01 0.90608455E-01 0.90467383E-01 0.90331010E-01 - 0.90198671E-01 0.90069670E-01 0.89943293E-01 0.89818810E-01 0.89695483E-01 - 0.89572578E-01 0.89449363E-01 0.89325125E-01 0.89199170E-01 0.89070830E-01 - 0.88939475E-01 0.88804513E-01 0.88665401E-01 0.88521645E-01 0.88372809E-01 - 0.88218518E-01 0.88058459E-01 0.87892386E-01 0.87720120E-01 0.87541552E-01 - 0.87356640E-01 0.87165411E-01 0.86967963E-01 0.86764459E-01 0.86555125E-01 - 0.86340254E-01 0.86120194E-01 0.85895352E-01 0.85666183E-01 0.85433191E-01 - 0.85196919E-01 0.84957943E-01 0.84716871E-01 0.84474330E-01 0.84230965E-01 - 0.83987427E-01 0.83744375E-01 0.83502460E-01 0.83262325E-01 0.83024599E-01 - 0.82789886E-01 0.82558766E-01 0.82331783E-01 0.82109442E-01 0.81892206E-01 - 0.81680487E-01 0.81474646E-01 0.81274989E-01 0.81081762E-01 0.80895153E-01 - 0.80715285E-01 0.80542223E-01 0.80375965E-01 0.80216452E-01 0.80063558E-01 - 0.79917102E-01 0.79776842E-01 0.79642482E-01 0.79513673E-01 0.79390016E-01 - 0.79271070E-01 0.79156353E-01 0.79045348E-01 0.78937510E-01 0.78832270E-01 - 0.78729044E-01 0.78627233E-01 0.78526236E-01 0.78425450E-01 0.78324279E-01 - 0.78222140E-01 0.78118466E-01 0.78012713E-01 0.77904366E-01 0.77792943E-01 - 0.77678000E-01 0.77559135E-01 0.77435994E-01 0.77308271E-01 0.77175713E-01 - 0.77038121E-01 0.76895353E-01 0.76747321E-01 0.76593998E-01 0.76435410E-01 - 0.76271640E-01 0.76102829E-01 0.75929167E-01 0.75750899E-01 0.75568317E-01 - gradient corrections used for XC - 5 - core charge-density (partial) - 0.13681949E+01 0.13676959E+01 0.13662000E+01 0.13637104E+01 0.13602324E+01 - 0.13557734E+01 0.13503428E+01 0.13439523E+01 0.13366153E+01 0.13283472E+01 - 0.13191654E+01 0.13090891E+01 0.12981392E+01 0.12863383E+01 0.12737106E+01 - 0.12602819E+01 0.12460793E+01 0.12311313E+01 0.12154677E+01 0.11991193E+01 - 0.11821181E+01 0.11644969E+01 0.11462894E+01 0.11275300E+01 0.11082538E+01 - 0.10884961E+01 0.10682931E+01 0.10476808E+01 0.10266956E+01 0.10053741E+01 - 0.98375262E+00 0.96186755E+00 0.93975498E+00 0.91745072E+00 0.89499016E+00 - 0.87240821E+00 0.84973922E+00 0.82701686E+00 0.80427411E+00 0.78154316E+00 - 0.75885532E+00 0.73624100E+00 0.71372964E+00 0.69134965E+00 0.66912839E+00 - 0.64709210E+00 0.62526589E+00 0.60367368E+00 0.58233822E+00 0.56128101E+00 - 0.54052235E+00 0.52008128E+00 0.49997557E+00 0.48022176E+00 0.46083511E+00 - 0.44182966E+00 0.42321818E+00 0.40501223E+00 0.38722214E+00 0.36985705E+00 - 0.35292494E+00 0.33643261E+00 0.32038576E+00 0.30478900E+00 0.28964586E+00 - 0.27495885E+00 0.26072948E+00 0.24695831E+00 0.23364497E+00 0.22078821E+00 - 0.20838595E+00 0.19643529E+00 0.18493257E+00 0.17387344E+00 0.16325283E+00 - 0.15306506E+00 0.14330386E+00 0.13396239E+00 0.12503331E+00 0.11650882E+00 - 0.10838067E+00 0.10064023E+00 0.93278518E-01 0.86286246E-01 0.79653843E-01 - 0.73371502E-01 0.67429213E-01 0.61816795E-01 0.56523931E-01 0.51540200E-01 - 0.46855106E-01 0.42458107E-01 0.38338646E-01 0.34486172E-01 0.30890172E-01 - 0.27540190E-01 0.24425849E-01 0.21536878E-01 0.18863124E-01 0.16394576E-01 - 0.14121382E-01 0.12033859E-01 0.10122515E-01 0.83780585E-02 0.67914089E-02 - 0.53537106E-02 0.40563407E-02 0.28909176E-02 0.18493085E-02 0.92363542E-03 - 0.10628041E-03 -0.61011040E-03 -0.12326236E-02 -0.17680763E-02 -0.22230145E-02 - -0.26037132E-02 -0.29161761E-02 -0.31661366E-02 -0.33590602E-02 -0.35001461E-02 - -0.35943308E-02 -0.36462914E-02 -0.36604499E-02 -0.36409778E-02 -0.35918010E-02 - -0.35166057E-02 -0.34188437E-02 -0.33017394E-02 -0.31682955E-02 -0.30213003E-02 - -0.28633343E-02 -0.26967779E-02 -0.25238181E-02 -0.23464567E-02 -0.21665170E-02 - -0.19856524E-02 -0.18053531E-02 -0.16269545E-02 -0.14516446E-02 -0.12804716E-02 - -0.11143515E-02 -0.95407586E-03 -0.80031865E-03 -0.65364404E-03 -0.51451333E-03 - -0.38329200E-03 -0.26025655E-03 -0.14560117E-03 -0.39444206E-04 0.58165511E-04 - 0.14724262E-03 0.22785936E-03 0.30013988E-03 0.36425486E-03 0.42041628E-03 - 0.46887249E-03 0.50990345E-03 0.54381626E-03 0.57094089E-03 0.59162618E-03 - 0.60623606E-03 0.61514598E-03 0.61873965E-03 0.61740590E-03 0.61153586E-03 - 0.60152029E-03 0.58774720E-03 0.57059962E-03 0.55045360E-03 0.52767638E-03 - 0.50262483E-03 0.47564398E-03 0.44706578E-03 0.41720804E-03 0.38637349E-03 - 0.35484905E-03 0.32290520E-03 0.29079554E-03 0.25875640E-03 0.22700668E-03 - 0.19574774E-03 0.16516339E-03 0.13542003E-03 0.10666685E-03 0.79036085E-04 - 0.52643449E-04 0.27588516E-04 0.39552468E-05 -0.18187463E-04 -0.38785176E-04 - -0.57797261E-04 -0.75196218E-04 -0.90966955E-04 -0.10510606E-03 -0.11762103E-03 - -0.12852953E-03 -0.13785857E-03 -0.14564378E-03 -0.15192857E-03 -0.15676342E-03 - -0.16020506E-03 -0.16231574E-03 -0.16316249E-03 -0.16281640E-03 -0.16135194E-03 - -0.15884627E-03 -0.15537858E-03 -0.15102952E-03 -0.14588055E-03 -0.14001345E-03 - -0.13350974E-03 -0.12645021E-03 -0.11891448E-03 -0.11098055E-03 -0.10272444E-03 - -0.94219805E-04 -0.85537640E-04 -0.76745997E-04 -0.67909725E-04 -0.59090261E-04 - -0.50345444E-04 -0.41729365E-04 -0.33292246E-04 -0.25080348E-04 -0.17135911E-04 - -0.94971158E-05 -0.21980783E-05 0.47311370E-05 0.11264478E-04 0.17379845E-04 - 0.23059011E-04 0.28287520E-04 0.33054576E-04 0.37352909E-04 0.41178630E-04 - 0.44531077E-04 0.47412643E-04 0.49828603E-04 0.51786928E-04 0.53298092E-04 - 0.54374882E-04 0.55032192E-04 0.55286830E-04 0.55157307E-04 0.54663644E-04 - 0.53827164E-04 0.52670300E-04 0.51216400E-04 0.49489532E-04 0.47514308E-04 - 0.45315698E-04 0.42918862E-04 0.40348985E-04 0.37631122E-04 0.34790049E-04 - 0.31850125E-04 0.28835161E-04 0.25768302E-04 0.22671916E-04 0.19567494E-04 - 0.16475560E-04 0.13415592E-04 0.10405948E-04 0.74638125E-05 0.46051404E-05 - 0.18446199E-05 -0.80435987E-06 -0.33297298E-05 -0.57207546E-05 -0.79680364E-05 - -0.10063512E-04 -0.12000439E-04 -0.13773380E-04 -0.15378176E-04 -0.16811912E-04 - -0.18072879E-04 -0.19160535E-04 -0.20075450E-04 -0.20819257E-04 -0.21394592E-04 - -0.21805037E-04 -0.22055053E-04 -0.22149916E-04 -0.22095648E-04 -0.21898950E-04 - -0.21567127E-04 -0.21108021E-04 -0.20529939E-04 -0.19841581E-04 -0.19051971E-04 - -0.18170386E-04 -0.17206292E-04 -0.16169276E-04 -0.15068979E-04 -0.13915042E-04 - -0.12717039E-04 -0.11484427E-04 -0.10226489E-04 -0.89522885E-05 -0.76706185E-05 - -0.63899616E-05 -0.51184500E-05 -0.38638298E-05 -0.26334298E-05 -0.14341327E-05 - -0.27235182E-06 0.84599001E-06 0.19154766E-05 0.29312116E-05 0.38888274E-05 - 0.47844916E-05 0.56149086E-05 0.63773189E-05 0.70694951E-05 0.76897347E-05 - 0.82368496E-05 0.87101542E-05 0.91094492E-05 0.94350049E-05 0.96875409E-05 - 0.98682048E-05 0.99785487E-05 0.10020504E-04 0.99963566E-05 0.99087169E-05 - 0.97604941E-05 0.95548658E-05 0.92952492E-05 0.89852707E-05 0.86287363E-05 - 0.82296016E-05 0.77919421E-05 0.73199235E-05 0.68177737E-05 0.62897538E-05 - 0.57401314E-05 0.51731542E-05 0.45930246E-05 0.40038757E-05 0.34097487E-05 - 0.28145712E-05 0.22221377E-05 0.16360907E-05 0.10599042E-05 0.49686836E-06 - -0.49924166E-07 -0.57759007E-06 -0.10834657E-05 -0.15651155E-05 -0.20203390E-05 - -0.24471762E-05 -0.28439108E-05 -0.32090727E-05 -0.35414380E-05 -0.38400284E-05 - -0.41041084E-05 -0.43331817E-05 -0.45269860E-05 -0.46854866E-05 -0.48088687E-05 - -0.48975292E-05 -0.49520669E-05 -0.49732720E-05 -0.49621150E-05 -0.49197346E-05 - -0.48474252E-05 -0.47466237E-05 -0.46188962E-05 -0.44659240E-05 -0.42894896E-05 - -0.40914625E-05 -0.38737847E-05 -0.36384570E-05 -0.33875242E-05 -0.31230617E-05 - -0.28471617E-05 -0.25619195E-05 -0.22694215E-05 -0.19717316E-05 -0.16708804E-05 - -0.13688531E-05 -0.10675792E-05 -0.76892254E-06 -0.47467176E-06 -0.18653194E-06 - 0.93883212E-07 0.36505831E-06 0.62558247E-06 0.87415464E-06 0.11095881E-05 - 0.13308142E-05 0.15368850E-05 0.17269756E-05 0.19003853E-05 0.20565378E-05 - 0.21949812E-05 0.23153867E-05 0.24175470E-05 0.25013740E-05 0.25668951E-05 - 0.26142502E-05 0.26436873E-05 0.26555573E-05 0.26503092E-05 0.26284843E-05 - 0.25907101E-05 0.25376938E-05 0.24702157E-05 0.23891225E-05 0.22953196E-05 - 0.21897645E-05 0.20734587E-05 0.19474410E-05 0.18127797E-05 0.16705653E-05 - 0.15219029E-05 0.13679056E-05 0.12096872E-05 0.10483551E-05 0.88500424E-06 - 0.72071045E-06 0.55652455E-06 0.39346662E-06 0.23252070E-06 0.74629781E-07 - -0.79308742E-07 -0.22844727E-06 -0.37199175E-06 -0.50920498E-06 -0.63940955E-06 - -0.76199023E-06 -0.87639600E-06 -0.98214161E-06 -0.10788087E-05 -0.11660465E-05 - -0.12435720E-05 -0.13111700E-05 -0.13686922E-05 -0.14160566E-05 -0.14532460E-05 - -0.14803064E-05 -0.14973448E-05 -0.15045271E-05 -0.15020754E-05 -0.14902650E-05 - -0.14694215E-05 -0.14399173E-05 -0.14021680E-05 -0.13566289E-05 -0.13037909E-05 - -0.12441768E-05 -0.11783371E-05 -0.11068459E-05 -0.10302970E-05 -0.94929927E-06 - -0.86447309E-06 -0.77644589E-06 -0.68584830E-06 -0.59331013E-06 -0.49945664E-06 - -0.40490479E-06 -0.31025970E-06 -0.21611131E-06 -0.12303113E-06 -0.31569266E-07 - 0.57748381E-07 0.14442270E-06 0.22798371E-06 0.30799268E-06 0.38404401E-06 - 0.45576680E-06 0.52282629E-06 0.58492494E-06 0.64180328E-06 0.69324054E-06 - 0.73905501E-06 0.77910415E-06 0.81328446E-06 0.84153115E-06 0.86381754E-06 - 0.88015423E-06 0.89058818E-06 0.89520142E-06 0.89410976E-06 0.88746116E-06 - 0.87543409E-06 0.85823563E-06 0.83609957E-06 0.80928424E-06 0.77807038E-06 - 0.74275888E-06 0.70366841E-06 0.66113309E-06 0.61550005E-06 0.56712697E-06 - 0.51637969E-06 0.46362977E-06 0.40925206E-06 0.35362237E-06 0.29711511E-06 - 0.24010109E-06 0.18294534E-06 0.12600498E-06 0.69627290E-07 0.14147785E-07 - -0.40111519E-07 -0.92843763E-07 -0.14375875E-06 -0.19258430E-06 -0.23906753E-06 - -0.28297589E-06 -0.32409810E-06 -0.36224497E-06 -0.39725004E-06 -0.42896999E-06 - -0.45728505E-06 -0.48209914E-06 -0.50333992E-06 -0.52095866E-06 -0.53492999E-06 - -0.54525153E-06 -0.55194336E-06 -0.55504738E-06 -0.55462655E-06 -0.55076401E-06 - -0.54356215E-06 -0.53314150E-06 -0.51963959E-06 -0.50320974E-06 -0.48401975E-06 - -0.46225056E-06 -0.43809484E-06 -0.41175556E-06 -0.38344448E-06 -0.35338074E-06 - -0.32178927E-06 -0.28889935E-06 -0.25494309E-06 -0.22015396E-06 -0.18476533E-06 - -0.14900905E-06 -0.11311407E-06 -0.77305131E-07 -0.41801454E-07 -0.68155504E-08 - 0.27447935E-07 0.60793293E-07 0.93034747E-07 0.12399737E-06 0.15351790E-06 - 0.18144551E-06 0.20764241E-06 0.23198445E-06 0.25436152E-06 0.27467799E-06 - 0.29285292E-06 0.30882025E-06 0.32252891E-06 0.33394274E-06 0.34304047E-06 - 0.34981548E-06 0.35427553E-06 0.35644240E-06 0.35635150E-06 0.35405130E-06 - 0.34960280E-06 0.34307888E-06 0.33456360E-06 0.32415144E-06 0.31194653E-06 - 0.29806180E-06 0.28261810E-06 0.26574328E-06 0.24757132E-06 0.22824131E-06 - 0.20789656E-06 0.18668361E-06 0.16475126E-06 0.14224963E-06 0.11932923E-06 - 0.96139992E-07 0.72830423E-07 0.49546690E-07 0.26431790E-07 0.36247474E-08 - -0.18740158E-07 -0.40534108E-07 -0.61634419E-07 -0.81925165E-07 -0.10129775E-06 - -0.11965140E-06 -0.13689367E-06 -0.15294075E-06 -0.16771790E-06 -0.18115965E-06 - -0.19321003E-06 -0.20382275E-06 -0.21296124E-06 -0.22059870E-06 -0.22671807E-06 - -0.23131193E-06 -0.23438237E-06 -0.23594075E-06 -0.23600750E-06 -0.23461175E-06 - -0.23179103E-06 -0.22759087E-06 -0.22206432E-06 -0.21527153E-06 -0.20727922E-06 - -0.19816015E-06 -0.18799254E-06 -0.17685950E-06 -0.16484846E-06 -0.15205046E-06 - -0.13855964E-06 -0.12447252E-06 -0.10988739E-06 -0.94903706E-07 -0.79621417E-07 - -0.64140371E-07 -0.48559705E-07 -0.32977250E-07 -0.17488964E-07 -0.21883816E-08 - 0.12833908E-07 0.27490758E-07 0.41698930E-07 0.55379522E-07 0.68458371E-07 - 0.80866414E-07 0.92540008E-07 0.10342122E-06 0.11345806E-06 0.12260471E-06 - 0.13082166E-06 0.13807583E-06 0.14434068E-06 0.14959621E-06 0.15382899E-06 - 0.15703210E-06 0.15920506E-06 0.16035369E-06 0.16048999E-06 0.15963194E-06 - 0.15780326E-06 0.15503318E-06 0.15135615E-06 0.14681155E-06 0.14144332E-06 - 0.13529964E-06 0.12843256E-06 0.12089760E-06 0.11275334E-06 0.10406101E-06 - 0.94884097E-07 0.85287872E-07 0.75338992E-07 0.65105052E-07 0.54654166E-07 - 0.44054534E-07 0.33374031E-07 0.22679796E-07 0.12037835E-07 0.15126439E-08 - -0.88331616E-08 -0.18939192E-07 -0.28747612E-07 -0.38203448E-07 -0.47254869E-07 - -0.55853449E-07 -0.63954402E-07 -0.71516784E-07 -0.78503682E-07 -0.84882360E-07 - -0.90624389E-07 -0.95705740E-07 -0.10010686E-06 -0.10381269E-06 -0.10681271E-06 - -0.10910090E-06 -0.11067571E-06 -0.11153996E-06 -0.11170080E-06 -0.11116954E-06 - -0.10996154E-06 -0.10809603E-06 -0.10559592E-06 -0.10248761E-06 -0.98800769E-07 - -0.94568084E-07 -0.89825018E-07 -0.84609542E-07 -0.78961861E-07 -0.72924130E-07 - -0.66540164E-07 -0.59855141E-07 -0.52915304E-07 -0.45767663E-07 -0.38459689E-07 - -0.31039024E-07 -0.23553181E-07 -0.16049264E-07 -0.85736810E-08 -0.11718762E-08 - 0.61119303E-08 0.13234992E-07 0.20156272E-07 0.26836663E-07 0.33239193E-07 - 0.39329216E-07 0.45074582E-07 0.50445791E-07 0.55416131E-07 0.59961790E-07 - 0.64061954E-07 0.67698885E-07 0.70857975E-07 0.73527785E-07 0.75700059E-07 - 0.77369725E-07 0.78534868E-07 0.79196695E-07 0.79359470E-07 0.79030440E-07 - 0.78219744E-07 0.76940297E-07 0.75207672E-07 0.73039955E-07 0.70457599E-07 - 0.67483257E-07 0.64141608E-07 0.60459175E-07 0.56464132E-07 0.52186103E-07 - 0.47655963E-07 0.42905624E-07 0.37967827E-07 0.32875928E-07 0.27663683E-07 - 0.22365039E-07 0.17013921E-07 0.11644026E-07 0.62886240E-08 0.98035848E-09 - -0.42489404E-08 -0.93684373E-08 -0.14348464E-07 -0.19160680E-07 -0.23778227E-07 - -0.28175866E-07 -0.32330105E-07 -0.36219319E-07 -0.39823844E-07 -0.43126075E-07 - -0.46110532E-07 -0.48763927E-07 -0.51075207E-07 -0.53035585E-07 -0.54638561E-07 - -0.55879923E-07 -0.56757739E-07 -0.57272329E-07 -0.57426230E-07 -0.57224148E-07 - -0.56672888E-07 -0.55781290E-07 -0.54560132E-07 -0.53022045E-07 -0.51181396E-07 - -0.49054183E-07 -0.46657907E-07 -0.44011441E-07 -0.41134899E-07 -0.38049487E-07 - -0.34777363E-07 -0.31341484E-07 -0.27765453E-07 -0.24073368E-07 -0.20289666E-07 - -0.16438967E-07 -0.12545925E-07 -0.86350721E-08 -0.47306767E-08 -0.85659471E-09 - 0.29638671E-08 0.67080858E-08 0.10354249E-07 0.13881476E-07 0.17269932E-07 - 0.20500930E-07 0.23557030E-07 0.26422126E-07 0.29081523E-07 0.31522008E-07 - 0.33731903E-07 0.35701119E-07 0.37421188E-07 0.38885294E-07 0.40088290E-07 - 0.41026701E-07 0.41698723E-07 0.42104208E-07 0.42244641E-07 0.42123104E-07 - 0.41744237E-07 0.41114183E-07 0.40240531E-07 0.39132246E-07 0.37799597E-07 - 0.36254071E-07 0.34508287E-07 0.32575904E-07 0.30471520E-07 0.28210569E-07 - 0.25809215E-07 0.23284245E-07 0.20652955E-07 0.17933036E-07 0.15142464E-07 - 0.12299384E-07 0.94219950E-08 0.65284431E-08 0.36367075E-08 0.76449490E-09 - -0.20708645E-08 -0.48525176E-08 -0.75641838E-08 -0.10190246E-07 -0.12715838E-07 - -0.15126919E-07 -0.17410355E-07 -0.19553980E-07 -0.21546658E-07 -0.23378335E-07 - -0.25040088E-07 -0.26524159E-07 -0.27823989E-07 -0.28934238E-07 -0.29850803E-07 - -0.30570824E-07 -0.31092680E-07 -0.31415990E-07 -0.31541588E-07 -0.31471507E-07 - -0.31208949E-07 -0.30758244E-07 -0.30124814E-07 -0.29315120E-07 -0.28336610E-07 - -0.27197660E-07 -0.25907508E-07 -0.24476186E-07 -0.22914450E-07 -0.21233701E-07 - -0.19445907E-07 -0.17563525E-07 -0.15599414E-07 -0.13566755E-07 -0.11478964E-07 - -0.93496047E-08 -0.71923085E-08 -0.50206872E-08 -0.28482512E-08 -0.68832844E-09 - 0.14460143E-08 0.35420475E-08 0.55874520E-08 0.75703881E-08 0.94795614E-08 - 0.11304284E-07 0.13034532E-07 0.14660996E-07 0.16175129E-07 0.17569189E-07 - 0.18836272E-07 0.19970348E-07 0.20966280E-07 0.21819847E-07 0.22527754E-07 - 0.23087645E-07 0.23498096E-07 0.23758621E-07 0.23869653E-07 0.23832537E-07 - 0.23649502E-07 0.23323641E-07 0.22858877E-07 0.22259931E-07 0.21532279E-07 - 0.20682110E-07 0.19716280E-07 0.18642260E-07 0.17468081E-07 0.16202279E-07 - 0.14853837E-07 0.13432120E-07 0.11946819E-07 0.10407883E-07 0.88254569E-08 - 0.72098151E-08 0.55712995E-08 0.39202539E-08 0.22669616E-08 0.62158331E-09 - -0.10059033E-08 -0.26057620E-08 -0.41685539E-08 -0.56851915E-08 -0.71469892E-08 - -0.85457109E-08 -0.98736147E-08 -0.11123493E-07 -0.12288710E-07 -0.13363235E-07 - -0.14341670E-07 -0.15219275E-07 -0.15991988E-07 -0.16656443E-07 -0.17209977E-07 - -0.17650642E-07 -0.17977205E-07 -0.18189145E-07 -0.18286649E-07 -0.18270603E-07 - -0.18142574E-07 -0.17904792E-07 -0.17560132E-07 -0.17112083E-07 -0.16564722E-07 - -0.15922679E-07 -0.15191107E-07 -0.14375635E-07 -0.13482335E-07 -0.12517678E-07 - -0.11488487E-07 -0.10401891E-07 -0.92652805E-08 -0.80862565E-08 -0.68725824E-08 - -0.56321344E-08 -0.43728519E-08 -0.31026886E-08 -0.18295637E-08 -0.56131359E-09 - 0.69435411E-09 0.19299065E-08 0.31380291E-08 0.43116674E-08 0.54440667E-08 - 0.65288098E-08 0.75598516E-08 0.85315512E-08 0.94387016E-08 0.10276556E-07 - 0.11040849E-07 0.11727821E-07 0.12334231E-07 0.12857370E-07 0.13295073E-07 - 0.13645724E-07 0.13908258E-07 0.14082165E-07 0.14167479E-07 0.14164779E-07 - 0.14075171E-07 0.13900282E-07 0.13642238E-07 0.13303647E-07 0.12887578E-07 - 0.12397536E-07 0.11837432E-07 0.11211561E-07 0.10524565E-07 0.97814013E-08 - 0.89873121E-08 0.81477851E-08 0.72685186E-08 0.63553840E-08 0.54143881E-08 - 0.44516340E-08 0.34732838E-08 0.24855193E-08 0.14945047E-08 0.50634894E-09 - -0.47293086E-09 -0.14374450E-08 -0.23814654E-08 -0.32994588E-08 -0.41861182E-08 - -0.50363921E-08 -0.58455128E-08 -0.66090213E-08 -0.73227917E-08 -0.79830514E-08 - -0.85864005E-08 -0.91298279E-08 -0.96107249E-08 -0.10026896E-07 -0.10376569E-07 - kinetic energy density (partial) - 0.20835504E+03 0.20828036E+03 0.20805646E+03 0.20768382E+03 0.20716321E+03 - 0.20649571E+03 0.20568271E+03 0.20472590E+03 0.20362725E+03 0.20238902E+03 - 0.20101375E+03 0.19950424E+03 0.19786357E+03 0.19609504E+03 0.19420220E+03 - 0.19218884E+03 0.19005893E+03 0.18781665E+03 0.18546639E+03 0.18301267E+03 - 0.18046020E+03 0.17781381E+03 0.17507846E+03 0.17225923E+03 0.16936128E+03 - 0.16638985E+03 0.16335026E+03 0.16024787E+03 0.15708805E+03 0.15387623E+03 - 0.15061780E+03 0.14731817E+03 0.14398270E+03 0.14061674E+03 0.13722555E+03 - 0.13381436E+03 0.13038829E+03 0.12695239E+03 0.12351160E+03 0.12007077E+03 - 0.11663459E+03 0.11320766E+03 0.10979442E+03 0.10639916E+03 0.10302604E+03 - 0.99679032E+02 0.96361963E+02 0.93078483E+02 0.89832071E+02 0.86626025E+02 - 0.83463466E+02 0.80347331E+02 0.77280372E+02 0.74265159E+02 0.71304074E+02 - 0.68399315E+02 0.65552897E+02 0.62766650E+02 0.60042224E+02 0.57381093E+02 - 0.54784551E+02 0.52253723E+02 0.49789562E+02 0.47392858E+02 0.45064240E+02 - 0.42804181E+02 0.40613001E+02 0.38490877E+02 0.36437844E+02 0.34453799E+02 - 0.32538514E+02 0.30691635E+02 0.28912689E+02 0.27201094E+02 0.25556161E+02 - 0.23977101E+02 0.22463033E+02 0.21012989E+02 0.19625917E+02 0.18300694E+02 - 0.17036125E+02 0.15830955E+02 0.14683869E+02 0.13593503E+02 0.12558445E+02 - 0.11577245E+02 0.10648418E+02 0.97704471E+01 0.89417938E+01 0.81608982E+01 - 0.74261859E+01 0.67360722E+01 0.60889662E+01 0.54832751E+01 0.49174080E+01 - 0.43897800E+01 0.38988154E+01 0.34429512E+01 0.30206401E+01 0.26303538E+01 - 0.22705855E+01 0.19398523E+01 0.16366980E+01 0.13596948E+01 0.11074456E+01 - 0.87858565E+00 0.67178422E+00 0.48574594E+00 0.31921212E+00 0.17096183E+00 - 0.39812801E-01 -0.75377769E-01 -0.17571264E+00 -0.26225402E+00 -0.33602329E+00 - -0.39800073E+00 -0.44912553E+00 -0.49029577E+00 -0.52236862E+00 -0.54616060E+00 - -0.56244795E+00 -0.57196711E+00 -0.57541524E+00 -0.57345089E+00 -0.56669468E+00 - -0.55573011E+00 -0.54110437E+00 -0.52332928E+00 -0.50288217E+00 -0.48020696E+00 - -0.45571513E+00 -0.42978682E+00 -0.40277192E+00 -0.37499119E+00 -0.34673741E+00 - -0.31827653E+00 -0.28984883E+00 -0.26167011E+00 -0.23393286E+00 -0.20680741E+00 - -0.18044314E+00 -0.15496957E+00 -0.13049761E+00 -0.10712057E+00 -0.84915366E-01 - -0.63943573E-01 -0.44252494E-01 -0.25876215E-01 -0.88366140E-02 0.68556445E-02 - 0.21200154E-01 0.34205840E-01 0.45890064E-01 0.56277765E-01 0.65400637E-01 - 0.73296336E-01 0.80007729E-01 0.85582175E-01 0.90070849E-01 0.93528093E-01 - 0.96010816E-01 0.97577916E-01 0.98289755E-01 0.98207654E-01 0.97393432E-01 - 0.95908984E-01 0.93815878E-01 0.91175003E-01 0.88046237E-01 0.84488154E-01 - 0.80557756E-01 0.76310236E-01 0.71798777E-01 0.67074366E-01 0.62185643E-01 - 0.57178774E-01 0.52097343E-01 0.46982274E-01 0.41871767E-01 0.36801261E-01 - 0.31803411E-01 0.26908081E-01 0.22142365E-01 0.17530606E-01 0.13094442E-01 - 0.88528613E-02 0.48222665E-02 0.10165515E-02 -0.25528120E-02 -0.58766813E-02 - -0.89481411E-02 -0.11762397E-01 -0.14316661E-01 -0.16610039E-01 -0.18643408E-01 - -0.20419294E-01 -0.21941748E-01 -0.23216224E-01 -0.24249453E-01 -0.25049318E-01 - -0.25624730E-01 -0.25985512E-01 -0.26142277E-01 -0.26106309E-01 -0.25889459E-01 - -0.25504030E-01 -0.24962671E-01 -0.24278285E-01 -0.23463926E-01 -0.22532710E-01 - -0.21497732E-01 -0.20371987E-01 -0.19168289E-01 -0.17899208E-01 -0.16577007E-01 - -0.15213577E-01 -0.13820394E-01 -0.12408464E-01 -0.10988288E-01 -0.95698218E-02 - -0.81624485E-02 -0.67749510E-02 -0.54154923E-02 -0.40916000E-02 -0.28101553E-02 - -0.15773864E-02 -0.39886651E-03 0.72048473E-03 0.17763955E-02 0.27652328E-02 - 0.36839902E-02 0.45302720E-02 0.53022756E-02 0.59987705E-02 0.66190753E-02 - 0.71630325E-02 0.76309821E-02 0.80237328E-02 0.83425325E-02 0.85890380E-02 - 0.87652830E-02 0.88736459E-02 0.89168179E-02 0.88977699E-02 0.88197199E-02 - 0.86861005E-02 0.85005271E-02 0.82667658E-02 0.79887032E-02 0.76703160E-02 - 0.73156421E-02 0.69287525E-02 0.65137246E-02 0.60746169E-02 0.56154446E-02 - 0.51401571E-02 0.46526166E-02 0.41565790E-02 0.36556751E-02 0.31533950E-02 - 0.26530725E-02 0.21578727E-02 0.16707797E-02 0.11945875E-02 0.73189104E-03 - 0.28508001E-03 -0.14366669E-03 -0.55238445E-03 -0.93932519E-03 -0.13029582E-02 - -0.16419696E-02 -0.19552605E-02 -0.22419442E-02 -0.25013417E-02 -0.27329770E-02 - -0.29365704E-02 -0.31120318E-02 -0.32594528E-02 -0.33790979E-02 -0.34713953E-02 - -0.35369271E-02 -0.35764189E-02 -0.35907293E-02 -0.35808384E-02 -0.35478369E-02 - -0.34929144E-02 -0.34173477E-02 -0.33224897E-02 -0.32097572E-02 -0.30806201E-02 - -0.29365895E-02 -0.27792070E-02 -0.26100340E-02 -0.24306410E-02 -0.22425978E-02 - -0.20474638E-02 -0.18467788E-02 -0.16420544E-02 -0.14347657E-02 -0.12263441E-02 - -0.10181700E-02 -0.81156627E-03 -0.60779290E-03 -0.40804141E-03 -0.21343046E-03 - -0.25001852E-04 0.15628276E-03 0.32954460E-03 0.49398988E-03 0.64891127E-03 - 0.79368888E-03 0.92779059E-03 0.10507719E-02 0.11622751E-02 0.12620285E-02 - 0.13498441E-02 0.14256161E-02 0.14893180E-02 0.15410000E-02 0.15807856E-02 - 0.16088679E-02 0.16255060E-02 0.16310209E-02 0.16257910E-02 0.16102478E-02 - 0.15848710E-02 0.15501843E-02 0.15067498E-02 0.14551636E-02 0.13960508E-02 - 0.13300607E-02 0.12578615E-02 0.11801361E-02 0.10975768E-02 0.10108810E-02 - 0.92074700E-03 0.82786904E-03 0.73293373E-03 0.63661585E-03 0.53957475E-03 - 0.44245079E-03 0.34586214E-03 0.25040173E-03 0.15663455E-03 0.65095164E-04 - -0.23714500E-04 -0.10932708E-03 -0.19131143E-03 -0.26927401E-03 -0.34285994E-03 - -0.41175391E-03 -0.47568067E-03 -0.53440538E-03 -0.58773362E-03 -0.63551122E-03 - -0.67762383E-03 -0.71399622E-03 -0.74459146E-03 -0.76940981E-03 -0.78848749E-03 - -0.80189519E-03 -0.80973655E-03 -0.81214635E-03 -0.80928869E-03 -0.80135496E-03 - -0.78856182E-03 -0.77114897E-03 -0.74937696E-03 -0.72352489E-03 -0.69388808E-03 - -0.66077574E-03 -0.62450863E-03 -0.58541671E-03 -0.54383679E-03 -0.50011029E-03 - -0.45458098E-03 -0.40759277E-03 -0.35948764E-03 -0.31060361E-03 -0.26127274E-03 - -0.21181934E-03 -0.16255822E-03 -0.11379303E-03 -0.65814813E-04 -0.18900553E-04 - 0.26688029E-04 0.70705622E-04 0.11292430E-03 0.15313440E-03 0.19114521E-03 - 0.22678554E-03 0.25990422E-03 0.29037033E-03 0.31807346E-03 0.34292368E-03 - 0.36485151E-03 0.38380771E-03 0.39976294E-03 0.41270738E-03 0.42265015E-03 - 0.42961871E-03 0.43365814E-03 0.43483030E-03 0.43321299E-03 0.42889897E-03 - 0.42199495E-03 0.41262052E-03 0.40090704E-03 0.38699647E-03 0.37104021E-03 - 0.35319789E-03 0.33363614E-03 0.31252735E-03 0.29004850E-03 0.26637987E-03 - 0.24170389E-03 0.21620396E-03 0.19006328E-03 0.16346372E-03 0.13658478E-03 - 0.10960256E-03 0.82688740E-04 0.56009692E-04 0.29725598E-04 0.39896497E-05 - -0.21052691E-04 -0.45264363E-04 -0.68517308E-04 -0.90692998E-04 -0.11168289E-03 - -0.13138883E-03 -0.14972332E-03 -0.16660980E-03 -0.18198279E-03 -0.19578798E-03 - -0.20798226E-03 -0.21853367E-03 -0.22742128E-03 -0.23463504E-03 -0.24017552E-03 - -0.24405359E-03 -0.24629014E-03 -0.24691561E-03 -0.24596960E-03 -0.24350031E-03 - -0.23956409E-03 -0.23422482E-03 -0.22755332E-03 -0.21962678E-03 -0.21052803E-03 - -0.20034496E-03 -0.18916981E-03 -0.17709849E-03 -0.16422991E-03 -0.15066529E-03 - -0.13650745E-03 -0.12186019E-03 -0.10682758E-03 -0.91513375E-04 -0.76020313E-04 - -0.60449570E-04 -0.44900164E-04 -0.29468403E-04 -0.14247369E-04 0.67356873E-06 - 0.15209201E-04 0.29278898E-04 0.42806978E-04 0.55723047E-04 0.67962294E-04 - 0.79465741E-04 0.90180456E-04 0.10005972E-03 0.10906317E-03 0.11715685E-03 - 0.12431330E-03 0.13051151E-03 0.13573694E-03 0.13998141E-03 0.14324297E-03 - 0.14552583E-03 0.14684010E-03 0.14720163E-03 0.14663173E-03 0.14515697E-03 - 0.14280879E-03 0.13962329E-03 0.13564081E-03 0.13090564E-03 0.12546560E-03 - 0.11937169E-03 0.11267772E-03 0.10543985E-03 0.97716257E-04 0.89566681E-04 - 0.81052046E-04 0.72234052E-04 0.63174778E-04 0.53936293E-04 0.44580280E-04 - 0.35167664E-04 0.25758258E-04 0.16410425E-04 0.71807582E-05 -0.18762292E-05 - -0.10708382E-04 -0.19266170E-04 -0.27502926E-04 -0.35375063E-04 -0.42842270E-04 - -0.49867681E-04 -0.56418019E-04 -0.62463713E-04 -0.67978995E-04 -0.72941966E-04 - -0.77334643E-04 -0.81142972E-04 -0.84356829E-04 -0.86969986E-04 -0.88980058E-04 - -0.90388434E-04 -0.91200176E-04 -0.91423905E-04 -0.91071668E-04 -0.90158782E-04 - -0.88703670E-04 -0.86727673E-04 -0.84254855E-04 -0.81311794E-04 -0.77927359E-04 - -0.74132487E-04 -0.69959941E-04 -0.65444073E-04 -0.60620576E-04 -0.55526235E-04 - -0.50198677E-04 -0.44676123E-04 -0.38997144E-04 -0.33200410E-04 -0.27324458E-04 - -0.21407454E-04 -0.15486974E-04 -0.95997806E-05 -0.37816209E-05 0.19329717E-05 - 0.75108618E-05 0.12920483E-04 0.18131998E-04 0.23117441E-04 0.27850849E-04 - 0.32308378E-04 0.36468400E-04 0.40311588E-04 0.43820986E-04 0.46982057E-04 - 0.49782725E-04 0.52213390E-04 0.54266937E-04 0.55938726E-04 0.57226565E-04 - 0.58130670E-04 0.58653618E-04 0.58800272E-04 0.58577711E-04 0.57995133E-04 - 0.57063756E-04 0.55796708E-04 0.54208903E-04 0.52316916E-04 0.50138843E-04 - 0.47694158E-04 0.45003568E-04 0.42088860E-04 0.38972742E-04 0.35678686E-04 - 0.32230771E-04 0.28653522E-04 0.24971746E-04 0.21210383E-04 0.17394344E-04 - 0.13548360E-04 0.96968386E-05 0.58637166E-05 0.20723263E-05 -0.16547360E-05 - -0.52957328E-05 -0.88299004E-05 -0.12237557E-04 -0.15500199E-04 -0.18600592E-04 - -0.21522849E-04 -0.24252503E-04 -0.26776561E-04 -0.29083560E-04 -0.31163602E-04 - -0.33008383E-04 -0.34611214E-04 -0.35967028E-04 -0.37072379E-04 -0.37925428E-04 - -0.38525927E-04 -0.38875183E-04 -0.38976024E-04 -0.38832747E-04 -0.38451068E-04 - -0.37838051E-04 -0.37002046E-04 -0.35952608E-04 -0.34700415E-04 -0.33257184E-04 - -0.31635572E-04 -0.29849087E-04 -0.27911987E-04 -0.25839175E-04 -0.23646098E-04 - -0.21348641E-04 -0.18963022E-04 -0.16505685E-04 -0.13993199E-04 -0.11442146E-04 - -0.88690290E-05 -0.62901661E-05 -0.37215965E-05 -0.11789878E-05 0.13224529E-05 - 0.37680632E-05 0.61438045E-05 0.84363361E-05 0.10633083E-04 0.12722299E-04 - 0.14693121E-04 0.16535621E-04 0.18240847E-04 0.19800862E-04 0.21208768E-04 - 0.22458735E-04 0.23546012E-04 0.24466937E-04 0.25218940E-04 0.25800535E-04 - 0.26211314E-04 0.26451924E-04 0.26524047E-04 0.26430366E-04 0.26174537E-04 - 0.25761142E-04 0.25195645E-04 0.24484347E-04 0.23634324E-04 0.22653376E-04 - 0.21549965E-04 0.20333146E-04 0.19012507E-04 0.17598097E-04 0.16100357E-04 - 0.14530050E-04 0.12898186E-04 0.11215955E-04 0.94946498E-05 0.77456005E-05 - 0.59801006E-05 0.42093402E-05 0.24443392E-05 0.69588305E-06 -0.10255391E-05 - -0.27097932E-05 -0.43471557E-05 -0.59283657E-05 -0.74446735E-05 -0.88878846E-05 - -0.10250400E-04 -0.11525252E-04 -0.12706136E-04 -0.13787435E-04 -0.14764247E-04 - -0.15632395E-04 -0.16388446E-04 -0.17029714E-04 -0.17554268E-04 -0.17960926E-04 - -0.18249251E-04 -0.18419541E-04 -0.18472814E-04 -0.18410788E-04 -0.18235860E-04 - -0.17951076E-04 -0.17560106E-04 -0.17067207E-04 -0.16477189E-04 -0.15795372E-04 - -0.15027551E-04 -0.14179945E-04 -0.13259159E-04 -0.12272129E-04 -0.11226083E-04 - -0.10128482E-04 -0.89869785E-05 -0.78093612E-05 -0.66035078E-05 -0.53773340E-05 - -0.41387451E-05 -0.28955872E-05 -0.16556008E-05 -0.42637512E-06 0.78469609E-06 - 0.19704560E-05 0.31240246E-05 0.42388361E-05 0.53086738E-05 0.63277021E-05 - 0.72904958E-05 0.81920664E-05 0.90278850E-05 0.97939027E-05 0.10486567E-04 - 0.11102835E-04 0.11640184E-04 0.12096617E-04 0.12470670E-04 0.12761405E-04 - 0.12968414E-04 0.13091808E-04 0.13132211E-04 0.13090746E-04 0.12969018E-04 - 0.12769101E-04 0.12493512E-04 0.12145193E-04 0.11727484E-04 0.11244095E-04 - 0.10699079E-04 0.10096800E-04 0.94419040E-05 0.87392820E-05 0.79940391E-05 - 0.72114589E-05 0.63969681E-05 0.55561008E-05 0.46944635E-05 0.38176993E-05 - 0.29314527E-05 0.20413352E-05 0.11528915E-05 0.27156677E-06 -0.59732538E-06 - -0.14486329E-05 -0.22773945E-05 -0.30788666E-05 -0.38485492E-05 -0.45822095E-05 - -0.52759030E-05 -0.59259938E-05 -0.65291711E-05 -0.70824650E-05 -0.75832587E-05 - -0.80292992E-05 -0.84187053E-05 -0.87499729E-05 -0.90219787E-05 -0.92339806E-05 - -0.93856165E-05 -0.94769006E-05 -0.95082170E-05 -0.94803119E-05 -0.93942833E-05 - -0.92515686E-05 -0.90539305E-05 -0.88034415E-05 -0.85024659E-05 -0.81536409E-05 - -0.77598566E-05 -0.73242339E-05 -0.68501020E-05 -0.63409749E-05 -0.58005271E-05 - -0.52325683E-05 -0.46410183E-05 -0.40298813E-05 -0.34032201E-05 -0.27651301E-05 - -0.21197141E-05 -0.14710570E-05 -0.82320100E-06 -0.18012145E-06 0.45429620E-06 - 0.10762789E-05 0.16821875E-05 0.22685368E-05 0.28320144E-05 0.33694988E-05 - 0.38780750E-05 0.43550498E-05 0.47979643E-05 0.52046059E-05 0.55730178E-05 - 0.59015075E-05 0.61886525E-05 0.64333056E-05 0.66345974E-05 0.67919376E-05 - 0.69050145E-05 0.69737925E-05 0.69985083E-05 0.69796653E-05 0.69180270E-05 - 0.68146078E-05 0.66706633E-05 0.64876794E-05 0.62673592E-05 0.60116097E-05 - 0.57225273E-05 0.54023816E-05 0.50535996E-05 0.46787481E-05 0.42805159E-05 - 0.38616961E-05 0.34251668E-05 0.29738728E-05 0.25108063E-05 0.20389878E-05 - 0.15614475E-05 0.10812068E-05 0.60125910E-06 0.12455292E-06 -0.34602614E-06 - -0.80767194E-06 -0.12576738E-05 -0.16934316E-05 -0.21124705E-05 -0.25124536E-05 - -0.28911947E-05 -0.32466697E-05 -0.35770259E-05 -0.38805915E-05 -0.41558832E-05 - -0.44016122E-05 -0.46166896E-05 -0.48002299E-05 -0.49515539E-05 -0.50701897E-05 - -0.51558725E-05 -0.52085437E-05 -0.52283475E-05 -0.52156282E-05 -0.51709243E-05 - -0.50949632E-05 -0.49886534E-05 -0.48530769E-05 -0.46894800E-05 -0.44992632E-05 - -0.42839704E-05 -0.40452779E-05 -0.37849817E-05 -0.35049853E-05 -0.32072862E-05 - -0.28939625E-05 -0.25671593E-05 -0.22290742E-05 -0.18819433E-05 -0.15280272E-05 - -0.11695966E-05 -0.80891805E-06 -0.44824079E-06 -0.89782651E-07 0.26428284E-06 - 0.61183918E-06 0.95083906E-06 0.12793159E-05 0.15953948E-05 0.18973026E-05 - 0.21833778E-05 0.24520784E-05 0.27019904E-05 0.29318344E-05 0.31404715E-05 - 0.33269085E-05 0.34903020E-05 0.36299613E-05 0.37453508E-05 0.38360910E-05 - 0.39019586E-05 0.39428861E-05 0.39589597E-05 0.39504170E-05 0.39176432E-05 - 0.38611669E-05 0.37816552E-05 0.36799070E-05 0.35568472E-05 0.34135188E-05 - 0.32510750E-05 0.30707708E-05 0.28739539E-05 0.26620552E-05 0.24365789E-05 - 0.21990924E-05 0.19512158E-05 0.16946112E-05 0.14309721E-05 0.11620127E-05 - 0.88945703E-06 0.61502809E-06 0.34043765E-06 0.67375715E-07 -0.20249943E-06 - -0.46757098E-06 -0.72627297E-06 -0.97709914E-06 -0.12186114E-05 -0.14494477E-05 - -0.16683294E-05 -0.18740679E-05 -0.20655709E-05 -0.22418477E-05 -0.24020138E-05 - -0.25452953E-05 -0.26710317E-05 -0.27786789E-05 -0.28678107E-05 -0.29381202E-05 - -0.29894195E-05 -0.30216403E-05 -0.30348315E-05 -0.30291587E-05 -0.30049005E-05 - -0.29624463E-05 -0.29022916E-05 -0.28250345E-05 -0.27313699E-05 -0.26220845E-05 - -0.24980506E-05 -0.23602199E-05 -0.22096162E-05 -0.20473288E-05 -0.18745043E-05 - -0.16923396E-05 -0.15020733E-05 -0.13049781E-05 -0.11023521E-05 -0.89551080E-06 - -0.68577898E-06 -0.47448212E-06 -0.26293851E-06 -0.52451208E-07 0.15569972E-06 - 0.36026474E-06 0.56003214E-06 0.75383492E-06 0.94055739E-06 0.11191414E-05 - 0.12885918E-05 0.14479824E-05 0.15964599E-05 0.17332489E-05 0.18576553E-05 - 0.19690697E-05 0.20669698E-05 0.21509228E-05 0.22205870E-05 0.22757124E-05 - 0.23161411E-05 0.23418077E-05 0.23527377E-05 0.23490468E-05 0.23309391E-05 - 0.22987043E-05 0.22527152E-05 0.21934244E-05 0.21213603E-05 0.20371233E-05 - 0.19413808E-05 0.18348627E-05 0.17183559E-05 0.15926988E-05 0.14587758E-05 - 0.13175109E-05 0.11698619E-05 0.10168141E-05 0.85937351E-06 0.69856113E-06 - 0.53540601E-06 0.37093900E-06 0.20618646E-06 0.42164007E-07 -0.12012960E-06 - -0.27971831E-06 -0.43565451E-06 -0.58702449E-06 -0.73295359E-06 -0.87261108E-06 - -0.10052148E-05 -0.11300351E-05 -0.12463990E-05 -0.13536936E-05 -0.14513687E-05 - -0.15389398E-05 -0.16159903E-05 -0.16821729E-05 -0.17372113E-05 -0.17809007E-05 - atomic pseudo charge-density - 0.40000000E+01 0.39865777E+01 0.39466377E+01 0.38811433E+01 0.37916420E+01 - 0.36801845E+01 0.35492225E+01 0.34014950E+01 0.32399100E+01 0.30674335E+01 - 0.28869885E+01 0.27013712E+01 0.25131849E+01 0.23247937E+01 0.21382919E+01 - 0.19554911E+01 0.17779180E+01 0.16068237E+01 0.14431992E+01 0.12877965E+01 - 0.11411517E+01 0.10036103E+01 0.87535141E+00 0.75641221E+00 0.64671002E+00 - 0.54606322E+00 0.45420995E+00 0.37082491E+00 0.29553413E+00 0.22792794E+00 - 0.16757211E+00 0.11401745E+00 0.66807917E-01 0.25487413E-01 -0.10394609E-01 - -0.41278559E-01 -0.67591068E-01 -0.89742136E-01 -0.10812299E+00 -0.12310456E+00 - -0.13503646E+00 -0.14424639E+00 -0.15104001E+00 -0.15570097E+00 -0.15849133E+00 - -0.15965211E+00 -0.15940402E+00 -0.15794831E+00 -0.15546772E+00 -0.15212749E+00 - -0.14807638E+00 -0.14344771E+00 -0.13836047E+00 -0.13292033E+00 -0.12722065E+00 - -0.12134348E+00 -0.11536052E+00 -0.10933401E+00 -0.10331755E+00 -0.97357004E-01 - -0.91491144E-01 -0.85752437E-01 -0.80167665E-01 -0.74758549E-01 -0.69542301E-01 - -0.64532148E-01 -0.59737800E-01 -0.55165888E-01 -0.50820352E-01 -0.46702805E-01 - -0.42812853E-01 -0.39148390E-01 -0.35705857E-01 -0.32480485E-01 -0.29466501E-01 - -0.26657315E-01 -0.24045694E-01 -0.21623903E-01 -0.19383840E-01 -0.17317147E-01 - -0.15415312E-01 -0.13669755E-01 -0.12071903E-01 -0.10613251E-01 -0.92854244E-02 - -0.80802145E-02 -0.69896237E-02 -0.60058937E-02 -0.51215308E-02 -0.43293255E-02 - -0.36223668E-02 -0.29940530E-02 -0.24380979E-02 -0.19485345E-02 -0.15197156E-02 - -0.11463116E-02 -0.82330733E-03 -0.54599558E-03 -0.30997057E-03 -0.11111936E-03 - 0.54387372E-04 0.19010550E-03 0.29932812E-03 0.38509675E-03 0.45021276E-03 - 0.49724905E-03 0.52856181E-03 0.54630218E-03 0.55242796E-03 0.54871497E-03 - 0.53676838E-03 0.51803363E-03 0.49380709E-03 0.46524638E-03 0.43338033E-03 - 0.39911848E-03 0.36326026E-03 0.32650372E-03 0.28945381E-03 0.25263029E-03 - 0.21647518E-03 0.18135979E-03 0.14759134E-03 0.11541921E-03 0.85040745E-04 - 0.56606693E-04 0.30226281E-04 0.59719222E-05 -0.16116422E-04 -0.36027206E-04 - -0.53773582E-04 -0.69389985E-04 -0.82929001E-04 -0.94458506E-04 -0.10405906E-03 - -0.11182154E-03 -0.11784501E-03 -0.12223480E-03 -0.12510074E-03 -0.12655568E-03 - -0.12671406E-03 -0.12569075E-03 -0.12359995E-03 -0.12055428E-03 -0.11666396E-03 - -0.11203618E-03 -0.10677445E-03 -0.10097817E-03 -0.94742200E-04 -0.88156578E-04 - -0.81306252E-04 -0.74270913E-04 -0.67124872E-04 -0.59936998E-04 -0.52770692E-04 - -0.45683909E-04 -0.38729216E-04 -0.31953881E-04 -0.25399987E-04 -0.19104573E-04 - -0.13099795E-04 -0.74131035E-05 -0.20674340E-05 0.29185877E-05 0.75304328E-05 - 0.11757450E-04 0.15592638E-04 0.19032418E-04 0.22076404E-04 0.24727164E-04 - 0.26989996E-04 0.28872695E-04 0.30385326E-04 0.31539999E-04 0.32350652E-04 - 0.32832837E-04 0.33003507E-04 0.32880817E-04 0.32483927E-04 0.31832814E-04 - 0.30948087E-04 0.29850816E-04 0.28562367E-04 0.27104239E-04 0.25497920E-04 - 0.23764744E-04 0.21925760E-04 0.20001608E-04 0.18012403E-04 0.15977634E-04 - 0.13916067E-04 0.11845655E-04 0.97834623E-05 0.77455969E-05 0.57471478E-05 - 0.38021359E-05 0.19234706E-05 0.12291697E-06 -0.15889304E-05 -0.32026639E-05 - -0.47100721E-05 -0.61041411E-05 -0.73790486E-05 -0.85301498E-05 -0.95539581E-05 - -0.10448118E-04 -0.11211371E-04 -0.11843520E-04 -0.12345381E-04 -0.12718740E-04 - -0.12966293E-04 -0.13091593E-04 -0.13098990E-04 -0.12993562E-04 -0.12781054E-04 - -0.12467805E-04 -0.12060682E-04 -0.11567004E-04 -0.10994473E-04 -0.10351102E-04 - -0.96451441E-05 -0.88850170E-05 -0.80792383E-05 -0.72363555E-05 -0.63648798E-05 - -0.54732233E-05 -0.45696380E-05 -0.36621584E-05 -0.27585476E-05 -0.18662470E-05 - -0.99232994E-06 -0.14345991E-06 0.67414724E-06 0.14547563E-05 0.21931413E-05 - 0.28846089E-05 0.35250176E-05 0.41107917E-05 0.46389308E-05 0.51070144E-05 - 0.55132020E-05 0.58562288E-05 0.61353970E-05 0.63505627E-05 0.65021194E-05 - 0.65909775E-05 0.66185401E-05 0.65866761E-05 0.64976905E-05 0.63542912E-05 - 0.61595549E-05 0.59168901E-05 0.56299985E-05 0.53028363E-05 0.49395727E-05 - 0.45445501E-05 0.41222417E-05 0.36772115E-05 0.32140730E-05 0.27374494E-05 - 0.22519349E-05 0.17620572E-05 0.12722412E-05 0.78677566E-06 0.30978034E-06 - -0.15482310E-06 -0.60333860E-06 -0.10323203E-05 -0.14385946E-05 -0.18192788E-05 - -0.21717975E-05 -0.24938958E-05 -0.27836489E-05 -0.30394696E-05 -0.32601119E-05 - -0.34446725E-05 -0.35925882E-05 -0.37036321E-05 -0.37779053E-05 -0.38158276E-05 - -0.38181247E-05 -0.37858135E-05 -0.37201857E-05 -0.36227886E-05 -0.34954051E-05 - -0.33400317E-05 -0.31588555E-05 -0.29542301E-05 -0.27286505E-05 -0.24847277E-05 - -0.22251630E-05 -0.19527218E-05 -0.16702079E-05 -0.13804382E-05 -0.10862172E-05 - -0.79031329E-06 -0.49543528E-06 -0.20421011E-06 0.80837975E-07 0.35730694E-06 - 0.62293496E-06 0.87561665E-06 0.11134176E-05 0.13345872E-05 0.15375689E-05 - 0.17210098E-05 0.18837666E-05 0.20249111E-05 0.21437323E-05 0.22397380E-05 - 0.23126529E-05 0.23624163E-05 0.23891767E-05 0.23932857E-05 0.23752896E-05 - 0.23359196E-05 0.22760813E-05 0.21968416E-05 0.20994157E-05 0.19851524E-05 - 0.18555189E-05 0.17120850E-05 0.15565060E-05 0.13905063E-05 0.12158623E-05 - 0.10343849E-05 0.84790272E-06 0.65824498E-06 0.46722542E-06 0.27662610E-06 - 0.88182235E-07 -0.96432316E-07 -0.27561869E-06 -0.44786563E-06 -0.61176105E-06 - -0.76600248E-06 -0.90940617E-06 -0.10409151E-05 -0.11596055E-05 -0.12646923E-05 - -0.13555325E-05 -0.14316286E-05 -0.14926287E-05 -0.15383273E-05 -0.15686631E-05 - -0.15837168E-05 -0.15837066E-05 -0.15689836E-05 -0.15400253E-05 -0.14974285E-05 - -0.14419012E-05 -0.13742537E-05 -0.12953888E-05 -0.12062916E-05 -0.11080189E-05 - -0.10016876E-05 -0.88846338E-06 -0.76954920E-06 -0.64617321E-06 -0.51957713E-06 - -0.39100463E-06 -0.26168993E-06 -0.13284673E-06 -0.56575985E-08 0.11873617E-06 - 0.23924409E-06 0.35483532E-06 0.46454683E-06 0.56749076E-06 0.66286088E-06 - 0.74993828E-06 0.82809602E-06 0.89680296E-06 0.95562658E-06 0.10042348E-05 - 0.10423970E-05 0.10699838E-05 0.10869664E-05 0.10934144E-05 0.10894933E-05 - 0.10754611E-05 0.10516635E-05 0.10185295E-05 0.97656513E-06 0.92634782E-06 - 0.86851908E-06 0.80377754E-06 0.73287127E-06 0.65658987E-06 0.57575636E-06 - 0.49121891E-06 0.40384252E-06 0.31450065E-06 0.22406708E-06 0.13340778E-06 - 0.43373085E-07 -0.45209802E-07 -0.13154400E-06 -0.21486970E-06 -0.29447041E-06 - -0.36967874E-06 -0.43988155E-06 -0.50452456E-06 -0.56311628E-06 -0.61523130E-06 - -0.66051294E-06 -0.69867516E-06 -0.72950384E-06 -0.75285729E-06 -0.76866619E-06 - -0.77693279E-06 -0.77772948E-06 -0.77119677E-06 -0.75754063E-06 -0.73702932E-06 - -0.70998970E-06 -0.67680308E-06 -0.63790058E-06 -0.59375822E-06 -0.54489163E-06 - -0.49185048E-06 -0.43521276E-06 -0.37557887E-06 -0.31356558E-06 -0.24980003E-06 - -0.18491372E-06 -0.11953651E-06 -0.54290839E-07 0.10213919E-07 0.73386891E-07 - 0.13466084E-06 0.19349698E-06 0.24938949E-06 0.30186957E-06 0.35050912E-06 - 0.39492396E-06 0.43477660E-06 0.46977849E-06 0.49969185E-06 0.52433087E-06 - 0.54356256E-06 0.55730693E-06 0.56553685E-06 0.56827727E-06 0.56560403E-06 - 0.55764226E-06 0.54456431E-06 0.52658722E-06 0.50396998E-06 0.47701026E-06 - 0.44604100E-06 0.41142663E-06 0.37355910E-06 0.33285375E-06 0.28974500E-06 - 0.24468193E-06 0.19812388E-06 0.15053591E-06 0.10238442E-06 0.54132716E-07 - 0.62367788E-08 -0.40858886E-07 -0.86725192E-07 -0.13095221E-06 -0.17315265E-06 - -0.21296511E-06 -0.25005694E-06 -0.28412693E-06 -0.31490750E-06 -0.34216665E-06 - -0.36570951E-06 -0.38537949E-06 -0.40105911E-06 -0.41267040E-06 -0.42017495E-06 - -0.42357360E-06 -0.42290573E-06 -0.41824822E-06 -0.40971413E-06 -0.39745100E-06 - -0.38163883E-06 -0.36248792E-06 -0.34023633E-06 -0.31514719E-06 -0.28750579E-06 - -0.25761658E-06 -0.22579994E-06 -0.19238893E-06 -0.15772595E-06 -0.12215937E-06 - -0.86040187E-07 -0.49718647E-07 -0.13541005E-07 0.22153667E-07 0.57036538E-07 - 0.90791928E-07 0.12312006E-06 0.15373959E-06 0.18238997E-06 0.20883351E-06 - 0.23285726E-06 0.25427458E-06 0.27292648E-06 0.28868261E-06 0.30144207E-06 - 0.31113384E-06 0.31771695E-06 0.32118039E-06 0.32154272E-06 0.31885140E-06 - 0.31318189E-06 0.30463646E-06 0.29334283E-06 0.27945252E-06 0.26313906E-06 - 0.24459602E-06 0.22403482E-06 0.20168251E-06 0.17777935E-06 0.15257633E-06 - 0.12633270E-06 0.99313318E-07 0.71786140E-07 0.44019613E-07 0.16280158E-07 - -0.11170310E-07 -0.38076764E-07 -0.64193365E-07 -0.89285620E-07 -0.11313241E-06 - -0.13552787E-06 -0.15628307E-06 -0.17522752E-06 -0.19221052E-06 -0.20710224E-06 - -0.21979462E-06 -0.23020204E-06 -0.23826179E-06 -0.24393430E-06 -0.24720317E-06 - -0.24807490E-06 -0.24657857E-06 -0.24276514E-06 -0.23670665E-06 -0.22849526E-06 - -0.21824199E-06 -0.20607547E-06 -0.19214040E-06 -0.17659596E-06 -0.15961408E-06 - -0.14137764E-06 -0.12207857E-06 -0.10191591E-06 -0.81093788E-07 -0.59819486E-07 - -0.38301386E-07 -0.16747007E-07 0.46389382E-08 0.25656431E-07 0.46111962E-07 - 0.65820256E-07 0.84605894E-07 0.10230482E-06 0.11876572E-06 0.13385124E-06 - 0.14743910E-06 0.15942300E-06 0.16971338E-06 0.17823805E-06 0.18494256E-06 - 0.18979049E-06 0.19276351E-06 0.19386128E-06 0.19310120E-06 0.19051798E-06 - 0.18616304E-06 0.18010382E-06 0.17242285E-06 0.16321682E-06 0.15259538E-06 - 0.14067999E-06 0.12760255E-06 0.11350404E-06 0.98533033E-07 0.82844184E-07 - 0.66596694E-07 0.49952735E-07 0.33075874E-07 0.16129513E-07 -0.72465324E-09 - -0.17328139E-07 -0.33527119E-07 -0.49173812E-07 -0.64127789E-07 -0.78257197E-07 - -0.91439880E-07 -0.10356439E-06 -0.11453090E-06 -0.12425196E-06 -0.13265316E-06 - -0.13967366E-06 -0.14526653E-06 -0.14939904E-06 -0.15205274E-06 -0.15322347E-06 - -0.15292112E-06 -0.15116943E-06 -0.14800550E-06 -0.14347929E-06 -0.13765294E-06 - -0.13060004E-06 -0.12240475E-06 -0.11316085E-06 -0.10297075E-06 -0.91944386E-07 - -0.80198063E-07 -0.67853283E-07 -0.55035515E-07 -0.41872961E-07 -0.28495302E-07 - -0.15032454E-07 -0.16133352E-08 0.11635336E-07 0.24590209E-07 0.37132420E-07 - 0.49148656E-07 0.60532143E-07 0.71183572E-07 0.81011928E-07 0.89935244E-07 - 0.97881244E-07 0.10478790E-06 0.11060388E-06 0.11528886E-06 0.11881382E-06 - 0.12116109E-06 0.12232439E-06 0.12230878E-06 0.12113038E-06 0.11881613E-06 - 0.11540336E-06 0.11093929E-06 0.10548050E-06 0.99092190E-07 0.91847517E-07 - 0.83826762E-07 0.75116481E-07 0.65808606E-07 0.55999498E-07 0.45788981E-07 - 0.35279347E-07 0.24574360E-07 0.13778250E-07 0.29947206E-08 -0.76740207E-08 - -0.18128214E-07 -0.28271459E-07 -0.38011579E-07 -0.47261440E-07 -0.55939707E-07 - -0.63971541E-07 -0.71289215E-07 -0.77832665E-07 -0.83549956E-07 -0.88397669E-07 - -0.92341195E-07 -0.95354951E-07 -0.97422500E-07 -0.98536589E-07 -0.98699097E-07 - -0.97920896E-07 -0.96221633E-07 -0.93629426E-07 -0.90180493E-07 -0.85918699E-07 - -0.80895042E-07 -0.75167082E-07 -0.68798307E-07 -0.61857456E-07 -0.54417806E-07 - -0.46556413E-07 -0.38353339E-07 -0.29890854E-07 -0.21252631E-07 -0.12522934E-07 - -0.37858156E-08 0.48756775E-08 0.13380276E-07 0.21649236E-07 0.29607050E-07 - 0.37182121E-07 0.44307392E-07 0.50920920E-07 0.56966402E-07 0.62393631E-07 - 0.67158897E-07 0.71225322E-07 0.74563117E-07 0.77149777E-07 0.78970206E-07 - 0.80016760E-07 0.80289232E-07 0.79794755E-07 0.78547650E-07 0.76569190E-07 - 0.73887320E-07 0.70536305E-07 0.66556328E-07 0.61993032E-07 0.56897024E-07 - 0.51323331E-07 0.45330824E-07 0.38981613E-07 0.32340417E-07 0.25473921E-07 - 0.18450117E-07 0.11337647E-07 0.42051412E-08 -0.28794280E-08 -0.98493841E-08 - -0.16639961E-07 -0.23188892E-07 -0.29436970E-07 -0.35328568E-07 -0.40812125E-07 - -0.45840579E-07 -0.50371762E-07 -0.54368736E-07 -0.57800081E-07 -0.60640121E-07 - -0.62869100E-07 -0.64473297E-07 -0.65445078E-07 -0.65782899E-07 -0.65491242E-07 - -0.64580506E-07 -0.63066827E-07 -0.60971865E-07 -0.58322529E-07 -0.55150654E-07 - -0.51492651E-07 -0.47389095E-07 -0.42884300E-07 -0.38025850E-07 -0.32864112E-07 - -0.27451723E-07 -0.21843067E-07 -0.16093737E-07 -0.10259996E-07 -0.43982344E-08 - 0.14355627E-08 0.71863441E-08 0.12800501E-07 0.18226358E-07 0.23414641E-07 - 0.28318911E-07 0.32895976E-07 0.37106258E-07 0.40914127E-07 0.44288191E-07 - 0.47201541E-07 0.49631959E-07 0.51562064E-07 0.52979429E-07 0.53876634E-07 - 0.54251276E-07 0.54105940E-07 0.53448106E-07 0.52290030E-07 0.50648562E-07 - 0.48544941E-07 0.46004535E-07 0.43056558E-07 0.39733743E-07 0.36071996E-07 - 0.32110014E-07 0.27888888E-07 0.23451687E-07 0.18843027E-07 0.14108626E-07 - 0.92948675E-08 0.44483443E-08 -0.38457840E-09 -0.51582018E-08 -0.98279172E-08 - -0.14350616E-07 -0.18685080E-07 -0.22792350E-07 -0.26636066E-07 -0.30182785E-07 - -0.33402261E-07 -0.36267695E-07 -0.38755949E-07 -0.40847725E-07 -0.42527701E-07 - -0.43784631E-07 -0.44611406E-07 -0.45005075E-07 -0.44966822E-07 -0.44501911E-07 - -0.43619591E-07 -0.42332959E-07 -0.40658796E-07 -0.38617367E-07 -0.36232185E-07 - -0.33529757E-07 -0.30539298E-07 -0.27292420E-07 -0.23822813E-07 -0.20165895E-07 - -0.16358462E-07 -0.12438321E-07 -0.84439245E-08 -0.44139948E-08 -0.38715706E-09 - 0.35984264E-08 0.75054122E-08 0.11297620E-07 0.14940364E-07 0.18400759E-07 - 0.21648020E-07 0.24653721E-07 0.27392046E-07 0.29839999E-07 0.31977589E-07 - 0.33787991E-07 0.35257666E-07 0.36376450E-07 0.37137619E-07 0.37537909E-07 - 0.37577509E-07 0.37260025E-07 0.36592399E-07 0.35584817E-07 0.34250568E-07 - 0.32605889E-07 0.30669776E-07 0.28463775E-07 0.26011753E-07 0.23339640E-07 - 0.20475168E-07 0.17447585E-07 0.14287364E-07 0.11025901E-07 0.76952074E-08 - 0.43276014E-08 0.95539922E-09 -0.23893908E-08 -0.56753669E-08 -0.88720247E-08 - -0.11950035E-07 -0.14881510E-07 -0.17640250E-07 -0.20201972E-07 -0.22544524E-07 - -0.24648063E-07 -0.26495225E-07 -0.28071260E-07 -0.29364144E-07 -0.30364661E-07 - -0.31066465E-07 -0.31466104E-07 -0.31563026E-07 -0.31359549E-07 -0.30860813E-07 - -0.30074696E-07 -0.29011715E-07 -0.27684894E-07 -0.26109618E-07 -0.24303457E-07 - -0.22285982E-07 -0.20078552E-07 -0.17704100E-07 -0.15186893E-07 -0.12552291E-07 - -0.98264954E-08 -0.70362916E-08 -0.42087888E-08 -0.13711612E-08 0.14496101E-08 - 0.42269901E-08 0.69351355E-08 0.95491319E-08 0.12045220E-07 0.14401006E-07 - 0.16595662E-07 0.18610102E-07 0.20427146E-07 0.22031661E-07 0.23410684E-07 - 0.24553519E-07 0.25451815E-07 0.26099624E-07 0.26493423E-07 0.26632132E-07 - 0.26517089E-07 0.26152020E-07 0.25542974E-07 0.24698245E-07 0.23628266E-07 - 0.22345494E-07 0.20864267E-07 0.19200649E-07 0.17372264E-07 0.15398111E-07 - 0.13298372E-07 0.11094206E-07 0.88075432E-08 0.64608682E-08 0.40770010E-08 - 0.16788790E-08 -0.71066103E-09 -0.30690973E-08 -0.53744330E-08 -0.76053985E-08 - -0.97416452E-08 -0.11763929E-07 -0.13654278E-07 -0.15396156E-07 -0.16974594E-07 - -0.18376325E-07 -0.19589886E-07 -0.20605711E-07 -0.21416199E-07 -0.22015771E-07 - -0.22400896E-07 -0.22570106E-07 -0.22523988E-07 -0.22265159E-07 -0.21798218E-07 - -0.21129684E-07 -0.20267914E-07 -0.19223007E-07 -0.18006686E-07 -0.16632178E-07 - -0.15114066E-07 -0.13468142E-07 -0.11711246E-07 -0.98610957E-08 -0.79361089E-08 - -0.59552253E-08 -0.39377210E-08 -0.19030235E-08 0.12947253E-09 0.21405877E-08 - 0.41115363E-08 0.60240990E-08 0.78607894E-08 0.96050108E-08 0.11241205E-07 - 0.12754986E-07 0.14133267E-07 0.15364369E-07 0.16438116E-07 0.17345916E-07 - 0.18080829E-07 0.18637614E-07 0.19012761E-07 0.19204509E-07 0.19212846E-07 - 0.19039488E-07 0.18687849E-07 0.18162994E-07 0.17471570E-07 0.16621732E-07 - 0.15623051E-07 0.14486408E-07 0.13223881E-07 0.11848618E-07 0.10374701E-07 - 0.88170072E-08 0.71910591E-08 0.55128719E-08 0.37987983E-08 0.20653706E-08 - 0.32914233E-09 -0.13934677E-08 -0.30863295E-08 -0.47337509E-08 -0.63206208E-08 - -0.78325462E-08 -0.92559808E-08 -0.10578344E-07 -0.11788132E-07 -0.12875014E-07 - -0.13829917E-07 -0.14645103E-07 -0.15314224E-07 -0.15832370E-07 -0.16196103E-07 - -0.16403471E-07 -0.16454014E-07 -0.16348755E-07 -0.16090177E-07 -0.15682179E-07 - -0.15130034E-07 -0.14440317E-07 -0.13620840E-07 -0.12680557E-07 -0.11629478E-07 - 19.8421228800787 T - Non local Part - 0 2 1.03170241276447 - 8.48191517218165 11.7757836767903 11.7757836767903 - 16.3515625171536 - Reciprocal Space Part - 0.59618802E+02 0.58728804E+02 0.56074024E+02 0.51699804E+02 0.45680746E+02 - 0.38119274E+02 0.29143671E+02 0.18905623E+02 0.75773282E+01 -0.46517683E+01 - -0.17578547E+02 -0.30990019E+02 -0.44667277E+02 -0.58389527E+02 -0.71938095E+02 - -0.85100345E+02 -0.97673425E+02 -0.10946776E+03 -0.12031025E+03 -0.13004704E+03 - -0.13854596E+03 -0.14569837E+03 -0.15142064E+03 -0.15565502E+03 -0.15837002E+03 - -0.15956027E+03 -0.15924591E+03 -0.15747139E+03 -0.15430398E+03 -0.14983176E+03 - -0.14416132E+03 -0.13741514E+03 -0.12972880E+03 -0.12124795E+03 -0.11212524E+03 - -0.10251722E+03 -0.92581277E+02 -0.82472653E+02 -0.72341667E+02 -0.62331121E+02 - -0.52573985E+02 -0.43191382E+02 -0.34290898E+02 -0.25965259E+02 -0.18291368E+02 - -0.11329718E+02 -0.51241765E+01 0.29787571E+00 0.49250688E+01 0.87612564E+01 - 0.11824451E+02 0.14145521E+02 0.15766686E+02 0.16739863E+02 0.17124899E+02 - 0.16987748E+02 0.16398621E+02 0.15430172E+02 0.14155746E+02 0.12647730E+02 - 0.10976046E+02 0.92067952E+01 0.74011008E+01 0.56141435E+01 0.38944113E+01 - 0.22831646E+01 0.81411644E+00 -0.48667978E+00 -0.16007453E+01 -0.25169103E+01 - -0.32308862E+01 -0.37446987E+01 -0.40660062E+01 -0.42073286E+01 -0.41852143E+01 - -0.40193711E+01 -0.37317875E+01 -0.33458677E+01 -0.28856034E+01 -0.23748024E+01 - -0.18363896E+01 -0.12917960E+01 -0.76044515E+00 -0.25934295E+00 0.19722322E+00 - 0.59787806E+00 0.93424642E+00 0.12009418E+01 0.13954638E+01 0.15180138E+01 - 0.15712429E+01 0.15599426E+01 0.14906942E+01 0.13714899E+01 0.12113407E+01 - 0.10198828E+01 0.80699697E+00 0.58245003E+00 0.35557023E+00 0.13496271E+00 - Real Space Part - -0.87257185E+03 -0.87217325E+03 -0.87097758E+03 -0.86898535E+03 -0.86619740E+03 - -0.86261493E+03 -0.85823952E+03 -0.85307319E+03 -0.84711839E+03 -0.84037811E+03 - -0.83285587E+03 -0.82455582E+03 -0.81548281E+03 -0.80564241E+03 -0.79504105E+03 - -0.78368604E+03 -0.77158570E+03 -0.75874942E+03 -0.74518776E+03 -0.73091251E+03 - -0.71593685E+03 -0.70027535E+03 -0.68394415E+03 -0.66696100E+03 -0.64934533E+03 - -0.63111840E+03 -0.61230331E+03 -0.59292512E+03 -0.57301089E+03 -0.55258976E+03 - -0.53169298E+03 -0.51035396E+03 -0.48860832E+03 -0.46649386E+03 -0.44405060E+03 - -0.42132076E+03 -0.39834870E+03 -0.37518093E+03 -0.35186599E+03 -0.32845440E+03 - -0.30499855E+03 -0.28155254E+03 -0.25817210E+03 -0.23491433E+03 -0.21183758E+03 - -0.18900121E+03 -0.16646531E+03 -0.14429049E+03 -0.12253758E+03 -0.10126731E+03 - -0.80539962E+02 -0.60415079E+02 -0.40951054E+02 -0.22204768E+02 -0.42311995E+01 - 0.12916980E+02 0.29189809E+02 0.44540445E+02 0.58925573E+02 0.72305804E+02 - 0.84646067E+02 0.95915987E+02 0.10609024E+03 0.11514887E+03 0.12307763E+03 - 0.12986821E+03 0.13551849E+03 0.14003271E+03 0.14342163E+03 0.14570263E+03 - 0.14689971E+03 0.14704353E+03 0.14617130E+03 0.14432667E+03 0.14155957E+03 - 0.13792593E+03 0.13348743E+03 0.12831116E+03 0.12246922E+03 0.11603831E+03 - 0.10909924E+03 0.10173648E+03 0.94037578E+02 0.86092680E+02 0.77993902E+02 - 0.69834817E+02 0.61709839E+02 0.53713688E+02 0.45940793E+02 0.38484762E+02 - 0.31437841E+02 0.24890391E+02 0.18930406E+02 0.13643042E+02 0.91101706E+01 - 0.54100403E+01 0.26168332E+01 0.79835325E+00 0.23427012E-01 0.00000000E+00 - Reciprocal Space Part - -0.36489549E+02 -0.35864156E+02 -0.33998755E+02 -0.30925471E+02 -0.26697153E+02 - -0.21386360E+02 -0.15083959E+02 -0.78973858E+01 0.51413298E-01 0.86282977E+01 - 0.17689472E+02 0.27084188E+02 0.36657552E+02 0.46253383E+02 0.55717063E+02 - 0.64898313E+02 0.73653855E+02 0.81849887E+02 0.89364344E+02 0.96088881E+02 - 0.10193057E+03 0.10681323E+03 0.11067846E+03 0.11348624E+03 0.11521523E+03 - 0.11586261E+03 0.11544365E+03 0.11399089E+03 0.11155304E+03 0.10819353E+03 - 0.10398888E+03 0.99026864E+02 0.93404459E+02 0.87225735E+02 0.80599650E+02 - 0.73637846E+02 0.66452460E+02 0.59154022E+02 0.51849465E+02 0.44640290E+02 - 0.37620922E+02 0.30877278E+02 0.24485577E+02 0.18511398E+02 0.13009009E+02 - 0.80209530E+01 0.35779043E+01 -0.30122499E+00 -0.36089364E+01 -0.63485770E+01 - -0.85335736E+01 -0.10186497E+02 -0.11337984E+02 -0.12025554E+02 -0.12292345E+02 - -0.12185805E+02 -0.11756382E+02 -0.11056219E+02 -0.10137909E+02 -0.90533175E+01 - -0.78525080E+01 -0.65827802E+01 -0.52878430E+01 -0.40071309E+01 -0.27752721E+01 - -0.16217097E+01 -0.57047779E+00 0.35987548E+00 0.11562232E+01 0.18106666E+01 - 0.23202264E+01 0.26864349E+01 0.29148468E+01 0.30144856E+01 0.29972475E+01 - 0.28772789E+01 0.26703472E+01 0.23932227E+01 0.20630867E+01 0.16969795E+01 - 0.13113025E+01 0.92138094E+00 0.54109837E+00 0.18260423E+00 -0.14390035E+00 - -0.43029978E+00 -0.67062595E+00 -0.86104898E+00 -0.99980292E+00 -0.10870535E+01 - -0.11247164E+01 -0.11162365E+01 -0.10663359E+01 -0.98074332E+00 -0.86591280E+00 - -0.72874294E+00 -0.57630417E+00 -0.41558297E+00 -0.25324963E+00 -0.95455204E-01 - Real Space Part - 0.63319797E+03 0.63291041E+03 0.63204781E+03 0.63061056E+03 0.62859924E+03 - 0.62601473E+03 0.62285817E+03 0.61913101E+03 0.61483503E+03 0.60997237E+03 - 0.60454559E+03 0.59855767E+03 0.59201211E+03 0.58491291E+03 0.57726472E+03 - 0.56907280E+03 0.56034314E+03 0.55108251E+03 0.54129849E+03 0.53099960E+03 - 0.52019530E+03 0.50889609E+03 0.49711358E+03 0.48486054E+03 0.47215095E+03 - 0.45900011E+03 0.44542463E+03 0.43144254E+03 0.41707331E+03 0.40233788E+03 - 0.38725874E+03 0.37185991E+03 0.35616701E+03 0.34020723E+03 0.32400934E+03 - 0.30760369E+03 0.29102220E+03 0.27429832E+03 0.25746694E+03 0.24056440E+03 - 0.22362837E+03 0.20669777E+03 0.18981266E+03 0.17301414E+03 0.15634416E+03 - 0.13984541E+03 0.12356113E+03 0.10753493E+03 0.91810566E+02 0.76431735E+02 - 0.61441833E+02 0.46883707E+02 0.32799393E+02 0.19229849E+02 0.62146659E+01 - -0.62082134E+01 -0.18002788E+02 -0.29135292E+02 -0.39574487E+02 -0.49291950E+02 - -0.58262356E+02 -0.66463744E+02 -0.73877778E+02 -0.80489982E+02 -0.86289957E+02 - -0.91271581E+02 -0.95433167E+02 -0.98777602E+02 -0.10131245E+03 -0.10305002E+03 - -0.10400739E+03 -0.10420642E+03 -0.10367366E+03 -0.10244030E+03 -0.10054203E+03 - -0.98018889E+02 -0.94915012E+02 -0.91278431E+02 -0.87160797E+02 -0.82617056E+02 - -0.77705128E+02 -0.72485556E+02 -0.67021113E+02 -0.61376433E+02 -0.55617588E+02 - -0.49811699E+02 -0.44026502E+02 -0.38329965E+02 -0.32789855E+02 -0.27473361E+02 - -0.22446704E+02 -0.17774762E+02 -0.13520724E+02 -0.97457493E+01 -0.65086493E+01 - -0.38656440E+01 -0.18700414E+01 -0.57059033E+00 -0.16714955E-01 0.00000000E+00 - Non local Part - 1 2 1.03170241276447 - 2.65859946877321 4.88238126795342 4.88238126795342 - 8.98435195357541 - Reciprocal Space Part - 0.00000000E+00 0.38861820E+01 0.76566506E+01 0.11198191E+02 0.14402530E+02 - 0.17168669E+02 0.19405054E+02 0.21031535E+02 0.21981079E+02 0.22201184E+02 - 0.21654974E+02 0.20321960E+02 0.18198431E+02 0.15297488E+02 0.11648712E+02 - 0.72974833E+01 0.23039596E+01 -0.32582469E+01 -0.93036735E+01 -0.15736848E+02 - -0.22454324E+02 -0.29346862E+02 -0.36301710E+02 -0.43204929E+02 -0.49943706E+02 - -0.56408622E+02 -0.62495800E+02 -0.68108907E+02 -0.73160964E+02 -0.77575922E+02 - -0.81289979E+02 -0.84252614E+02 -0.86427318E+02 -0.87792019E+02 -0.88339191E+02 - -0.88075654E+02 -0.87022084E+02 -0.85212244E+02 -0.82691957E+02 -0.79517866E+02 - -0.75756000E+02 -0.71480183E+02 -0.66770345E+02 -0.61710747E+02 -0.56388189E+02 - -0.50890222E+02 -0.45303418E+02 -0.39711722E+02 -0.34194931E+02 -0.28827322E+02 - -0.23676453E+02 -0.18802168E+02 -0.14255806E+02 -0.10079632E+02 -0.63064936E+01 - -0.29596904E+01 -0.53068427E-01 0.24086920E+01 0.44295991E+01 0.60217087E+01 - 0.72043400E+01 0.80031754E+01 0.84492707E+01 0.85780039E+01 0.84279913E+01 - 0.80399981E+01 0.74558696E+01 0.67175093E+01 0.58659233E+01 0.49403528E+01 - 0.39775094E+01 0.30109262E+01 0.20704331E+01 0.11817624E+01 0.36628567E+00 - -0.35911990E+00 -0.98208097E+00 -0.14946211E+01 -0.18929483E+01 -0.21771504E+01 - -0.23508116E+01 -0.24205674E+01 -0.23956133E+01 -0.22871843E+01 -0.21080214E+01 - -0.18718417E+01 -0.15928247E+01 -0.12851305E+01 -0.96245910E+00 -0.63766179E+00 - -0.32241090E+00 -0.26933951E-01 0.24018535E+00 0.47213831E+00 0.66395584E+00 - 0.81250966E+00 0.91645487E+00 0.97612004E+00 0.99335246E+00 0.97132649E+00 - Real Space Part - 0.00000000E+00 -0.22186814E+02 -0.44301839E+02 -0.66273522E+02 -0.88030777E+02 - -0.10950322E+03 -0.13062140E+03 -0.15131703E+03 -0.17152321E+03 -0.19117468E+03 - -0.21020799E+03 -0.22856176E+03 -0.24617690E+03 -0.26299679E+03 -0.27896751E+03 - -0.29403802E+03 -0.30816039E+03 -0.32128994E+03 -0.33338548E+03 -0.34440943E+03 - -0.35432799E+03 -0.36311136E+03 -0.37073379E+03 -0.37717380E+03 -0.38241426E+03 - -0.38644250E+03 -0.38925046E+03 -0.39083470E+03 -0.39119654E+03 -0.39034209E+03 - -0.38828229E+03 -0.38503293E+03 -0.38061467E+03 -0.37505301E+03 -0.36837826E+03 - -0.36062551E+03 -0.35183450E+03 -0.34204958E+03 -0.33131954E+03 -0.31969747E+03 - -0.30724063E+03 -0.29401017E+03 -0.28007097E+03 -0.26549136E+03 -0.25034283E+03 - -0.23469974E+03 -0.21863898E+03 -0.20223959E+03 -0.18558239E+03 -0.16874958E+03 - -0.15182426E+03 -0.13489000E+03 -0.11803038E+03 -0.10132845E+03 -0.84866250E+02 - -0.68724282E+02 -0.52980979E+02 -0.37712173E+02 -0.22990566E+02 -0.88851973E+01 - 0.45390663E+01 0.17222034E+02 0.29108671E+02 0.40149550E+02 0.50301273E+02 - 0.59526856E+02 0.67796071E+02 0.75085739E+02 0.81379975E+02 0.86670373E+02 - 0.90956134E+02 0.94244139E+02 0.96548944E+02 0.97892719E+02 0.98305121E+02 - 0.97823113E+02 0.96490698E+02 0.94358601E+02 0.91483927E+02 0.87929701E+02 - 0.83764422E+02 0.79061549E+02 0.73898935E+02 0.68358280E+02 0.62524490E+02 - 0.56485105E+02 0.50329625E+02 0.44148922E+02 0.38034576E+02 0.32078283E+02 - 0.26371234E+02 0.21003526E+02 0.16063609E+02 0.11637728E+02 0.78094126E+01 - 0.46590409E+01 0.22633368E+01 0.69369210E+00 0.17747648E-01 0.00000000E+00 - Reciprocal Space Part - 0.00000000E+00 -0.17868362E+01 -0.35144269E+01 -0.51248277E+01 -0.65626676E+01 - -0.77763629E+01 -0.87192468E+01 -0.93505888E+01 -0.96364809E+01 -0.95505728E+01 - -0.90746384E+01 -0.81989629E+01 -0.69225426E+01 -0.52530936E+01 -0.32068705E+01 - -0.80830207E+00 0.19105496E+01 0.49106895E+01 0.81470351E+01 0.11569385E+02 - 0.15123488E+02 0.18752190E+02 0.22396628E+02 0.25997448E+02 0.29496018E+02 - 0.32835608E+02 0.35962514E+02 0.38827099E+02 0.41384736E+02 0.43596618E+02 - 0.45430445E+02 0.46860950E+02 0.47870266E+02 0.48448142E+02 0.48591976E+02 - 0.48306705E+02 0.47604523E+02 0.46504469E+02 0.45031872E+02 0.43217686E+02 - 0.41097721E+02 0.38711805E+02 0.36102874E+02 0.33316045E+02 0.30397658E+02 - 0.27394336E+02 0.24352070E+02 0.21315355E+02 0.18326389E+02 0.15424354E+02 - 0.12644790E+02 0.10019079E+02 0.75740326E+01 0.53316035E+01 0.33087123E+01 - 0.15171909E+01 -0.36159418E-01 -0.13494048E+01 -0.24252171E+01 -0.32705252E+01 - -0.38960921E+01 -0.43160309E+01 -0.45472742E+01 -0.46090120E+01 -0.45221135E+01 - -0.43085463E+01 -0.39908088E+01 -0.35913863E+01 -0.31322454E+01 -0.26343746E+01 - -0.21173800E+01 -0.15991441E+01 -0.10955499E+01 -0.62027500E+00 -0.18465580E+00 - 0.20237976E+00 0.53431322E+00 0.80697500E+00 0.10184279E+01 0.11688016E+01 - 0.12600861E+01 0.12958921E+01 0.12811875E+01 0.12220191E+01 0.11252269E+01 - 0.99816176E+00 0.84841124E+00 0.68354309E+00 0.51087098E+00 0.33724795E+00 - 0.16889148E+00 0.11242749E-01 -0.13113830E+00 -0.25464171E+00 -0.35664077E+00 - -0.43549155E+00 -0.49050126E+00 -0.52186870E+00 -0.53060095E+00 -0.51841055E+00 - Real Space Part - 0.00000000E+00 0.12161252E+02 0.24283469E+02 0.36327742E+02 0.48255417E+02 - 0.60028220E+02 0.71608383E+02 0.82958767E+02 0.94042991E+02 0.10482555E+03 - 0.11527193E+03 0.12534873E+03 0.13502381E+03 0.14426635E+03 0.15304699E+03 - 0.16133795E+03 0.16911311E+03 0.17634814E+03 0.18302057E+03 0.18910990E+03 - 0.19459770E+03 0.19946767E+03 0.20370573E+03 0.20730009E+03 0.21024135E+03 - 0.21252251E+03 0.21413905E+03 0.21508898E+03 0.21537287E+03 0.21499392E+03 - 0.21395789E+03 0.21227322E+03 0.20995096E+03 0.20700481E+03 0.20345105E+03 - 0.19930856E+03 0.19459875E+03 0.18934552E+03 0.18357519E+03 0.17731641E+03 - 0.17060007E+03 0.16345921E+03 0.15592887E+03 0.14804598E+03 0.13984918E+03 - 0.13137868E+03 0.12267606E+03 0.11378408E+03 0.10474648E+03 0.95607747E+02 - 0.86412883E+02 0.77207157E+02 0.68035844E+02 0.58943963E+02 0.49976000E+02 - 0.41175622E+02 0.32585400E+02 0.24246514E+02 0.16198468E+02 0.84788075E+01 - 0.11228377E+01 -0.58366451E+01 -0.12369614E+02 -0.18449055E+02 -0.24051197E+02 - -0.29155721E+02 -0.33745943E+02 -0.37808973E+02 -0.41335852E+02 -0.44321651E+02 - -0.46765541E+02 -0.48670833E+02 -0.50044982E+02 -0.50899549E+02 -0.51250136E+02 - -0.51116295E+02 -0.50521380E+02 -0.49492378E+02 -0.48059729E+02 -0.46257077E+02 - -0.44121031E+02 -0.41690892E+02 -0.39008342E+02 -0.36117160E+02 -0.33062867E+02 - -0.29892426E+02 -0.26653874E+02 -0.23396010E+02 -0.20168034E+02 -0.17019231E+02 - -0.13998631E+02 -0.11154698E+02 -0.85350264E+01 -0.61860453E+01 -0.41527388E+01 - -0.24784108E+01 -0.12044157E+01 -0.36926989E+00 -0.93979993E-02 0.00000000E+00 - PAW radial sets - 351 0.989218471734280 -(5E20.12) - augmentation charges (non sperical) - -0.118612820244E+00 -0.166184190184E+00 -0.532145144881E-01 -0.101222164724E+00 -0.166184190184E+00 - -0.232848406670E+00 -0.743582353105E-01 -0.141431561143E+00 -0.532145144881E-01 -0.743582353105E-01 - -0.179689550405E-01 -0.343505440708E-01 -0.101222164724E+00 -0.141431561143E+00 -0.343505440708E-01 - -0.655142723317E-01 - uccopancies in atom - 0.200000000235E+01 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.666666668254E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 - grid - 0.353278105438E-04 0.364765828105E-04 0.376627102856E-04 0.388874076671E-04 0.401519291522E-04 - 0.414575697215E-04 0.428056664648E-04 0.441975999512E-04 0.456347956421E-04 0.471187253514E-04 - 0.486509087530E-04 0.502329149365E-04 0.518663640144E-04 0.535529287814E-04 0.552943364272E-04 - 0.570923703053E-04 0.589488717596E-04 0.608657420098E-04 0.628449440985E-04 0.648885049016E-04 - 0.669985172040E-04 0.691771418426E-04 0.714266099194E-04 0.737492250864E-04 0.761473659044E-04 - 0.786234882792E-04 0.811801279764E-04 0.838199032186E-04 0.865455173661E-04 0.893597616862E-04 - 0.922655182109E-04 0.952657626888E-04 0.983635676325E-04 0.101562105465E-03 0.104864651768E-03 - 0.108274588638E-03 0.111795408149E-03 0.115430715926E-03 0.119184234844E-03 0.123059808833E-03 - 0.127061406819E-03 0.131193126789E-03 0.135459199986E-03 0.139863995240E-03 0.144412023447E-03 - 0.149107942186E-03 0.153956560487E-03 0.158962843759E-03 0.164131918874E-03 0.169469079417E-03 - 0.174979791106E-03 0.180669697392E-03 0.186544625235E-03 0.192610591076E-03 0.198873806993E-03 - 0.205340687067E-03 0.212017853948E-03 0.218912145639E-03 0.226030622496E-03 0.233380574462E-03 - 0.240969528530E-03 0.248805256452E-03 0.256895782699E-03 0.265249392676E-03 0.273874641209E-03 - 0.282780361308E-03 0.291975673207E-03 0.301469993709E-03 0.311273045829E-03 0.321394868748E-03 - 0.331845828098E-03 0.342636626573E-03 0.353778314896E-03 0.365282303127E-03 0.377160372357E-03 - 0.389424686766E-03 0.402087806084E-03 0.415162698451E-03 0.428662753701E-03 0.442601797068E-03 - 0.456994103352E-03 0.471854411533E-03 0.487197939863E-03 0.503040401457E-03 0.519398020380E-03 - 0.536287548263E-03 0.553726281459E-03 0.571732078754E-03 0.590323379658E-03 0.609519223288E-03 - 0.629339267864E-03 0.649803810846E-03 0.670933809712E-03 0.692750903429E-03 0.715277434606E-03 - 0.738536472381E-03 0.762551836040E-03 0.787348119413E-03 0.812950716063E-03 0.839385845286E-03 - 0.866680578963E-03 0.894862869287E-03 0.923961577387E-03 0.954006502881E-03 0.985028414399E-03 - 0.101705908109E-02 0.105013130516E-02 0.108427895544E-02 0.111953700213E-02 0.115594155253E-02 - 0.119352988810E-02 0.123234050257E-02 0.127241314140E-02 0.131378884247E-02 0.135650997813E-02 - 0.140062029854E-02 0.144616497653E-02 0.149319065382E-02 0.154174548883E-02 0.159187920594E-02 - 0.164364314647E-02 0.169709032120E-02 0.175227546473E-02 0.180925509145E-02 0.186808755348E-02 - 0.192883310041E-02 0.199155394099E-02 0.205631430683E-02 0.212318051821E-02 0.219222105197E-02 - 0.226350661166E-02 0.233711019991E-02 0.241310719324E-02 0.249157541920E-02 0.257259523611E-02 - 0.265624961535E-02 0.274262422631E-02 0.283180752413E-02 0.292389084032E-02 0.301896847624E-02 - 0.311713779968E-02 0.321849934462E-02 0.332315691413E-02 0.343121768671E-02 0.354279232604E-02 - 0.365799509430E-02 0.377694396919E-02 0.389976076474E-02 0.402657125610E-02 0.415750530828E-02 - 0.429269700920E-02 0.443228480697E-02 0.457641165169E-02 0.472522514185E-02 0.487887767547E-02 - 0.503752660617E-02 0.520133440431E-02 0.537046882339E-02 0.554510307185E-02 0.572541599040E-02 - 0.591159223525E-02 0.610382246712E-02 0.630230354657E-02 0.650723873558E-02 0.671883790569E-02 - 0.693731775293E-02 0.716290201977E-02 0.739582172419E-02 0.763631539635E-02 0.788462932276E-02 - 0.814101779859E-02 0.840574338805E-02 0.867907719326E-02 0.896129913194E-02 0.925269822400E-02 - 0.955357288758E-02 0.986423124464E-02 0.101849914365E-01 0.105161819495E-01 0.108581419519E-01 - 0.112112216404E-01 0.115757825996E-01 0.119521981717E-01 0.123408538391E-01 0.127421476193E-01 - 0.131564904721E-01 0.135843067208E-01 0.140260344867E-01 0.144821261375E-01 0.149530487510E-01 - 0.154392845931E-01 0.159413316118E-01 0.164597039470E-01 0.169949324574E-01 0.175475652638E-01 - 0.181181683104E-01 0.187073259445E-01 0.193156415151E-01 0.199437379906E-01 0.205922585965E-01 - 0.212618674747E-01 0.219532503631E-01 0.226671152982E-01 0.234041933402E-01 0.241652393213E-01 - 0.249510326191E-01 0.257623779547E-01 0.266001062165E-01 0.274650753115E-01 0.283581710436E-01 - 0.292803080210E-01 0.302324305924E-01 0.312155138146E-01 0.322305644506E-01 0.332786220011E-01 - 0.343607597686E-01 0.354780859567E-01 0.366317448050E-01 0.378229177610E-01 0.390528246900E-01 - 0.403227251240E-01 0.416339195521E-01 0.429877507520E-01 0.443856051652E-01 0.458289143166E-01 - 0.473191562810E-01 0.488578571964E-01 0.504465928270E-01 0.520869901769E-01 0.537807291563E-01 - 0.555295443019E-01 0.573352265534E-01 0.591996250870E-01 0.611246492098E-01 0.631122703148E-01 - 0.651645238996E-01 0.672835116512E-01 0.694714035982E-01 0.717304403333E-01 0.740629353074E-01 - 0.764712771991E-01 0.789579323611E-01 0.815254473456E-01 0.841764515121E-01 0.869136597208E-01 - 0.897398751118E-01 0.926579919768E-01 0.956709987225E-01 0.987819809310E-01 0.101994124520E+00 - 0.105310719005E+00 0.108735160869E+00 0.112270957040E+00 0.115921728481E+00 0.119691213902E+00 - 0.123583273585E+00 0.127601893339E+00 0.131751188583E+00 0.136035408556E+00 0.140458940676E+00 - 0.145026315024E+00 0.149742208992E+00 0.154611452068E+00 0.159639030781E+00 0.164830093811E+00 - 0.170189957261E+00 0.175724110099E+00 0.181438219781E+00 0.187338138056E+00 0.193429906954E+00 - 0.199719764979E+00 0.206214153496E+00 0.212919723327E+00 0.219843341560E+00 0.226992098585E+00 - 0.234373315355E+00 0.241994550880E+00 0.249863609972E+00 0.257988551235E+00 0.266377695319E+00 - 0.275039633439E+00 0.283983236179E+00 0.293217662568E+00 0.302752369466E+00 0.312597121246E+00 - 0.322761999795E+00 0.333257414835E+00 0.344094114590E+00 0.355283196787E+00 0.366836120023E+00 - 0.378764715502E+00 0.391081199148E+00 0.403798184116E+00 0.416928693710E+00 0.430486174720E+00 - 0.444484511190E+00 0.458938038641E+00 0.473861558747E+00 0.489270354497E+00 0.505180205845E+00 - 0.521607405869E+00 0.538568777456E+00 0.556081690535E+00 0.574164079857E+00 0.592834463369E+00 - 0.612111961177E+00 0.632016315122E+00 0.652567909002E+00 0.673787789445E+00 0.695697687463E+00 - 0.718320040705E+00 0.741678016440E+00 0.765795535274E+00 0.790697295655E+00 0.816408799161E+00 - 0.842956376619E+00 0.870367215068E+00 0.898669385601E+00 0.927891872115E+00 0.958064600989E+00 - 0.989218471734E+00 0.102138538864E+01 0.105459829343E+01 0.108889119903E+01 0.112429922437E+01 - 0.116085863037E+01 0.119860685704E+01 0.123758256187E+01 0.127782565939E+01 0.131937736205E+01 - 0.136228022242E+01 0.140657817678E+01 0.145231659010E+01 0.149954230253E+01 0.154830367731E+01 - 0.159865065035E+01 0.165063478135E+01 0.170430930661E+01 0.175972919353E+01 0.181695119691E+01 - 0.187603391710E+01 0.193703785995E+01 0.200002549884E+01 0.206506133861E+01 0.213221198164E+01 - 0.220154619607E+01 0.227313498618E+01 0.234705166515E+01 0.242337193012E+01 0.250217393971E+01 - 0.258353839407E+01 - aepotential - 0.249296947974E+05 0.247076561026E+05 0.244443313547E+05 0.241414253640E+05 0.238012309740E+05 - 0.234259813678E+05 0.230181043480E+05 0.225809961616E+05 0.221166800594E+05 0.216286109490E+05 - 0.211196171324E+05 0.205927134571E+05 0.200508445329E+05 0.194969963033E+05 0.189339771311E+05 - 0.183645026186E+05 0.177912890443E+05 0.172167064663E+05 0.166431270260E+05 0.160726862779E+05 - 0.155073402250E+05 0.149488848247E+05 0.143989117768E+05 0.138588393857E+05 0.133298963598E+05 - 0.128131392341E+05 0.123094542557E+05 0.118195679465E+05 0.113440551010E+05 0.108833506728E+05 - 0.104377601662E+05 0.100074714858E+05 0.959256622382E+04 0.919303113611E+04 0.880876923400E+04 - 0.843961046952E+04 0.808532192877E+04 0.774561726178E+04 0.742016531312E+04 0.710859980992E+04 - 0.681052312014E+04 0.652551855157E+04 0.625315022209E+04 0.599297269034E+04 0.574453507071E+04 - 0.550738155297E+04 0.528105943435E+04 0.506512283908E+04 0.485912669119E+04 0.466263692016E+04 - 0.447523241993E+04 0.429649920031E+04 0.412604217009E+04 0.396347085357E+04 0.380841736718E+04 - 0.366052059215E+04 0.351943998366E+04 0.338484550824E+04 0.325642075114E+04 0.313386784458E+04 - 0.301689601724E+04 0.290523717951E+04 0.279862371984E+04 0.269681383135E+04 0.259956791482E+04 - 0.250666289932E+04 0.241788601403E+04 0.233303530109E+04 0.225191811134E+04 0.217435346238E+04 - 0.210016749107E+04 0.202919681943E+04 0.196128648905E+04 0.189628781912E+04 0.183406428064E+04 - 0.177447815378E+04 0.171741030060E+04 0.166273352875E+04 0.161034457319E+04 0.156012716492E+04 - 0.151198483922E+04 0.146582052759E+04 0.142154086395E+04 0.137906097148E+04 0.133829797002E+04 - 0.129917207490E+04 0.126161128619E+04 0.122554324807E+04 0.119090137554E+04 0.115762273768E+04 - 0.112564519634E+04 0.109491260230E+04 0.106536958529E+04 0.103696376209E+04 0.100964626874E+04 - 0.983369386517E+03 0.958088259270E+03 0.933760491469E+03 0.910344918809E+03 0.887802999165E+03 - 0.866097688492E+03 0.845193722659E+03 0.825057486722E+03 0.805656949235E+03 0.786961460988E+03 - 0.768941860254E+03 0.751570280175E+03 0.734820058649E+03 0.718665839953E+03 0.703083242035E+03 - 0.688049142095E+03 0.673541285748E+03 0.659538404365E+03 0.646020181485E+03 0.632967306499E+03 - 0.620361030724E+03 0.608183507242E+03 0.596417880177E+03 0.585047538181E+03 0.574057005309E+03 - 0.563431167138E+03 0.553155596176E+03 0.543216472219E+03 0.533600577089E+03 0.524295096628E+03 - 0.515287864004E+03 0.506567108698E+03 0.498121462758E+03 0.489940202519E+03 0.482012792804E+03 - 0.474329218497E+03 0.466879807157E+03 0.459655277491E+03 0.452646665244E+03 0.445845326919E+03 - 0.439242971301E+03 0.432831556708E+03 0.426603373307E+03 0.420550954576E+03 0.414667100840E+03 - 0.408944864593E+03 0.403377536754E+03 0.397958629585E+03 0.392681872509E+03 0.387541205851E+03 - 0.382530758167E+03 0.377644862756E+03 0.372878015893E+03 0.368224898594E+03 0.363680350460E+03 - 0.359239371823E+03 0.354897097109E+03 0.350648847402E+03 0.346490028292E+03 0.342416224923E+03 - 0.338423107351E+03 0.334506504401E+03 0.330662319969E+03 0.326886614027E+03 0.323175521883E+03 - 0.319525291137E+03 0.315932256738E+03 0.312392876939E+03 0.308903660561E+03 0.305461237806E+03 - 0.302062300261E+03 0.298703633981E+03 0.295382095296E+03 0.292094620152E+03 0.288838215885E+03 - 0.285609957558E+03 0.282406996490E+03 0.279226542503E+03 0.276065876855E+03 0.272922339966E+03 - 0.269793336953E+03 0.266676334707E+03 0.263568857552E+03 0.260468492389E+03 0.257372882929E+03 - 0.254279731763E+03 0.251186799954E+03 0.248091907223E+03 0.244992930370E+03 0.241887805727E+03 - 0.238774528474E+03 0.235651153907E+03 0.232515797559E+03 0.229366637373E+03 0.226201914200E+03 - 0.223019933142E+03 0.219819066345E+03 0.216597753942E+03 0.213354506528E+03 0.210087907601E+03 - 0.206796615859E+03 0.203479368741E+03 0.200134985511E+03 0.196762369494E+03 0.193360514156E+03 - 0.189928504953E+03 0.186465523853E+03 0.182970859195E+03 0.179443901362E+03 0.175884158862E+03 - 0.172291258360E+03 0.168664954571E+03 0.165005135639E+03 0.161311835637E+03 0.157585240450E+03 - 0.153825700617E+03 0.150033742009E+03 0.146210080614E+03 0.142355636015E+03 0.138471547517E+03 - 0.134559194204E+03 0.130620213523E+03 0.126656524279E+03 0.122670351092E+03 0.118664249894E+03 - 0.114641136764E+03 0.110604316344E+03 0.106557510777E+03 0.102504888129E+03 0.984510866673E+02 - 0.944012339330E+02 0.903609560496E+02 0.863363727654E+02 0.823340720461E+02 0.783610582152E+02 - 0.744246635302E+02 0.705324163825E+02 0.666918556889E+02 0.629102857223E+02 0.591944670209E+02 - 0.555502478406E+02 0.519821533052E+02 0.484929672889E+02 0.450833747727E+02 0.417517679667E+02 - 0.384943736779E+02 0.353058990617E+02 0.321809409707E+02 0.291163273793E+02 0.261144332192E+02 - 0.231871490568E+02 0.203596536608E+02 0.176726023843E+02 0.151810750610E+02 0.129490421852E+02 - 0.110393419844E+02 0.950085960920E+01 0.835601618665E+01 0.759220066468E+01 0.716026999428E+01 - 0.698171626746E+01 0.696337605625E+01 0.701521103097E+01 0.706481640894E+01 0.706399982574E+01 - 0.698712687429E+01 0.682461396109E+01 0.657568919727E+01 0.624298320013E+01 0.582958771078E+01 - 0.533805431630E+01 0.477041539462E+01 0.412860168459E+01 0.341481102932E+01 0.263171179070E+01 - 0.178246356837E+01 0.870634857094E+00 -0.999435655628E-01 -0.112531912148E+01 -0.220152950825E+01 - -0.332469240511E+01 -0.449104627091E+01 -0.569696900609E+01 -0.693897492393E+01 -0.821369212013E+01 - -0.951782569431E+01 -0.108481106853E+02 -0.122012544483E+02 -0.135738672328E+02 -0.149623731041E+02 - -0.163629117637E+02 -0.177712183098E+02 -0.191824857852E+02 -0.205912204531E+02 -0.219910948777E+02 - -0.233748286767E+02 -0.247341319073E+02 -0.260597523206E+02 -0.273417023275E+02 -0.285696934068E+02 - -0.297338412853E+02 -0.308256304858E+02 -0.318391158428E+02 -0.327722938273E+02 -0.336286100568E+02 - -0.344184456425E+02 -0.351602597585E+02 -0.358801414497E+02 -0.366068311139E+02 -0.373574865876E+02 - -0.381118852062E+02 -0.387867275622E+02 -0.392441858557E+02 -0.393634905455E+02 -0.391349287148E+02 - -0.386756213346E+02 -0.381417405135E+02 -0.376357899161E+02 -0.371876084550E+02 -0.367857306580E+02 - -0.364089018964E+02 -0.360404071207E+02 -0.356705439437E+02 -0.352947736228E+02 -0.349113814513E+02 - -0.345198082255E+02 -0.341197553652E+02 -0.337108267652E+02 -0.332925072698E+02 -0.328642387355E+02 - -0.324255741088E+02 -0.319762374262E+02 -0.315162189773E+02 -0.310457677673E+02 -0.305653859428E+02 - -0.300757994671E+02 -0.295778859655E+02 -0.290726427811E+02 -0.285611001088E+02 -0.280442533230E+02 - -0.275230192225E+02 -0.269981679715E+02 -0.264702980396E+02 -0.259398216884E+02 -0.254069826533E+02 - -0.248718994035E+02 -0.243346314120E+02 -0.237952586509E+02 -0.232539624826E+02 -0.227110955590E+02 - -0.221672327097E+02 -0.216231877485E+02 -0.210800244667E+02 -0.205390174919E+02 -0.200016158604E+02 - -0.194693867508E+02 - core charge-density - 0.566100408154E-04 0.603274793074E-04 0.642889044283E-04 0.685103166568E-04 0.730087644875E-04 - 0.778024129493E-04 0.829106165872E-04 0.883539971990E-04 0.941545266560E-04 0.100335615085E-03 - 0.106922204822E-03 0.113940870473E-03 0.121419925474E-03 0.129389535603E-03 0.137881839861E-03 - 0.146931079213E-03 0.156573733698E-03 0.166848668447E-03 0.177797289174E-03 0.189463707768E-03 - 0.201894918628E-03 0.215140986428E-03 0.229255246056E-03 0.244294515512E-03 0.260319322588E-03 - 0.277394146221E-03 0.295587673465E-03 0.314973073077E-03 0.335628286780E-03 0.357636339348E-03 - 0.381085668699E-03 0.406070477287E-03 0.432691106152E-03 0.461054433064E-03 0.491274296314E-03 - 0.523471945761E-03 0.557776522887E-03 0.594325571702E-03 0.633265582435E-03 0.674752570119E-03 - 0.718952690251E-03 0.766042893893E-03 0.816211624674E-03 0.869659560374E-03 0.926600401855E-03 - 0.987261712339E-03 0.105188581019E-02 0.112073071853E-02 0.119407117529E-02 0.127219970739E-02 - 0.135542777315E-02 0.144408697705E-02 0.153853036146E-02 0.163913378001E-02 0.174629735768E-02 - 0.186044704296E-02 0.198203625776E-02 0.211154765105E-02 0.224949496259E-02 0.239642500343E-02 - 0.255291976042E-02 0.271959863213E-02 0.289712080418E-02 0.308618777247E-02 0.328754602307E-02 - 0.350198987838E-02 0.373036451926E-02 0.397356919385E-02 0.423256062398E-02 0.450835662100E-02 - 0.480203992329E-02 0.511476226845E-02 0.544774871395E-02 0.580230222061E-02 0.617980851421E-02 - 0.658174124108E-02 0.700966743491E-02 0.746525331166E-02 0.795027041269E-02 0.846660211345E-02 - 0.901625052094E-02 0.960134377894E-02 0.102241438053E-01 0.108870544842E-01 0.115926303386E-01 - 0.123435857075E-01 0.131428044576E-01 0.139933502544E-01 0.148984774251E-01 0.158616424421E-01 - 0.168865160593E-01 0.179769961364E-01 0.191372211820E-01 0.203715846550E-01 0.216847500588E-01 - 0.230816668678E-01 0.245675873256E-01 0.261480841549E-01 0.278290692218E-01 0.296168131969E-01 - 0.315179662576E-01 0.335395798759E-01 0.356891297379E-01 0.379745398409E-01 0.404042078155E-01 - 0.429870315194E-01 0.457324369501E-01 0.486504075246E-01 0.517515147717E-01 0.550469504838E-01 - 0.585485603728E-01 0.622688792735E-01 0.662211679360E-01 0.704194514465E-01 0.748785593126E-01 - 0.796141672432E-01 0.846428406544E-01 0.899820799218E-01 0.956503673943E-01 0.101667216183E+00 - 0.108053220723E+00 0.114830109098E+00 0.122020797119E+00 0.129649444112E+00 0.137741510382E+00 - 0.146323816290E+00 0.155424602864E+00 0.165073593849E+00 0.175302059090E+00 0.186142879083E+00 - 0.197630610560E+00 0.209801552888E+00 0.222693815062E+00 0.236347383047E+00 0.250804187144E+00 - 0.266108169064E+00 0.282305348315E+00 0.299443887476E+00 0.317574155859E+00 0.336748791043E+00 - 0.357022757646E+00 0.378453402702E+00 0.401100506871E+00 0.425026330694E+00 0.450295654993E+00 - 0.476975814429E+00 0.505136723178E+00 0.534850891530E+00 0.566193432183E+00 0.599242054859E+00 - 0.634077047765E+00 0.670781244335E+00 0.709439973560E+00 0.750140992075E+00 0.792974396116E+00 - 0.838032511244E+00 0.885409757763E+00 0.935202489441E+00 0.987508803254E+00 0.104242831757E+01 - 0.110006191624E+01 0.116051145584E+01 0.122387943346E+01 0.129026861193E+01 0.135978160000E+01 - 0.143252038425E+01 0.150858581009E+01 0.158807700896E+01 0.167109076893E+01 0.175772084621E+01 - 0.184805721494E+01 0.194218525332E+01 0.204018486382E+01 0.214212952620E+01 0.224808528199E+01 - 0.235810964998E+01 0.247225047259E+01 0.259054469391E+01 0.271301707089E+01 0.283967882006E+01 - 0.297052620347E+01 0.310553905827E+01 0.324467927606E+01 0.338788923929E+01 0.353509022378E+01 - 0.368618077801E+01 0.384103509149E+01 0.399950136699E+01 0.416140021288E+01 0.432652307442E+01 - 0.449463072490E+01 0.466545184006E+01 0.483868168119E+01 0.501398091525E+01 0.519097460207E+01 - 0.536925138135E+01 0.554836289419E+01 0.572782347558E+01 0.590711015619E+01 0.608566301289E+01 - 0.626288590845E+01 0.643814766086E+01 0.661078368325E+01 0.678009813289E+01 0.694536660815E+01 - 0.710583942709E+01 0.726074551989E+01 0.740929696058E+01 0.755069415923E+01 0.768413172658E+01 - 0.780880501609E+01 0.792391733716E+01 0.802868782233E+01 0.812235991814E+01 0.820421045514E+01 - 0.827355923749E+01 0.832977907498E+01 0.837230616406E+01 0.840065070529E+01 0.841440762640E+01 - 0.841326726196E+01 0.839702582182E+01 0.836559546387E+01 0.831901377096E+01 0.825745241774E+01 - 0.818122480318E+01 0.809079241634E+01 0.798676970036E+01 0.786992718088E+01 0.774119263277E+01 - 0.760165007160E+01 0.745253637713E+01 0.729523538244E+01 0.713126929714E+01 0.696228737445E+01 - 0.679005178084E+01 0.661642068236E+01 0.644332862226E+01 0.627276433103E+01 0.610674617685E+01 - 0.594729553438E+01 0.579640841756E+01 0.565602577058E+01 0.552800288253E+01 0.541407841905E+01 - 0.531584356652E+01 0.523471181475E+01 0.517188988547E+01 0.512835031200E+01 0.510480618875E+01 - 0.510168864942E+01 0.511912769994E+01 0.515693711116E+01 0.521460413879E+01 0.529128484590E+01 - 0.538580571386E+01 0.549667199677E+01 0.562208291152E+01 0.575995334559E+01 0.590794145701E+01 - 0.606348143532E+01 0.622382075557E+01 0.638606135679E+01 0.654720421042E+01 0.670419668250E+01 - 0.685398197262E+01 0.699354974359E+01 0.711998698134E+01 0.723052801287E+01 0.732260261476E+01 - 0.739388115435E+01 0.744231577198E+01 0.746617668668E+01 0.746408283670E+01 0.743502617611E+01 - 0.737838909120E+01 0.729395456103E+01 0.718190882688E+01 0.704283651135E+01 0.687770827554E+01 - 0.668786127413E+01 0.647497281612E+01 0.624102777379E+01 0.598828043165E+01 0.571921157955E+01 - 0.543648173800E+01 0.514288151879E+01 0.484128016393E+01 0.453457335746E+01 0.422563144517E+01 - 0.391724918675E+01 0.361209817668E+01 0.331268302840E+01 0.302130238260E+01 0.274001574527E+01 - 0.247061699977E+01 0.221461533156E+01 0.197322403846E+01 0.174735751206E+01 0.153763599448E+01 - 0.134439801278E+01 0.116771925944E+01 0.100743683521E+01 0.863177139249E+00 0.734385165797E+00 - 0.620352459116E+00 0.520241286779E+00 0.433104991178E+00 0.357908936157E+00 0.293558957004E+00 - 0.238939321053E+00 0.192952592630E+00 0.154550553295E+00 0.122752018839E+00 0.966509407173E-01 - 0.754197350591E-01 0.583105682925E-01 0.446553131657E-01 0.338641471811E-01 0.254227490621E-01 - 0.188882113068E-01 0.138839137385E-01 0.100936590292E-01 0.725537509450E-02 0.515466309716E-02 - 0.361842930266E-02 0.250879268147E-02 0.171741309607E-02 0.116033964584E-02 0.773437417305E-03 - 0.508414784907E-03 0.329443540380E-03 0.210340978410E-03 0.132266603092E-03 0.818759218551E-04 - 0.498690344207E-04 0.298714443437E-04 0.175875916060E-04 0.101729842592E-04 0.577752330293E-05 - 0.321986957660E-05 0.175987680434E-05 0.942778890139E-06 0.494707936977E-06 0.254108048780E-06 - 0.127681574306E-06 0.627159654587E-07 0.300924318689E-07 0.140943050873E-07 0.643877401768E-08 - 0.286674509825E-08 - kinetic energy-density - 0.667225310700E+01 0.673712425060E+01 0.680446712075E+01 0.687435266836E+01 0.694687887984E+01 - 0.702214082218E+01 0.710025487509E+01 0.718131997169E+01 0.726545491156E+01 0.735279351610E+01 - 0.744345621126E+01 0.753759398662E+01 0.763534237524E+01 0.773686628737E+01 0.784232168333E+01 - 0.795188785384E+01 0.806574492238E+01 0.818408265740E+01 0.830710733444E+01 0.843502925423E+01 - 0.856807463323E+01 0.870648022710E+01 0.885049499152E+01 0.900038141274E+01 0.915641556867E+01 - 0.931888763846E+01 0.948810409274E+01 0.966438660997E+01 0.984807420966E+01 0.100395236808E+02 - 0.102391106259E+02 0.104472306217E+02 0.106643002129E+02 0.108907581658E+02 0.111270667128E+02 - 0.113737129173E+02 0.116312100590E+02 0.119000995563E+02 0.121809512045E+02 0.124743676751E+02 - 0.127809824020E+02 0.131014661587E+02 0.134365246352E+02 0.137869037259E+02 0.141533910827E+02 - 0.145368181700E+02 0.149380606607E+02 0.153580481103E+02 0.157977592006E+02 0.162582301660E+02 - 0.167405536651E+02 0.172458849260E+02 0.177754489910E+02 0.183305360512E+02 0.189125147524E+02 - 0.195228293783E+02 0.201630086713E+02 0.208346731063E+02 0.215395302738E+02 0.222793960940E+02 - 0.230561817871E+02 0.238719218157E+02 0.247287599035E+02 0.256289715181E+02 0.265749638623E+02 - 0.275692850611E+02 0.286146340209E+02 0.297138695051E+02 0.308700182642E+02 0.320862867047E+02 - 0.333660717388E+02 0.347129698754E+02 0.361307926843E+02 0.376235764044E+02 0.391956004794E+02 - 0.408513960801E+02 0.425957666062E+02 0.444337942387E+02 0.463708781148E+02 0.484127240612E+02 - 0.505653883328E+02 0.528352819288E+02 0.552292043731E+02 0.577543512402E+02 0.604183612533E+02 - 0.632293138540E+02 0.661957773286E+02 0.693268340532E+02 0.726320939962E+02 0.761217542966E+02 - 0.798066090433E+02 0.836980910427E+02 0.878083249418E+02 0.921501355979E+02 0.967371238079E+02 - 0.101583684198E+03 0.106705064907E+03 0.112117414382E+03 0.117837828166E+03 0.123884409252E+03 - 0.130276320038E+03 0.137033846315E+03 0.144178456681E+03 0.151732871301E+03 0.159721130996E+03 - 0.168168668145E+03 0.177102387622E+03 0.186550741797E+03 0.196543819667E+03 0.207113432275E+03 - 0.218293201433E+03 0.230118672341E+03 0.242627385569E+03 0.255859010707E+03 0.269855440106E+03 - 0.284660908978E+03 0.300322108546E+03 0.316888318424E+03 0.334411529406E+03 0.352946573140E+03 - 0.372551274533E+03 0.393286576958E+03 0.415216695313E+03 0.438409280187E+03 0.462935557123E+03 - 0.488870494758E+03 0.516292978425E+03 0.545285959475E+03 0.575936645981E+03 0.608336673183E+03 - 0.642582272764E+03 0.678774465893E+03 0.717019234895E+03 0.757427713585E+03 0.800116365041E+03 - 0.845207163513E+03 0.892827774214E+03 0.943111730250E+03 0.996198604607E+03 0.105223417127E+04 - 0.111137056935E+04 0.117376644321E+04 0.123958708989E+04 0.130900456745E+04 0.138219781473E+04 - 0.145935272609E+04 0.154066222765E+04 0.162632630841E+04 0.171655203448E+04 0.181155353272E+04 - 0.191155192916E+04 0.201677522893E+04 0.212745822677E+04 0.224384225326E+04 0.236617496487E+04 - 0.249470999554E+04 0.262970664059E+04 0.277142932316E+04 0.292014710882E+04 0.307613311454E+04 - 0.323966371690E+04 0.341101780179E+04 0.359047581918E+04 0.377831874540E+04 0.397482689787E+04 - 0.418027868224E+04 0.439494910623E+04 0.461910823060E+04 0.485301940433E+04 0.509693739042E+04 - 0.535110624785E+04 0.561575714459E+04 0.589110587709E+04 0.617735029231E+04 0.647466747894E+04 - 0.678321077715E+04 0.710310660594E+04 0.743445110603E+04 0.777730657840E+04 0.813169778138E+04 - 0.849760803098E+04 0.887497515215E+04 0.926368732206E+04 0.966357873788E+04 0.100744252544E+05 - 0.104959398994E+05 0.109277683923E+05 0.113694846547E+05 0.118205863594E+05 0.122804905999E+05 - 0.127485296775E+05 0.132239471319E+05 0.137058940235E+05 0.141934255844E+05 0.146854982713E+05 - 0.151809673402E+05 0.156785850245E+05 0.161769993384E+05 0.166747537999E+05 0.171702877851E+05 - 0.176619381795E+05 0.181479417092E+05 0.186264388693E+05 0.190954787135E+05 0.195530252077E+05 - 0.199969649781E+05 0.204251165206E+05 0.208352408440E+05 0.212250539761E+05 0.215922406971E+05 - 0.219344702466E+05 0.222494132725E+05 0.225347605700E+05 0.227882431008E+05 0.230076533887E+05 - 0.231908681093E+05 0.233358716618E+05 0.234407805422E+05 0.235038681649E+05 0.235235900533E+05 - 0.234986087273E+05 0.234278183452E+05 0.233103682573E+05 0.231456854657E+05 0.229334951930E+05 - 0.226738393464E+05 0.223670922687E+05 0.220139733481E+05 0.216155560262E+05 0.211732727643E+05 - 0.206889155309E+05 0.201646315262E+05 0.196029137234E+05 0.190065861750E+05 0.183787838063E+05 - 0.177229268159E+05 0.170426897439E+05 0.163419655306E+05 0.156248249789E+05 0.148954722996E+05 - 0.141581974366E+05 0.134173260145E+05 0.126771691922E+05 0.119419712654E+05 0.112158605109E+05 - 0.105028006708E+05 0.980654494715E+04 0.913059233055E+04 0.847814609089E+04 0.785207437181E+04 - 0.725487344241E+04 0.668863511465E+04 0.615502070274E+04 0.565524418908E+04 0.519006674861E+04 - 0.475980360345E+04 0.436434269355E+04 0.400317323797E+04 0.367542134980E+04 0.337988976738E+04 - 0.311509951930E+04 0.287933230868E+04 0.267067322820E+04 0.248705367813E+04 0.232629416860E+04 - 0.218614663490E+04 0.206433550064E+04 0.195859682514E+04 0.186671473352E+04 0.178655448544E+04 - 0.171609162157E+04 0.165343661643E+04 0.159685503183E+04 0.154478263373E+04 0.149583588863E+04 - 0.144881771439E+04 0.140271881257E+04 0.135671510542E+04 0.131016154370E+04 0.126258277404E+04 - 0.121366142449E+04 0.116322423992E+04 0.111122677010E+04 0.105773711321E+04 0.100291904396E+04 - 0.947015115197E+03 0.890329984746E+03 0.833214293651E+03 0.776049439334E+03 0.719233318915E+03 - 0.663167296841E+03 0.608244410857E+03 0.554838974842E+03 0.503297365412E+03 0.453930342998E+03 - 0.407006607762E+03 0.362747812430E+03 0.321325009274E+03 0.282856892955E+03 0.247409239962E+03 - 0.214996579478E+03 0.185585236387E+03 0.159098170048E+03 0.135421626809E+03 0.114413852193E+03 - 0.959157052909E+02 0.797613940556E+02 0.657850483488E+02 0.538188496239E+02 0.436847852099E+02 - 0.351899899383E+02 0.281331189963E+02 0.223174718914E+02 0.175614959268E+02 0.137029350510E+02 - 0.105987314150E+02 0.812353702200E+01 0.616817828814E+01 0.463832179287E+01 0.345324799301E+01 - 0.254463529365E+01 0.185531917238E+01 0.133803581425E+01 0.954179117826E+00 0.672602600260E+00 - 0.468493246228E+00 0.322336920783E+00 0.218987948819E+00 0.146848889647E+00 0.971612490355E-01 - 0.634035384763E-01 0.407898120808E-01 0.258595430908E-01 0.161483043720E-01 0.992815188083E-02 - 0.600670032098E-02 0.357447468542E-02 0.209108518893E-02 0.120193474515E-02 0.678419162901E-03 - 0.375814762046E-03 0.204197625257E-03 0.108758556747E-03 0.567464770105E-04 0.289864948733E-04 - 0.144858436153E-04 0.707756608346E-05 0.337836787752E-05 0.157431967128E-05 0.715666075556E-06 - 0.317114132527E-06 - mkinetic energy-density pseudized - 0.246124888537E-05 0.262391872705E-05 0.279733980862E-05 0.298222270529E-05 0.317932495590E-05 - 0.338945416684E-05 0.361347132112E-05 0.385229430620E-05 0.410690167493E-05 0.437833665502E-05 - 0.466771142364E-05 0.497621166438E-05 0.530510142550E-05 0.565572829925E-05 0.602952894345E-05 - 0.642803496809E-05 0.685287921091E-05 0.730580242782E-05 0.778866042541E-05 0.830343166499E-05 - 0.885222536910E-05 0.943729016382E-05 0.100610232923E-04 0.107259804372E-04 0.114348861924E-04 - 0.121906452265E-04 0.129963541849E-04 0.138553143773E-04 0.147710453050E-04 0.157472990816E-04 - 0.167880758067E-04 0.178976399559E-04 0.190805378543E-04 0.203416163043E-04 0.216860424450E-04 - 0.231193249240E-04 0.246473364685E-04 0.262763379479E-04 0.280130040275E-04 0.298644505170E-04 - 0.318382635266E-04 0.339425305504E-04 0.361858736043E-04 0.385774845531E-04 0.411271627738E-04 - 0.438453553070E-04 0.467431996630E-04 0.498325694557E-04 0.531261230543E-04 0.566373554487E-04 - 0.603806535444E-04 0.643713551105E-04 0.686258116250E-04 0.731614552728E-04 0.779968703717E-04 - 0.831518695202E-04 0.886475747764E-04 0.945065042035E-04 0.100752664135E-03 0.107411647536E-03 - 0.114510738871E-03 0.122079025894E-03 0.130147518834E-03 0.138749277455E-03 0.147919546515E-03 - 0.157695900175E-03 0.168118395953E-03 0.179229738859E-03 0.191075456367E-03 0.203704084960E-03 - 0.217167368999E-03 0.231520472741E-03 0.246822206364E-03 0.263135266932E-03 0.280526495285E-03 - 0.299067149912E-03 0.318833198917E-03 0.339905631285E-03 0.362370788723E-03 0.386320719425E-03 - 0.411853555224E-03 0.439073913670E-03 0.468093326673E-03 0.499030697487E-03 0.532012787889E-03 - 0.567174737555E-03 0.604660617766E-03 0.644624021702E-03 0.687228693756E-03 0.732649200428E-03 - 0.781071645568E-03 0.832694432884E-03 0.887729078840E-03 0.946401079282E-03 0.100895083333E-02 - 0.107563462835E-02 0.114672568997E-02 0.122251530154E-02 0.130331399757E-02 0.138945283600E-02 - 0.148128475454E-02 0.157918601675E-02 0.168355775354E-02 0.179482760663E-02 0.191345148057E-02 - 0.203991541061E-02 0.217473755390E-02 0.231847031234E-02 0.247170259569E-02 0.263506223425E-02 - 0.280921855092E-02 0.299488510327E-02 0.319282260676E-02 0.340384205113E-02 0.362880802269E-02 - 0.386864224621E-02 0.412432736067E-02 0.439691094468E-02 0.468750980769E-02 0.499731456482E-02 - 0.532759451388E-02 0.567970283464E-02 0.605508213156E-02 0.645527034270E-02 0.688190703890E-02 - 0.733674013923E-02 0.782163306985E-02 0.833857239587E-02 0.888967595726E-02 0.947720154207E-02 - 0.101035561326E-01 0.107713057620E-01 0.114831860220E-01 0.122421132645E-01 0.130511965425E-01 - 0.139137503401E-01 0.148333081420E-01 0.158136368999E-01 0.168587524534E-01 0.179729359683E-01 - 0.191607514608E-01 0.204270644777E-01 0.217770620094E-01 0.232162737168E-01 0.247505945587E-01 - 0.263863089121E-01 0.281301162830E-01 0.299891587134E-01 0.319710499956E-01 0.340839068117E-01 - 0.363363819276E-01 0.387376995728E-01 0.412976931539E-01 0.440268454506E-01 0.469363314614E-01 - 0.500380640699E-01 0.533447427184E-01 0.568699052850E-01 0.606279833752E-01 0.646343612502E-01 - 0.689054386321E-01 0.734586976370E-01 0.783127741083E-01 0.834875336360E-01 0.890041525683E-01 - 0.948852043423E-01 0.101154751478E+00 0.107838443608E+00 0.114963621928E+00 0.122559430498E+00 - 0.130656934825E+00 0.139289248207E+00 0.148491666340E+00 0.158301810719E+00 0.168759781404E+00 - 0.179908319753E+00 0.191792981758E+00 0.204462322680E+00 0.217968093682E+00 0.232365451255E+00 - 0.247713180229E+00 0.264073931251E+00 0.281514473635E+00 0.300105964567E+00 0.319924235688E+00 - 0.341050098153E+00 0.363569667326E+00 0.387574708325E+00 0.413163003732E+00 0.440438744815E+00 - 0.469512947739E+00 0.500503896272E+00 0.533537612614E+00 0.568748358052E+00 0.606279165231E+00 - 0.646282403941E+00 0.688920382391E+00 0.734365986076E+00 0.782803356419E+00 0.834428611485E+00 - 0.889450611177E+00 0.948091769412E+00 0.101058891592E+01 0.107719421035E+01 0.114817611159E+01 - 0.122382040517E+01 0.130443129174E+01 0.139033253996E+01 0.148186870671E+01 0.157940642816E+01 - 0.168333578488E+01 0.179407174448E+01 0.191205568510E+01 0.203775700321E+01 0.217167480907E+01 - 0.231433971309E+01 0.246631570642E+01 0.262820213858E+01 0.280063579506E+01 0.298429307738E+01 - 0.317989228767E+01 0.338819601939E+01 0.361001365538E+01 0.384620397342E+01 0.409767785897E+01 - 0.436540112340E+01 0.465039742506E+01 0.495375128900E+01 0.527661121944E+01 0.562019289718E+01 - 0.598578245173E+01 0.637473979546E+01 0.678850200382E+01 0.722858672211E+01 0.769659557535E+01 - 0.819421755303E+01 0.872323233508E+01 0.928551351943E+01 0.988303170455E+01 0.105178573724E+02 - 0.111921635088E+02 0.119082278868E+02 0.126684349296E+02 0.134752770548E+02 0.143313553888E+02 - 0.152393797229E+02 0.162021675676E+02 0.172226421397E+02 0.183038290954E+02 0.194488518008E+02 - 0.206609249026E+02 0.219433459362E+02 0.232994846736E+02 0.247327698867E+02 0.262466731579E+02 - 0.278446893409E+02 0.295303132271E+02 0.313070119380E+02 0.331781925189E+02 0.351471641660E+02 - 0.372170944806E+02 0.393909591027E+02 0.416714840392E+02 0.440610799768E+02 0.465617678440E+02 - 0.491750948813E+02 0.519020404840E+02 0.547429111112E+02 0.576972236099E+02 0.607635763904E+02 - 0.639395080244E+02 0.672213430148E+02 0.706040247369E+02 0.740809358673E+02 0.776437070298E+02 - 0.812820149008E+02 0.849833716541E+02 0.887329083984E+02 0.925131561972E+02 0.963038293663E+02 - 0.100081617042E+03 0.103819990523E+03 0.107489035592E+03 0.111055320957E+03 0.114481816082E+03 - 0.117727873934E+03 0.120749296619E+03 0.123498504282E+03 0.125924830007E+03 0.127974965523E+03 - 0.129593584125E+03 0.130724168028E+03 0.131310066960E+03 0.131295812806E+03 0.130628710801E+03 - 0.129260720552E+03 0.127150629186E+03 0.124266503311E+03 0.120588385239E+03 0.116111171158E+03 - 0.110847573781E+03 0.104831028760E+03 0.981183527514E+02 0.907919018702E+02 0.829609139133E+02 - 0.747616492584E+02 0.663558786143E+02 0.579272084376E+02 0.496746974883E+02 0.418032150738E+02 - 0.345100415462E+02 0.279673372239E+02 0.223003334013E+02 0.175614569257E+02 0.137029350510E+02 - 0.105987314150E+02 0.812353702200E+01 0.616817828814E+01 0.463832179287E+01 0.345324799301E+01 - 0.254463529365E+01 0.185531917238E+01 0.133803581425E+01 0.954179117826E+00 0.672602600260E+00 - 0.468493246228E+00 0.322336920783E+00 0.218987948819E+00 0.146848889647E+00 0.971612490355E-01 - 0.634035384763E-01 0.407898120808E-01 0.258595430908E-01 0.161483043720E-01 0.992815188083E-02 - 0.600670032098E-02 0.357447468542E-02 0.209108518893E-02 0.120193474515E-02 0.678419162901E-03 - 0.375814762046E-03 0.204197625257E-03 0.108758556747E-03 0.567464770105E-04 0.289864948733E-04 - 0.144858436153E-04 0.707756608346E-05 0.337836787752E-05 0.157431967128E-05 0.715666075556E-06 - 0.317114132527E-06 - local pseudopotential core - -0.170419537102E+03 -0.170420949717E+03 -0.170422317842E+03 -0.170423642879E+03 -0.170424926184E+03 - -0.170426169071E+03 -0.170427372814E+03 -0.170428538644E+03 -0.170429667756E+03 -0.170430761305E+03 - -0.170431820413E+03 -0.170432846163E+03 -0.170433839605E+03 -0.170434801757E+03 -0.170435733605E+03 - -0.170436636101E+03 -0.170437510171E+03 -0.170438356709E+03 -0.170439176583E+03 -0.170439970631E+03 - -0.170440739667E+03 -0.170441484477E+03 -0.170442205825E+03 -0.170442904450E+03 -0.170443581066E+03 - -0.170444236365E+03 -0.170444871020E+03 -0.170445485679E+03 -0.170446080973E+03 -0.170446657509E+03 - -0.170447215878E+03 -0.170447756653E+03 -0.170448280385E+03 -0.170448787612E+03 -0.170449278852E+03 - -0.170449754608E+03 -0.170450215366E+03 -0.170450661599E+03 -0.170451093762E+03 -0.170451512298E+03 - -0.170451917634E+03 -0.170452310186E+03 -0.170452690353E+03 -0.170453058526E+03 -0.170453415081E+03 - -0.170453760381E+03 -0.170454094779E+03 -0.170454418618E+03 -0.170454732227E+03 -0.170455035927E+03 - -0.170455330028E+03 -0.170455614830E+03 -0.170455890623E+03 -0.170456157689E+03 -0.170456416298E+03 - -0.170456666716E+03 -0.170456909196E+03 -0.170457143986E+03 -0.170457371323E+03 -0.170457591439E+03 - -0.170457804557E+03 -0.170458010893E+03 -0.170458210657E+03 -0.170458404049E+03 -0.170458591266E+03 - -0.170458772496E+03 -0.170458947923E+03 -0.170459117722E+03 -0.170459282063E+03 -0.170459441112E+03 - -0.170459595027E+03 -0.170459743963E+03 -0.170459888066E+03 -0.170460027479E+03 -0.170460162341E+03 - -0.170460292785E+03 -0.170460418937E+03 -0.170460540921E+03 -0.170460658856E+03 -0.170460772854E+03 - -0.170460883026E+03 -0.170460989477E+03 -0.170461092306E+03 -0.170461191611E+03 -0.170461287482E+03 - -0.170461380009E+03 -0.170461469275E+03 -0.170461555360E+03 -0.170461638339E+03 -0.170461718285E+03 - -0.170461795264E+03 -0.170461869342E+03 -0.170461940576E+03 -0.170462009025E+03 -0.170462074739E+03 - -0.170462137766E+03 -0.170462198149E+03 -0.170462255930E+03 -0.170462311143E+03 -0.170462363820E+03 - -0.170462413988E+03 -0.170462461669E+03 -0.170462506883E+03 -0.170462549643E+03 -0.170462589958E+03 - -0.170462627833E+03 -0.170462663267E+03 -0.170462696254E+03 -0.170462726785E+03 -0.170462754841E+03 - -0.170462780402E+03 -0.170462803439E+03 -0.170462823919E+03 -0.170462841800E+03 -0.170462857035E+03 - -0.170462869571E+03 -0.170462879345E+03 -0.170462886288E+03 -0.170462890322E+03 -0.170462891361E+03 - -0.170462889311E+03 -0.170462884065E+03 -0.170462875510E+03 -0.170462863519E+03 -0.170462847957E+03 - -0.170462828675E+03 -0.170462805511E+03 -0.170462778292E+03 -0.170462746828E+03 -0.170462710916E+03 - -0.170462670337E+03 -0.170462624855E+03 -0.170462574215E+03 -0.170462518144E+03 -0.170462456350E+03 - -0.170462388516E+03 -0.170462314306E+03 -0.170462233358E+03 -0.170462145285E+03 -0.170462049670E+03 - -0.170461946071E+03 -0.170461834011E+03 -0.170461712983E+03 -0.170461582443E+03 -0.170461441809E+03 - -0.170461290461E+03 -0.170461127736E+03 -0.170460952924E+03 -0.170460765269E+03 -0.170460563963E+03 - -0.170460348143E+03 -0.170460116887E+03 -0.170459869212E+03 -0.170459604070E+03 -0.170459320339E+03 - -0.170459016825E+03 -0.170458692253E+03 -0.170458345262E+03 -0.170457974402E+03 -0.170457578124E+03 - -0.170457154777E+03 -0.170456702599E+03 -0.170456219712E+03 -0.170455704112E+03 -0.170455153662E+03 - -0.170454566083E+03 -0.170453938945E+03 -0.170453269655E+03 -0.170452555451E+03 -0.170451793386E+03 - -0.170450980315E+03 -0.170450112890E+03 -0.170449187536E+03 -0.170448200444E+03 -0.170447147552E+03 - -0.170446024529E+03 -0.170444826756E+03 -0.170443549310E+03 -0.170442186941E+03 -0.170440734052E+03 - -0.170439184675E+03 -0.170437532448E+03 -0.170435770588E+03 -0.170433891862E+03 -0.170431888560E+03 - -0.170429752461E+03 -0.170427474801E+03 -0.170425046235E+03 -0.170422456803E+03 -0.170419695882E+03 - -0.170416752151E+03 -0.170413613538E+03 -0.170410267173E+03 -0.170406699335E+03 -0.170402895397E+03 - -0.170398839763E+03 -0.170394515809E+03 -0.170389905809E+03 -0.170384990867E+03 -0.170379750838E+03 - -0.170374164243E+03 -0.170368208187E+03 -0.170361858259E+03 -0.170355088435E+03 -0.170347870972E+03 - -0.170340176292E+03 -0.170331972863E+03 -0.170323227069E+03 -0.170313903070E+03 -0.170303962662E+03 - -0.170293365112E+03 -0.170282066997E+03 -0.170270022024E+03 -0.170257180840E+03 -0.170243490832E+03 - -0.170228895910E+03 -0.170213336276E+03 -0.170196748182E+03 -0.170179063668E+03 -0.170160210283E+03 - -0.170140110788E+03 -0.170118682843E+03 -0.170095838665E+03 -0.170071484675E+03 -0.170045521109E+03 - -0.170017841614E+03 -0.169988332811E+03 -0.169956873833E+03 -0.169923335828E+03 -0.169887581435E+03 - -0.169849464223E+03 -0.169808828091E+03 -0.169765506630E+03 -0.169719322446E+03 -0.169670086435E+03 - -0.169617597009E+03 -0.169561639280E+03 -0.169501984178E+03 -0.169438387525E+03 -0.169370589035E+03 - -0.169298311258E+03 -0.169221258457E+03 -0.169139115401E+03 -0.169051546092E+03 -0.168958192406E+03 - -0.168858672641E+03 -0.168752579981E+03 -0.168639480855E+03 -0.168518913198E+03 -0.168390384594E+03 - -0.168253370309E+03 -0.168107311204E+03 -0.167951611505E+03 -0.167785636454E+03 -0.167608709804E+03 - -0.167420111171E+03 -0.167219073226E+03 -0.167004778724E+03 -0.166776357364E+03 -0.166532882470E+03 - -0.166273367495E+03 -0.165996762340E+03 -0.165701949473E+03 -0.165387739872E+03 -0.165052868771E+03 - -0.164695991214E+03 -0.164315677437E+03 -0.163910408071E+03 -0.163478569188E+03 -0.163018447220E+03 - -0.162528223759E+03 -0.162005970291E+03 -0.161449642903E+03 -0.160857077032E+03 -0.160225982313E+03 - -0.159553937637E+03 -0.158838386520E+03 -0.158076632925E+03 -0.157265837697E+03 -0.156403015813E+03 - -0.155485034673E+03 -0.154508613712E+03 -0.153470325644E+03 -0.152366599705E+03 -0.151193727316E+03 - -0.149947870610E+03 -0.148625074366E+03 -0.147221281861E+03 -0.145732355265E+03 -0.144154101132E+03 - -0.142482301607E+03 -0.140712751934E+03 -0.138841304820E+03 -0.136863922328E+03 -0.134776736105E+03 - -0.132576117318E+03 -0.130258758637E+03 -0.127821772487E+03 -0.125262812538E+03 -0.122580229059E+03 - -0.119773271760E+03 -0.116842352862E+03 -0.113789370936E+03 -0.110618060917E+03 -0.107334264143E+03 - -0.103945902820E+03 -0.100462343388E+03 -0.968929062687E+02 -0.932448542294E+02 -0.895225733479E+02 - -0.857313037065E+02 -0.818880617199E+02 -0.780358333759E+02 -0.742473243088E+02 -0.706440350099E+02 - -0.673676290197E+02 -0.644619319781E+02 -0.619683360442E+02 -0.599284345816E+02 -0.583796623680E+02 - -0.573483530309E+02 -0.564049902626E+02 -0.546210802335E+02 -0.528996689676E+02 -0.512328440922E+02 - -0.496187824367E+02 -0.480557353139E+02 -0.465420447751E+02 -0.450761138085E+02 -0.436564113209E+02 - -0.422814579106E+02 -0.409498296230E+02 -0.396601648757E+02 -0.384111132984E+02 -0.372014105719E+02 - -0.360298090009E+02 -0.348951073700E+02 -0.337961428427E+02 -0.327317889712E+02 -0.317009553098E+02 - -0.307025861793E+02 -0.297356589850E+02 -0.287991834979E+02 -0.278922006685E+02 -0.270137815368E+02 - -0.261630366058E+02 -0.253390651934E+02 -0.245410526913E+02 -0.237681721638E+02 -0.230196321567E+02 - -0.222946661137E+02 - pspotential valence only - -0.284683254939E+02 -0.284697381271E+02 -0.284711062722E+02 -0.284724313301E+02 -0.284737146577E+02 - -0.284749575687E+02 -0.284761613369E+02 -0.284773271946E+02 -0.284784563356E+02 -0.284795499162E+02 - -0.284806090569E+02 -0.284816348417E+02 -0.284826283219E+02 -0.284835905140E+02 -0.284845224041E+02 - -0.284854249463E+02 -0.284862990646E+02 -0.284871456545E+02 -0.284879655834E+02 -0.284887596902E+02 - -0.284895287887E+02 -0.284902736661E+02 -0.284909950857E+02 -0.284916937861E+02 -0.284923704830E+02 - -0.284930258689E+02 -0.284936606159E+02 -0.284942753734E+02 -0.284948707712E+02 -0.284954474190E+02 - -0.284960059074E+02 -0.284965468085E+02 -0.284970706761E+02 -0.284975780470E+02 -0.284980694407E+02 - -0.284985453605E+02 -0.284990062936E+02 -0.284994527124E+02 -0.284998850740E+02 -0.285003038213E+02 - -0.285007093833E+02 -0.285011021753E+02 -0.285014825995E+02 -0.285018510457E+02 -0.285022078914E+02 - -0.285025535021E+02 -0.285028882318E+02 -0.285032124235E+02 -0.285035264093E+02 -0.285038305108E+02 - -0.285041250397E+02 -0.285044102975E+02 -0.285046865768E+02 -0.285049541606E+02 -0.285052133230E+02 - -0.285054643298E+02 -0.285057074382E+02 -0.285059428973E+02 -0.285061709484E+02 -0.285063918256E+02 - -0.285066057551E+02 -0.285068129563E+02 -0.285070136418E+02 -0.285072080174E+02 -0.285073962824E+02 - -0.285075786302E+02 -0.285077552478E+02 -0.285079263164E+02 -0.285080920118E+02 -0.285082525042E+02 - -0.285084079583E+02 -0.285085585338E+02 -0.285087043857E+02 -0.285088456639E+02 -0.285089825138E+02 - -0.285091150761E+02 -0.285092434873E+02 -0.285093678798E+02 -0.285094883820E+02 -0.285096051179E+02 - -0.285097182082E+02 -0.285098277696E+02 -0.285099339157E+02 -0.285100367561E+02 -0.285101363974E+02 - -0.285102329431E+02 -0.285103264934E+02 -0.285104171457E+02 -0.285105049943E+02 -0.285105901311E+02 - -0.285106726449E+02 -0.285107526223E+02 -0.285108301473E+02 -0.285109053014E+02 -0.285109781641E+02 - -0.285110488125E+02 -0.285111173215E+02 -0.285111837644E+02 -0.285112482121E+02 -0.285113107339E+02 - -0.285113713974E+02 -0.285114302684E+02 -0.285114874111E+02 -0.285115428883E+02 -0.285115967613E+02 - -0.285116490901E+02 -0.285116999333E+02 -0.285117493486E+02 -0.285117973923E+02 -0.285118441197E+02 - -0.285118895855E+02 -0.285119338432E+02 -0.285119769456E+02 -0.285120189449E+02 -0.285120598926E+02 - -0.285120998398E+02 -0.285121388371E+02 -0.285121769347E+02 -0.285122141827E+02 -0.285122506311E+02 - -0.285122863296E+02 -0.285123213283E+02 -0.285123556771E+02 -0.285123894265E+02 -0.285124226271E+02 - -0.285124553304E+02 -0.285124875881E+02 -0.285125194530E+02 -0.285125509785E+02 -0.285125822193E+02 - -0.285126132311E+02 -0.285126440711E+02 -0.285126747978E+02 -0.285127054715E+02 -0.285127361543E+02 - -0.285127669105E+02 -0.285127978063E+02 -0.285128289107E+02 -0.285128602951E+02 -0.285128920341E+02 - -0.285129242051E+02 -0.285129568892E+02 -0.285129901712E+02 -0.285130241396E+02 -0.285130588877E+02 - -0.285130945129E+02 -0.285131311181E+02 -0.285131688112E+02 -0.285132077060E+02 -0.285132479227E+02 - -0.285132895879E+02 -0.285133328354E+02 -0.285133778067E+02 -0.285134246515E+02 -0.285134735280E+02 - -0.285135246042E+02 -0.285135780579E+02 -0.285136340776E+02 -0.285136928634E+02 -0.285137546275E+02 - -0.285138195953E+02 -0.285138880062E+02 -0.285139601145E+02 -0.285140361905E+02 -0.285141165216E+02 - -0.285142014133E+02 -0.285142911905E+02 -0.285143861989E+02 -0.285144868063E+02 -0.285145934040E+02 - -0.285147064088E+02 -0.285148262640E+02 -0.285149534418E+02 -0.285150884449E+02 -0.285152318086E+02 - -0.285153841030E+02 -0.285155459353E+02 -0.285157179524E+02 -0.285159008433E+02 -0.285160953418E+02 - -0.285163022301E+02 -0.285165223414E+02 -0.285167565633E+02 -0.285170058417E+02 -0.285172711848E+02 - -0.285175536665E+02 -0.285178544315E+02 -0.285181746997E+02 -0.285185157709E+02 -0.285188790308E+02 - -0.285192659558E+02 -0.285196781195E+02 -0.285201171993E+02 -0.285205849827E+02 -0.285210833750E+02 - -0.285216144068E+02 -0.285221802425E+02 -0.285227831889E+02 -0.285234257049E+02 -0.285241104112E+02 - -0.285248401009E+02 -0.285256177512E+02 -0.285264465355E+02 -0.285273298358E+02 -0.285282712570E+02 - -0.285292746413E+02 -0.285303440839E+02 -0.285314839493E+02 -0.285326988895E+02 -0.285339938624E+02 - -0.285353741521E+02 -0.285368453903E+02 -0.285384135789E+02 -0.285400851146E+02 -0.285418668143E+02 - -0.285437659428E+02 -0.285457902421E+02 -0.285479479626E+02 -0.285502478960E+02 -0.285526994109E+02 - -0.285553124903E+02 -0.285580977715E+02 -0.285610665889E+02 -0.285642310189E+02 -0.285676039282E+02 - -0.285711990254E+02 -0.285750309147E+02 -0.285791151548E+02 -0.285834683196E+02 -0.285881080644E+02 - -0.285930531950E+02 -0.285983237421E+02 -0.286039410397E+02 -0.286099278085E+02 -0.286163082449E+02 - -0.286231081153E+02 -0.286303548557E+02 -0.286380776784E+02 -0.286463076848E+02 -0.286550779846E+02 - -0.286644238234E+02 -0.286743827172E+02 -0.286849945954E+02 -0.286963019524E+02 -0.287083500084E+02 - -0.287211868802E+02 -0.287348637617E+02 -0.287494351160E+02 -0.287649588782E+02 -0.287814966715E+02 - -0.287991140354E+02 -0.288178806686E+02 -0.288378706855E+02 -0.288591628897E+02 -0.288818410637E+02 - -0.289059942762E+02 -0.289317172092E+02 -0.289591105063E+02 -0.289882811421E+02 -0.290193428172E+02 - -0.290524163789E+02 -0.290876302704E+02 -0.291251210119E+02 -0.291650337155E+02 -0.292075226382E+02 - -0.292527517763E+02 -0.293008955057E+02 -0.293521392725E+02 -0.294066803402E+02 -0.294647285981E+02 - -0.295265074381E+02 -0.295922547054E+02 -0.296622237316E+02 -0.297366844547E+02 -0.298159246331E+02 - -0.299002511574E+02 -0.299899914619E+02 -0.300854950331E+02 -0.301871350062E+02 -0.302953098315E+02 - -0.304104449766E+02 -0.305329946137E+02 -0.306634432095E+02 -0.308023068980E+02 -0.309501344631E+02 - -0.311075076831E+02 -0.312750406914E+02 -0.314533778800E+02 -0.316431897005E+02 -0.318451655059E+02 - -0.320600023150E+02 -0.322883880770E+02 -0.325309776931E+02 -0.327883597634E+02 -0.330610118773E+02 - -0.333492424278E+02 -0.336531176630E+02 -0.339723743103E+02 -0.343063208921E+02 -0.346537346687E+02 - -0.350127648590E+02 -0.353808530483E+02 -0.357546715064E+02 -0.361300480027E+02 -0.365017784125E+02 - -0.368631247339E+02 -0.372047035859E+02 -0.375125522453E+02 -0.377657441997E+02 -0.379353184151E+02 - -0.379879389853E+02 -0.378969925173E+02 -0.376572676136E+02 -0.372895576797E+02 -0.368609888139E+02 - -0.364580391316E+02 -0.360705398812E+02 -0.356893344048E+02 -0.353112759769E+02 -0.349371692593E+02 - -0.345669009289E+02 -0.341257389052E+02 -0.337108267652E+02 -0.332925072698E+02 -0.328642387355E+02 - -0.324255741088E+02 -0.319762374261E+02 -0.315162189773E+02 -0.310457677673E+02 -0.305653859428E+02 - -0.300757994671E+02 -0.295778859655E+02 -0.290726427811E+02 -0.285611001088E+02 -0.280442533230E+02 - -0.275230192225E+02 -0.269981679715E+02 -0.264702980396E+02 -0.259398216884E+02 -0.254069826533E+02 - -0.248718994035E+02 -0.243346314120E+02 -0.237952586509E+02 -0.232539624826E+02 -0.227110955590E+02 - -0.221672327097E+02 -0.216231877485E+02 -0.210800244667E+02 -0.205390174919E+02 -0.200016158604E+02 - -0.194693867508E+02 - core charge-density (pseudized) - 0.158850899689E-07 0.169349736624E-07 0.180542466866E-07 0.192474951488E-07 0.205196082630E-07 - 0.218757983829E-07 0.233216223594E-07 0.248630043089E-07 0.265062598867E-07 0.282581221651E-07 - 0.301257692211E-07 0.321168535480E-07 0.342395334107E-07 0.365025062734E-07 0.389150444365E-07 - 0.414870330287E-07 0.442290105108E-07 0.471522118554E-07 0.502686145814E-07 0.535909878306E-07 - 0.571329446878E-07 0.609089979589E-07 0.649346196361E-07 0.692263042922E-07 0.738016366658E-07 - 0.786793637127E-07 0.838794714198E-07 0.894232666954E-07 0.953334646720E-07 0.101634281779E-06 - 0.108351534969E-06 0.115512747495E-06 0.123147261690E-06 0.131286359191E-06 0.139963389111E-06 - 0.149213904686E-06 0.159075808951E-06 0.169589510039E-06 0.180798086755E-06 0.192747465083E-06 - 0.205486606363E-06 0.219067707906E-06 0.233546416867E-06 0.248982058249E-06 0.265437877988E-06 - 0.282981302089E-06 0.301684212902E-06 0.321623243648E-06 0.342880092415E-06 0.365541856912E-06 - 0.389701391334E-06 0.415457686827E-06 0.442916277090E-06 0.472189670789E-06 0.503397812546E-06 - 0.536668574398E-06 0.572138279738E-06 0.609952261885E-06 0.650265459564E-06 0.693243051754E-06 - 0.739061134484E-06 0.787907442368E-06 0.839982117817E-06 0.895498531103E-06 0.954684154611E-06 - 0.101778149488E-05 0.108504908622E-05 0.115676255007E-05 0.123321572426E-05 0.131472186699E-05 - 0.140161494038E-05 0.149425097881E-05 0.159300954771E-05 0.169829529876E-05 0.181053962793E-05 - 0.193020244302E-05 0.205777404804E-05 0.219377715217E-05 0.233876901146E-05 0.249334371211E-05 - 0.265813460454E-05 0.283381689853E-05 0.302111042968E-05 0.322078260881E-05 0.343365156625E-05 - 0.366058950397E-05 0.390252626918E-05 0.416045316422E-05 0.443542700810E-05 0.472857446660E-05 - 0.504109666844E-05 0.537427412660E-05 0.572947198482E-05 0.610814561085E-05 0.651184655941E-05 - 0.694222892908E-05 0.740105613948E-05 0.789020815618E-05 0.841168919314E-05 0.896763592422E-05 - 0.956032623723E-05 0.101921885666E-04 0.108658118429E-04 0.115839560993E-04 0.123495637799E-04 - 0.131657717945E-04 0.140359243703E-04 0.149635867530E-04 0.159525598129E-04 0.170068956168E-04 - 0.181309140281E-04 0.193292204042E-04 0.206067244636E-04 0.219686603991E-04 0.234206083207E-04 - 0.249685171141E-04 0.266187288110E-04 0.283780045687E-04 0.302535523666E-04 0.322530565318E-04 - 0.343847092167E-04 0.366572439552E-04 0.390799714357E-04 0.416628176384E-04 0.444163644905E-04 - 0.473518932085E-04 0.504814305022E-04 0.538177978316E-04 0.573746639168E-04 0.611666007161E-04 - 0.652091431015E-04 0.695188524752E-04 0.741133845867E-04 0.790115618292E-04 0.842334503093E-04 - 0.898004420053E-04 0.957353423503E-04 0.102062463597E-03 0.108807724347E-03 0.115998755648E-03 - 0.123665014094E-03 0.131837902394E-03 0.140550897889E-03 0.149839689558E-03 0.159742324059E-03 - 0.170299361400E-03 0.181554040889E-03 0.193552458017E-03 0.206343753015E-03 0.219980311839E-03 - 0.234517980402E-03 0.250016292931E-03 0.266538715372E-03 0.284152904832E-03 0.302930986106E-03 - 0.322949846427E-03 0.344291449616E-03 0.367043170908E-03 0.391298153825E-03 0.417155690514E-03 - 0.444721627115E-03 0.474108795766E-03 0.505437475018E-03 0.538835880487E-03 0.574440687740E-03 - 0.612397589507E-03 0.652861889451E-03 0.695999134887E-03 0.741985790982E-03 0.791009959124E-03 - 0.843272142330E-03 0.898986060746E-03 0.958379520474E-03 0.102169533918E-02 0.108919233216E-02 - 0.116114636271E-02 0.123785146100E-02 0.131962101582E-02 0.140678904387E-02 0.149971154154E-02 - 0.159876792447E-02 0.170436256045E-02 0.181692640162E-02 0.193691872220E-02 0.206482896848E-02 - 0.220117872808E-02 0.234652382591E-02 0.250145655494E-02 0.266660804989E-02 0.284265081290E-02 - 0.303030140045E-02 0.323032328145E-02 0.344352987696E-02 0.367078779245E-02 0.391302025435E-02 - 0.417121076298E-02 0.444640697483E-02 0.473972482752E-02 0.505235292185E-02 0.538555717557E-02 - 0.574068576454E-02 0.611917436752E-02 0.652255173154E-02 0.695244557544E-02 0.741058885008E-02 - 0.789882637401E-02 0.841912186447E-02 0.897356538393E-02 0.956438122282E-02 0.101939362401E-01 - 0.108647486828E-01 0.115794975075E-01 0.123410322246E-01 0.131523832881E-01 0.140167730542E-01 - 0.149376273269E-01 0.159185875152E-01 0.169635234187E-01 0.180765466621E-01 0.192620247931E-01 - 0.205245960596E-01 0.218691848759E-01 0.233010179867E-01 0.248256413302E-01 0.264489375990E-01 - 0.281771444893E-01 0.300168736216E-01 0.319751301066E-01 0.340593327208E-01 0.362773346404E-01 - 0.386374446716E-01 0.411484488953E-01 0.438196326246E-01 0.466608025533E-01 0.496823089450E-01 - 0.528950676826E-01 0.563105819652E-01 0.599409633995E-01 0.637989521880E-01 0.678979360665E-01 - 0.722519675861E-01 0.768757792715E-01 0.817847961130E-01 0.869951447703E-01 0.925236587738E-01 - 0.983878789095E-01 0.104606047856E+00 0.111197098026E+00 0.118180631410E+00 0.125576890089E+00 - 0.133406715896E+00 0.141691497534E+00 0.150453103263E+00 0.159713797041E+00 0.169496135792E+00 - 0.179822845227E+00 0.190716671374E+00 0.202200204726E+00 0.214295673644E+00 0.227024703349E+00 - 0.240408036607E+00 0.254465211927E+00 0.269214194858E+00 0.284670957798E+00 0.300849003557E+00 - 0.317758827869E+00 0.335407316081E+00 0.353797069410E+00 0.372925656495E+00 0.392784786538E+00 - 0.413359401140E+00 0.434626683071E+00 0.456554981802E+00 0.479102657604E+00 0.502216848651E+00 - 0.525832168810E+00 0.549869347839E+00 0.574233830650E+00 0.598814358233E+00 0.623481559867E+00 - 0.648086594575E+00 0.672459889336E+00 0.696410032536E+00 0.719722893489E+00 0.742161052417E+00 - 0.763463640029E+00 0.783346701288E+00 0.801504213699E+00 0.817609905660E+00 0.831320034071E+00 - 0.842277290994E+00 0.850116014825E+00 0.854468879722E+00 0.854975224885E+00 0.851291158898E+00 - 0.843101529429E+00 0.830133779791E+00 0.812173615663E+00 0.789082271343E+00 0.760814989247E+00 - 0.727440103218E+00 0.689157841522E+00 0.646317637620E+00 0.599432358703E+00 0.549187442962E+00 - 0.496442495144E+00 0.442222457189E+00 0.387695093883E+00 0.334131279817E+00 0.282844534494E+00 - 0.235106544671E+00 0.192036181160E+00 0.154460929685E+00 0.122751894914E+00 0.966509407173E-01 - 0.754197350591E-01 0.583105682925E-01 0.446553131657E-01 0.338641471811E-01 0.254227490621E-01 - 0.188882113068E-01 0.138839137385E-01 0.100936590292E-01 0.725537509450E-02 0.515466309716E-02 - 0.361842930266E-02 0.250879268147E-02 0.171741309607E-02 0.116033964584E-02 0.773437417305E-03 - 0.508414784907E-03 0.329443540380E-03 0.210340978410E-03 0.132266603092E-03 0.818759218551E-04 - 0.498690344207E-04 0.298714443437E-04 0.175875916060E-04 0.101729842592E-04 0.577752330293E-05 - 0.321986957660E-05 0.175987680434E-05 0.942778890139E-06 0.494707936977E-06 0.254108048780E-06 - 0.127681574306E-06 0.627159654587E-07 0.300924318689E-07 0.140943050873E-07 0.643877401768E-08 - 0.286674509825E-08 - pseudo wavefunction - 0.575355347414E-04 0.594064467959E-04 0.613381962429E-04 0.633327613625E-04 0.653921847638E-04 - 0.675185754763E-04 0.697141111098E-04 0.719810400846E-04 0.743216839340E-04 0.767384396819E-04 - 0.792337822973E-04 0.818102672291E-04 0.844705330228E-04 0.872173040230E-04 0.900533931633E-04 - 0.929817048467E-04 0.960052379201E-04 0.991270887455E-04 0.102350454371E-03 0.105678635804E-03 - 0.109115041394E-03 0.112663190320E-03 0.116326716196E-03 0.120109370793E-03 0.124015027879E-03 - 0.128047687189E-03 0.132211478517E-03 0.136510665951E-03 0.140949652232E-03 0.145532983271E-03 - 0.150265352799E-03 0.155151607175E-03 0.160196750351E-03 0.165405948993E-03 0.170784537776E-03 - 0.176338024845E-03 0.182072097454E-03 0.187992627794E-03 0.194105679006E-03 0.200417511387E-03 - 0.206934588804E-03 0.213663585314E-03 0.220611391995E-03 0.227785124008E-03 0.235192127880E-03 - 0.242839989029E-03 0.250736539531E-03 0.258889866144E-03 0.267308318583E-03 0.276000518079E-03 - 0.284975366201E-03 0.294242053974E-03 0.303810071296E-03 0.313689216649E-03 0.323889607138E-03 - 0.334421688850E-03 0.345296247555E-03 0.356524419746E-03 0.368117704048E-03 0.380087972993E-03 - 0.392447485177E-03 0.405208897814E-03 0.418385279700E-03 0.431990124593E-03 0.446037365035E-03 - 0.460541386620E-03 0.475517042723E-03 0.490979669714E-03 0.506945102664E-03 0.523429691556E-03 - 0.540450318037E-03 0.558024412700E-03 0.576169972936E-03 0.594905581365E-03 0.614250424868E-03 - 0.634224314231E-03 0.654847704436E-03 0.676141715611E-03 0.698128154653E-03 0.720829537562E-03 - 0.744269112502E-03 0.768470883604E-03 0.793459635553E-03 0.819260958964E-03 0.845901276593E-03 - 0.873407870393E-03 0.901808909454E-03 0.931133478849E-03 0.961411609419E-03 0.992674308528E-03 - 0.102495359182E-02 0.105828251598E-02 0.109269521264E-02 0.112822692327E-02 0.116491403530E-02 - 0.120279411939E-02 0.124190596788E-02 0.128228963454E-02 0.132398647555E-02 0.136703919190E-02 - 0.141149187307E-02 0.145739004223E-02 0.150478070279E-02 0.155371238660E-02 0.160423520360E-02 - 0.165640089318E-02 0.171026287709E-02 0.176587631424E-02 0.182329815708E-02 0.188258721000E-02 - 0.194380418952E-02 0.200701178645E-02 0.207227473008E-02 0.213965985449E-02 0.220923616697E-02 - 0.228107491867E-02 0.235524967759E-02 0.243183640385E-02 0.251091352754E-02 0.259256202897E-02 - 0.267686552162E-02 0.276391033773E-02 0.285378561671E-02 0.294658339640E-02 0.304239870730E-02 - 0.314132966987E-02 0.324347759500E-02 0.334894708771E-02 0.345784615425E-02 0.357028631269E-02 - 0.368638270708E-02 0.380625422530E-02 0.393002362081E-02 0.405781763827E-02 0.418976714331E-02 - 0.432600725647E-02 0.446667749152E-02 0.461192189824E-02 0.476188920988E-02 0.491673299540E-02 - 0.507661181657E-02 0.524168939035E-02 0.541213475631E-02 0.558812244973E-02 0.576983268009E-02 - 0.595745151554E-02 0.615117107322E-02 0.635118971583E-02 0.655771225459E-02 0.677095015869E-02 - 0.699112177165E-02 0.721845253461E-02 0.745317521688E-02 0.769553015401E-02 0.794576549352E-02 - 0.820413744857E-02 0.847091055996E-02 0.874635796647E-02 0.903076168407E-02 0.932441289409E-02 - 0.962761224075E-02 0.994067013832E-02 0.102639070882E-01 0.105976540061E-01 0.109422525603E-01 - 0.112980555200E-01 0.116654271158E-01 0.120447434111E-01 0.124363926859E-01 0.128407758327E-01 - 0.132583067657E-01 0.136894128422E-01 0.141345352983E-01 0.145941296989E-01 0.150686664009E-01 - 0.155586310324E-01 0.160645249872E-01 0.165868659342E-01 0.171261883444E-01 0.176830440334E-01 - 0.182580027226E-01 0.188516526171E-01 0.194646010028E-01 0.200974748619E-01 0.207509215086E-01 - 0.214256092443E-01 0.221222280340E-01 0.228414902039E-01 0.235841311607E-01 0.243509101343E-01 - 0.251426109430E-01 0.259600427831E-01 0.268040410429E-01 0.276754681426E-01 0.285752143996E-01 - 0.295041989207E-01 0.304633705225E-01 0.314537086791E-01 0.324762244998E-01 0.335319617355E-01 - 0.346219978170E-01 0.357474449230E-01 0.369094510819E-01 0.381092013054E-01 0.393479187556E-01 - 0.406268659478E-01 0.419473459869E-01 0.433107038408E-01 0.447183276504E-01 0.461716500761E-01 - 0.476721496838E-01 0.492213523683E-01 0.508208328170E-01 0.524722160129E-01 0.541771787786E-01 - 0.559374513607E-01 0.577548190562E-01 0.596311238795E-01 0.615682662727E-01 0.635682068564E-01 - 0.656329682232E-01 0.677646367731E-01 0.699653645899E-01 0.722373713589E-01 0.745829463247E-01 - 0.770044502882E-01 0.795043176423E-01 0.820850584435E-01 0.847492605189E-01 0.874995916053E-01 - 0.903388015194E-01 0.932697243532E-01 0.962952806953E-01 0.994184798691E-01 0.102642422187E+00 - 0.105970301215E+00 0.109405406033E+00 0.112951123506E+00 0.116610940521E+00 0.120388446224E+00 - 0.124287334210E+00 0.128311404677E+00 0.132464566515E+00 0.136750839334E+00 0.141174355391E+00 - 0.145739361417E+00 0.150450220313E+00 0.155311412692E+00 0.160327538239E+00 0.165503316871E+00 - 0.170843589644E+00 0.176353319387E+00 0.182037591014E+00 0.187901611466E+00 0.193950709243E+00 - 0.200190333447E+00 0.206626052301E+00 0.213263551044E+00 0.220108629141E+00 0.227167196717E+00 - 0.234445270112E+00 0.241948966447E+00 0.249684497087E+00 0.257658159858E+00 0.265876329868E+00 - 0.274345448775E+00 0.283072012294E+00 0.292062555759E+00 0.301323637496E+00 0.310861819756E+00 - 0.320683646913E+00 0.330795620636E+00 0.341204171652E+00 0.351915627735E+00 0.362936177473E+00 - 0.374271829350E+00 0.385928365590E+00 0.397911290193E+00 0.410225770506E+00 0.422876571603E+00 - 0.435867982691E+00 0.449203734658E+00 0.462886907800E+00 0.476919828678E+00 0.491303954927E+00 - 0.506039746769E+00 0.521126523834E+00 0.536562305798E+00 0.552343635204E+00 0.568465380740E+00 - 0.584920519078E+00 0.601699893320E+00 0.618791945934E+00 0.636182424019E+00 0.653854054636E+00 - 0.671786187949E+00 0.689954405900E+00 0.708330094256E+00 0.726879976015E+00 0.745565604418E+00 - 0.764342814255E+00 0.783161130667E+00 0.801963135452E+00 0.820683791893E+00 0.839249730401E+00 - 0.857578498949E+00 0.875577784285E+00 0.893144612430E+00 0.910164539979E+00 0.926510851286E+00 - 0.942043780836E+00 0.956609784897E+00 0.970040892017E+00 0.982154168004E+00 0.992751337569E+00 - 0.100161861173E+01 0.100852677707E+01 0.101323160973E+01 0.101547468295E+01 0.101498464155E+01 - 0.101147901924E+01 0.100466111497E+01 0.994375950448E+00 0.980707901591E+00 0.963803639329E+00 - 0.943836461172E+00 0.921004116597E+00 0.895526241359E+00 0.867641520215E+00 0.837604615000E+00 - 0.805682938619E+00 0.772153342384E+00 0.737298744582E+00 0.701404760904E+00 0.664756390336E+00 - 0.627634741711E+00 0.590313873286E+00 0.553057756365E+00 0.516117401327E+00 0.479728188657E+00 - 0.444107456577E+00 0.409452404308E+00 0.375938372688E+00 0.343717559055E+00 0.312918210115E+00 - 0.283644316716E+00 0.255975805584E+00 0.229969207138E+00 0.205658740221E+00 0.183057753088E+00 - 0.162160440109E+00 - ae wavefunction - 0.723210491956E-03 0.746578583770E-03 0.770700951993E-03 0.795601892227E-03 0.821306479369E-03 - 0.847840592488E-03 0.875230940375E-03 0.903505087772E-03 0.932691482807E-03 0.962819484716E-03 - 0.993919392949E-03 0.102602247680E-02 0.105916100625E-02 0.109336828344E-02 0.112867867548E-02 - 0.116512764795E-02 0.120275179956E-02 0.124158889799E-02 0.128167791659E-02 0.132305907237E-02 - 0.136577386512E-02 0.140986511768E-02 0.145537701746E-02 0.150235515925E-02 0.155084658925E-02 - 0.160089985052E-02 0.165256502971E-02 0.170589380529E-02 0.176093949712E-02 0.181775711760E-02 - 0.187640342429E-02 0.193693697409E-02 0.199941817908E-02 0.206390936390E-02 0.213047482499E-02 - 0.219918089139E-02 0.227009598742E-02 0.234329069717E-02 0.241883783086E-02 0.249681249307E-02 - 0.257729215303E-02 0.266035671683E-02 0.274608860172E-02 0.283457281255E-02 0.292589702034E-02 - 0.302015164303E-02 0.311742992854E-02 0.321782804011E-02 0.332144514397E-02 0.342838349946E-02 - 0.353874855160E-02 0.365264902612E-02 0.377019702712E-02 0.389150813722E-02 0.401670152049E-02 - 0.414590002798E-02 0.427923030608E-02 0.441682290755E-02 0.455881240553E-02 0.470533751028E-02 - 0.485654118900E-02 0.501257078845E-02 0.517357816061E-02 0.533971979149E-02 0.551115693279E-02 - 0.568805573680E-02 0.587058739433E-02 0.605892827584E-02 0.625326007563E-02 0.645376995923E-02 - 0.666065071397E-02 0.687410090265E-02 0.709432502044E-02 0.732153365478E-02 0.755594364866E-02 - 0.779777826652E-02 0.804726736390E-02 0.830464755909E-02 0.857016240900E-02 0.884406258640E-02 - 0.912660606132E-02 0.941805828423E-02 0.971869237195E-02 0.100287892964E-01 0.103486380750E-01 - 0.106785359635E-01 0.110187886512E-01 0.113697104566E-01 0.117316245259E-01 0.121048630323E-01 - 0.124897673751E-01 0.128866883819E-01 0.132959865080E-01 0.137180320384E-01 0.141532052878E-01 - 0.146018967997E-01 0.150645075448E-01 0.155414491166E-01 0.160331439246E-01 0.165400253848E-01 - 0.170625381058E-01 0.176011380701E-01 0.181562928105E-01 0.187284815799E-01 0.193181955137E-01 - 0.199259377844E-01 0.205522237464E-01 0.211975810705E-01 0.218625498668E-01 0.225476827944E-01 - 0.232535451563E-01 0.239807149791E-01 0.247297830737E-01 0.255013530778E-01 0.262960414763E-01 - 0.271144775983E-01 0.279573035892E-01 0.288251743544E-01 0.297187574729E-01 0.306387330781E-01 - 0.315857937031E-01 0.325606440870E-01 0.335640009402E-01 0.345965926640E-01 0.356591590220E-01 - 0.367524507593E-01 0.378772291652E-01 0.390342655757E-01 0.402243408110E-01 0.414482445436E-01 - 0.427067745919E-01 0.440007361337E-01 0.453309408352E-01 0.466982058883E-01 0.481033529513E-01 - 0.495472069857E-01 0.510305949826E-01 0.525543445725E-01 0.541192825097E-01 0.557262330238E-01 - 0.573760160317E-01 0.590694451993E-01 0.608073258461E-01 0.625904526816E-01 0.644196073662E-01 - 0.662955558838E-01 0.682190457186E-01 0.701908028232E-01 0.722115283678E-01 0.742818952600E-01 - 0.764025444212E-01 0.785740808093E-01 0.807970691759E-01 0.830720295419E-01 0.853994323839E-01 - 0.877796935126E-01 0.902131686350E-01 0.927001475833E-01 0.952408481994E-01 0.978354098603E-01 - 0.100483886632E+00 0.103186240036E+00 0.105942331426E+00 0.108751913939E+00 0.111614624041E+00 - 0.114529972627E+00 0.117497335686E+00 0.120515944498E+00 0.123584875391E+00 0.126703039005E+00 - 0.129869169099E+00 0.133081810868E+00 0.136339308790E+00 0.139639793985E+00 0.142981171101E+00 - 0.146361104728E+00 0.149777005361E+00 0.153226014901E+00 0.156704991737E+00 0.160210495417E+00 - 0.163738770941E+00 0.167285732697E+00 0.170846948100E+00 0.174417620953E+00 0.177992574602E+00 - 0.181566234937E+00 0.185132613303E+00 0.188685289405E+00 0.192217394292E+00 0.195721593504E+00 - 0.199190070517E+00 0.202614510568E+00 0.205986085027E+00 0.209295436430E+00 0.212532664351E+00 - 0.215687312275E+00 0.218748355654E+00 0.221704191362E+00 0.224542628748E+00 0.227250882534E+00 - 0.229815567796E+00 0.232222697309E+00 0.234457681522E+00 0.236505331475E+00 0.238349864969E+00 - 0.239974916327E+00 0.241363550084E+00 0.242498278978E+00 0.243361086622E+00 0.243933455227E+00 - 0.244196398801E+00 0.244130502213E+00 0.243715966544E+00 0.242932661134E+00 0.241760182755E+00 - 0.240177922301E+00 0.238165139411E+00 0.235701045406E+00 0.232764894903E+00 0.229336086443E+00 - 0.225394272444E+00 0.220919478726E+00 0.215892233821E+00 0.210293708226E+00 0.204105863651E+00 - 0.197311612280E+00 0.189894985941E+00 0.181841314980E+00 0.173137416545E+00 0.163771791829E+00 - 0.153734831728E+00 0.143019030191E+00 0.131619204408E+00 0.119532720834E+00 0.106759725824E+00 - 0.933033795277E-01 0.791700914244E-01 0.643697556857E-01 0.489159842511E-01 0.328263352024E-01 - 0.161225336665E-01 -0.116931789868E-02 -0.190185435298E-01 -0.373897154028E-01 -0.562424969619E-01 - -0.755315106483E-01 -0.952062321565E-01 -0.115210911281E+00 -0.135484517197E+00 -0.155960704264E+00 - -0.176567794242E+00 -0.197228773280E+00 -0.217861307379E+00 -0.238377787431E+00 -0.258685422148E+00 - -0.278686401205E+00 -0.298278148790E+00 -0.317353678585E+00 -0.335802047802E+00 -0.353508896551E+00 - -0.370357054796E+00 -0.386227202908E+00 -0.400998578759E+00 -0.414549729935E+00 -0.426759311736E+00 - -0.437506930652E+00 -0.446674029444E+00 -0.454144808540E+00 -0.459807175066E+00 -0.463553710585E+00 - -0.465282647871E+00 -0.464898847424E+00 -0.462314764450E+00 -0.457451398192E+00 -0.450239215408E+00 - -0.440619040450E+00 -0.428542905017E+00 -0.413974850508E+00 -0.396891676564E+00 -0.377283628995E+00 - -0.355155020605E+00 -0.330524777997E+00 -0.303426906672E+00 -0.273910866599E+00 -0.242041849499E+00 - -0.207900947753E+00 -0.171585205060E+00 -0.133207537940E+00 -0.928965176587E-01 -0.507960038343E-01 - -0.706462318874E-02 0.381249082892E-01 0.845866157243E-01 0.132122236961E+00 0.180522317286E+00 - 0.229567404370E+00 0.279029321687E+00 0.328672508685E+00 0.378255438406E+00 0.427532163428E+00 - 0.476254096171E+00 0.524172207141E+00 0.571039903275E+00 0.616616865042E+00 0.660673900196E+00 - 0.702998097735E+00 0.743396073735E+00 0.781691755158E+00 0.817716506082E+00 0.851295262156E+00 - 0.882238303138E+00 0.910345746539E+00 0.935421494478E+00 0.957287301571E+00 0.975791215624E+00 - 0.990810703323E+00 0.100225314176E+01 0.101005569110E+01 0.101418535318E+01 0.101463927790E+01 - 0.101144510801E+01 0.100466111497E+01 0.994375950448E+00 0.980707901591E+00 0.963803639329E+00 - 0.943836461172E+00 0.921004116597E+00 0.895526241359E+00 0.867641520215E+00 0.837604615000E+00 - 0.805682938619E+00 0.772153342384E+00 0.737298744582E+00 0.701404760904E+00 0.664756390336E+00 - 0.627634741711E+00 0.590313873286E+00 0.553057756365E+00 0.516117401327E+00 0.479728188657E+00 - 0.444107456577E+00 0.409452404308E+00 0.375938372688E+00 0.343717559055E+00 0.312918210115E+00 - 0.283644316716E+00 0.255975805584E+00 0.229969207138E+00 0.205658740221E+00 0.183057753088E+00 - 0.162160440109E+00 - pseudo wavefunction - 0.815477257113E-04 0.841994543121E-04 0.869374105116E-04 0.897643982160E-04 0.926833125078E-04 - 0.956971426101E-04 0.988089749485E-04 0.102021996311E-03 0.105339497112E-03 0.108764874763E-03 - 0.112301637149E-03 0.115953406224E-03 0.119723921719E-03 0.123617044970E-03 0.127636762877E-03 - 0.131787191981E-03 0.136072582684E-03 0.140497323602E-03 0.145065946054E-03 0.149783128710E-03 - 0.154653702376E-03 0.159682654946E-03 0.164875136506E-03 0.170236464611E-03 0.175772129729E-03 - 0.181487800864E-03 0.187389331363E-03 0.193482764908E-03 0.199774341708E-03 0.206270504887E-03 - 0.212977907082E-03 0.219903417260E-03 0.227054127747E-03 0.234437361494E-03 0.242060679579E-03 - 0.249931888944E-03 0.258059050395E-03 0.266450486854E-03 0.275114791884E-03 0.284060838489E-03 - 0.293297788203E-03 0.302835100467E-03 0.312682542321E-03 0.322850198405E-03 0.333348481284E-03 - 0.344188142117E-03 0.355380281661E-03 0.366936361642E-03 0.378868216494E-03 0.391188065475E-03 - 0.403908525185E-03 0.417042622483E-03 0.430603807827E-03 0.444605969054E-03 0.459063445594E-03 - 0.473991043162E-03 0.489404048917E-03 0.505318247117E-03 0.521749935286E-03 0.538715940900E-03 - 0.556233638623E-03 0.574320968101E-03 0.592996452329E-03 0.612279216623E-03 0.632189008208E-03 - 0.652746216436E-03 0.673971893671E-03 0.695887776845E-03 0.718516309720E-03 0.741880665872E-03 - 0.766004772421E-03 0.790913334538E-03 0.816631860740E-03 0.843186689018E-03 0.870605013804E-03 - 0.898914913826E-03 0.928145380856E-03 0.958326349405E-03 0.989488727376E-03 0.102166442772E-02 - 0.105488640110E-02 0.108918866967E-02 0.112460636188E-02 0.116117574846E-02 0.119893427959E-02 - 0.123792062320E-02 0.127817470463E-02 0.131973774745E-02 0.136265231574E-02 0.140696235763E-02 - 0.145271325033E-02 0.149995184660E-02 0.154872652270E-02 0.159908722798E-02 0.165108553598E-02 - 0.170477469728E-02 0.176020969402E-02 0.181744729618E-02 0.187654611977E-02 0.193756668679E-02 - 0.200057148727E-02 0.206562504320E-02 0.213279397464E-02 0.220214706796E-02 0.227375534621E-02 - 0.234769214192E-02 0.242403317215E-02 0.250285661605E-02 0.258424319488E-02 0.266827625473E-02 - 0.275504185178E-02 0.284462884051E-02 0.293712896461E-02 0.303263695098E-02 0.313125060667E-02 - 0.323307091910E-02 0.333820215938E-02 0.344675198915E-02 0.355883157079E-02 0.367455568123E-02 - 0.379404282948E-02 0.391741537797E-02 0.404479966782E-02 0.417632614823E-02 0.431212951000E-02 - 0.445234882345E-02 0.459712768080E-02 0.474661434319E-02 0.490096189243E-02 0.506032838775E-02 - 0.522487702760E-02 0.539477631671E-02 0.557020023861E-02 0.575132843370E-02 0.593834638314E-02 - 0.613144559871E-02 0.633082381885E-02 0.653668521102E-02 0.674924058070E-02 0.696870758709E-02 - 0.719531096592E-02 0.742928275939E-02 0.767086255365E-02 0.792029772395E-02 0.817784368774E-02 - 0.844376416603E-02 0.871833145320E-02 0.900182669552E-02 0.929454017879E-02 0.959677162528E-02 - 0.990883050022E-02 0.102310363284E-01 0.105637190207E-01 0.109072192118E-01 0.112618886081E-01 - 0.116280903475E-01 0.120061993704E-01 0.123966028031E-01 0.127997003533E-01 0.132159047187E-01 - 0.136456420083E-01 0.140893521779E-01 0.145474894794E-01 0.150205229247E-01 0.155089367643E-01 - 0.160132309818E-01 0.165339218041E-01 0.170715422280E-01 0.176266425639E-01 0.181997909971E-01 - 0.187915741665E-01 0.194025977634E-01 0.200334871477E-01 0.206848879853E-01 0.213574669051E-01 - 0.220519121780E-01 0.227689344162E-01 0.235092672965E-01 0.242736683058E-01 0.250629195101E-01 - 0.258778283492E-01 0.267192284556E-01 0.275879804997E-01 0.284849730625E-01 0.294111235351E-01 - 0.303673790471E-01 0.313547174243E-01 0.323741481766E-01 0.334267135170E-01 0.345134894124E-01 - 0.356355866676E-01 0.367941520430E-01 0.379903694070E-01 0.392254609246E-01 0.405006882824E-01 - 0.418173539515E-01 0.431768024894E-01 0.445804218815E-01 0.460296449237E-01 0.475259506471E-01 - 0.490708657851E-01 0.506659662856E-01 0.523128788672E-01 0.540132826228E-01 0.557689106695E-01 - 0.575815518475E-01 0.594530524680E-01 0.613853181118E-01 0.633803154786E-01 0.654400742896E-01 - 0.675666892425E-01 0.697623220209E-01 0.720292033587E-01 0.743696351597E-01 0.767859926737E-01 - 0.792807267296E-01 0.818563660248E-01 0.845155194735E-01 0.872608786111E-01 0.900952200573E-01 - 0.930214080363E-01 0.960423969533E-01 0.991612340281E-01 0.102381061984E+00 0.105705121788E+00 - 0.109136755450E+00 0.112679408862E+00 0.116336634695E+00 0.120112095333E+00 0.124009565855E+00 - 0.128032937043E+00 0.132186218436E+00 0.136473541398E+00 0.140899162210E+00 0.145467465175E+00 - 0.150182965720E+00 0.155050313498E+00 0.160074295467E+00 0.165259838936E+00 0.170612014565E+00 - 0.176136039306E+00 0.181837279254E+00 0.187721252402E+00 0.193793631261E+00 0.200060245332E+00 - 0.206527083389E+00 0.213200295545E+00 0.220086195059E+00 0.227191259846E+00 0.234522133635E+00 - 0.242085626730E+00 0.249888716306E+00 0.257938546178E+00 0.266242425963E+00 0.274807829557E+00 - 0.283642392830E+00 0.292753910429E+00 0.302150331579E+00 0.311839754749E+00 0.321830421030E+00 - 0.332130706074E+00 0.342749110390E+00 0.353694247819E+00 0.364974831931E+00 0.376599660114E+00 - 0.388577595046E+00 0.400917543253E+00 0.413628430375E+00 0.426719172761E+00 0.440198644933E+00 - 0.454075642432E+00 0.468358839485E+00 0.483056740881E+00 0.498177627359E+00 0.513729493745E+00 - 0.529719978972E+00 0.546156287045E+00 0.563045097863E+00 0.580392466756E+00 0.598203711387E+00 - 0.616483284604E+00 0.635234631621E+00 0.654460029761E+00 0.674160408817E+00 0.694335149890E+00 - 0.714981860357E+00 0.736096122431E+00 0.757671212527E+00 0.779697788452E+00 0.802163541211E+00 - 0.825052808008E+00 0.848346142836E+00 0.872019840898E+00 0.896045412985E+00 0.920389005890E+00 - 0.945010765014E+00 0.969864135460E+00 0.994895098271E+00 0.102004133900E+01 0.104523134664E+01 - 0.107038344193E+01 0.109540473587E+01 0.112019002089E+01 0.114462060011E+01 0.116856306300E+01 - 0.119186802024E+01 0.121436881520E+01 0.123588023556E+01 0.125619725611E+01 0.127509385139E+01 - 0.129232192750E+01 0.130761043238E+01 0.132066471690E+01 0.133116623170E+01 0.133877265895E+01 - 0.134311859223E+01 0.134381689101E+01 0.134046084846E+01 0.133262731962E+01 0.131988096144E+01 - 0.130177973302E+01 0.127788310298E+01 0.124791979347E+01 0.121195594119E+01 0.117012955700E+01 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 - ae wavefunction - 0.100411206031E-02 0.103655653257E-02 0.107004824907E-02 0.110462094206E-02 0.114030942574E-02 - 0.117714963085E-02 0.121517864012E-02 0.125443472469E-02 0.129495738219E-02 0.133678737526E-02 - 0.137996677192E-02 0.142453898676E-02 0.147054882368E-02 0.151804251972E-02 0.156706779045E-02 - 0.161767387661E-02 0.166991159220E-02 0.172383337419E-02 0.177949333353E-02 0.183694730789E-02 - 0.189625291594E-02 0.195746961331E-02 0.202065875021E-02 0.208588363083E-02 0.215320957455E-02 - 0.222270397899E-02 0.229443638492E-02 0.236847854317E-02 0.244490448353E-02 0.252379058570E-02 - 0.260521565239E-02 0.268926098453E-02 0.277601045878E-02 0.286555060725E-02 0.295797069967E-02 - 0.305336282787E-02 0.315182199281E-02 0.325344619405E-02 0.335833652195E-02 0.346659725239E-02 - 0.357833594433E-02 0.369366354008E-02 0.381269446850E-02 0.393554675108E-02 0.406234211102E-02 - 0.419320608539E-02 0.432826814042E-02 0.446766179001E-02 0.461152471742E-02 0.475999890040E-02 - 0.491323073976E-02 0.507137119125E-02 0.523457590114E-02 0.540300534538E-02 0.557682497239E-02 - 0.575620534963E-02 0.594132231403E-02 0.613235712617E-02 0.632949662853E-02 0.653293340765E-02 - 0.674286596037E-02 0.695949886420E-02 0.718304295172E-02 0.741371548940E-02 0.765174036045E-02 - 0.789734825209E-02 0.815077684708E-02 0.841227101959E-02 0.868208303549E-02 0.896047275693E-02 - 0.924770785137E-02 0.954406400502E-02 0.984982514059E-02 0.101652836394E-01 0.104907405678E-01 - 0.108265059081E-01 0.111728987934E-01 0.115302477462E-01 0.118988909230E-01 0.122791763595E-01 - 0.126714622234E-01 0.130761170679E-01 0.134935200904E-01 0.139240613943E-01 0.143681422534E-01 - 0.148261753797E-01 0.152985851939E-01 0.157858080982E-01 0.162882927509E-01 0.168065003429E-01 - 0.173409048755E-01 0.178919934394E-01 0.184602664931E-01 0.190462381422E-01 0.196504364175E-01 - 0.202734035517E-01 0.209156962543E-01 0.215778859835E-01 0.222605592145E-01 0.229643177037E-01 - 0.236897787473E-01 0.244375754327E-01 0.252083568839E-01 0.260027884966E-01 0.268215521644E-01 - 0.276653464924E-01 0.285348869993E-01 0.294309063035E-01 0.303541542937E-01 0.313053982813E-01 - 0.322854231324E-01 0.332950313775E-01 0.343350432971E-01 0.354062969793E-01 0.365096483489E-01 - 0.376459711628E-01 0.388161569716E-01 0.400211150407E-01 0.412617722306E-01 0.425390728318E-01 - 0.438539783489E-01 0.452074672324E-01 0.466005345522E-01 0.480341916085E-01 0.495094654752E-01 - 0.510273984710E-01 0.525890475524E-01 0.541954836221E-01 0.558477907490E-01 0.575470652892E-01 - 0.592944149057E-01 0.610909574763E-01 0.629378198839E-01 0.648361366790E-01 0.667870486093E-01 - 0.687917010039E-01 0.708512420048E-01 0.729668206361E-01 0.751395846992E-01 0.773706784836E-01 - 0.796612402836E-01 0.820123997070E-01 0.844252747640E-01 0.869009687250E-01 0.894405667320E-01 - 0.920451321506E-01 0.947157026489E-01 0.974532859865E-01 0.100258855501E+00 0.103133345273E+00 - 0.106077644955E+00 0.109092594252E+00 0.112178977021E+00 0.115337514994E+00 0.118568861090E+00 - 0.121873592304E+00 0.125252202151E+00 0.128705092655E+00 0.132232565851E+00 0.135834814797E+00 - 0.139511914062E+00 0.143263809684E+00 0.147090308573E+00 0.150991067341E+00 0.154965580557E+00 - 0.159013168383E+00 0.163132963604E+00 0.167323898026E+00 0.171584688232E+00 0.175913820679E+00 - 0.180309536147E+00 0.184769813516E+00 0.189292352876E+00 0.193874557970E+00 0.198513517983E+00 - 0.203205988665E+00 0.207948372826E+00 0.212736700201E+00 0.217566606727E+00 0.222433313240E+00 - 0.227331603655E+00 0.232255802656E+00 0.237199752953E+00 0.242156792174E+00 0.247119729454E+00 - 0.252080821818E+00 0.257031750433E+00 0.261963596859E+00 0.266866819394E+00 0.271731229666E+00 - 0.276545969613E+00 0.281299489013E+00 0.285979523755E+00 0.290573075042E+00 0.295066389746E+00 - 0.299444942160E+00 0.303693417394E+00 0.307795696707E+00 0.311734845064E+00 0.315493101250E+00 - 0.319051870893E+00 0.322391722743E+00 0.325492388640E+00 0.328332767535E+00 0.330890934065E+00 - 0.333144152089E+00 0.335068893704E+00 0.336640864237E+00 0.337835033734E+00 0.338625675483E+00 - 0.338986412149E+00 0.338890270056E+00 0.338309742224E+00 0.337216860715E+00 0.335583278886E+00 - 0.333380364113E+00 0.330579301540E+00 0.327151209410E+00 0.323067266460E+00 0.318298851881E+00 - 0.312817698250E+00 0.306596057794E+00 0.299606882299E+00 0.291824016847E+00 0.283222407513E+00 - 0.273778323006E+00 0.263469590136E+00 0.252275842824E+00 0.240178784252E+00 0.227162461529E+00 - 0.213213552127E+00 0.198321661083E+00 0.182479627800E+00 0.165683841030E+00 0.147934560383E+00 - 0.129236242450E+00 0.109597869328E+00 0.890332769980E-01 0.675614806516E-01 0.452069935885E-01 - 0.220001358488E-01 -0.202267179947E-02 -0.268186333727E-01 -0.523383278445E-01 -0.785254890752E-01 - -0.105316819838E+00 -0.132641842628E+00 -0.160422787359E+00 -0.188574512968E+00 -0.217004457521E+00 - -0.245612611158E+00 -0.274291509649E+00 -0.302926253778E+00 -0.331394570066E+00 -0.359566938358E+00 - -0.387306817382E+00 -0.414470996398E+00 -0.440910088322E+00 -0.466469161094E+00 -0.490988488296E+00 - -0.514304394485E+00 -0.536250175925E+00 -0.556657087061E+00 -0.575355390912E+00 -0.592175474576E+00 - -0.606949029250E+00 -0.619510290028E+00 -0.629697327922E+00 -0.637353382408E+00 -0.642328222232E+00 - -0.644479521263E+00 -0.643674236664E+00 -0.639789976727E+00 -0.632716347271E+00 -0.622356265451E+00 - -0.608627230644E+00 -0.591462542992E+00 -0.570812459931E+00 -0.546645281908E+00 -0.518948357967E+00 - -0.487729002212E+00 -0.453015311567E+00 -0.414856874075E+00 -0.373325356806E+00 -0.328514960967E+00 - -0.280542730002E+00 -0.229548696640E+00 -0.175695853364E+00 -0.119169931471E+00 -0.601789761144E-01 - 0.104729211497E-02 0.642583319031E-01 0.129183846977E+00 0.195535047578E+00 0.263006083041E+00 - 0.331275614340E+00 0.400008497553E+00 0.468857563600E+00 0.537465511386E+00 0.605466987922E+00 - 0.672491005793E+00 0.738163955532E+00 0.802113577363E+00 0.863974274314E+00 0.923393832698E+00 - 0.980040530008E+00 0.103360754150E+01 0.108380973017E+01 0.113036987200E+01 0.117299954389E+01 - 0.121138806437E+01 0.124520914300E+01 0.127414049534E+01 0.129788344230E+01 0.131617463165E+01 - 0.132879041294E+01 0.133554760128E+01 0.133630339312E+01 0.133095550857E+01 0.131944261315E+01 - 0.130174470424E+01 0.127788310298E+01 0.124791979347E+01 0.121195594119E+01 0.117012955700E+01 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 - pseudo wavefunction - 0.291401325933E-08 0.310660738532E-08 0.331193051905E-08 0.353082395115E-08 0.376418457520E-08 - 0.401296856259E-08 0.427819528042E-08 0.456095146815E-08 0.486239569049E-08 0.518376308447E-08 - 0.552637042028E-08 0.589162149659E-08 0.628101289253E-08 0.669614009969E-08 0.713870405956E-08 - 0.761051813289E-08 0.811351552983E-08 0.864975723099E-08 0.922144043214E-08 0.983090754699E-08 - 0.104806558050E-07 0.111733474834E-07 0.119118208158E-07 0.126991016216E-07 0.135384157035E-07 - 0.144332020655E-07 0.153871270036E-07 0.164040991280E-07 0.174882853783E-07 0.186441280971E-07 - 0.198763632322E-07 0.211900397413E-07 0.225905402799E-07 0.240836032562E-07 0.256753463434E-07 - 0.273722915463E-07 0.291813919246E-07 0.311100600826E-07 0.331661985409E-07 0.353582321169E-07 - 0.376951424442E-07 0.401865047739E-07 0.428425272086E-07 0.456740925285E-07 0.486928027827E-07 - 0.519110268276E-07 0.553419510066E-07 0.589996331800E-07 0.628990603257E-07 0.670562099465E-07 - 0.714881155364E-07 0.762129363737E-07 0.812500319263E-07 0.866200411755E-07 0.923449671818E-07 - 0.984482672403E-07 0.104954948994E-06 0.111891672901E-06 0.119286861471E-06 0.127170815725E-06 - 0.135575839351E-06 0.144536371063E-06 0.154089125712E-06 0.164273244717E-06 0.175130456449E-06 - 0.186705247203E-06 0.199045043477E-06 0.212200406300E-06 0.226225238396E-06 0.241177005044E-06 - 0.257116969541E-06 0.274110444214E-06 0.292227058035E-06 0.311541041917E-06 0.332131532864E-06 - 0.354082898227E-06 0.377485081388E-06 0.402433970297E-06 0.429031790352E-06 0.457387523265E-06 - 0.487617353594E-06 0.519845144797E-06 0.554202946748E-06 0.590831536791E-06 0.629880996560E-06 - 0.671511326916E-06 0.715893103531E-06 0.763208175796E-06 0.813650411926E-06 0.867426493305E-06 - 0.924756761334E-06 0.985876120248E-06 0.105103499960E-05 0.112050038036E-05 0.119455688881E-05 - 0.127350796280E-05 0.135767709497E-05 0.144740915825E-05 0.154307181892E-05 0.164505704302E-05 - 0.175378270241E-05 0.186969428687E-05 0.199326672943E-05 0.212500635232E-05 0.226545294151E-05 - 0.241518195837E-05 0.257480689749E-05 0.274498180033E-05 0.292640393503E-05 0.311981665323E-05 - 0.332601243583E-05 0.354583613993E-05 0.378018846037E-05 0.403002962015E-05 0.429638330457E-05 - 0.458034085547E-05 0.488306574269E-05 0.520579833094E-05 0.554986096174E-05 0.591666337125E-05 - 0.630770846606E-05 0.672459848077E-05 0.716904154238E-05 0.764285866862E-05 0.814799122864E-05 - 0.868650889678E-05 0.926061813200E-05 0.987267121753E-05 0.105251758980E-04 0.112208056533E-04 - 0.119624106514E-04 0.127530294250E-04 0.135959013195E-04 0.144944797639E-04 0.154524464180E-04 - 0.164737262553E-04 0.175625036414E-04 0.187232394758E-04 0.199606894654E-04 0.212799236061E-04 - 0.226863469507E-04 0.241857217502E-04 0.257841910562E-04 0.274883038842E-04 0.293050420377E-04 - 0.312418487062E-04 0.333066589503E-04 0.355079322021E-04 0.378546869117E-04 0.403565374822E-04 - 0.430237336441E-04 0.458672024310E-04 0.488985929263E-04 0.521303239658E-04 0.555756349902E-04 - 0.592486402555E-04 0.631643866225E-04 0.673389151628E-04 0.717893268303E-04 0.765338524702E-04 - 0.815919274473E-04 0.869842712009E-04 0.927329720510E-04 0.988615775995E-04 0.105395191097E-03 - 0.112360574171E-03 0.119786256323E-03 0.127702651659E-03 0.136142183310E-03 0.145139416064E-03 - 0.154731197738E-03 0.164956809872E-03 0.175858128359E-03 0.187479794654E-03 0.199869398273E-03 - 0.213077671305E-03 0.227158695740E-03 0.242170124453E-03 0.258173416722E-03 0.275234089266E-03 - 0.293421983779E-03 0.312811552079E-03 0.333482159993E-03 0.355518411228E-03 0.379010492513E-03 - 0.404054541419E-03 0.430753038322E-03 0.459215224092E-03 0.489557545183E-03 0.521904127897E-03 - 0.556387283727E-03 0.593148047805E-03 0.632336752571E-03 0.674113638984E-03 0.718649507668E-03 - 0.766126412590E-03 0.816738400009E-03 0.870692295607E-03 0.928208542896E-03 0.989522096188E-03 - 0.105488337163E-02 0.112455925999E-02 0.119883420516E-02 0.127801135246E-02 0.136241377136E-02 - 0.145238575706E-02 0.154829421610E-02 0.165053014116E-02 0.175951018072E-02 0.187567830934E-02 - 0.199950760496E-02 0.213150213972E-02 0.227219899132E-02 0.242217038221E-02 0.258202595451E-02 - 0.275241518875E-02 0.293402997498E-02 0.312760734560E-02 0.333393237906E-02 0.355384128481E-02 - 0.378822467978E-02 0.403803106746E-02 0.430427053110E-02 0.458801865300E-02 0.489042067249E-02 - 0.521269589551E-02 0.555614236937E-02 0.592214183672E-02 0.631216498309E-02 0.672777699290E-02 - 0.717064342923E-02 0.764253645283E-02 0.814534139630E-02 0.868106370932E-02 0.925183629115E-02 - 0.985992722626E-02 0.105077479391E-01 0.111978617832E-01 0.119329930798E-01 0.127160366201E-01 - 0.135500676430E-01 0.144383523024E-01 0.153843586304E-01 0.163917680069E-01 0.174644871387E-01 - 0.186066605500E-01 0.198226835816E-01 0.211172158919E-01 0.224951954467E-01 0.239618529776E-01 - 0.255227268840E-01 0.271836785399E-01 0.289509079627E-01 0.308309697832E-01 0.328307894472E-01 - 0.349576795609E-01 0.372193562769E-01 0.396239555947E-01 0.421800494316E-01 0.448966612888E-01 - 0.477832813164E-01 0.508498805412E-01 0.541069239936E-01 0.575653824260E-01 0.612367422726E-01 - 0.651330134572E-01 0.692667345974E-01 0.736509751040E-01 0.782993336099E-01 0.832259320992E-01 - 0.884454050389E-01 0.939728827426E-01 0.998239681194E-01 0.106014705886E+00 0.112561543240E+00 - 0.119481280915E+00 0.126791013465E+00 0.134508057562E+00 0.142649867018E+00 0.151233933217E+00 - 0.160277669592E+00 0.169798278812E+00 0.179812601329E+00 0.190336944040E+00 0.201386887920E+00 - 0.212977073642E+00 0.225120964474E+00 0.237830586025E+00 0.251116242866E+00 0.264986212540E+00 - 0.279446418120E+00 0.294500081243E+00 0.310147358389E+00 0.326384964240E+00 0.343205787016E+00 - 0.360598501964E+00 0.378547190403E+00 0.397030973061E+00 0.416023667596E+00 0.435493481275E+00 - 0.455402750406E+00 0.475707738325E+00 0.496358503092E+00 0.517298844394E+00 0.538466336048E+00 - 0.559792445588E+00 0.581202735249E+00 0.602617128756E+00 0.623950215269E+00 0.645111545175E+00 - 0.666005852012E+00 0.686533110574E+00 0.706588313742E+00 0.726060820598E+00 0.744833097986E+00 - 0.762778649522E+00 0.779758904490E+00 0.795618829863E+00 0.810181039602E+00 0.823238215978E+00 - 0.834543739524E+00 0.843795380124E+00 0.850759109567E+00 0.855372161004E+00 0.857616014165E+00 - 0.857488659294E+00 0.855004671928E+00 0.850195047582E+00 0.843106865393E+00 0.833802777701E+00 - 0.822360362211E+00 0.808871363829E+00 0.793440811751E+00 0.776186049690E+00 0.757235651501E+00 - 0.736728220047E+00 0.714811061352E+00 0.691638711026E+00 0.667371314245E+00 0.642172855225E+00 - 0.616209261527E+00 0.589646422683E+00 0.562648185013E+00 0.535374398390E+00 0.507979097643E+00 - 0.480608903449E+00 0.453401664511E+00 0.426485488588E+00 0.399978037834E+00 0.373986187145E+00 - 0.348605971935E+00 - ae wavefunction - -0.227640005255E-06 -0.241553109009E-06 -0.256238555862E-06 -0.271743807437E-06 -0.288119059293E-06 - -0.305417409476E-06 -0.323695040919E-06 -0.343011420299E-06 -0.363429495424E-06 -0.385015922933E-06 - -0.407841293332E-06 -0.431980383418E-06 -0.457512411824E-06 -0.484521323890E-06 -0.513096081958E-06 - -0.543330981917E-06 -0.575325988290E-06 -0.609187085051E-06 -0.645026656992E-06 -0.682963888178E-06 - -0.723125189070E-06 -0.765644650027E-06 -0.810664522779E-06 -0.858335733716E-06 -0.908818428675E-06 - -0.962282552862E-06 -0.101890846811E-05 -0.107888760881E-05 -0.114242318028E-05 -0.120973090157E-05 - -0.128103979581E-05 -0.135659303138E-05 -0.143664881687E-05 -0.152148135361E-05 -0.161138184934E-05 - -0.170665959719E-05 -0.180764312397E-05 -0.191468141270E-05 -0.202814520350E-05 -0.214842837891E-05 - -0.227594943811E-05 -0.241115306706E-05 -0.255451180887E-05 -0.270652784384E-05 -0.286773488278E-05 - -0.303870018477E-05 -0.322002670283E-05 -0.341235537132E-05 -0.361636753939E-05 -0.383278756495E-05 - -0.406238557063E-05 -0.430598038508E-05 -0.456444267121E-05 -0.483869825746E-05 -0.512973168541E-05 - -0.543858998763E-05 -0.576638670788E-05 -0.611430618730E-05 -0.648360812039E-05 -0.687563241512E-05 - -0.729180435834E-05 -0.773364012362E-05 -0.820275263428E-05 -0.870085779960E-05 -0.922978116481E-05 - -0.979146498304E-05 -0.103879757491E-04 -0.110215122190E-04 -0.116944139499E-04 -0.124091703881E-04 - -0.131684305501E-04 -0.139750133231E-04 -0.148319184326E-04 -0.157423381229E-04 -0.167096695752E-04 - -0.177375281589E-04 -0.188297614847E-04 -0.199904644287E-04 -0.212239950522E-04 -0.225349916140E-04 - -0.239283906270E-04 -0.254094460867E-04 -0.269837499631E-04 -0.286572539548E-04 -0.304362926902E-04 - -0.323276083849E-04 -0.343383770722E-04 -0.364762365488E-04 -0.387493160542E-04 -0.411662678874E-04 - -0.437363010297E-04 -0.464692168819E-04 -0.493754473295E-04 -0.524660951748E-04 -0.557529771750E-04 - -0.592486697906E-04 -0.629665578455E-04 -0.669208862560E-04 -0.711268150389E-04 -0.756004777882E-04 - -0.803590438379E-04 -0.854207843420E-04 -0.908051425042E-04 -0.965328082167E-04 -0.102625797380E-03 - -0.109107536177E-03 -0.116002950618E-03 -0.123338561665E-03 -0.131142586270E-03 -0.139445044700E-03 - -0.148277874493E-03 -0.157675051496E-03 -0.167672718328E-03 -0.178309320794E-03 -0.189625752649E-03 - -0.201665509253E-03 -0.214474850611E-03 -0.228102974334E-03 -0.242602199160E-03 -0.258028159541E-03 - -0.274440012047E-03 -0.291900654171E-03 -0.310476956278E-03 -0.330240007458E-03 -0.351265376027E-03 - -0.373633385506E-03 -0.397429406962E-03 -0.422744168568E-03 -0.449674083345E-03 -0.478321596087E-03 - -0.508795550462E-03 -0.541211577400E-03 -0.575692505860E-03 -0.612368797176E-03 -0.651379004183E-03 - -0.692870256384E-03 -0.736998772502E-03 -0.783930401758E-03 -0.833841195310E-03 -0.886918009300E-03 - -0.943359141058E-03 -0.100337499998E-02 -0.106718881472E-02 -0.113503737838E-02 -0.120717183327E-02 - -0.128385849716E-02 -0.136537973262E-02 -0.145203486135E-02 -0.154414112528E-02 -0.164203469631E-02 - -0.174607173654E-02 -0.185662951080E-02 -0.197410755347E-02 -0.209892889124E-02 -0.223154132379E-02 - -0.237241876400E-02 -0.252206263958E-02 -0.268100335757E-02 -0.284980183333E-02 -0.302905108546E-02 - -0.321937789790E-02 -0.342144455020E-02 -0.363595061718E-02 -0.386363483818E-02 -0.410527705682E-02 - -0.436170023094E-02 -0.463377251275E-02 -0.492240939836E-02 -0.522857594571E-02 -0.555328905920E-02 - -0.589761983893E-02 -0.626269599168E-02 -0.664970430008E-02 -0.705989314585E-02 -0.749457508176E-02 - -0.795512944634E-02 -0.844300501417E-02 -0.895972267341E-02 -0.950687812113E-02 -0.100861445655E-01 - -0.106992754225E-01 -0.113481069932E-01 -0.120345611055E-01 -0.127606477039E-01 -0.135284673660E-01 - -0.143402137256E-01 -0.151981757772E-01 -0.161047400361E-01 -0.170623925242E-01 -0.180737205506E-01 - -0.191414142506E-01 -0.202682678463E-01 -0.214571805872E-01 -0.227111573254E-01 -0.240333086777E-01 - -0.254268507220E-01 -0.268951041730E-01 -0.284414929755E-01 -0.300695422528E-01 -0.317828755418E-01 - -0.335852112415E-01 -0.354803581988E-01 -0.374722103520E-01 -0.395647403441E-01 -0.417619920209E-01 - -0.440680717173E-01 -0.464871382391E-01 -0.490233914373E-01 -0.516810592766E-01 -0.544643832889E-01 - -0.573776023104E-01 -0.604249343909E-01 -0.636105567713E-01 -0.669385838208E-01 -0.704130428314E-01 - -0.740378475674E-01 -0.778167694752E-01 -0.817534064637E-01 -0.858511491736E-01 -0.901131446652E-01 - -0.945422574669E-01 -0.991410279388E-01 -0.103911627924E+00 -0.108855813683E+00 -0.113974876119E+00 - -0.119269588339E+00 -0.124740150612E+00 -0.130386132824E+00 -0.136206414546E+00 -0.142199122910E+00 - -0.148361568461E+00 -0.154690179275E+00 -0.161180433629E+00 -0.167826791604E+00 -0.174622626074E+00 - -0.181560153598E+00 -0.188630365898E+00 -0.195822962666E+00 -0.203126286646E+00 -0.210527262067E+00 - -0.218011337617E+00 -0.225562435281E+00 -0.233162906153E+00 -0.240793494317E+00 -0.248433308678E+00 - -0.256059802119E+00 -0.263648755337E+00 -0.271174261340E+00 -0.278608705431E+00 -0.285922735843E+00 - -0.293085222680E+00 -0.300063207642E+00 -0.306821853310E+00 -0.313324406907E+00 -0.319532197048E+00 - -0.325404681082E+00 -0.330899553728E+00 -0.335972916269E+00 -0.340579493605E+00 -0.344672879916E+00 - -0.348205795435E+00 -0.351130344647E+00 -0.353398274582E+00 -0.354961236968E+00 -0.355771059088E+00 - -0.355780026627E+00 -0.354941178528E+00 -0.353208612936E+00 -0.350537800929E+00 -0.346885905093E+00 - -0.342212099803E+00 -0.336477890709E+00 -0.329647431117E+00 -0.321687833920E+00 -0.312569477680E+00 - -0.302266305867E+00 -0.290756118642E+00 -0.278020856318E+00 -0.264046873976E+00 -0.248825206279E+00 - -0.232351821598E+00 -0.214627864069E+00 -0.195659881552E+00 -0.175460037164E+00 -0.154046301204E+00 - -0.131442619230E+00 -0.107679051851E+00 -0.827918807912E-01 -0.568236757344E-01 -0.298233169267E-01 - -0.184596946076E-02 0.270469925154E-01 0.567881103967E-01 0.873040093439E-01 0.118515601298E+00 - 0.150338311437E+00 0.182682317934E+00 0.215452801690E+00 0.248550217585E+00 0.281870625756E+00 - 0.315306159094E+00 0.348745757632E+00 0.382076358511E+00 0.415184751019E+00 0.447960169991E+00 - 0.480297184335E+00 0.512097402331E+00 0.543267514780E+00 0.573711997895E+00 0.603322786216E+00 - 0.631972550728E+00 0.659516794793E+00 0.685802869564E+00 0.710679507526E+00 0.734002706130E+00 - 0.755638016764E+00 0.775461068324E+00 0.793357768618E+00 0.809224794004E+00 0.822970440992E+00 - 0.834515701559E+00 0.843795380124E+00 0.850759109567E+00 0.855372161004E+00 0.857616014165E+00 - 0.857488659294E+00 0.855004671928E+00 0.850195047582E+00 0.843106865393E+00 0.833802777701E+00 - 0.822360362211E+00 0.808871363829E+00 0.793440811751E+00 0.776186049690E+00 0.757235651501E+00 - 0.736728220047E+00 0.714811061352E+00 0.691638711026E+00 0.667371314245E+00 0.642172855225E+00 - 0.616209261527E+00 0.589646422683E+00 0.562648185013E+00 0.535374398390E+00 0.507979097643E+00 - 0.480608903449E+00 0.453401664511E+00 0.426485488588E+00 0.399978037834E+00 0.373986187145E+00 - 0.348605971935E+00 - pseudo wavefunction - 0.571859011433E-08 0.609654545181E-08 0.649948076408E-08 0.692904703757E-08 0.738700437639E-08 - 0.787522921411E-08 0.839572200233E-08 0.895061540728E-08 0.954218304822E-08 0.101728488133E-07 - 0.108451967914E-07 0.115619818599E-07 0.123261409726E-07 0.131408051936E-07 0.140093125268E-07 - 0.149352215925E-07 0.159223262088E-07 0.169746709363E-07 0.180965676505E-07 0.192926132088E-07 - 0.205677082861E-07 0.219270774547E-07 0.233762905912E-07 0.249212856987E-07 0.265683932374E-07 - 0.283243620625E-07 0.301963870773E-07 0.321921387136E-07 0.343197943601E-07 0.365880718690E-07 - 0.390062652761E-07 0.415842828822E-07 0.443326878514E-07 0.472627414928E-07 0.503864494020E-07 - 0.537166106534E-07 0.572668702427E-07 0.610517749960E-07 0.650868331739E-07 0.693885780150E-07 - 0.739746354790E-07 0.788637964676E-07 0.840760938177E-07 0.896328843845E-07 0.955569365484E-07 - 0.101872523507E-06 0.108605522730E-06 0.115783521992E-06 0.123435932409E-06 0.131594108947E-06 - 0.140291478898E-06 0.149563678840E-06 0.159448700661E-06 0.169987047220E-06 0.181221898307E-06 - 0.193199287567E-06 0.205968291118E-06 0.219581228634E-06 0.234093877719E-06 0.249565702454E-06 - 0.266060097037E-06 0.283644645541E-06 0.302391398829E-06 0.322377169775E-06 0.343683847994E-06 - 0.366398735382E-06 0.390614903817E-06 0.416431576518E-06 0.443954534597E-06 0.473296550488E-06 - 0.504577850014E-06 0.537926605003E-06 0.573479458457E-06 0.611382084429E-06 0.651789784903E-06 - 0.694868126132E-06 0.740793617018E-06 0.789754432343E-06 0.841951183784E-06 0.897597741903E-06 - 0.956922112447E-06 0.102016737058E-05 0.108759265684E-05 0.115947423894E-05 0.123610664376E-05 - 0.131780386409E-05 0.140490064518E-05 0.149775385636E-05 0.159674395320E-05 0.170227653640E-05 - 0.181478401368E-05 0.193472737149E-05 0.206259806386E-05 0.219892002601E-05 0.234425182114E-05 - 0.249918892902E-05 0.266436618586E-05 0.284046038547E-05 0.302819305228E-05 0.322833339765E-05 - 0.344170147158E-05 0.366917152271E-05 0.391167558034E-05 0.417020727329E-05 0.444582590101E-05 - 0.473966077387E-05 0.505291584021E-05 0.538687461930E-05 0.574290546014E-05 0.612246714800E-05 - 0.652711488135E-05 0.695850664388E-05 0.741840999759E-05 0.790870932483E-05 0.843141354897E-05 - 0.898866436529E-05 0.958274501585E-05 0.102160896442E-04 0.108912932684E-04 0.116111224130E-04 - 0.123785264438E-04 0.131966496516E-04 0.140688441343E-04 0.149986835308E-04 0.159899776617E-04 - 0.170467881383E-04 0.181734450030E-04 0.193745644683E-04 0.206550678297E-04 0.220202016265E-04 - 0.234755591366E-04 0.250271032906E-04 0.266811910997E-04 0.284445996990E-04 0.303245541103E-04 - 0.323287568402E-04 0.344654194334E-04 0.367432961109E-04 0.391717196309E-04 0.417606395186E-04 - 0.445206628220E-04 0.474630975606E-04 0.505999990437E-04 0.539442192495E-04 0.575094594655E-04 - 0.613103264068E-04 0.653623920412E-04 0.696822573653E-04 0.742876203945E-04 0.791973486423E-04 - 0.844315563875E-04 0.900116870435E-04 0.959606009681E-04 0.102302669071E-03 0.109063872604E-03 - 0.116271909536E-03 0.123956307960E-03 0.132148546974E-03 0.140882185556E-03 0.150192999936E-03 - 0.160119130038E-03 0.170701235589E-03 0.181982662526E-03 0.194009620390E-03 0.206831371412E-03 - 0.220500432088E-03 0.235072788041E-03 0.250608123054E-03 0.267170063210E-03 0.284826437132E-03 - 0.303649553366E-03 0.323716496065E-03 0.345109440149E-03 0.367915987228E-03 0.392229523661E-03 - 0.418149602194E-03 0.445782348728E-03 0.475240895864E-03 0.506645844977E-03 0.540125758697E-03 - 0.575817685783E-03 0.613867720498E-03 0.654431598766E-03 0.697675333489E-03 0.743775891598E-03 - 0.792921915550E-03 0.845314492176E-03 0.901167971957E-03 0.960710842010E-03 0.102418665628E-02 - 0.109185502664E-02 0.116399267887E-02 0.124089457767E-02 0.132287512520E-02 0.141026943799E-02 - 0.150343470699E-02 0.160275164648E-02 0.170862603724E-02 0.182149037022E-02 0.194180559698E-02 - 0.207006299390E-02 0.220678614720E-02 0.235253306661E-02 0.250789843574E-02 0.267351600781E-02 - 0.285006115589E-02 0.303825358734E-02 0.323886023271E-02 0.345269831994E-02 0.368063864538E-02 - 0.392360905372E-02 0.418259813975E-02 0.445865918532E-02 0.475291434595E-02 0.506655910210E-02 - 0.540086699085E-02 0.575719463481E-02 0.613698708574E-02 0.654178350122E-02 0.697322317377E-02 - 0.743305193259E-02 0.792312893904E-02 0.844543389789E-02 0.900207470735E-02 0.959529557187E-02 - 0.102274856023E-01 0.109011879290E-01 0.116191093550E-01 0.123841305757E-01 0.131993169929E-01 - 0.140679301526E-01 0.149934398347E-01 0.159795368236E-01 0.170301463902E-01 0.181494425124E-01 - 0.193418628649E-01 0.206121246052E-01 0.219652409824E-01 0.234065387949E-01 0.249416767187E-01 - 0.265766645285E-01 0.283178832261E-01 0.301721060910E-01 0.321465206585E-01 0.342487516267E-01 - 0.364868846862E-01 0.388694912556E-01 0.414056540969E-01 0.441049937706E-01 0.469776958777E-01 - 0.500345390167E-01 0.532869233644E-01 0.567468997676E-01 0.604271992062E-01 0.643412624556E-01 - 0.685032697477E-01 0.729281701846E-01 0.776317106202E-01 0.826304636723E-01 0.879418544734E-01 - 0.935841857035E-01 0.995766603828E-01 0.105939401817E+00 0.112693470009E+00 0.119860873746E+00 - 0.127464577480E+00 0.135528501980E+00 0.144077517640E+00 0.153137429157E+00 0.162734950176E+00 - 0.172897666316E+00 0.183653984845E+00 0.195033069077E+00 0.207064755401E+00 0.219779450647E+00 - 0.233208007313E+00 0.247381573997E+00 0.262331418175E+00 0.278088718331E+00 0.294684322281E+00 - 0.312148468415E+00 0.330510466534E+00 0.349798334922E+00 0.370038390351E+00 0.391254787886E+00 - 0.413469007585E+00 0.436699285593E+00 0.460959987670E+00 0.486260923917E+00 0.512606604396E+00 - 0.539995436506E+00 0.568418866379E+00 0.597860468269E+00 0.628294987815E+00 0.659687347350E+00 - 0.691991623811E+00 0.725150012533E+00 0.759091792928E+00 0.793732314786E+00 0.828972026523E+00 - 0.864695568746E+00 0.900770957920E+00 0.937048885087E+00 0.973362153233E+00 0.100952527329E+01 - 0.104533423238E+01 0.108056643801E+01 0.111498082794E+01 0.114831811634E+01 0.118030112315E+01 - 0.121063510361E+01 0.123900796074E+01 0.126509018530E+01 0.128853432805E+01 0.130897377168E+01 - 0.132602054064E+01 0.133926187406E+01 0.134825530053E+01 0.135252200835E+01 0.135153841669E+01 - 0.134472604132E+01 0.133144140608E+01 0.131113985918E+01 0.128359157702E+01 0.124863807676E+01 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 - ae wavefunction - -0.419591673479E-06 -0.445236660954E-06 -0.472305250687E-06 -0.500884925173E-06 -0.531068206171E-06 - -0.562952965377E-06 -0.596642760627E-06 -0.632247202450E-06 -0.669882317876E-06 -0.709670970157E-06 - -0.751743273549E-06 -0.796237058589E-06 -0.843298343123E-06 -0.893081857389E-06 -0.945751579095E-06 - -0.100148131688E-05 -0.106045532797E-05 -0.112286896479E-05 -0.118892937793E-05 -0.125885625057E-05 - -0.133288258579E-05 -0.141125554245E-05 -0.149423732267E-05 -0.158210611794E-05 -0.167515711327E-05 - -0.177370355614E-05 -0.187807789421E-05 -0.198863298444E-05 -0.210574338018E-05 -0.222980670049E-05 - -0.236124508686E-05 -0.250050675369E-05 -0.264806763788E-05 -0.280443315455E-05 -0.297014006557E-05 - -0.314575846823E-05 -0.333189391186E-05 -0.352918965126E-05 -0.373832904445E-05 -0.396003810621E-05 - -0.419508822539E-05 -0.444429905901E-05 -0.470854161112E-05 -0.498874151386E-05 -0.528588251725E-05 - -0.560101020839E-05 -0.593523596640E-05 -0.628974117866E-05 -0.666578172614E-05 -0.706469276477E-05 - -0.748789380520E-05 -0.793689413439E-05 -0.841329858154E-05 -0.891881365848E-05 -0.945525409868E-05 - -0.100245498207E-04 -0.106287533383E-04 -0.112700476609E-04 -0.119507546913E-04 -0.126733441838E-04 - -0.134404432655E-04 -0.142548465883E-04 -0.151195271357E-04 -0.160376477180E-04 -0.170125732291E-04 - -0.180478836822E-04 -0.191473880961E-04 -0.203151392770E-04 -0.215554495594E-04 -0.228729075576E-04 - -0.242723960099E-04 -0.257591107648E-04 -0.273385809967E-04 -0.290166907359E-04 -0.307997017578E-04 - -0.326942780077E-04 -0.347075115045E-04 -0.368469500336E-04 -0.391206264891E-04 -0.415370902338E-04 - -0.441054403792E-04 -0.468353612296E-04 -0.497371600521E-04 -0.528218071755E-04 -0.561009787541E-04 - -0.595871022176E-04 -0.632934046196E-04 -0.672339641473E-04 -0.714237648284E-04 -0.758787548062E-04 - -0.806159083152E-04 -0.856532915520E-04 -0.910101328385E-04 -0.967068971463E-04 -0.102765365426E-03 - -0.109208718933E-03 -0.116061628915E-03 -0.123350351965E-03 -0.131102831408E-03 -0.139348805092E-03 - -0.148119919962E-03 -0.157449853863E-03 -0.167374444991E-03 -0.177931829462E-03 -0.189162587521E-03 - -0.201109898875E-03 -0.213819707747E-03 -0.227340898203E-03 -0.241725480395E-03 -0.257028788379E-03 - -0.273309690165E-03 -0.290630810807E-03 -0.309058769194E-03 -0.328664429493E-03 -0.349523167966E-03 - -0.371715156200E-03 -0.395325661615E-03 -0.420445366275E-03 -0.447170705143E-03 -0.475604224753E-03 - -0.505854963647E-03 -0.538038855679E-03 -0.572279157541E-03 -0.608706901916E-03 -0.647461377632E-03 - -0.688690638341E-03 -0.732552041357E-03 -0.779212818233E-03 -0.828850678842E-03 -0.881654450829E-03 - -0.937824756232E-03 -0.997574727349E-03 -0.106113076385E-02 -0.112873333333E-02 -0.120063781758E-02 - -0.127711540679E-02 -0.135845404425E-02 -0.144495942405E-02 -0.153695604425E-02 -0.163478831840E-02 - -0.173882174815E-02 -0.184944415970E-02 -0.196706700731E-02 -0.209212674663E-02 -0.222508628119E-02 - -0.236643648517E-02 -0.251669780565E-02 -0.267642194774E-02 -0.284619364598E-02 -0.302663252522E-02 - -0.321839505467E-02 -0.342217659827E-02 -0.363871356504E-02 -0.386878566260E-02 -0.411321825733E-02 - -0.437288484423E-02 -0.464870962994E-02 -0.494167023156E-02 -0.525280049427E-02 -0.558319343043E-02 - -0.593400428227E-02 -0.630645371041E-02 -0.670183110977E-02 -0.712149805405E-02 -0.756689186960E-02 - -0.803952933881E-02 -0.854101053252E-02 -0.907302277019E-02 -0.963734470585E-02 -0.102358505368E-01 - -0.108705143308E-01 -0.115434144674E-01 -0.122567381851E-01 -0.130127862287E-01 -0.138139775855E-01 - -0.146628542996E-01 -0.155620863515E-01 -0.165144765871E-01 -0.175229656786E-01 -0.185906370974E-01 - -0.197207220752E-01 -0.209166045292E-01 -0.221818259204E-01 -0.235200900133E-01 -0.249352675010E-01 - -0.264314004551E-01 -0.280127065554E-01 -0.296835830510E-01 -0.314486103986E-01 -0.333125555181E-01 - -0.352803746019E-01 -0.373572154063E-01 -0.395484189494E-01 -0.418595205316E-01 -0.442962499902E-01 - -0.468645310902E-01 -0.495704799483E-01 -0.524204023797E-01 -0.554207900464E-01 -0.585783152843E-01 - -0.618998244709E-01 -0.653923297941E-01 -0.690629992720E-01 -0.729191448635E-01 -0.769682085087E-01 - -0.812177459231E-01 -0.856754079688E-01 -0.903489194169E-01 -0.952460549124E-01 -0.100374611944E+00 - -0.105742380623E+00 -0.111357110073E+00 -0.117226471226E+00 -0.123358015834E+00 -0.129759131487E+00 - -0.136436992474E+00 -0.143398506273E+00 -0.150650255536E+00 -0.158198435396E+00 -0.166048785958E+00 - -0.174206519886E+00 -0.182676244971E+00 -0.191461881646E+00 -0.200566575426E+00 -0.209992604279E+00 - -0.219741281025E+00 -0.229812850838E+00 -0.240206384069E+00 -0.250919664576E+00 -0.261949073908E+00 - -0.273289471686E+00 -0.284934072667E+00 -0.296874321046E+00 -0.309099762684E+00 -0.321597916090E+00 - -0.334354143130E+00 -0.347351520683E+00 -0.360570714663E+00 -0.373989858114E+00 -0.387584435372E+00 - -0.401327174519E+00 -0.415187950525E+00 -0.429133701186E+00 -0.443128357849E+00 -0.457132790725E+00 - -0.471104767691E+00 -0.484998921703E+00 -0.498766719483E+00 -0.512356422012E+00 -0.525713027989E+00 - -0.538778196066E+00 -0.551490150480E+00 -0.563783586393E+00 -0.575589602443E+00 -0.586835694684E+00 - -0.597445844316E+00 -0.607340718942E+00 -0.616437986018E+00 -0.624652715226E+00 -0.631897834510E+00 - -0.638084607775E+00 -0.643123116697E+00 -0.646922744393E+00 -0.649392668142E+00 -0.650442370237E+00 - -0.649982173269E+00 -0.647923800099E+00 -0.644180957099E+00 -0.638669934919E+00 -0.631310221677E+00 - -0.622025123195E+00 -0.610742386067E+00 -0.597394819740E+00 -0.581920915553E+00 -0.564265460645E+00 - -0.544380145344E+00 -0.522224163384E+00 -0.497764803837E+00 -0.470978034224E+00 -0.441849073512E+00 - -0.410372953728E+00 -0.376555068042E+00 -0.340411701856E+00 -0.301970542890E+00 -0.261271164549E+00 - -0.218365474934E+00 -0.173318123362E+00 -0.126206854463E+00 -0.771227997739E-01 -0.261706977118E-01 - 0.265309653866E-01 0.808498960908E-01 0.136640013874E+00 0.193741540297E+00 0.251981141124E+00 - 0.311172100787E+00 0.371114512072E+00 0.431595477829E+00 0.492389350851E+00 0.553258089799E+00 - 0.613951880087E+00 0.674210269543E+00 0.733764172796E+00 0.792339126527E+00 0.849659904580E+00 - 0.905455616556E+00 0.959462488428E+00 0.101141974424E+01 0.106105567211E+01 0.110806847980E+01 - 0.115211437455E+01 0.119281223958E+01 0.122976095463E+01 0.126255737984E+01 0.129080748835E+01 - 0.131413097418E+01 0.133216280306E+01 0.134455435589E+01 0.135097524786E+01 0.135111592592E+01 - 0.134469077582E+01 0.133144140608E+01 0.131113985918E+01 0.128359157702E+01 0.124863807676E+01 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 0.000000000000E+00 - 0.000000000000E+00 - End of Dataset diff --git a/docs.it4i/salomon/software/chemistry/gofree-cond1.sh b/docs.it4i/salomon/software/chemistry/gofree-cond1.sh deleted file mode 100644 index 8dbf6d276..000000000 --- a/docs.it4i/salomon/software/chemistry/gofree-cond1.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -#PBS -A OPEN-6-23 -#PBS -N Si-test1 -#PBS -q qfree -#PBS -l select=1:ncpus=24:mpiprocs=24:ompthreads=1 -#PBS -l walltime=01:59:59 -##PBS-l mem=6gb -#PBS -j oe -#PBS -S /bin/bash -module purge -module load phono3py/0.9.14-ictce-7.3.5-Python-2.7.9 -export OMP_NUM_THREADS=1 -export I_MPI_COMPATIBILITY=4 -##export OMP_STACKSIZE=10gb -##0 1 2 3 4 10 11 12 13 20 21 22 30 31 40 91 92 93 94 101 102 103 111 112 121 182 183 184 192 193 202 273 274 283 364 -cd $PBS_O_WORKDIR -phono3py --fc3 --fc2 --dim="2 2 2" --mesh="9 9 9" -c POSCAR --sigma 0.1 --br --write_gamma --gp="0 1 3 4 10" -#phono3py --fc3 --fc2 --dim="2 2 2" --mesh="9 9 9" -c POSCAR --sigma 0.1 --br --write_gamma --gp="11 12 13 20 21" -#phono3py --fc3 --fc2 --dim="2 2 2" --mesh="9 9 9" -c POSCAR --sigma 0.1 --br --write_gamma --gp="21 22 30 31 40" -#phono3py --fc3 --fc2 --dim="2 2 2" --mesh="9 9 9" -c POSCAR --sigma 0.1 --br --write_gamma --gp="91 92 93 94 101" -#phono3py --fc3 --fc2 --dim="2 2 2" --mesh="9 9 9" -c POSCAR --sigma 0.1 --br --write_gamma --gp="102 103 111 112 121" -#phono3py --fc3 --fc2 --dim="2 2 2" --mesh="9 9 9" -c POSCAR --sigma 0.1 --br --write_gamma --gp="182 183 184 192 193" -#phono3py --fc3 --fc2 --dim="2 2 2" --mesh="9 9 9" -c POSCAR --sigma 0.1 --br --write_gamma --gp="202 273 274 283 364" diff --git a/docs.it4i/salomon/software/chemistry/prepare.sh b/docs.it4i/salomon/software/chemistry/prepare.sh deleted file mode 100644 index a057dbbad..000000000 --- a/docs.it4i/salomon/software/chemistry/prepare.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -P=`pwd` - -# number of displacements -poc=9 - - for i in `seq 1 $poc `; - do - cd $P - mkdir disp-0000"$i" - cd disp-0000"$i" - cp ../KPOINTS . - cp ../INCAR . - cp ../POTCAR . - cp ../POSCAR-0000"$i" POSCAR - echo $i - done - -poc=99 - - for i in `seq 10 $poc `; - do - cd $P - mkdir disp-000"$i" - cd disp-000"$i" - cp ../KPOINTS . - cp ../INCAR . - cp ../POTCAR . - cp ../POSCAR-000"$i" POSCAR - echo $i - done -poc=111 - - for i in `seq 100 $poc `; - do - cd $P - mkdir disp-00"$i" - cd disp-00"$i" - cp ../KPOINTS . - cp ../INCAR . - cp ../POTCAR . - cp ../POSCAR-00"$i" POSCAR - echo $i - done diff --git a/docs.it4i/salomon/software/chemistry/run.sh b/docs.it4i/salomon/software/chemistry/run.sh deleted file mode 100644 index 63faf63ba..000000000 --- a/docs.it4i/salomon/software/chemistry/run.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#PBS -A IT4I-9-11 -#PBS -N Si-test -#PBS -q qprod -#PBS -l select=8:ncpus=16:mpiprocs=16:ompthreads=1 -#PBS -l walltime=23:59:59 -##PBS-l mem=6gb -#PBS -j oe -#PBS -S /bin/bash -module load impi/4.1.1.036 intel/13.5.192 fftw3-mpi/3.3.3-icc -export OMP_NUM_THREADS=1 -export I_MPI_COMPATIBILITY=4 -##export OMP_STACKSIZE=10gb -b=`basename $PBS_O_WORKDIR` -echo $b >log.vasp -SCRDIR=/scratch/$USER/$b -mkdir -p $SCRDIR -cd $SCRDIR || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/* . - -mpirun ~/bin/vasp5.4.1 > log.exc - -# copy output file to home -cp * $PBS_O_WORKDIR/. && cd .. - -rm -rf "$SCRDIR" - -#exit -exit diff --git a/docs.it4i/salomon/software/chemistry/submit.sh b/docs.it4i/salomon/software/chemistry/submit.sh deleted file mode 100644 index 12fa02ecc..000000000 --- a/docs.it4i/salomon/software/chemistry/submit.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -P=`pwd` - -# number of displacements -poc=9 - - for i in `seq 1 $poc `; - do - cd $P - cd disp-0000"$i" - cp ../run.sh . - qsub run.sh - echo $i - done - -poc=99 - - for i in `seq 10 $poc `; - do - cd $P - cd disp-000"$i" - cp ../run.sh . - qsub run.sh - echo $i - done -poc=111 - - for i in `seq 100 $poc `; - do - cd $P - cd disp-00"$i" - cp ../run.sh . - qsub run.sh - echo $i - done diff --git a/docs.it4i/salomon/software/compilers.md b/docs.it4i/salomon/software/compilers.md deleted file mode 100644 index 76bd63261..000000000 --- a/docs.it4i/salomon/software/compilers.md +++ /dev/null @@ -1,193 +0,0 @@ -# Compilers - -## Available compilers, including GNU, INTEL and UPC compilers - -There are several compilers for different programming languages available on the cluster: - -* C/C++ -* Fortran 77/90/95/HPF -* Unified Parallel C -* Java - -The C/C++ and Fortran compilers are provided by: - -Opensource: - -* GNU GCC -* Clang/LLVM - -Commercial licenses: - -* Intel -* PGI - -## Intel Compilers - -For information about the usage of Intel Compilers and other Intel products, please read the [Intel Parallel studio](intel-suite/) page. - -## PGI Compilers - -The Portland Group Cluster Development Kit (PGI CDK) is available. - -```console -$ module load PGI -$ pgcc -v -$ pgc++ -v -$ pgf77 -v -$ pgf90 -v -$ pgf95 -v -$ pghpf -v -``` - -The PGI CDK also incudes tools for debugging and profiling. - -PGDBG OpenMP/MPI debugger and PGPROF OpenMP/MPI profiler are available - -```console -$ module load PGI -$ module load Java -$ pgdbg & -$ pgprof & -``` - -For more information, see the [PGI page](http://www.pgroup.com/products/pgicdk.htm). - -## GNU - -For compatibility reasons there are still available the original (old 4.4.7-11) versions of GNU compilers as part of the OS. These are accessible in the search path by default. - -It is strongly recommended to use the up to date version which comes with the module GCC: - -```console -$ module load GCC -$ gcc -v -$ g++ -v -$ gfortran -v -``` - -With the module loaded two environment variables are predefined. One for maximum optimizations on the cluster's architecture, and the other for debugging purposes: - -```console -$ echo $OPTFLAGS --O3 -march=native - -$ echo $DEBUGFLAGS --O0 -g -``` - -For more information about the possibilities of the compilers, please see the man pages. - -## Unified Parallel C - -UPC is supported by two compiler/runtime implementations: - -* GNU - SMP/multi-threading support only -* Berkley - multi-node support as well as SMP/multi-threading support - -### GNU UPC Compiler - -To use the GNU UPC compiler and run the compiled binaries use the module gupc - -```console -$ module add gupc -$ gupc -v -$ g++ -v -``` - -Simple program to test the compiler - -```cpp -$ cat count.upc - -/* hello.upc - a simple UPC example */ -#include <upc.h> -#include <stdio.h> - -int main() { - if (MYTHREAD == 0) { - printf("Welcome to GNU UPC!!!n"); - } - upc_barrier; - printf(" - Hello from thread %in", MYTHREAD); - return 0; -} -``` - -To compile the example use - -```console -$ gupc -o count.upc.x count.upc -``` - -To run the example with 5 threads issue - -```console -$ ./count.upc.x -fupc-threads-5 -``` - -For more information see the man pages. - -### Berkley UPC Compiler - -To use the Berkley UPC compiler and runtime environment to run the binaries use the module bupc - -```console -$ module add BerkeleyUPC/2.16.2-gompi-2015b -$ upcc -version -``` - -As default UPC network the "smp" is used. This is very quick and easy way for testing/debugging, but limited to one node only. - -For production runs, it is recommended to use the native InfiniBand implementation of UPC network "ibv". For testing/debugging using multiple nodes, the "mpi" UPC network is recommended. - -!!! warning - Selection of the network is done at the compile time and not at runtime (as expected)! - -Example UPC code: - -```cpp -$ cat hello.upc - -/* hello.upc - a simple UPC example */ -#include <upc.h> -#include <stdio.h> - -int main() { - if (MYTHREAD == 0) { - printf("Welcome to Berkeley UPC!!!n"); - } - upc_barrier; - printf(" - Hello from thread %in", MYTHREAD); - return 0; -} -``` - -To compile the example with the "ibv" UPC network use - -```console -$ upcc -network=ibv -o hello.upc.x hello.upc -``` - -To run the example with 5 threads issue - -```console -$ upcrun -n 5 ./hello.upc.x -``` - -To run the example on two compute nodes using all 48 cores, with 48 threads, issue - -```console -$ qsub -I -q qprod -A PROJECT_ID -l select=2:ncpus=24 -$ module add bupc -$ upcrun -n 48 ./hello.upc.x -``` - -For more information see the man pages. - -## Java - -For information how to use Java (runtime and/or compiler), please read the [Java page](java/). - -## NVIDIA CUDA - -For information how to work with NVIDIA CUDA, please read the [NVIDIA CUDA page](../../anselm/software/nvidia-cuda/). diff --git a/docs.it4i/salomon/software/comsol/comsol-multiphysics.md b/docs.it4i/salomon/software/comsol/comsol-multiphysics.md deleted file mode 100644 index 431294469..000000000 --- a/docs.it4i/salomon/software/comsol/comsol-multiphysics.md +++ /dev/null @@ -1,118 +0,0 @@ -# COMSOL Multiphysics - -## Introduction - -[COMSOL](http://www.comsol.com) is a powerful environment for modelling and solving various engineering and scientific problems based on partial differential equations. COMSOL is designed to solve coupled or multiphysics phenomena. For many standard engineering problems COMSOL provides add-on products such as electrical, mechanical, fluid flow, and chemical applications. - -* [Structural Mechanics Module](http://www.comsol.com/structural-mechanics-module), -* [Heat Transfer Module](http://www.comsol.com/heat-transfer-module), -* [CFD Module](http://www.comsol.com/cfd-module), -* [Acoustics Module](http://www.comsol.com/acoustics-module), -* and [many others](http://www.comsol.com/products) - -COMSOL also allows an interface support for equation-based modelling of partial differential equations. - -## Execution - -On the clusters COMSOL is available in the latest stable version. There are two variants of the release: - -* **Non commercial** or so called >**EDU variant**>, which can be used for research and educational purposes. - -* **Commercial** or so called **COM variant**, which can used also for commercial activities. **COM variant** has only subset of features compared to the **EDU variant** available. More about licensing will be posted here soon. - -To load the of COMSOL load the module - -```console -$ ml COMSOL/51-EDU -``` - -By default the **EDU variant** will be loaded. If user needs other version or variant, load the particular version. To obtain the list of available versions use - -```console -$ ml av COMSOL -``` - -If user needs to prepare COMSOL jobs in the interactive mode it is recommend to use COMSOL on the compute nodes via PBS Pro scheduler. In order run the COMSOL Desktop GUI on Windows is recommended to use the [Virtual Network Computing (VNC)](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/). - -```console -$ xhost + -$ qsub -I -X -A PROJECT_ID -q qprod -l select=1:ppn=24 -$ ml COMSOL -$ comsol -``` - -To run COMSOL in batch mode, without the COMSOL Desktop GUI environment, user can utilized the default (comsol.pbs) job script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l select=3:ppn=24 -#PBS -q qprod -#PBS -N JOB_NAME -#PBS -A PROJECT_ID - -cd /scratch/work/user/$USER/ || exit - -echo Time is `date` -echo Directory is `pwd` -echo '**PBS_NODEFILE***START*******' -cat $PBS_NODEFILE -echo '**PBS_NODEFILE***END*********' - -text_nodes < cat $PBS_NODEFILE - -module load COMSOL -# module load COMSOL/51-EDU - -ntask=$(wc -l $PBS_NODEFILE) - -comsol -nn ${ntask} batch -configuration /tmp –mpiarg –rmk –mpiarg pbs -tmpdir /scratch/$USER/ -inputfile name_input_f.mph -outputfile name_output_f.mph -batchlog name_log_f.log -``` - -Working directory has to be created before sending the (comsol.pbs) job script into the queue. Input file (name_input_f.mph) has to be in working directory or full path to input file has to be specified. The appropriate path to the temp directory of the job has to be set by command option (-tmpdir). - -## LiveLink for MATLAB - -COMSOL is the software package for the numerical solution of the partial differential equations. LiveLink for MATLAB allows connection to the COMSOL API (Application Programming Interface) with the benefits of the programming language and computing environment of the MATLAB. - -LiveLink for MATLAB is available in both **EDU** and **COM** **variant** of the COMSOL release. On the clusters 1 commercial (**COM**) license and the 5 educational (**EDU**) licenses of LiveLink for MATLAB (please see the [ISV Licenses](../../../anselm/software/isv_licenses/)) are available. Following example shows how to start COMSOL model from MATLAB via LiveLink in the interactive mode. - -```console -$ xhost + -$ qsub -I -X -A PROJECT_ID -q qexp -l select=1:ppn=24 -$ ml MATLAB -$ ml COMSOL -$ comsol server MATLAB -``` - -At the first time to launch the LiveLink for MATLAB (client-MATLAB/server-COMSOL connection) the login and password is requested and this information is not requested again. - -To run LiveLink for MATLAB in batch mode with (comsol_matlab.pbs) job script you can utilize/modify the following script and execute it via the qsub command. - -```bash -#!/bin/bash -#PBS -l select=3:ppn=24 -#PBS -q qprod -#PBS -N JOB_NAME -#PBS -A PROJECT_ID - -cd /scratch/work/user/$USER || exit - -echo Time is `date` -echo Directory is `pwd` -echo '**PBS_NODEFILE***START*******' -cat $PBS_NODEFILE -echo '**PBS_NODEFILE***END*********' - -text_nodes < cat $PBS_NODEFILE - -module load MATLAB -module load COMSOL/51-EDU - -ntask=$(wc -l $PBS_NODEFILE) - -comsol -nn ${ntask} server -configuration /tmp -mpiarg -rmk -mpiarg pbs -tmpdir /scratch/work/user/$USER/work & -cd /apps/cae/COMSOL/51/mli -matlab -nodesktop -nosplash -r "mphstart; addpath /scratch/work/user/$USER/work; test_job" -``` - -This example shows how to run Livelink for MATLAB with following configuration: 3 nodes and 16 cores per node. Working directory has to be created before submitting (comsol_matlab.pbs) job script into the queue. Input file (test_job.m) has to be in working directory or full path to input file has to be specified. The Matlab command option (-r ”mphstart”) created a connection with a COMSOL server using the default port number. diff --git a/docs.it4i/salomon/software/comsol/licensing-and-available-versions.md b/docs.it4i/salomon/software/comsol/licensing-and-available-versions.md deleted file mode 100644 index 4358b930f..000000000 --- a/docs.it4i/salomon/software/comsol/licensing-and-available-versions.md +++ /dev/null @@ -1,19 +0,0 @@ -# Licensing and Available Versions - -## Comsol Licence Can Be Used By: - -* all persons in the carrying out of the CE IT4Innovations Project (In addition to the primary licensee, which is VSB - Technical University of Ostrava, users are CE IT4Innovations third parties - CE IT4Innovations project partners, particularly the University of Ostrava, the Brno University of Technology - Faculty of Informatics, the Silesian University in Opava, Institute of Geonics AS CR.) -* all persons who have a valid license -* students of the Technical University - -## Comsol EDU Network Licence - -The licence intended to be used for science and research, publications, students’ projects, teaching (academic licence). - -## Comsol COM Network Licence - -The licence intended to be used for science and research, publications, students’ projects, commercial research with no commercial use restrictions. Enables the solution of at least one job by one user in one program start. - -## Available Versions - -* ver. 51 diff --git a/docs.it4i/salomon/software/debuggers/Introduction.md b/docs.it4i/salomon/software/debuggers/Introduction.md deleted file mode 100644 index 4ce2fc77b..000000000 --- a/docs.it4i/salomon/software/debuggers/Introduction.md +++ /dev/null @@ -1,62 +0,0 @@ -# Debuggers and profilers summary - -## Introduction - -We provide state of the art programms and tools to develop, profile and debug HPC codes at IT4Innovations. On these pages, we provide an overview of the profiling and debugging tools available on Anslem at IT4I. - -## Intel Debugger - -Intel debugger is no longer available since Parallel Studio version 2015 - -The intel debugger version 13.0 is available, via module intel. The debugger works for applications compiled with C and C++ compiler and the ifort fortran 77/90/95 compiler. The debugger provides java GUI environment. - -```console -$ ml intel -$ idb -``` - -Read more at the [Intel Debugger](../intel-suite/intel-debugger/) page. - -## Allinea Forge (DDT/MAP) - -Allinea DDT, is a commercial debugger primarily for debugging parallel MPI or OpenMP programs. It also has a support for GPU (CUDA) and Intel Xeon Phi accelerators. DDT provides all the standard debugging features (stack trace, breakpoints, watches, view variables, threads etc.) for every thread running as part of your program, or for every process - even if these processes are distributed across a cluster using an MPI implementation. - -```console -$ ml Forge -$ forge -``` - -Read more at the [Allinea DDT](allinea-ddt/) page. - -## Allinea Performance Reports - -Allinea Performance Reports characterize the performance of HPC application runs. After executing your application through the tool, a synthetic HTML report is generated automatically, containing information about several metrics along with clear behavior statements and hints to help you improve the efficiency of your runs. Our license is limited to 64 MPI processes. - -```console -$ ml PerformanceReports/6.0 -$ perf-report mpirun -n 64 ./my_application argument01 argument02 -``` - -Read more at the [Allinea Performance Reports](allinea-performance-reports/) page. - -## RougeWave Totalview - -TotalView is a source- and machine-level debugger for multi-process, multi-threaded programs. Its wide range of tools provides ways to analyze, organize, and test programs, making it easy to isolate and identify problems in individual threads and processes in programs of great complexity. - -```console -$ ml TotalView/8.15.4-6-linux-x86-64 -$ totalview -``` - -Read more at the [Totalview](total-view/) page. - -## Vampir Trace Analyzer - -Vampir is a GUI trace analyzer for traces in OTF format. - -```console - $ ml Vampir/8.5.0 - $ vampir -``` - -Read more at the [Vampir](vampir/) page. diff --git a/docs.it4i/salomon/software/debuggers/aislinn.md b/docs.it4i/salomon/software/debuggers/aislinn.md deleted file mode 100644 index 89cf75380..000000000 --- a/docs.it4i/salomon/software/debuggers/aislinn.md +++ /dev/null @@ -1,102 +0,0 @@ -# Aislinn - -* Aislinn is a dynamic verifier for MPI programs. For a fixed input it covers all possible runs with respect to nondeterminism introduced by MPI. It allows to detect bugs (for sure) that occurs very rare in normal runs. -* Aislinn detects problems like invalid memory accesses, deadlocks, misuse of MPI, and resource leaks. -* Aislinn is open-source software; you can use it without any licensing limitations. -* Web page of the project: <http://verif.cs.vsb.cz/aislinn/> - -!!! note - Aislinn is software developed at IT4Innovations and some parts are still considered experimental. If you have any questions or experienced any problems, please contact the author: <mailto:stanislav.bohm@vsb.cz>. - -## Usage - -Let us have the following program that contains a bug that is not manifested in all runs: - -```cpp -#include <mpi.h> -#include <stdlib.h> - -int main(int argc, char **argv) { - int rank; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (rank == 0) { - int *mem1 = (int*) malloc(sizeof(int) * 2); - int *mem2 = (int*) malloc(sizeof(int) * 3); - int data; - MPI_Recv(&data, 1, MPI_INT, MPI_ANY_SOURCE, 1, - MPI_COMM_WORLD, MPI_STATUS_IGNORE); - mem1[data] = 10; // <---------- Possible invalid memory write - MPI_Recv(&data, 1, MPI_INT, MPI_ANY_SOURCE, 1, - MPI_COMM_WORLD, MPI_STATUS_IGNORE); - mem2[data] = 10; - free(mem1); - free(mem2); - } - - if (rank == 1 || rank == 2) { - MPI_Send(&rank, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); - } - - MPI_Finalize(); - return 0; -} -``` - -The program does the following: process 0 receives two messages from anyone and processes 1 and 2 send a message to process 0. If a message from process 1 is received first, then the run does not expose the error. If a message from process 2 is received first, then invalid memory write occurs at line 16. - -To verify this program by Aislinn, we first load Aislinn itself: - -```console -$ ml aislinn -``` - -Now we compile the program by Aislinn implementation of MPI. There are `mpicc` for C programs and `mpicxx` for C++ programs. Only MPI parts of the verified application has to be recompiled; non-MPI parts may remain untouched. Let us assume that our program is in `test.cpp`. - -```console -$ mpicc -g test.cpp -o test -``` - -The `-g` flag is not necessary, but it puts more debugging information into the program, hence Aislinn may provide more detailed report. The command produces executable file `test`. - -Now we run the Aislinn itself. The argument `-p 3` specifies that we want to verify our program for the case of three MPI processes - -```console -$ aislinn -p 3 ./test -==AN== INFO: Aislinn v0.3.0 -==AN== INFO: Found error 'Invalid write' -==AN== INFO: 1 error(s) found -==AN== INFO: Report written into 'report.html' -``` - -Aislinn found an error and produced HTML report. To view it, we can use any browser, e.g.: - -```console -$ firefox report.html -``` - -At the beginning of the report there are some basic summaries of the verification. In the second part (depicted in the following picture), the error is described. - - - -It shows us: - -* Error occurs in process 0 in test.cpp on line 16. -* Stdout and stderr streams are empty. (The program does not write anything). -* The last part shows MPI calls for each process that occurs in the invalid run. The more detailed information about each call can be obtained by mouse cursor. - -### Limitations - -Since the verification is a non-trivial process there are some of limitations. - -* The verified process has to terminate in all runs, i.e. we cannot answer the halting problem. -* The verification is a computationally and memory demanding process. We put an effort to make it efficient and it is an important point for further research. However covering all runs will be always more demanding than techniques that examines only a single run. The good practise is to start with small instances and when it is feasible, make them bigger. The Aislinn is good to find bugs that are hard to find because they occur very rarely (only in a rare scheduling). Such bugs often do not need big instances. -* Aislinn expects that your program is a "standard MPI" program, i.e. processes communicate only through MPI, the verified program does not interacts with the system in some unusual ways (e.g. opening sockets). - -There are also some limitations bounded to the current version and they will be removed in the future: - -* All files containing MPI calls have to be recompiled by MPI implementation provided by Aislinn. The files that does not contain MPI calls, they do not have to recompiled. Aislinn MPI implementation supports many commonly used calls from MPI-2 and MPI-3 related to point-to-point communication, collective communication, and communicator management. Unfortunately, MPI-IO and one-side communication is not implemented yet. -* Each MPI can use only one thread (if you use OpenMP, set OMP_NUM_THREADS to 1). -* There are some limitations for using files, but if the program just reads inputs and writes results, it is ok. diff --git a/docs.it4i/salomon/software/debuggers/allinea-ddt.md b/docs.it4i/salomon/software/debuggers/allinea-ddt.md deleted file mode 100644 index 6e1f046f1..000000000 --- a/docs.it4i/salomon/software/debuggers/allinea-ddt.md +++ /dev/null @@ -1,94 +0,0 @@ -# Allinea Forge (DDT,MAP) - -Allinea Forge consist of two tools - debugger DDT and profiler MAP. - -Allinea DDT, is a commercial debugger primarily for debugging parallel MPI or OpenMP programs. It also has a support for GPU (CUDA) and Intel Xeon Phi accelerators. DDT provides all the standard debugging features (stack trace, breakpoints, watches, view variables, threads etc.) for every thread running as part of your program, or for every process - even if these processes are distributed across a cluster using an MPI implementation. - -Allinea MAP is a profiler for C/C++/Fortran HPC codes. It is designed for profiling parallel code, which uses pthreads, OpenMP or MPI. - -## License and Limitations for Anselm Users - -On Anselm users can debug OpenMP or MPI code that runs up to 64 parallel processes. In case of debugging GPU or Xeon Phi accelerated codes the limit is 8 accelerators. These limitation means that: - -* 1 user can debug up 64 processes, or -* 32 users can debug 2 processes, etc. - -In case of debugging on accelerators: - -* 1 user can debug on up to 8 accelerators, or -* 8 users can debug on single accelerator. - -## Compiling Code to Run With DDT - -### Modules - -Load all necessary modules to compile the code. For example: - -```console -$ ml intel -$ ml impi **or** ml OpenMPI/X.X.X-icc -``` - -Load the Allinea DDT module: - -```console -$ ml Forge -``` - -Compile the code: - -```console -$ mpicc -g -O0 -o test_debug test.c -$ mpif90 -g -O0 -o test_debug test.f -``` - -### Compiler Flags - -Before debugging, you need to compile your code with theses flags: - -!!! note - \- **g** : Generates extra debugging information usable by GDB. -g3 includes even more debugging information. This option is available for GNU and INTEL C/C++ and Fortran compilers. - - - - **O0** : Suppress all optimizations. - -## Starting a Job With DDT - -Be sure to log in with an X window forwarding enabled. This could mean using the -X in the ssh: - -```console -$ ssh -X username@anselm.it4i.cz -``` - -Other options is to access login node using VNC. Please see the detailed information on how to [use graphic user interface on Anselm](/general/accessing-the-clusters/graphical-user-interface/x-window-system/) - -From the login node an interactive session **with X windows forwarding** (-X option) can be started by following command: - -```console -$ qsub -I -X -A NONE-0-0 -q qexp -lselect=1:ncpus=16:mpiprocs=16,walltime=01:00:00 -``` - -Then launch the debugger with the ddt command followed by the name of the executable to debug: - -```console -$ ddt test_debug -``` - -A submission window that appears have a prefilled path to the executable to debug. You can select the number of MPI processors and/or OpenMP threads on which to run and press run. Command line arguments to a program can be entered to the "Arguments " box. - - - -To start the debugging directly without the submission window, user can specify the debugging and execution parameters from the command line. For example the number of MPI processes is set by option "-np 4". Skipping the dialog is done by "-start" option. To see the list of the "ddt" command line parameters, run "ddt --help". - -```console -ddt -start -np 4 ./hello_debug_impi -``` - -## Documentation - -Users can find original User Guide after loading the DDT module: - -```console -$DDTPATH/doc/userguide.pdf -``` - -[1] Discipline, Magic, Inspiration and Science: Best Practice Debugging with Allinea DDT, Workshop conducted at LLNL by Allinea on May 10, 2013, [link](https://computing.llnl.gov/tutorials/allineaDDT/index.html) diff --git a/docs.it4i/salomon/software/debuggers/allinea-performance-reports.md b/docs.it4i/salomon/software/debuggers/allinea-performance-reports.md deleted file mode 100644 index b43ed9ab2..000000000 --- a/docs.it4i/salomon/software/debuggers/allinea-performance-reports.md +++ /dev/null @@ -1,59 +0,0 @@ -# Allinea Performance Reports - -## Introduction - -Allinea Performance Reports characterize the performance of HPC application runs. After executing your application through the tool, a synthetic HTML report is generated automatically, containing information about several metrics along with clear behavior statements and hints to help you improve the efficiency of your runs. - -The Allinea Performance Reports is most useful in profiling MPI rograms. - -Our license is limited to 64 MPI processes. - -## Modules - -Allinea Performance Reports version 6.0 is available - -```console -$ ml PerformanceReports/6.0 -``` - -The module sets up environment variables, required for using the Allinea Performance Reports. - -## Usage - -Use the the perf-report wrapper on your (MPI) program. - -Instead of [running your MPI program the usual way](../mpi/mpi/), use the the perf report wrapper: - -```console -$ perf-report mpirun ./mympiprog.x -``` - -The mpi program will run as usual. The perf-report creates two additional files, in \*.txt and \*.html format, containing the performance report. Note that demanding MPI codes should be run within [the queue system](../../job-submission-and-execution/). - -## Example - -In this example, we will be profiling the mympiprog.x MPI program, using Allinea performance reports. Assume that the code is compiled with intel compilers and linked against intel MPI library: - -First, we allocate some nodes via the express queue: - -```console -$ qsub -q qexp -l select=2:ppn=24:mpiprocs=24:ompthreads=1 -I - qsub: waiting for job 262197.dm2 to start - qsub: job 262197.dm2 ready -``` - -Then we load the modules and run the program the usual way: - -```console -$ ml intel -$ ml PerfReports/6.0 -$ mpirun ./mympiprog.x -``` - -Now lets profile the code: - -```console -$ perf-report mpirun ./mympiprog.x -``` - -Performance report files [mympiprog_32p\*.txt](mympiprog_32p_2014-10-15_16-56.txt) and [mympiprog_32p\*.html](mympiprog_32p_2014-10-15_16-56.html) were created. We can see that the code is very efficient on MPI and is CPU bounded. diff --git a/docs.it4i/salomon/software/debuggers/intel-vtune-amplifier.md b/docs.it4i/salomon/software/debuggers/intel-vtune-amplifier.md deleted file mode 100644 index 192aece7e..000000000 --- a/docs.it4i/salomon/software/debuggers/intel-vtune-amplifier.md +++ /dev/null @@ -1,94 +0,0 @@ -# Intel VTune Amplifier XE - -## Introduction - -Intel *®* VTune™ Amplifier, part of Intel Parallel studio, is a GUI profiling tool designed for Intel processors. It offers a graphical performance analysis of single core and multithreaded applications. A highlight of the features: - -* Hotspot analysis -* Locks and waits analysis -* Low level specific counters, such as branch analysis and memory bandwidth -* Power usage analysis - frequency and sleep states. - - - -## Usage - -To profile an application with VTune Amplifier, special kernel modules need to be loaded. The modules are not loaded on the login nodes, thus direct profiling on login nodes is not possible. By default, the kernel modules ale not loaded on compute nodes neither. In order to have the modules loaded, you need to specify vtune=version PBS resource at job submit. The version is the same as for environment module. For example to use VTune/2016_update1: - -```console -$ qsub -q qexp -A OPEN-0-0 -I -l select=1,vtune=2016_update1 -``` - -After that, you can verify the modules sep\*, pax and vtsspp are present in the kernel : - -```console -$ lsmod | grep -e sep -e pax -e vtsspp - vtsspp 362000 0 - sep3_15 546657 0 - pax 4312 0 -``` - -To launch the GUI, first load the module: - -```console -$ module add VTune/2016_update1 -``` - -and launch the GUI : - -```console -$ amplxe-gui -``` - -The GUI will open in new window. Click on "New Project..." to create a new project. After clicking OK, a new window with project properties will appear. At "Application:", select the bath to your binary you want to profile (the binary should be compiled with -g flag). Some additional options such as command line arguments can be selected. At "Managed code profiling mode:" select "Native" (unless you want to profile managed mode .NET/Mono applications). After clicking OK, your project is created. - -To run a new analysis, click "New analysis...". You will see a list of possible analysis. Some of them will not be possible on the current CPU (eg. Intel Atom analysis is not possible on Sandy bridge CPU), the GUI will show an error box if you select the wrong analysis. For example, select "Advanced Hotspots". Clicking on Start will start profiling of the application. - -## Remote Analysis - -VTune Amplifier also allows a form of remote analysis. In this mode, data for analysis is collected from the command line without GUI, and the results are then loaded to GUI on another machine. This allows profiling without interactive graphical jobs. To perform a remote analysis, launch a GUI somewhere, open the new analysis window and then click the button "Command line" in bottom right corner. It will show the command line needed to perform the selected analysis. - -The command line will look like this: - -```console -/apps/all/VTune/2016_update1/vtune_amplifier_xe_2016.1.1.434111/bin64/amplxe-cl -collect advanced-hotspots -app-working-dir /home/sta545/tmp -- /home/sta545/tmp/sgemm -``` - -Copy the line to clipboard and then you can paste it in your jobscript or in command line. After the collection is run, open the GUI once again, click the menu button in the upper right corner, and select "Open > Result...". The GUI will load the results from the run. - -## Xeon Phi - -It is possible to analyze both native and offloaded Xeon Phi applications. - -### Native Mode - -This mode is useful for native Xeon Phi applications launched directly on the card. In *Analysis Target* window, select *Intel Xeon Phi coprocessor (native)*, choose path to the binary and MIC card to run on. - -### Offload Mode - -This mode is useful for applications that are launched from the host and use offload, OpenCL or mpirun. In *Analysis Target* window, select *Intel Xeon Phi coprocessor (native)*, choose path to the binaryand MIC card to run on. - -!!! note - If the analysis is interrupted or aborted, further analysis on the card might be impossible and you will get errors like "ERROR connecting to MIC card". In this case please contact our support to reboot the MIC card. - -You may also use remote analysis to collect data from the MIC and then analyze it in the GUI later : - -Native launch: - -```console -$ /apps/all/VTune/2016_update1/vtune_amplifier_xe_2016.1.1.434111/bin64/amplxe-cl -target-system mic-native:0 -collect advanced-hotspots -- /home/sta545/tmp/vect-add-mic -``` - -Host launch: - -```console -$ /apps/all/VTune/2016_update1/vtune_amplifier_xe_2016.1.1.434111/bin64/amplxe-cl -target-system mic-host-launch:0 -collect advanced-hotspots -- /home/sta545/tmp/sgemm -``` - -You can obtain this command line by pressing the "Command line..." button on Analysis Type screen. - -## References - -1. [Performance Tuning for Intel® Xeon Phi™ Coprocessors](https://www.rcac.purdue.edu/tutorials/phi/PerformanceTuningXeonPhi-Tullos.pdf) -1. [Intel® VTune™ Amplifier Support](https://software.intel.com/en-us/intel-vtune-amplifier-xe-support/documentation) -1. [https://software.intel.com/en-us/amplifier_help_linux](https://software.intel.com/en-us/amplifier_help_linux) diff --git a/docs.it4i/salomon/software/debuggers/mympiprog_32p_2014-10-15_16-56.html b/docs.it4i/salomon/software/debuggers/mympiprog_32p_2014-10-15_16-56.html deleted file mode 100644 index ce60070a9..000000000 --- a/docs.it4i/salomon/software/debuggers/mympiprog_32p_2014-10-15_16-56.html +++ /dev/null @@ -1,610 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> -<head> -<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> -<title>mympiprog.x - Performance Report</title> -<style type="text/css"> -body { - font-family: "Lucida Grande", "Lucida Sans Unicode", "Bitstream Vera Sans",sans-serif; - font-size: 1em; - color: #2c1a0a; -} -div#content { - width: 900px; - margin-left: auto; - margin-right: auto; -} -.header { - padding-top: 16px; -} -.header_left { - float:left; - width: 532px; -} -.logo { - float: left; -} -.logo img { height: 150px; } -#time_radar { - float: right; -} -#time_radar .legend_CPU { fill: #4fd32e; font-size: 1.5em; } -#time_radar .legend_MPI { fill: #409ded; font-size: 1.5em; } -#time_radar .legend_IO { fill: #ed8140; font-size: 1.5em; } -.clear { - clear: both; -} -table { - border-spacing: 0; -} -td { - padding-left: 0; - padding-right: 16px; - padding-top: 1px; - padding-bottom: 1px; -} -#error { - border: 1px solid; - margin: 16px 0px; - padding: 16px 16px 16px 16px; - color: #C80000; - background-color: #FFC0C0; -} -#error p { - margin: 8px; -} -.application_details { - margin-top: 8px; - margin-left: 16px; - margin-right: 16px; -} -.application_details .details_key { width: 100px; color: #bdc4d5;} -.application_details table { - table-layout: fixed; - width: 100%; - color: #37537b; -} -.application_details #cmdline { - word-wrap: break-word; -} -.application_details #exe_path { - word-wrap: break-word; -} -.summary { -} -hr { - margin-top: 32px; - margin-bottom: 32px; - visibility: hidden; -} -.summary .heading { - font-family: inherit; - font-size: 1.8em; - padding-bottom: 8px; -} -#summary_cpu_class { - font-weight: bold; -} -#summary_mpi_class { - font-weight: bold; -} -#summary_io_class { - font-weight: bold; -} -.overview_general_advice p { - margin-top: 8px; - margin-bottom: 8px; -} -.subsections { - margin-bottom: 32px; -} -.subsections .heading { - font-family: inherit; - font-size: 2em; - padding-bottom: 8px; -} -.subsections .heading_cpu { - padding-bottom: 8px; -} -.subsections .heading_mpi { - padding-bottom: 8px; -} -.subsections .heading_ram { - padding-bottom: 8px; -} -.subsections .heading_io { - padding-bottom: 8px; -} -.subsections .explanation { - font-size: 0.9em; - color: #404040; - padding-top: 8px; -} - -.ltcol, .ctcol { float: left; width: 436px; padding-right: 16px; } -.rtcol { float: right; width: 436px; } - -.heading_cpu { color: #4fd32e; font-size: 1.5em; } -.heading_mpi { color: #409ded; font-size: 1.5em; } -.heading_ram { color: #ed4040; font-size: 1.5em; } -.heading_io { color: #ed8140; font-size: 1.5em; } -.cpu_span { color: #4fd32e; } -.mpi_span { color: #409ded; } -.ram { color: #ed4040; } -.io_span { color: #ed8140; } -.bar_graph { width: 200px; } -#cpu_bar { background-color: #4fd32e; width: 0; height: 2em; } -#mpi_bar { background-color: #409ded; width: 0; height: 2em; } -#io_bar { background-color: #ed8140; width: 0; height: 2em; } -.summary_table { padding-top: 16px; padding-bottom: 8px; } -.summary_table td p { margin: 0px; margin-bottom: 4px; } -.summary_table td { padding-bottom: 8px; } -.summary_table .details { font-size: 0.9em; } -.balanced_span { color: #bb58d6; } -.right_cell { text-align: right; } - -#cpu_chart { padding-top: 8px; } -#cpu_chart td { font-size: 0.9em; padding-bottom: 8px; } -.mini_bar_graph { width: 50px; } -#cpu_num_bar { background-color: #4fd32e; width: 0; height: 1em; } -#cpu_vec_bar { background-color: #3c9f23; width: 0; height: 1em; } -#cpu_mem_bar { background-color: #266516; width: 0; height: 1em; } -#cpu_other_bar { background-color: #808080; width: 0; height: 1em; } -.cpu_num_span { color: #4fd32e; } -.cpu_vec_span { color: #3c9f23; } -.cpu_mem_span { color: #266516; } -.cpu_other_span { color: #808080; } - -#mpi_chart { padding-top: 8px; } -#mpi_chart td { font-size: 0.9em; padding-bottom: 8px; } -#mpi_col_bar { background-color: #409ded; width: 0; height: 1em; } -#mpi_p2p_bar { background-color: #2f73ad; width: 0; height: 1em; } -#mpi_colrate_bar { background-color: #255a87; width: 0; height: 1em; } -#mpi_p2prate_bar { background-color: #1f4a70; width: 0; height: 1em; } -.mpi_col_span { color: #409ded; } -.mpi_p2p_span { color: #2f73ad; } -.mpi_colrate_span { color: #255a87; } -.mpi_p2prate_span { color: #1f4a70; } - -#ram_chart { padding-top: 8px; } -#ram_chart td { font-size: 0.9em; padding-bottom: 8px; } -#ram_mean_bar { background-color: #ed4040; width: 0; height: 1em; } -#ram_peak_bar { background-color: #b53131; width: 0; height: 1em; } -#ram_node_bar { background-color: #742020; width: 0; height: 1em; } -.ram_mean_span { color: #ed4040; } -.ram_peak_span { color: #b53131; } -.ram_node_span { color: #742020; } - -#io_chart { padding-top: 8px; } -#io_chart td { font-size: 0.9em; padding-bottom: 8px; } -#io_read_bar { background-color: #ed8140; width: 0; height: 1em; } -#io_write_bar { background-color: #a95e0b; width: 0; height: 1em; } -#io_readrate_bar { background-color: #9b7c14; width: 0; height: 1em; } -#io_writerate_bar { background-color: #7b6210; width: 0; height: 1em; } -.io_read_span { color: #ed8140; } -.io_write_span { color: #a95e0b; } -.io_readrate_span { color: #9b7c14; } -.io_writerate_span { color: #7b6210; } - -</style> -<script type="text/javascript">d3=function(){function n(n){return null!=n&&!isNaN(n)}function t(n){return n.length}function e(n){for(var t=1;n*t%1;)t*=10;return t}function r(n,t){try{for(var e in t)Object.defineProperty(n.prototype,e,{value:t[e],enumerable:!1})}catch(r){n.prototype=t}}function u(){}function i(){}function o(n,t,e){return function(){var r=e.apply(t,arguments);return r===t?n:r}}function a(n,t){if(t in n)return t;t=t.charAt(0).toUpperCase()+t.substring(1);for(var e=0,r=la.length;r>e;++e){var u=la[e]+t;if(u in n)return u}}function c(){}function s(){}function l(n){function t(){for(var t,r=e,u=-1,i=r.length;++u<i;)(t=r[u].on)&&t.apply(this,arguments);return n}var e=[],r=new u;return t.on=function(t,u){var i,o=r.get(t);return arguments.length<2?o&&o.on:(o&&(o.on=null,e=e.slice(0,i=e.indexOf(o)).concat(e.slice(i+1)),r.remove(t)),u&&e.push(r.set(t,{on:u})),n)},t}function f(){$o.event.preventDefault()}function h(){for(var n,t=$o.event;n=t.sourceEvent;)t=n;return t}function g(n){for(var t=new s,e=0,r=arguments.length;++e<r;)t[arguments[e]]=l(t);return t.of=function(e,r){return function(u){try{var i=u.sourceEvent=$o.event;u.target=n,$o.event=u,t[u.type].apply(e,r)}finally{$o.event=i}}},t}function p(n){return ha(n,ma),n}function v(n){return"function"==typeof n?n:function(){return ga(n,this)}}function d(n){return"function"==typeof n?n:function(){return pa(n,this)}}function m(n,t){function e(){this.removeAttribute(n)}function r(){this.removeAttributeNS(n.space,n.local)}function u(){this.setAttribute(n,t)}function i(){this.setAttributeNS(n.space,n.local,t)}function o(){var e=t.apply(this,arguments);null==e?this.removeAttribute(n):this.setAttribute(n,e)}function a(){var e=t.apply(this,arguments);null==e?this.removeAttributeNS(n.space,n.local):this.setAttributeNS(n.space,n.local,e)}return n=$o.ns.qualify(n),null==t?n.local?r:e:"function"==typeof t?n.local?a:o:n.local?i:u}function y(n){return n.trim().replace(/\s+/g," ")}function x(n){return new RegExp("(?:^|\\s+)"+$o.requote(n)+"(?:\\s+|$)","g")}function M(n,t){function e(){for(var e=-1;++e<u;)n[e](this,t)}function r(){for(var e=-1,r=t.apply(this,arguments);++e<u;)n[e](this,r)}n=n.trim().split(/\s+/).map(_);var u=n.length;return"function"==typeof t?r:e}function _(n){var t=x(n);return function(e,r){if(u=e.classList)return r?u.add(n):u.remove(n);var u=e.getAttribute("class")||"";r?(t.lastIndex=0,t.test(u)||e.setAttribute("class",y(u+" "+n))):e.setAttribute("class",y(u.replace(t," ")))}}function b(n,t,e){function r(){this.style.removeProperty(n)}function u(){this.style.setProperty(n,t,e)}function i(){var r=t.apply(this,arguments);null==r?this.style.removeProperty(n):this.style.setProperty(n,r,e)}return null==t?r:"function"==typeof t?i:u}function w(n,t){function e(){delete this[n]}function r(){this[n]=t}function u(){var e=t.apply(this,arguments);null==e?delete this[n]:this[n]=e}return null==t?e:"function"==typeof t?u:r}function S(n){return"function"==typeof n?n:(n=$o.ns.qualify(n)).local?function(){return this.ownerDocument.createElementNS(n.space,n.local)}:function(){return this.ownerDocument.createElementNS(this.namespaceURI,n)}}function k(n){return{__data__:n}}function E(n){return function(){return da(this,n)}}function A(n){return arguments.length||(n=$o.ascending),function(t,e){return t&&e?n(t.__data__,e.__data__):!t-!e}}function C(n,t){for(var e=0,r=n.length;r>e;e++)for(var u,i=n[e],o=0,a=i.length;a>o;o++)(u=i[o])&&t(u,o,e);return n}function N(n){return ha(n,xa),n}function L(n){var t,e;return function(r,u,i){var o,a=n[i].update,c=a.length;for(i!=e&&(e=i,t=0),u>=t&&(t=u+1);!(o=a[t])&&++t<c;);return o}}function T(){var n=this.__transition__;n&&++n.active}function q(n,t,e){function r(){var t=this[o];t&&(this.removeEventListener(n,t,t.$),delete this[o])}function u(){var u=s(t,Wo(arguments));r.call(this),this.addEventListener(n,this[o]=u,u.$=e),u._=t}function i(){var t,e=new RegExp("^__on([^.]+)"+$o.requote(n)+"$");for(var r in this)if(t=r.match(e)){var u=this[r];this.removeEventListener(t[1],u,u.$),delete this[r]}}var o="__on"+n,a=n.indexOf("."),s=z;a>0&&(n=n.substring(0,a));var l=_a.get(n);return l&&(n=l,s=R),a?t?u:r:t?c:i}function z(n,t){return function(e){var r=$o.event;$o.event=e,t[0]=this.__data__;try{n.apply(this,t)}finally{$o.event=r}}}function R(n,t){var e=z(n,t);return function(n){var t=this,r=n.relatedTarget;r&&(r===t||8&r.compareDocumentPosition(t))||e.call(t,n)}}function D(){var n=".dragsuppress-"+ ++wa,t="click"+n,e=$o.select(Ko).on("touchmove"+n,f).on("dragstart"+n,f).on("selectstart"+n,f);if(ba){var r=Go.style,u=r[ba];r[ba]="none"}return function(i){function o(){e.on(t,null)}e.on(n,null),ba&&(r[ba]=u),i&&(e.on(t,function(){f(),o()},!0),setTimeout(o,0))}}function P(n,t){t.changedTouches&&(t=t.changedTouches[0]);var e=n.ownerSVGElement||n;if(e.createSVGPoint){var r=e.createSVGPoint();if(0>Sa&&(Ko.scrollX||Ko.scrollY)){e=$o.select("body").append("svg").style({position:"absolute",top:0,left:0,margin:0,padding:0,border:"none"},"important");var u=e[0][0].getScreenCTM();Sa=!(u.f||u.e),e.remove()}return Sa?(r.x=t.pageX,r.y=t.pageY):(r.x=t.clientX,r.y=t.clientY),r=r.matrixTransform(n.getScreenCTM().inverse()),[r.x,r.y]}var i=n.getBoundingClientRect();return[t.clientX-i.left-n.clientLeft,t.clientY-i.top-n.clientTop]}function U(n){return n>0?1:0>n?-1:0}function j(n){return n>1?0:-1>n?ka:Math.acos(n)}function H(n){return n>1?Aa:-1>n?-Aa:Math.asin(n)}function F(n){return((n=Math.exp(n))-1/n)/2}function O(n){return((n=Math.exp(n))+1/n)/2}function Y(n){return((n=Math.exp(2*n))-1)/(n+1)}function I(n){return(n=Math.sin(n/2))*n}function Z(){}function V(n,t,e){return new X(n,t,e)}function X(n,t,e){this.h=n,this.s=t,this.l=e}function $(n,t,e){function r(n){return n>360?n-=360:0>n&&(n+=360),60>n?i+(o-i)*n/60:180>n?o:240>n?i+(o-i)*(240-n)/60:i}function u(n){return Math.round(255*r(n))}var i,o;return n=isNaN(n)?0:(n%=360)<0?n+360:n,t=isNaN(t)?0:0>t?0:t>1?1:t,e=0>e?0:e>1?1:e,o=.5>=e?e*(1+t):e+t-e*t,i=2*e-o,ot(u(n+120),u(n),u(n-120))}function B(n,t,e){return new W(n,t,e)}function W(n,t,e){this.h=n,this.c=t,this.l=e}function J(n,t,e){return isNaN(n)&&(n=0),isNaN(t)&&(t=0),G(e,Math.cos(n*=La)*t,Math.sin(n)*t)}function G(n,t,e){return new K(n,t,e)}function K(n,t,e){this.l=n,this.a=t,this.b=e}function Q(n,t,e){var r=(n+16)/116,u=r+t/500,i=r-e/200;return u=tt(u)*Oa,r=tt(r)*Ya,i=tt(i)*Ia,ot(rt(3.2404542*u-1.5371385*r-.4985314*i),rt(-.969266*u+1.8760108*r+.041556*i),rt(.0556434*u-.2040259*r+1.0572252*i))}function nt(n,t,e){return n>0?B(Math.atan2(e,t)*Ta,Math.sqrt(t*t+e*e),n):B(0/0,0/0,n)}function tt(n){return n>.206893034?n*n*n:(n-4/29)/7.787037}function et(n){return n>.008856?Math.pow(n,1/3):7.787037*n+4/29}function rt(n){return Math.round(255*(.00304>=n?12.92*n:1.055*Math.pow(n,1/2.4)-.055))}function ut(n){return ot(n>>16,255&n>>8,255&n)}function it(n){return ut(n)+""}function ot(n,t,e){return new at(n,t,e)}function at(n,t,e){this.r=n,this.g=t,this.b=e}function ct(n){return 16>n?"0"+Math.max(0,n).toString(16):Math.min(255,n).toString(16)}function st(n,t,e){var r,u,i,o=0,a=0,c=0;if(r=/([a-z]+)\((.*)\)/i.exec(n))switch(u=r[2].split(","),r[1]){case"hsl":return e(parseFloat(u[0]),parseFloat(u[1])/100,parseFloat(u[2])/100);case"rgb":return t(gt(u[0]),gt(u[1]),gt(u[2]))}return(i=Xa.get(n))?t(i.r,i.g,i.b):(null!=n&&"#"===n.charAt(0)&&(4===n.length?(o=n.charAt(1),o+=o,a=n.charAt(2),a+=a,c=n.charAt(3),c+=c):7===n.length&&(o=n.substring(1,3),a=n.substring(3,5),c=n.substring(5,7)),o=parseInt(o,16),a=parseInt(a,16),c=parseInt(c,16)),t(o,a,c))}function lt(n,t,e){var r,u,i=Math.min(n/=255,t/=255,e/=255),o=Math.max(n,t,e),a=o-i,c=(o+i)/2;return a?(u=.5>c?a/(o+i):a/(2-o-i),r=n==o?(t-e)/a+(e>t?6:0):t==o?(e-n)/a+2:(n-t)/a+4,r*=60):(r=0/0,u=c>0&&1>c?0:r),V(r,u,c)}function ft(n,t,e){n=ht(n),t=ht(t),e=ht(e);var r=et((.4124564*n+.3575761*t+.1804375*e)/Oa),u=et((.2126729*n+.7151522*t+.072175*e)/Ya),i=et((.0193339*n+.119192*t+.9503041*e)/Ia);return G(116*u-16,500*(r-u),200*(u-i))}function ht(n){return(n/=255)<=.04045?n/12.92:Math.pow((n+.055)/1.055,2.4)}function gt(n){var t=parseFloat(n);return"%"===n.charAt(n.length-1)?Math.round(2.55*t):t}function pt(n){return"function"==typeof n?n:function(){return n}}function vt(n){return n}function dt(n){return function(t,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=null),mt(t,e,n,r)}}function mt(n,t,e,r){function u(){var n,t=c.status;if(!t&&c.responseText||t>=200&&300>t||304===t){try{n=e.call(i,c)}catch(r){return o.error.call(i,r),void 0}o.load.call(i,n)}else o.error.call(i,c)}var i={},o=$o.dispatch("beforesend","progress","load","error"),a={},c=new XMLHttpRequest,s=null;return!Ko.XDomainRequest||"withCredentials"in c||!/^(http(s)?:)?\/\//.test(n)||(c=new XDomainRequest),"onload"in c?c.onload=c.onerror=u:c.onreadystatechange=function(){c.readyState>3&&u()},c.onprogress=function(n){var t=$o.event;$o.event=n;try{o.progress.call(i,c)}finally{$o.event=t}},i.header=function(n,t){return n=(n+"").toLowerCase(),arguments.length<2?a[n]:(null==t?delete a[n]:a[n]=t+"",i)},i.mimeType=function(n){return arguments.length?(t=null==n?null:n+"",i):t},i.responseType=function(n){return arguments.length?(s=n,i):s},i.response=function(n){return e=n,i},["get","post"].forEach(function(n){i[n]=function(){return i.send.apply(i,[n].concat(Wo(arguments)))}}),i.send=function(e,r,u){if(2===arguments.length&&"function"==typeof r&&(u=r,r=null),c.open(e,n,!0),null==t||"accept"in a||(a.accept=t+",*/*"),c.setRequestHeader)for(var l in a)c.setRequestHeader(l,a[l]);return null!=t&&c.overrideMimeType&&c.overrideMimeType(t),null!=s&&(c.responseType=s),null!=u&&i.on("error",u).on("load",function(n){u(null,n)}),o.beforesend.call(i,c),c.send(null==r?null:r),i},i.abort=function(){return c.abort(),i},$o.rebind(i,o,"on"),null==r?i:i.get(yt(r))}function yt(n){return 1===n.length?function(t,e){n(null==t?e:null)}:n}function xt(){var n=Mt(),t=_t()-n;t>24?(isFinite(t)&&(clearTimeout(Ja),Ja=setTimeout(xt,t)),Wa=0):(Wa=1,Ka(xt))}function Mt(){var n=Date.now();for(Ga=$a;Ga;)n>=Ga.t&&(Ga.f=Ga.c(n-Ga.t)),Ga=Ga.n;return n}function _t(){for(var n,t=$a,e=1/0;t;)t.f?t=n?n.n=t.n:$a=t.n:(t.t<e&&(e=t.t),t=(n=t).n);return Ba=n,e}function bt(n,t){var e=Math.pow(10,3*aa(8-t));return{scale:t>8?function(n){return n/e}:function(n){return n*e},symbol:n}}function wt(n,t){return t-(n?Math.ceil(Math.log(n)/Math.LN10):1)}function St(n){return n+""}function kt(){}function Et(n,t,e){var r=e.s=n+t,u=r-n,i=r-u;e.t=n-i+(t-u)}function At(n,t){n&&lc.hasOwnProperty(n.type)&&lc[n.type](n,t)}function Ct(n,t,e){var r,u=-1,i=n.length-e;for(t.lineStart();++u<i;)r=n[u],t.point(r[0],r[1],r[2]);t.lineEnd()}function Nt(n,t){var e=-1,r=n.length;for(t.polygonStart();++e<r;)Ct(n[e],t,1);t.polygonEnd()}function Lt(){function n(n,t){n*=La,t=t*La/2+ka/4;var e=n-r,o=Math.cos(t),a=Math.sin(t),c=i*a,s=u*o+c*Math.cos(e),l=c*Math.sin(e);hc.add(Math.atan2(l,s)),r=n,u=o,i=a}var t,e,r,u,i;gc.point=function(o,a){gc.point=n,r=(t=o)*La,u=Math.cos(a=(e=a)*La/2+ka/4),i=Math.sin(a)},gc.lineEnd=function(){n(t,e)}}function Tt(n){var t=n[0],e=n[1],r=Math.cos(e);return[r*Math.cos(t),r*Math.sin(t),Math.sin(e)]}function qt(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]}function zt(n,t){return[n[1]*t[2]-n[2]*t[1],n[2]*t[0]-n[0]*t[2],n[0]*t[1]-n[1]*t[0]]}function Rt(n,t){n[0]+=t[0],n[1]+=t[1],n[2]+=t[2]}function Dt(n,t){return[n[0]*t,n[1]*t,n[2]*t]}function Pt(n){var t=Math.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2]);n[0]/=t,n[1]/=t,n[2]/=t}function Ut(n){return[Math.atan2(n[1],n[0]),H(n[2])]}function jt(n,t){return aa(n[0]-t[0])<Ca&&aa(n[1]-t[1])<Ca}function Ht(n,t){n*=La;var e=Math.cos(t*=La);Ft(e*Math.cos(n),e*Math.sin(n),Math.sin(t))}function Ft(n,t,e){++pc,dc+=(n-dc)/pc,mc+=(t-mc)/pc,yc+=(e-yc)/pc}function Ot(){function n(n,u){n*=La;var i=Math.cos(u*=La),o=i*Math.cos(n),a=i*Math.sin(n),c=Math.sin(u),s=Math.atan2(Math.sqrt((s=e*c-r*a)*s+(s=r*o-t*c)*s+(s=t*a-e*o)*s),t*o+e*a+r*c);vc+=s,xc+=s*(t+(t=o)),Mc+=s*(e+(e=a)),_c+=s*(r+(r=c)),Ft(t,e,r)}var t,e,r;kc.point=function(u,i){u*=La;var o=Math.cos(i*=La);t=o*Math.cos(u),e=o*Math.sin(u),r=Math.sin(i),kc.point=n,Ft(t,e,r)}}function Yt(){kc.point=Ht}function It(){function n(n,t){n*=La;var e=Math.cos(t*=La),o=e*Math.cos(n),a=e*Math.sin(n),c=Math.sin(t),s=u*c-i*a,l=i*o-r*c,f=r*a-u*o,h=Math.sqrt(s*s+l*l+f*f),g=r*o+u*a+i*c,p=h&&-j(g)/h,v=Math.atan2(h,g);bc+=p*s,wc+=p*l,Sc+=p*f,vc+=v,xc+=v*(r+(r=o)),Mc+=v*(u+(u=a)),_c+=v*(i+(i=c)),Ft(r,u,i)}var t,e,r,u,i;kc.point=function(o,a){t=o,e=a,kc.point=n,o*=La;var c=Math.cos(a*=La);r=c*Math.cos(o),u=c*Math.sin(o),i=Math.sin(a),Ft(r,u,i)},kc.lineEnd=function(){n(t,e),kc.lineEnd=Yt,kc.point=Ht}}function Zt(){return!0}function Vt(n,t,e,r,u){var i=[],o=[];if(n.forEach(function(n){if(!((t=n.length-1)<=0)){var t,e=n[0],r=n[t];if(jt(e,r)){u.lineStart();for(var a=0;t>a;++a)u.point((e=n[a])[0],e[1]);return u.lineEnd(),void 0}var c=new $t(e,n,null,!0),s=new $t(e,null,c,!1);c.o=s,i.push(c),o.push(s),c=new $t(r,n,null,!1),s=new $t(r,null,c,!0),c.o=s,i.push(c),o.push(s)}}),o.sort(t),Xt(i),Xt(o),i.length){for(var a=0,c=e,s=o.length;s>a;++a)o[a].e=c=!c;for(var l,f,h=i[0];;){for(var g=h,p=!0;g.v;)if((g=g.n)===h)return;l=g.z,u.lineStart();do{if(g.v=g.o.v=!0,g.e){if(p)for(var a=0,s=l.length;s>a;++a)u.point((f=l[a])[0],f[1]);else r(g.x,g.n.x,1,u);g=g.n}else{if(p){l=g.p.z;for(var a=l.length-1;a>=0;--a)u.point((f=l[a])[0],f[1])}else r(g.x,g.p.x,-1,u);g=g.p}g=g.o,l=g.z,p=!p}while(!g.v);u.lineEnd()}}}function Xt(n){if(t=n.length){for(var t,e,r=0,u=n[0];++r<t;)u.n=e=n[r],e.p=u,u=e;u.n=e=n[0],e.p=u}}function $t(n,t,e,r){this.x=n,this.z=t,this.o=e,this.e=r,this.v=!1,this.n=this.p=null}function Bt(n,t,e,r){return function(u,i){function o(t,e){var r=u(t,e);n(t=r[0],e=r[1])&&i.point(t,e)}function a(n,t){var e=u(n,t);d.point(e[0],e[1])}function c(){y.point=a,d.lineStart()}function s(){y.point=o,d.lineEnd()}function l(n,t){v.push([n,t]);var e=u(n,t);M.point(e[0],e[1])}function f(){M.lineStart(),v=[]}function h(){l(v[0][0],v[0][1]),M.lineEnd();var n,t=M.clean(),e=x.buffer(),r=e.length;if(v.pop(),p.push(v),v=null,r){if(1&t){n=e[0];var u,r=n.length-1,o=-1;for(i.lineStart();++o<r;)i.point((u=n[o])[0],u[1]);return i.lineEnd(),void 0}r>1&&2&t&&e.push(e.pop().concat(e.shift())),g.push(e.filter(Wt))}}var g,p,v,d=t(i),m=u.invert(r[0],r[1]),y={point:o,lineStart:c,lineEnd:s,polygonStart:function(){y.point=l,y.lineStart=f,y.lineEnd=h,g=[],p=[],i.polygonStart()},polygonEnd:function(){y.point=o,y.lineStart=c,y.lineEnd=s,g=$o.merge(g);var n=Kt(m,p);g.length?Vt(g,Gt,n,e,i):n&&(i.lineStart(),e(null,null,1,i),i.lineEnd()),i.polygonEnd(),g=p=null},sphere:function(){i.polygonStart(),i.lineStart(),e(null,null,1,i),i.lineEnd(),i.polygonEnd()}},x=Jt(),M=t(x);return y}}function Wt(n){return n.length>1}function Jt(){var n,t=[];return{lineStart:function(){t.push(n=[])},point:function(t,e){n.push([t,e])},lineEnd:c,buffer:function(){var e=t;return t=[],n=null,e},rejoin:function(){t.length>1&&t.push(t.pop().concat(t.shift()))}}}function Gt(n,t){return((n=n.x)[0]<0?n[1]-Aa-Ca:Aa-n[1])-((t=t.x)[0]<0?t[1]-Aa-Ca:Aa-t[1])}function Kt(n,t){var e=n[0],r=n[1],u=[Math.sin(e),-Math.cos(e),0],i=0,o=0;hc.reset();for(var a=0,c=t.length;c>a;++a){var s=t[a],l=s.length;if(l)for(var f=s[0],h=f[0],g=f[1]/2+ka/4,p=Math.sin(g),v=Math.cos(g),d=1;;){d===l&&(d=0),n=s[d];var m=n[0],y=n[1]/2+ka/4,x=Math.sin(y),M=Math.cos(y),_=m-h,b=aa(_)>ka,w=p*x;if(hc.add(Math.atan2(w*Math.sin(_),v*M+w*Math.cos(_))),i+=b?_+(_>=0?Ea:-Ea):_,b^h>=e^m>=e){var S=zt(Tt(f),Tt(n));Pt(S);var k=zt(u,S);Pt(k);var E=(b^_>=0?-1:1)*H(k[2]);(r>E||r===E&&(S[0]||S[1]))&&(o+=b^_>=0?1:-1)}if(!d++)break;h=m,p=x,v=M,f=n}}return(-Ca>i||Ca>i&&0>hc)^1&o}function Qt(n){var t,e=0/0,r=0/0,u=0/0;return{lineStart:function(){n.lineStart(),t=1},point:function(i,o){var a=i>0?ka:-ka,c=aa(i-e);aa(c-ka)<Ca?(n.point(e,r=(r+o)/2>0?Aa:-Aa),n.point(u,r),n.lineEnd(),n.lineStart(),n.point(a,r),n.point(i,r),t=0):u!==a&&c>=ka&&(aa(e-u)<Ca&&(e-=u*Ca),aa(i-a)<Ca&&(i-=a*Ca),r=ne(e,r,i,o),n.point(u,r),n.lineEnd(),n.lineStart(),n.point(a,r),t=0),n.point(e=i,r=o),u=a},lineEnd:function(){n.lineEnd(),e=r=0/0},clean:function(){return 2-t}}}function ne(n,t,e,r){var u,i,o=Math.sin(n-e);return aa(o)>Ca?Math.atan((Math.sin(t)*(i=Math.cos(r))*Math.sin(e)-Math.sin(r)*(u=Math.cos(t))*Math.sin(n))/(u*i*o)):(t+r)/2}function te(n,t,e,r){var u;if(null==n)u=e*Aa,r.point(-ka,u),r.point(0,u),r.point(ka,u),r.point(ka,0),r.point(ka,-u),r.point(0,-u),r.point(-ka,-u),r.point(-ka,0),r.point(-ka,u);else if(aa(n[0]-t[0])>Ca){var i=n[0]<t[0]?ka:-ka;u=e*i/2,r.point(-i,u),r.point(0,u),r.point(i,u)}else r.point(t[0],t[1])}function ee(n){function t(n,t){return Math.cos(n)*Math.cos(t)>i}function e(n){var e,i,c,s,l;return{lineStart:function(){s=c=!1,l=1},point:function(f,h){var g,p=[f,h],v=t(f,h),d=o?v?0:u(f,h):v?u(f+(0>f?ka:-ka),h):0;if(!e&&(s=c=v)&&n.lineStart(),v!==c&&(g=r(e,p),(jt(e,g)||jt(p,g))&&(p[0]+=Ca,p[1]+=Ca,v=t(p[0],p[1]))),v!==c)l=0,v?(n.lineStart(),g=r(p,e),n.point(g[0],g[1])):(g=r(e,p),n.point(g[0],g[1]),n.lineEnd()),e=g;else if(a&&e&&o^v){var m;d&i||!(m=r(p,e,!0))||(l=0,o?(n.lineStart(),n.point(m[0][0],m[0][1]),n.point(m[1][0],m[1][1]),n.lineEnd()):(n.point(m[1][0],m[1][1]),n.lineEnd(),n.lineStart(),n.point(m[0][0],m[0][1])))}!v||e&&jt(e,p)||n.point(p[0],p[1]),e=p,c=v,i=d},lineEnd:function(){c&&n.lineEnd(),e=null},clean:function(){return l|(s&&c)<<1}}}function r(n,t,e){var r=Tt(n),u=Tt(t),o=[1,0,0],a=zt(r,u),c=qt(a,a),s=a[0],l=c-s*s;if(!l)return!e&&n;var f=i*c/l,h=-i*s/l,g=zt(o,a),p=Dt(o,f),v=Dt(a,h);Rt(p,v);var d=g,m=qt(p,d),y=qt(d,d),x=m*m-y*(qt(p,p)-1);if(!(0>x)){var M=Math.sqrt(x),_=Dt(d,(-m-M)/y);if(Rt(_,p),_=Ut(_),!e)return _;var b,w=n[0],S=t[0],k=n[1],E=t[1];w>S&&(b=w,w=S,S=b);var A=S-w,C=aa(A-ka)<Ca,N=C||Ca>A;if(!C&&k>E&&(b=k,k=E,E=b),N?C?k+E>0^_[1]<(aa(_[0]-w)<Ca?k:E):k<=_[1]&&_[1]<=E:A>ka^(w<=_[0]&&_[0]<=S)){var L=Dt(d,(-m+M)/y);return Rt(L,p),[_,Ut(L)]}}}function u(t,e){var r=o?n:ka-n,u=0;return-r>t?u|=1:t>r&&(u|=2),-r>e?u|=4:e>r&&(u|=8),u}var i=Math.cos(n),o=i>0,a=aa(i)>Ca,c=Le(n,6*La);return Bt(t,e,c,o?[0,-n]:[-ka,n-ka])}function re(n,t,e,r){return function(u){var i,o=u.a,a=u.b,c=o.x,s=o.y,l=a.x,f=a.y,h=0,g=1,p=l-c,v=f-s;if(i=n-c,p||!(i>0)){if(i/=p,0>p){if(h>i)return;g>i&&(g=i)}else if(p>0){if(i>g)return;i>h&&(h=i)}if(i=e-c,p||!(0>i)){if(i/=p,0>p){if(i>g)return;i>h&&(h=i)}else if(p>0){if(h>i)return;g>i&&(g=i)}if(i=t-s,v||!(i>0)){if(i/=v,0>v){if(h>i)return;g>i&&(g=i)}else if(v>0){if(i>g)return;i>h&&(h=i)}if(i=r-s,v||!(0>i)){if(i/=v,0>v){if(i>g)return;i>h&&(h=i)}else if(v>0){if(h>i)return;g>i&&(g=i)}return h>0&&(u.a={x:c+h*p,y:s+h*v}),1>g&&(u.b={x:c+g*p,y:s+g*v}),u}}}}}}function ue(n,t,e,r){function u(r,u){return aa(r[0]-n)<Ca?u>0?0:3:aa(r[0]-e)<Ca?u>0?2:1:aa(r[1]-t)<Ca?u>0?1:0:u>0?3:2}function i(n,t){return o(n.x,t.x)}function o(n,t){var e=u(n,1),r=u(t,1);return e!==r?e-r:0===e?t[1]-n[1]:1===e?n[0]-t[0]:2===e?n[1]-t[1]:t[0]-n[0]}return function(a){function c(n){for(var t=0,e=m.length,r=n[1],u=0;e>u;++u)for(var i,o=1,a=m[u],c=a.length,l=a[0];c>o;++o)i=a[o],l[1]<=r?i[1]>r&&s(l,i,n)>0&&++t:i[1]<=r&&s(l,i,n)<0&&--t,l=i;return 0!==t}function s(n,t,e){return(t[0]-n[0])*(e[1]-n[1])-(e[0]-n[0])*(t[1]-n[1])}function l(i,a,c,s){var l=0,f=0;if(null==i||(l=u(i,c))!==(f=u(a,c))||o(i,a)<0^c>0){do s.point(0===l||3===l?n:e,l>1?r:t);while((l=(l+c+4)%4)!==f)}else s.point(a[0],a[1])}function f(u,i){return u>=n&&e>=u&&i>=t&&r>=i}function h(n,t){f(n,t)&&a.point(n,t)}function g(){L.point=v,m&&m.push(y=[]),k=!0,S=!1,b=w=0/0}function p(){d&&(v(x,M),_&&S&&C.rejoin(),d.push(C.buffer())),L.point=h,S&&a.lineEnd()}function v(n,t){n=Math.max(-Ac,Math.min(Ac,n)),t=Math.max(-Ac,Math.min(Ac,t));var e=f(n,t);if(m&&y.push([n,t]),k)x=n,M=t,_=e,k=!1,e&&(a.lineStart(),a.point(n,t));else if(e&&S)a.point(n,t);else{var r={a:{x:b,y:w},b:{x:n,y:t}};N(r)?(S||(a.lineStart(),a.point(r.a.x,r.a.y)),a.point(r.b.x,r.b.y),e||a.lineEnd(),E=!1):e&&(a.lineStart(),a.point(n,t),E=!1)}b=n,w=t,S=e}var d,m,y,x,M,_,b,w,S,k,E,A=a,C=Jt(),N=re(n,t,e,r),L={point:h,lineStart:g,lineEnd:p,polygonStart:function(){a=C,d=[],m=[],E=!0},polygonEnd:function(){a=A,d=$o.merge(d);var t=c([n,r]),e=E&&t,u=d.length;(e||u)&&(a.polygonStart(),e&&(a.lineStart(),l(null,null,1,a),a.lineEnd()),u&&Vt(d,i,t,l,a),a.polygonEnd()),d=m=y=null}};return L}}function ie(n,t){function e(e,r){return e=n(e,r),t(e[0],e[1])}return n.invert&&t.invert&&(e.invert=function(e,r){return e=t.invert(e,r),e&&n.invert(e[0],e[1])}),e}function oe(n){var t=0,e=ka/3,r=be(n),u=r(t,e);return u.parallels=function(n){return arguments.length?r(t=n[0]*ka/180,e=n[1]*ka/180):[180*(t/ka),180*(e/ka)]},u}function ae(n,t){function e(n,t){var e=Math.sqrt(i-2*u*Math.sin(t))/u;return[e*Math.sin(n*=u),o-e*Math.cos(n)]}var r=Math.sin(n),u=(r+Math.sin(t))/2,i=1+r*(2*u-r),o=Math.sqrt(i)/u;return e.invert=function(n,t){var e=o-t;return[Math.atan2(n,e)/u,H((i-(n*n+e*e)*u*u)/(2*u))]},e}function ce(){function n(n,t){Nc+=u*n-r*t,r=n,u=t}var t,e,r,u;Rc.point=function(i,o){Rc.point=n,t=r=i,e=u=o},Rc.lineEnd=function(){n(t,e)}}function se(n,t){Lc>n&&(Lc=n),n>qc&&(qc=n),Tc>t&&(Tc=t),t>zc&&(zc=t)}function le(){function n(n,t){o.push("M",n,",",t,i)}function t(n,t){o.push("M",n,",",t),a.point=e}function e(n,t){o.push("L",n,",",t)}function r(){a.point=n}function u(){o.push("Z")}var i=fe(4.5),o=[],a={point:n,lineStart:function(){a.point=t},lineEnd:r,polygonStart:function(){a.lineEnd=u},polygonEnd:function(){a.lineEnd=r,a.point=n},pointRadius:function(n){return i=fe(n),a},result:function(){if(o.length){var n=o.join("");return o=[],n}}};return a}function fe(n){return"m0,"+n+"a"+n+","+n+" 0 1,1 0,"+-2*n+"a"+n+","+n+" 0 1,1 0,"+2*n+"z"}function he(n,t){dc+=n,mc+=t,++yc}function ge(){function n(n,r){var u=n-t,i=r-e,o=Math.sqrt(u*u+i*i);xc+=o*(t+n)/2,Mc+=o*(e+r)/2,_c+=o,he(t=n,e=r)}var t,e;Pc.point=function(r,u){Pc.point=n,he(t=r,e=u)}}function pe(){Pc.point=he}function ve(){function n(n,t){var e=n-r,i=t-u,o=Math.sqrt(e*e+i*i);xc+=o*(r+n)/2,Mc+=o*(u+t)/2,_c+=o,o=u*n-r*t,bc+=o*(r+n),wc+=o*(u+t),Sc+=3*o,he(r=n,u=t)}var t,e,r,u;Pc.point=function(i,o){Pc.point=n,he(t=r=i,e=u=o)},Pc.lineEnd=function(){n(t,e)}}function de(n){function t(t,e){n.moveTo(t,e),n.arc(t,e,o,0,Ea)}function e(t,e){n.moveTo(t,e),a.point=r}function r(t,e){n.lineTo(t,e)}function u(){a.point=t}function i(){n.closePath()}var o=4.5,a={point:t,lineStart:function(){a.point=e},lineEnd:u,polygonStart:function(){a.lineEnd=i},polygonEnd:function(){a.lineEnd=u,a.point=t},pointRadius:function(n){return o=n,a},result:c};return a}function me(n){function t(n){return(a?r:e)(n)}function e(t){return Me(t,function(e,r){e=n(e,r),t.point(e[0],e[1])})}function r(t){function e(e,r){e=n(e,r),t.point(e[0],e[1])}function r(){x=0/0,S.point=i,t.lineStart()}function i(e,r){var i=Tt([e,r]),o=n(e,r);u(x,M,y,_,b,w,x=o[0],M=o[1],y=e,_=i[0],b=i[1],w=i[2],a,t),t.point(x,M)}function o(){S.point=e,t.lineEnd()}function c(){r(),S.point=s,S.lineEnd=l}function s(n,t){i(f=n,h=t),g=x,p=M,v=_,d=b,m=w,S.point=i}function l(){u(x,M,y,_,b,w,g,p,f,v,d,m,a,t),S.lineEnd=o,o()}var f,h,g,p,v,d,m,y,x,M,_,b,w,S={point:e,lineStart:r,lineEnd:o,polygonStart:function(){t.polygonStart(),S.lineStart=c},polygonEnd:function(){t.polygonEnd(),S.lineStart=r}};return S}function u(t,e,r,a,c,s,l,f,h,g,p,v,d,m){var y=l-t,x=f-e,M=y*y+x*x;if(M>4*i&&d--){var _=a+g,b=c+p,w=s+v,S=Math.sqrt(_*_+b*b+w*w),k=Math.asin(w/=S),E=aa(aa(w)-1)<Ca?(r+h)/2:Math.atan2(b,_),A=n(E,k),C=A[0],N=A[1],L=C-t,T=N-e,q=x*L-y*T;(q*q/M>i||aa((y*L+x*T)/M-.5)>.3||o>a*g+c*p+s*v)&&(u(t,e,r,a,c,s,C,N,E,_/=S,b/=S,w,d,m),m.point(C,N),u(C,N,E,_,b,w,l,f,h,g,p,v,d,m))}}var i=.5,o=Math.cos(30*La),a=16;return t.precision=function(n){return arguments.length?(a=(i=n*n)>0&&16,t):Math.sqrt(i)},t}function ye(n){var t=me(function(t,e){return n([t*Ta,e*Ta])});return function(n){return we(t(n))}}function xe(n){this.stream=n}function Me(n,t){return{point:t,sphere:function(){n.sphere()},lineStart:function(){n.lineStart()},lineEnd:function(){n.lineEnd()},polygonStart:function(){n.polygonStart()},polygonEnd:function(){n.polygonEnd()}}}function _e(n){return be(function(){return n})()}function be(n){function t(n){return n=a(n[0]*La,n[1]*La),[n[0]*h+c,s-n[1]*h]}function e(n){return n=a.invert((n[0]-c)/h,(s-n[1])/h),n&&[n[0]*Ta,n[1]*Ta]}function r(){a=ie(o=Ee(m,y,x),i);var n=i(v,d);return c=g-n[0]*h,s=p+n[1]*h,u()}function u(){return l&&(l.valid=!1,l=null),t}var i,o,a,c,s,l,f=me(function(n,t){return n=i(n,t),[n[0]*h+c,s-n[1]*h]}),h=150,g=480,p=250,v=0,d=0,m=0,y=0,x=0,M=Ec,_=vt,b=null,w=null;return t.stream=function(n){return l&&(l.valid=!1),l=we(M(o,f(_(n)))),l.valid=!0,l},t.clipAngle=function(n){return arguments.length?(M=null==n?(b=n,Ec):ee((b=+n)*La),u()):b},t.clipExtent=function(n){return arguments.length?(w=n,_=n?ue(n[0][0],n[0][1],n[1][0],n[1][1]):vt,u()):w},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return arguments.length?(g=+n[0],p=+n[1],r()):[g,p]},t.center=function(n){return arguments.length?(v=n[0]%360*La,d=n[1]%360*La,r()):[v*Ta,d*Ta]},t.rotate=function(n){return arguments.length?(m=n[0]%360*La,y=n[1]%360*La,x=n.length>2?n[2]%360*La:0,r()):[m*Ta,y*Ta,x*Ta]},$o.rebind(t,f,"precision"),function(){return i=n.apply(this,arguments),t.invert=i.invert&&e,r()}}function we(n){return Me(n,function(t,e){n.point(t*La,e*La)})}function Se(n,t){return[n,t]}function ke(n,t){return[n>ka?n-Ea:-ka>n?n+Ea:n,t]}function Ee(n,t,e){return n?t||e?ie(Ce(n),Ne(t,e)):Ce(n):t||e?Ne(t,e):ke}function Ae(n){return function(t,e){return t+=n,[t>ka?t-Ea:-ka>t?t+Ea:t,e]}}function Ce(n){var t=Ae(n);return t.invert=Ae(-n),t}function Ne(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,s=Math.sin(t),l=s*r+a*u;return[Math.atan2(c*i-l*o,a*r-s*u),H(l*i+c*o)]}var r=Math.cos(n),u=Math.sin(n),i=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,s=Math.sin(t),l=s*i-c*o;return[Math.atan2(c*i+s*o,a*r+l*u),H(l*r-a*u)]},e}function Le(n,t){var e=Math.cos(n),r=Math.sin(n);return function(u,i,o,a){var c=o*t;null!=u?(u=Te(e,u),i=Te(e,i),(o>0?i>u:u>i)&&(u+=o*Ea)):(u=n+o*Ea,i=n-.5*c);for(var s,l=u;o>0?l>i:i>l;l-=c)a.point((s=Ut([e,-r*Math.cos(l),-r*Math.sin(l)]))[0],s[1])}}function Te(n,t){var e=Tt(t);e[0]-=n,Pt(e);var r=j(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-Ca)%(2*Math.PI)}function qe(n,t,e){var r=$o.range(n,t-Ca,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function ze(n,t,e){var r=$o.range(n,t-Ca,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function Re(n){return n.source}function De(n){return n.target}function Pe(n,t,e,r){var u=Math.cos(t),i=Math.sin(t),o=Math.cos(r),a=Math.sin(r),c=u*Math.cos(n),s=u*Math.sin(n),l=o*Math.cos(e),f=o*Math.sin(e),h=2*Math.asin(Math.sqrt(I(r-t)+u*o*I(e-n))),g=1/Math.sin(h),p=h?function(n){var t=Math.sin(n*=h)*g,e=Math.sin(h-n)*g,r=e*c+t*l,u=e*s+t*f,o=e*i+t*a;return[Math.atan2(u,r)*Ta,Math.atan2(o,Math.sqrt(r*r+u*u))*Ta]}:function(){return[n*Ta,t*Ta]};return p.distance=h,p}function Ue(){function n(n,u){var i=Math.sin(u*=La),o=Math.cos(u),a=aa((n*=La)-t),c=Math.cos(a);Uc+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*i-e*o*c)*a),e*i+r*o*c),t=n,e=i,r=o}var t,e,r;jc.point=function(u,i){t=u*La,e=Math.sin(i*=La),r=Math.cos(i),jc.point=n},jc.lineEnd=function(){jc.point=jc.lineEnd=c}}function je(n,t){function e(t,e){var r=Math.cos(t),u=Math.cos(e),i=n(r*u);return[i*u*Math.sin(t),i*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),u=t(r),i=Math.sin(u),o=Math.cos(u);return[Math.atan2(n*i,r*o),Math.asin(r&&e*i/r)]},e}function He(n,t){function e(n,t){var e=aa(aa(t)-Aa)<Ca?0:o/Math.pow(u(t),i);return[e*Math.sin(i*n),o-e*Math.cos(i*n)]}var r=Math.cos(n),u=function(n){return Math.tan(ka/4+n/2)},i=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(u(t)/u(n)),o=r*Math.pow(u(n),i)/i;return i?(e.invert=function(n,t){var e=o-t,r=U(i)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/i,2*Math.atan(Math.pow(o/r,1/i))-Aa]},e):Oe}function Fe(n,t){function e(n,t){var e=i-t;return[e*Math.sin(u*n),i-e*Math.cos(u*n)]}var r=Math.cos(n),u=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),i=r/u+n;return aa(u)<Ca?Se:(e.invert=function(n,t){var e=i-t;return[Math.atan2(n,e)/u,i-U(u)*Math.sqrt(n*n+e*e)]},e)}function Oe(n,t){return[n,Math.log(Math.tan(ka/4+t/2))]}function Ye(n){var t,e=_e(n),r=e.scale,u=e.translate,i=e.clipExtent;return e.scale=function(){var n=r.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.translate=function(){var n=u.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.clipExtent=function(n){var o=i.apply(e,arguments);if(o===e){if(t=null==n){var a=ka*r(),c=u();i([[c[0]-a,c[1]-a],[c[0]+a,c[1]+a]])}}else t&&(o=null);return o},e.clipExtent(null)}function Ie(n,t){var e=Math.cos(t)*Math.sin(n);return[Math.log((1+e)/(1-e))/2,Math.atan2(Math.tan(t),Math.cos(n))]}function Ze(n){return n[0]}function Ve(n){return n[1]}function Xe(n,t,e,r){var u,i,o,a,c,s,l;return u=r[n],i=u[0],o=u[1],u=r[t],a=u[0],c=u[1],u=r[e],s=u[0],l=u[1],(l-o)*(a-i)-(c-o)*(s-i)>0}function $e(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function Be(n,t,e,r){var u=n[0],i=e[0],o=t[0]-u,a=r[0]-i,c=n[1],s=e[1],l=t[1]-c,f=r[1]-s,h=(a*(c-s)-f*(u-i))/(f*o-a*l);return[u+h*o,c+h*l]}function We(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function Je(){mr(this),this.edge=this.site=this.circle=null}function Ge(n){var t=Jc.pop()||new Je;return t.site=n,t}function Ke(n){cr(n),$c.remove(n),Jc.push(n),mr(n)}function Qe(n){var t=n.circle,e=t.x,r=t.cy,u={x:e,y:r},i=n.P,o=n.N,a=[n];Ke(n);for(var c=i;c.circle&&aa(e-c.circle.x)<Ca&&aa(r-c.circle.cy)<Ca;)i=c.P,a.unshift(c),Ke(c),c=i;a.unshift(c),cr(c);for(var s=o;s.circle&&aa(e-s.circle.x)<Ca&&aa(r-s.circle.cy)<Ca;)o=s.N,a.push(s),Ke(s),s=o;a.push(s),cr(s);var l,f=a.length;for(l=1;f>l;++l)s=a[l],c=a[l-1],pr(s.edge,c.site,s.site,u);c=a[0],s=a[f-1],s.edge=hr(c.site,s.site,null,u),ar(c),ar(s)}function nr(n){for(var t,e,r,u,i=n.x,o=n.y,a=$c._;a;)if(r=tr(a,o)-i,r>Ca)a=a.L;else{if(u=i-er(a,o),!(u>Ca)){r>-Ca?(t=a.P,e=a):u>-Ca?(t=a,e=a.N):t=e=a;break}if(!a.R){t=a;break}a=a.R}var c=Ge(n);if($c.insert(t,c),t||e){if(t===e)return cr(t),e=Ge(t.site),$c.insert(c,e),c.edge=e.edge=hr(t.site,c.site),ar(t),ar(e),void 0;if(!e)return c.edge=hr(t.site,c.site),void 0;cr(t),cr(e);var s=t.site,l=s.x,f=s.y,h=n.x-l,g=n.y-f,p=e.site,v=p.x-l,d=p.y-f,m=2*(h*d-g*v),y=h*h+g*g,x=v*v+d*d,M={x:(d*y-g*x)/m+l,y:(h*x-v*y)/m+f};pr(e.edge,s,p,M),c.edge=hr(s,n,null,M),e.edge=hr(n,p,null,M),ar(t),ar(e)}}function tr(n,t){var e=n.site,r=e.x,u=e.y,i=u-t;if(!i)return r;var o=n.P;if(!o)return-1/0;e=o.site;var a=e.x,c=e.y,s=c-t;if(!s)return a;var l=a-r,f=1/i-1/s,h=l/s;return f?(-h+Math.sqrt(h*h-2*f*(l*l/(-2*s)-c+s/2+u-i/2)))/f+r:(r+a)/2}function er(n,t){var e=n.N;if(e)return tr(e,t);var r=n.site;return r.y===t?r.x:1/0}function rr(n){this.site=n,this.edges=[]}function ur(n){for(var t,e,r,u,i,o,a,c,s,l,f=n[0][0],h=n[1][0],g=n[0][1],p=n[1][1],v=Xc,d=v.length;d--;)if(i=v[d],i&&i.prepare())for(a=i.edges,c=a.length,o=0;c>o;)l=a[o].end(),r=l.x,u=l.y,s=a[++o%c].start(),t=s.x,e=s.y,(aa(r-t)>Ca||aa(u-e)>Ca)&&(a.splice(o,0,new vr(gr(i.site,l,aa(r-f)<Ca&&p-u>Ca?{x:f,y:aa(t-f)<Ca?e:p}:aa(u-p)<Ca&&h-r>Ca?{x:aa(e-p)<Ca?t:h,y:p}:aa(r-h)<Ca&&u-g>Ca?{x:h,y:aa(t-h)<Ca?e:g}:aa(u-g)<Ca&&r-f>Ca?{x:aa(e-g)<Ca?t:f,y:g}:null),i.site,null)),++c)}function ir(n,t){return t.angle-n.angle}function or(){mr(this),this.x=this.y=this.arc=this.site=this.cy=null}function ar(n){var t=n.P,e=n.N;if(t&&e){var r=t.site,u=n.site,i=e.site;if(r!==i){var o=u.x,a=u.y,c=r.x-o,s=r.y-a,l=i.x-o,f=i.y-a,h=2*(c*f-s*l);if(!(h>=-Na)){var g=c*c+s*s,p=l*l+f*f,v=(f*g-s*p)/h,d=(c*p-l*g)/h,f=d+a,m=Gc.pop()||new or;m.arc=n,m.site=u,m.x=v+o,m.y=f+Math.sqrt(v*v+d*d),m.cy=f,n.circle=m;for(var y=null,x=Wc._;x;)if(m.y<x.y||m.y===x.y&&m.x<=x.x){if(!x.L){y=x.P;break}x=x.L}else{if(!x.R){y=x;break}x=x.R}Wc.insert(y,m),y||(Bc=m)}}}}function cr(n){var t=n.circle;t&&(t.P||(Bc=t.N),Wc.remove(t),Gc.push(t),mr(t),n.circle=null)}function sr(n){for(var t,e=Vc,r=re(n[0][0],n[0][1],n[1][0],n[1][1]),u=e.length;u--;)t=e[u],(!lr(t,n)||!r(t)||aa(t.a.x-t.b.x)<Ca&&aa(t.a.y-t.b.y)<Ca)&&(t.a=t.b=null,e.splice(u,1))}function lr(n,t){var e=n.b;if(e)return!0;var r,u,i=n.a,o=t[0][0],a=t[1][0],c=t[0][1],s=t[1][1],l=n.l,f=n.r,h=l.x,g=l.y,p=f.x,v=f.y,d=(h+p)/2,m=(g+v)/2; -if(v===g){if(o>d||d>=a)return;if(h>p){if(i){if(i.y>=s)return}else i={x:d,y:c};e={x:d,y:s}}else{if(i){if(i.y<c)return}else i={x:d,y:s};e={x:d,y:c}}}else if(r=(h-p)/(v-g),u=m-r*d,-1>r||r>1)if(h>p){if(i){if(i.y>=s)return}else i={x:(c-u)/r,y:c};e={x:(s-u)/r,y:s}}else{if(i){if(i.y<c)return}else i={x:(s-u)/r,y:s};e={x:(c-u)/r,y:c}}else if(v>g){if(i){if(i.x>=a)return}else i={x:o,y:r*o+u};e={x:a,y:r*a+u}}else{if(i){if(i.x<o)return}else i={x:a,y:r*a+u};e={x:o,y:r*o+u}}return n.a=i,n.b=e,!0}function fr(n,t){this.l=n,this.r=t,this.a=this.b=null}function hr(n,t,e,r){var u=new fr(n,t);return Vc.push(u),e&&pr(u,n,t,e),r&&pr(u,t,n,r),Xc[n.i].edges.push(new vr(u,n,t)),Xc[t.i].edges.push(new vr(u,t,n)),u}function gr(n,t,e){var r=new fr(n,null);return r.a=t,r.b=e,Vc.push(r),r}function pr(n,t,e,r){n.a||n.b?n.l===e?n.b=r:n.a=r:(n.a=r,n.l=t,n.r=e)}function vr(n,t,e){var r=n.a,u=n.b;this.edge=n,this.site=t,this.angle=e?Math.atan2(e.y-t.y,e.x-t.x):n.l===t?Math.atan2(u.x-r.x,r.y-u.y):Math.atan2(r.x-u.x,u.y-r.y)}function dr(){this._=null}function mr(n){n.U=n.C=n.L=n.R=n.P=n.N=null}function yr(n,t){var e=t,r=t.R,u=e.U;u?u.L===e?u.L=r:u.R=r:n._=r,r.U=u,e.U=r,e.R=r.L,e.R&&(e.R.U=e),r.L=e}function xr(n,t){var e=t,r=t.L,u=e.U;u?u.L===e?u.L=r:u.R=r:n._=r,r.U=u,e.U=r,e.L=r.R,e.L&&(e.L.U=e),r.R=e}function Mr(n){for(;n.L;)n=n.L;return n}function _r(n,t){var e,r,u,i=n.sort(br).pop();for(Vc=[],Xc=new Array(n.length),$c=new dr,Wc=new dr;;)if(u=Bc,i&&(!u||i.y<u.y||i.y===u.y&&i.x<u.x))(i.x!==e||i.y!==r)&&(Xc[i.i]=new rr(i),nr(i),e=i.x,r=i.y),i=n.pop();else{if(!u)break;Qe(u.arc)}t&&(sr(t),ur(t));var o={cells:Xc,edges:Vc};return $c=Wc=Vc=Xc=null,o}function br(n,t){return t.y-n.y||t.x-n.x}function wr(n,t,e){return(n.x-e.x)*(t.y-n.y)-(n.x-t.x)*(e.y-n.y)}function Sr(n){return n.x}function kr(n){return n.y}function Er(){return{leaf:!0,nodes:[],point:null,x:null,y:null}}function Ar(n,t,e,r,u,i){if(!n(t,e,r,u,i)){var o=.5*(e+u),a=.5*(r+i),c=t.nodes;c[0]&&Ar(n,c[0],e,r,o,a),c[1]&&Ar(n,c[1],o,r,u,a),c[2]&&Ar(n,c[2],e,a,o,i),c[3]&&Ar(n,c[3],o,a,u,i)}}function Cr(n,t){n=$o.rgb(n),t=$o.rgb(t);var e=n.r,r=n.g,u=n.b,i=t.r-e,o=t.g-r,a=t.b-u;return function(n){return"#"+ct(Math.round(e+i*n))+ct(Math.round(r+o*n))+ct(Math.round(u+a*n))}}function Nr(n,t){var e,r={},u={};for(e in n)e in t?r[e]=qr(n[e],t[e]):u[e]=n[e];for(e in t)e in n||(u[e]=t[e]);return function(n){for(e in r)u[e]=r[e](n);return u}}function Lr(n,t){return t-=n=+n,function(e){return n+t*e}}function Tr(n,t){var e,r,u,i,o,a=0,c=0,s=[],l=[];for(n+="",t+="",Qc.lastIndex=0,r=0;e=Qc.exec(t);++r)e.index&&s.push(t.substring(a,c=e.index)),l.push({i:s.length,x:e[0]}),s.push(null),a=Qc.lastIndex;for(a<t.length&&s.push(t.substring(a)),r=0,i=l.length;(e=Qc.exec(n))&&i>r;++r)if(o=l[r],o.x==e[0]){if(o.i)if(null==s[o.i+1])for(s[o.i-1]+=o.x,s.splice(o.i,1),u=r+1;i>u;++u)l[u].i--;else for(s[o.i-1]+=o.x+s[o.i+1],s.splice(o.i,2),u=r+1;i>u;++u)l[u].i-=2;else if(null==s[o.i+1])s[o.i]=o.x;else for(s[o.i]=o.x+s[o.i+1],s.splice(o.i+1,1),u=r+1;i>u;++u)l[u].i--;l.splice(r,1),i--,r--}else o.x=Lr(parseFloat(e[0]),parseFloat(o.x));for(;i>r;)o=l.pop(),null==s[o.i+1]?s[o.i]=o.x:(s[o.i]=o.x+s[o.i+1],s.splice(o.i+1,1)),i--;return 1===s.length?null==s[0]?(o=l[0].x,function(n){return o(n)+""}):function(){return t}:function(n){for(r=0;i>r;++r)s[(o=l[r]).i]=o.x(n);return s.join("")}}function qr(n,t){for(var e,r=$o.interpolators.length;--r>=0&&!(e=$o.interpolators[r](n,t)););return e}function zr(n,t){var e,r=[],u=[],i=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e;++e)r.push(qr(n[e],t[e]));for(;i>e;++e)u[e]=n[e];for(;o>e;++e)u[e]=t[e];return function(n){for(e=0;a>e;++e)u[e]=r[e](n);return u}}function Rr(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function Dr(n){return function(t){return 1-n(1-t)}}function Pr(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function Ur(n){return n*n}function jr(n){return n*n*n}function Hr(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function Fr(n){return function(t){return Math.pow(t,n)}}function Or(n){return 1-Math.cos(n*Aa)}function Yr(n){return Math.pow(2,10*(n-1))}function Ir(n){return 1-Math.sqrt(1-n*n)}function Zr(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/Ea*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*Ea/t)}}function Vr(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function Xr(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)*n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function $r(n,t){n=$o.hcl(n),t=$o.hcl(t);var e=n.h,r=n.c,u=n.l,i=t.h-e,o=t.c-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return J(e+i*n,r+o*n,u+a*n)+""}}function Br(n,t){n=$o.hsl(n),t=$o.hsl(t);var e=n.h,r=n.s,u=n.l,i=t.h-e,o=t.s-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return $(e+i*n,r+o*n,u+a*n)+""}}function Wr(n,t){n=$o.lab(n),t=$o.lab(t);var e=n.l,r=n.a,u=n.b,i=t.l-e,o=t.a-r,a=t.b-u;return function(n){return Q(e+i*n,r+o*n,u+a*n)+""}}function Jr(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function Gr(n){var t=[n.a,n.b],e=[n.c,n.d],r=Qr(t),u=Kr(t,e),i=Qr(nu(e,t,-u))||0;t[0]*e[1]<e[0]*t[1]&&(t[0]*=-1,t[1]*=-1,r*=-1,u*=-1),this.rotate=(r?Math.atan2(t[1],t[0]):Math.atan2(-e[0],e[1]))*Ta,this.translate=[n.e,n.f],this.scale=[r,i],this.skew=i?Math.atan2(u,i)*Ta:0}function Kr(n,t){return n[0]*t[0]+n[1]*t[1]}function Qr(n){var t=Math.sqrt(Kr(n,n));return t&&(n[0]/=t,n[1]/=t),t}function nu(n,t,e){return n[0]+=e*t[0],n[1]+=e*t[1],n}function tu(n,t){var e,r=[],u=[],i=$o.transform(n),o=$o.transform(t),a=i.translate,c=o.translate,s=i.rotate,l=o.rotate,f=i.skew,h=o.skew,g=i.scale,p=o.scale;return a[0]!=c[0]||a[1]!=c[1]?(r.push("translate(",null,",",null,")"),u.push({i:1,x:Lr(a[0],c[0])},{i:3,x:Lr(a[1],c[1])})):c[0]||c[1]?r.push("translate("+c+")"):r.push(""),s!=l?(s-l>180?l+=360:l-s>180&&(s+=360),u.push({i:r.push(r.pop()+"rotate(",null,")")-2,x:Lr(s,l)})):l&&r.push(r.pop()+"rotate("+l+")"),f!=h?u.push({i:r.push(r.pop()+"skewX(",null,")")-2,x:Lr(f,h)}):h&&r.push(r.pop()+"skewX("+h+")"),g[0]!=p[0]||g[1]!=p[1]?(e=r.push(r.pop()+"scale(",null,",",null,")"),u.push({i:e-4,x:Lr(g[0],p[0])},{i:e-2,x:Lr(g[1],p[1])})):(1!=p[0]||1!=p[1])&&r.push(r.pop()+"scale("+p+")"),e=u.length,function(n){for(var t,i=-1;++i<e;)r[(t=u[i]).i]=t.x(n);return r.join("")}}function eu(n,t){return t=t-(n=+n)?1/(t-n):0,function(e){return(e-n)*t}}function ru(n,t){return t=t-(n=+n)?1/(t-n):0,function(e){return Math.max(0,Math.min(1,(e-n)*t))}}function uu(n){for(var t=n.source,e=n.target,r=ou(t,e),u=[t];t!==r;)t=t.parent,u.push(t);for(var i=u.length;e!==r;)u.splice(i,0,e),e=e.parent;return u}function iu(n){for(var t=[],e=n.parent;null!=e;)t.push(n),n=e,e=e.parent;return t.push(n),t}function ou(n,t){if(n===t)return n;for(var e=iu(n),r=iu(t),u=e.pop(),i=r.pop(),o=null;u===i;)o=u,u=e.pop(),i=r.pop();return o}function au(n){n.fixed|=2}function cu(n){n.fixed&=-7}function su(n){n.fixed|=4,n.px=n.x,n.py=n.y}function lu(n){n.fixed&=-5}function fu(n,t,e){var r=0,u=0;if(n.charge=0,!n.leaf)for(var i,o=n.nodes,a=o.length,c=-1;++c<a;)i=o[c],null!=i&&(fu(i,t,e),n.charge+=i.charge,r+=i.charge*i.cx,u+=i.charge*i.cy);if(n.point){n.leaf||(n.point.x+=Math.random()-.5,n.point.y+=Math.random()-.5);var s=t*e[n.point.index];n.charge+=n.pointCharge=s,r+=s*n.point.x,u+=s*n.point.y}n.cx=r/n.charge,n.cy=u/n.charge}function hu(n,t){return $o.rebind(n,t,"sort","children","value"),n.nodes=n,n.links=du,n}function gu(n){return n.children}function pu(n){return n.value}function vu(n,t){return t.value-n.value}function du(n){return $o.merge(n.map(function(n){return(n.children||[]).map(function(t){return{source:n,target:t}})}))}function mu(n){return n.x}function yu(n){return n.y}function xu(n,t,e){n.y0=t,n.y=e}function Mu(n){return $o.range(n.length)}function _u(n){for(var t=-1,e=n[0].length,r=[];++t<e;)r[t]=0;return r}function bu(n){for(var t,e=1,r=0,u=n[0][1],i=n.length;i>e;++e)(t=n[e][1])>u&&(r=e,u=t);return r}function wu(n){return n.reduce(Su,0)}function Su(n,t){return n+t[1]}function ku(n,t){return Eu(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function Eu(n,t){for(var e=-1,r=+n[0],u=(n[1]-r)/t,i=[];++e<=t;)i[e]=u*e+r;return i}function Au(n){return[$o.min(n),$o.max(n)]}function Cu(n,t){return n.parent==t.parent?1:2}function Nu(n){var t=n.children;return t&&t.length?t[0]:n._tree.thread}function Lu(n){var t,e=n.children;return e&&(t=e.length)?e[t-1]:n._tree.thread}function Tu(n,t){var e=n.children;if(e&&(u=e.length))for(var r,u,i=-1;++i<u;)t(r=Tu(e[i],t),n)>0&&(n=r);return n}function qu(n,t){return n.x-t.x}function zu(n,t){return t.x-n.x}function Ru(n,t){return n.depth-t.depth}function Du(n,t){function e(n,r){var u=n.children;if(u&&(o=u.length))for(var i,o,a=null,c=-1;++c<o;)i=u[c],e(i,a),a=i;t(n,r)}e(n,null)}function Pu(n){for(var t,e=0,r=0,u=n.children,i=u.length;--i>=0;)t=u[i]._tree,t.prelim+=e,t.mod+=e,e+=t.shift+(r+=t.change)}function Uu(n,t,e){n=n._tree,t=t._tree;var r=e/(t.number-n.number);n.change+=r,t.change-=r,t.shift+=e,t.prelim+=e,t.mod+=e}function ju(n,t,e){return n._tree.ancestor.parent==t.parent?n._tree.ancestor:e}function Hu(n,t){return n.value-t.value}function Fu(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function Ou(n,t){n._pack_next=t,t._pack_prev=n}function Yu(n,t){var e=t.x-n.x,r=t.y-n.y,u=n.r+t.r;return.999*u*u>e*e+r*r}function Iu(n){function t(n){l=Math.min(n.x-n.r,l),f=Math.max(n.x+n.r,f),h=Math.min(n.y-n.r,h),g=Math.max(n.y+n.r,g)}if((e=n.children)&&(s=e.length)){var e,r,u,i,o,a,c,s,l=1/0,f=-1/0,h=1/0,g=-1/0;if(e.forEach(Zu),r=e[0],r.x=-r.r,r.y=0,t(r),s>1&&(u=e[1],u.x=u.r,u.y=0,t(u),s>2))for(i=e[2],$u(r,u,i),t(i),Fu(r,i),r._pack_prev=i,Fu(i,u),u=r._pack_next,o=3;s>o;o++){$u(r,u,i=e[o]);var p=0,v=1,d=1;for(a=u._pack_next;a!==u;a=a._pack_next,v++)if(Yu(a,i)){p=1;break}if(1==p)for(c=r._pack_prev;c!==a._pack_prev&&!Yu(c,i);c=c._pack_prev,d++);p?(d>v||v==d&&u.r<r.r?Ou(r,u=a):Ou(r=c,u),o--):(Fu(r,i),u=i,t(i))}var m=(l+f)/2,y=(h+g)/2,x=0;for(o=0;s>o;o++)i=e[o],i.x-=m,i.y-=y,x=Math.max(x,i.r+Math.sqrt(i.x*i.x+i.y*i.y));n.r=x,e.forEach(Vu)}}function Zu(n){n._pack_next=n._pack_prev=n}function Vu(n){delete n._pack_next,delete n._pack_prev}function Xu(n,t,e,r){var u=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,u)for(var i=-1,o=u.length;++i<o;)Xu(u[i],t,e,r)}function $u(n,t,e){var r=n.r+e.r,u=t.x-n.x,i=t.y-n.y;if(r&&(u||i)){var o=t.r+e.r,a=u*u+i*i;o*=o,r*=r;var c=.5+(r-o)/(2*a),s=Math.sqrt(Math.max(0,2*o*(r+a)-(r-=a)*r-o*o))/(2*a);e.x=n.x+c*u+s*i,e.y=n.y+c*i-s*u}else e.x=n.x+r,e.y=n.y}function Bu(n){return 1+$o.max(n,function(n){return n.y})}function Wu(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Ju(n){var t=n.children;return t&&t.length?Ju(t[0]):n}function Gu(n){var t,e=n.children;return e&&(t=e.length)?Gu(e[t-1]):n}function Ku(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function Qu(n,t){var e=n.x+t[3],r=n.y+t[0],u=n.dx-t[1]-t[3],i=n.dy-t[0]-t[2];return 0>u&&(e+=u/2,u=0),0>i&&(r+=i/2,i=0),{x:e,y:r,dx:u,dy:i}}function ni(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function ti(n){return n.rangeExtent?n.rangeExtent():ni(n.range())}function ei(n,t,e,r){var u=e(n[0],n[1]),i=r(t[0],t[1]);return function(n){return i(u(n))}}function ri(n,t){var e,r=0,u=n.length-1,i=n[r],o=n[u];return i>o&&(e=r,r=u,u=e,e=i,i=o,o=e),n[r]=t.floor(i),n[u]=t.ceil(o),n}function ui(n){return n?{floor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:ss}function ii(n,t,e,r){var u=[],i=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]<n[0]&&(n=n.slice().reverse(),t=t.slice().reverse());++o<=a;)u.push(e(n[o-1],n[o])),i.push(r(t[o-1],t[o]));return function(t){var e=$o.bisect(n,t,1,a)-1;return i[e](u[e](t))}}function oi(n,t,e,r){function u(){var u=Math.min(n.length,t.length)>2?ii:ei,c=r?ru:eu;return o=u(n,t,c,e),a=u(t,n,c,qr),i}function i(n){return o(n)}var o,a;return i.invert=function(n){return a(n)},i.domain=function(t){return arguments.length?(n=t.map(Number),u()):n},i.range=function(n){return arguments.length?(t=n,u()):t},i.rangeRound=function(n){return i.range(n).interpolate(Jr)},i.clamp=function(n){return arguments.length?(r=n,u()):r},i.interpolate=function(n){return arguments.length?(e=n,u()):e},i.ticks=function(t){return li(n,t)},i.tickFormat=function(t,e){return fi(n,t,e)},i.nice=function(t){return ci(n,t),u()},i.copy=function(){return oi(n,t,e,r)},u()}function ai(n,t){return $o.rebind(n,t,"range","rangeRound","interpolate","clamp")}function ci(n,t){return ri(n,ui(si(n,t)[2]))}function si(n,t){null==t&&(t=10);var e=ni(n),r=e[1]-e[0],u=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),i=t/r*u;return.15>=i?u*=10:.35>=i?u*=5:.75>=i&&(u*=2),e[0]=Math.ceil(e[0]/u)*u,e[1]=Math.floor(e[1]/u)*u+.5*u,e[2]=u,e}function li(n,t){return $o.range.apply($o,si(n,t))}function fi(n,t,e){var r=si(n,t);return $o.format(e?e.replace(uc,function(n,t,e,u,i,o,a,c,s,l){return[t,e,u,i,o,a,c,s||"."+gi(l,r),l].join("")}):",."+hi(r[2])+"f")}function hi(n){return-Math.floor(Math.log(n)/Math.LN10+.01)}function gi(n,t){var e=hi(t[2]);return n in ls?Math.abs(e-hi(Math.max(Math.abs(t[0]),Math.abs(t[1]))))+ +("e"!==n):e-2*("%"===n)}function pi(n,t,e,r){function u(n){return(e?Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function i(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(u(t))}return o.invert=function(t){return i(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(u)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(u)),o):t},o.nice=function(){var t=ri(r.map(u),e?Math:hs);return n.domain(t),r=t.map(i),o},o.ticks=function(){var n=ni(r),o=[],a=n[0],c=n[1],s=Math.floor(u(a)),l=Math.ceil(u(c)),f=t%1?2:t;if(isFinite(l-s)){if(e){for(;l>s;s++)for(var h=1;f>h;h++)o.push(i(s)*h);o.push(i(s))}else for(o.push(i(s));s++<l;)for(var h=f-1;h>0;h--)o.push(i(s)*h);for(s=0;o[s]<a;s++);for(l=o.length;o[l-1]>c;l--);o=o.slice(s,l)}return o},o.tickFormat=function(n,t){if(!arguments.length)return fs;arguments.length<2?t=fs:"function"!=typeof t&&(t=$o.format(t));var r,a=Math.max(.1,n/o.ticks().length),c=e?(r=1e-12,Math.ceil):(r=-1e-12,Math.floor);return function(n){return n/i(c(u(n)+r))<=a?t(n):""}},o.copy=function(){return pi(n.copy(),t,e,r)},ai(o,n)}function vi(n,t,e){function r(t){return n(u(t))}var u=di(t),i=di(1/t);return r.invert=function(t){return i(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(u)),r):e},r.ticks=function(n){return li(e,n)},r.tickFormat=function(n,t){return fi(e,n,t)},r.nice=function(n){return r.domain(ci(e,n))},r.exponent=function(o){return arguments.length?(u=di(t=o),i=di(1/t),n.domain(e.map(u)),r):t},r.copy=function(){return vi(n.copy(),t,e)},ai(r,n)}function di(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function mi(n,t){function e(e){return o[((i.get(e)||"range"===t.t&&i.set(e,n.push(e)))-1)%o.length]}function r(t,e){return $o.range(n.length).map(function(n){return t+e*n})}var i,o,a;return e.domain=function(r){if(!arguments.length)return n;n=[],i=new u;for(var o,a=-1,c=r.length;++a<c;)i.has(o=r[a])||i.set(o,n.push(o));return e[t.t].apply(e,t.a)},e.range=function(n){return arguments.length?(o=n,a=0,t={t:"range",a:arguments},e):o},e.rangePoints=function(u,i){arguments.length<2&&(i=0);var c=u[0],s=u[1],l=(s-c)/(Math.max(1,n.length-1)+i);return o=r(n.length<2?(c+s)/2:c+l*i/2,l),a=0,t={t:"rangePoints",a:arguments},e},e.rangeBands=function(u,i,c){arguments.length<2&&(i=0),arguments.length<3&&(c=i);var s=u[1]<u[0],l=u[s-0],f=u[1-s],h=(f-l)/(n.length-i+2*c);return o=r(l+h*c,h),s&&o.reverse(),a=h*(1-i),t={t:"rangeBands",a:arguments},e},e.rangeRoundBands=function(u,i,c){arguments.length<2&&(i=0),arguments.length<3&&(c=i);var s=u[1]<u[0],l=u[s-0],f=u[1-s],h=Math.floor((f-l)/(n.length-i+2*c)),g=f-l-(n.length-i)*h;return o=r(l+Math.round(g/2),h),s&&o.reverse(),a=Math.round(h*(1-i)),t={t:"rangeRoundBands",a:arguments},e},e.rangeBand=function(){return a},e.rangeExtent=function(){return ni(t.a[0])},e.copy=function(){return mi(n,t)},e.domain(n)}function yi(n,t){function e(){var e=0,i=t.length;for(u=[];++e<i;)u[e-1]=$o.quantile(n,e/i);return r}function r(n){return isNaN(n=+n)?void 0:t[$o.bisect(u,n)]}var u;return r.domain=function(t){return arguments.length?(n=t.filter(function(n){return!isNaN(n)}).sort($o.ascending),e()):n},r.range=function(n){return arguments.length?(t=n,e()):t},r.quantiles=function(){return u},r.invertExtent=function(e){return e=t.indexOf(e),0>e?[0/0,0/0]:[e>0?u[e-1]:n[0],e<u.length?u[e]:n[n.length-1]]},r.copy=function(){return yi(n,t)},e()}function xi(n,t,e){function r(t){return e[Math.max(0,Math.min(o,Math.floor(i*(t-n))))]}function u(){return i=e.length/(t-n),o=e.length-1,r}var i,o;return r.domain=function(e){return arguments.length?(n=+e[0],t=+e[e.length-1],u()):[n,t]},r.range=function(n){return arguments.length?(e=n,u()):e},r.invertExtent=function(t){return t=e.indexOf(t),t=0>t?0/0:t/i+n,[t,t+1/i]},r.copy=function(){return xi(n,t,e)},u()}function Mi(n,t){function e(e){return e>=e?t[$o.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return Mi(n,t)},e}function _i(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return li(n,t)},t.tickFormat=function(t,e){return fi(n,t,e)},t.copy=function(){return _i(n)},t}function bi(n){return n.innerRadius}function wi(n){return n.outerRadius}function Si(n){return n.startAngle}function ki(n){return n.endAngle}function Ei(n){function t(t){function o(){s.push("M",i(n(l),a))}for(var c,s=[],l=[],f=-1,h=t.length,g=pt(e),p=pt(r);++f<h;)u.call(this,c=t[f],f)?l.push([+g.call(this,c,f),+p.call(this,c,f)]):l.length&&(o(),l=[]);return l.length&&o(),s.length?s.join(""):null}var e=Ze,r=Ve,u=Zt,i=Ai,o=i.key,a=.7;return t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t.defined=function(n){return arguments.length?(u=n,t):u},t.interpolate=function(n){return arguments.length?(o="function"==typeof n?i=n:(i=xs.get(n)||Ai).key,t):o},t.tension=function(n){return arguments.length?(a=n,t):a},t}function Ai(n){return n.join("L")}function Ci(n){return Ai(n)+"Z"}function Ni(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("H",(r[0]+(r=n[t])[0])/2,"V",r[1]);return e>1&&u.push("H",r[0]),u.join("")}function Li(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("V",(r=n[t])[1],"H",r[0]);return u.join("")}function Ti(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("H",(r=n[t])[0],"V",r[1]);return u.join("")}function qi(n,t){return n.length<4?Ai(n):n[1]+Di(n.slice(1,n.length-1),Pi(n,t))}function zi(n,t){return n.length<3?Ai(n):n[0]+Di((n.push(n[0]),n),Pi([n[n.length-2]].concat(n,[n[1]]),t))}function Ri(n,t){return n.length<3?Ai(n):n[0]+Di(n,Pi(n,t))}function Di(n,t){if(t.length<1||n.length!=t.length&&n.length!=t.length+2)return Ai(n);var e=n.length!=t.length,r="",u=n[0],i=n[1],o=t[0],a=o,c=1;if(e&&(r+="Q"+(i[0]-2*o[0]/3)+","+(i[1]-2*o[1]/3)+","+i[0]+","+i[1],u=n[1],c=2),t.length>1){a=t[1],i=n[c],c++,r+="C"+(u[0]+o[0])+","+(u[1]+o[1])+","+(i[0]-a[0])+","+(i[1]-a[1])+","+i[0]+","+i[1];for(var s=2;s<t.length;s++,c++)i=n[c],a=t[s],r+="S"+(i[0]-a[0])+","+(i[1]-a[1])+","+i[0]+","+i[1]}if(e){var l=n[c];r+="Q"+(i[0]+2*a[0]/3)+","+(i[1]+2*a[1]/3)+","+l[0]+","+l[1]}return r}function Pi(n,t){for(var e,r=[],u=(1-t)/2,i=n[0],o=n[1],a=1,c=n.length;++a<c;)e=i,i=o,o=n[a],r.push([u*(o[0]-e[0]),u*(o[1]-e[1])]);return r}function Ui(n){if(n.length<3)return Ai(n);var t=1,e=n.length,r=n[0],u=r[0],i=r[1],o=[u,u,u,(r=n[1])[0]],a=[i,i,i,r[1]],c=[u,",",i,"L",Oi(bs,o),",",Oi(bs,a)];for(n.push(n[e-1]);++t<=e;)r=n[t],o.shift(),o.push(r[0]),a.shift(),a.push(r[1]),Yi(c,o,a);return n.pop(),c.push("L",r),c.join("")}function ji(n){if(n.length<4)return Ai(n);for(var t,e=[],r=-1,u=n.length,i=[0],o=[0];++r<3;)t=n[r],i.push(t[0]),o.push(t[1]);for(e.push(Oi(bs,i)+","+Oi(bs,o)),--r;++r<u;)t=n[r],i.shift(),i.push(t[0]),o.shift(),o.push(t[1]),Yi(e,i,o);return e.join("")}function Hi(n){for(var t,e,r=-1,u=n.length,i=u+4,o=[],a=[];++r<4;)e=n[r%u],o.push(e[0]),a.push(e[1]);for(t=[Oi(bs,o),",",Oi(bs,a)],--r;++r<i;)e=n[r%u],o.shift(),o.push(e[0]),a.shift(),a.push(e[1]),Yi(t,o,a);return t.join("")}function Fi(n,t){var e=n.length-1;if(e)for(var r,u,i=n[0][0],o=n[0][1],a=n[e][0]-i,c=n[e][1]-o,s=-1;++s<=e;)r=n[s],u=s/e,r[0]=t*r[0]+(1-t)*(i+u*a),r[1]=t*r[1]+(1-t)*(o+u*c);return Ui(n)}function Oi(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]+n[3]*t[3]}function Yi(n,t,e){n.push("C",Oi(Ms,t),",",Oi(Ms,e),",",Oi(_s,t),",",Oi(_s,e),",",Oi(bs,t),",",Oi(bs,e))}function Ii(n,t){return(t[1]-n[1])/(t[0]-n[0])}function Zi(n){for(var t=0,e=n.length-1,r=[],u=n[0],i=n[1],o=r[0]=Ii(u,i);++t<e;)r[t]=(o+(o=Ii(u=i,i=n[t+1])))/2;return r[t]=o,r}function Vi(n){for(var t,e,r,u,i=[],o=Zi(n),a=-1,c=n.length-1;++a<c;)t=Ii(n[a],n[a+1]),aa(t)<Ca?o[a]=o[a+1]=0:(e=o[a]/t,r=o[a+1]/t,u=e*e+r*r,u>9&&(u=3*t/Math.sqrt(u),o[a]=u*e,o[a+1]=u*r));for(a=-1;++a<=c;)u=(n[Math.min(c,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),i.push([u||0,o[a]*u||0]);return i}function Xi(n){return n.length<3?Ai(n):n[0]+Di(n,Vi(n))}function $i(n){for(var t,e,r,u=-1,i=n.length;++u<i;)t=n[u],e=t[0],r=t[1]+ms,t[0]=e*Math.cos(r),t[1]=e*Math.sin(r);return n}function Bi(n){function t(t){function c(){v.push("M",a(n(m),f),l,s(n(d.reverse()),f),"Z")}for(var h,g,p,v=[],d=[],m=[],y=-1,x=t.length,M=pt(e),_=pt(u),b=e===r?function(){return g}:pt(r),w=u===i?function(){return p}:pt(i);++y<x;)o.call(this,h=t[y],y)?(d.push([g=+M.call(this,h,y),p=+_.call(this,h,y)]),m.push([+b.call(this,h,y),+w.call(this,h,y)])):d.length&&(c(),d=[],m=[]);return d.length&&c(),v.length?v.join(""):null}var e=Ze,r=Ze,u=0,i=Ve,o=Zt,a=Ai,c=a.key,s=a,l="L",f=.7;return t.x=function(n){return arguments.length?(e=r=n,t):r},t.x0=function(n){return arguments.length?(e=n,t):e},t.x1=function(n){return arguments.length?(r=n,t):r},t.y=function(n){return arguments.length?(u=i=n,t):i},t.y0=function(n){return arguments.length?(u=n,t):u},t.y1=function(n){return arguments.length?(i=n,t):i},t.defined=function(n){return arguments.length?(o=n,t):o},t.interpolate=function(n){return arguments.length?(c="function"==typeof n?a=n:(a=xs.get(n)||Ai).key,s=a.reverse||a,l=a.closed?"M":"L",t):c},t.tension=function(n){return arguments.length?(f=n,t):f},t}function Wi(n){return n.radius}function Ji(n){return[n.x,n.y]}function Gi(n){return function(){var t=n.apply(this,arguments),e=t[0],r=t[1]+ms;return[e*Math.cos(r),e*Math.sin(r)]}}function Ki(){return 64}function Qi(){return"circle"}function no(n){var t=Math.sqrt(n/ka);return"M0,"+t+"A"+t+","+t+" 0 1,1 0,"+-t+"A"+t+","+t+" 0 1,1 0,"+t+"Z"}function to(n,t){return ha(n,Cs),n.id=t,n}function eo(n,t,e,r){var u=n.id;return C(n,"function"==typeof e?function(n,i,o){n.__transition__[u].tween.set(t,r(e.call(n,n.__data__,i,o)))}:(e=r(e),function(n){n.__transition__[u].tween.set(t,e)}))}function ro(n){return null==n&&(n=""),function(){this.textContent=n}}function uo(n,t,e,r){var i=n.__transition__||(n.__transition__={active:0,count:0}),o=i[e];if(!o){var a=r.time;o=i[e]={tween:new u,time:a,ease:r.ease,delay:r.delay,duration:r.duration},++i.count,$o.timer(function(r){function u(r){return i.active>e?s():(i.active=e,o.event&&o.event.start.call(n,l,t),o.tween.forEach(function(e,r){(r=r.call(n,l,t))&&v.push(r)}),$o.timer(function(){return p.c=c(r||1)?Zt:c,1},0,a),void 0)}function c(r){if(i.active!==e)return s();for(var u=r/g,a=f(u),c=v.length;c>0;)v[--c].call(n,a);return u>=1?(o.event&&o.event.end.call(n,l,t),s()):void 0}function s(){return--i.count?delete i[e]:delete n.__transition__,1}var l=n.__data__,f=o.ease,h=o.delay,g=o.duration,p=Ga,v=[];return p.t=h+a,r>=h?u(r-h):(p.c=u,void 0)},0,a)}}function io(n,t){n.attr("transform",function(n){return"translate("+t(n)+",0)"})}function oo(n,t){n.attr("transform",function(n){return"translate(0,"+t(n)+")"})}function ao(){this._=new Date(arguments.length>1?Date.UTC.apply(this,arguments):arguments[0])}function co(n,t,e){function r(t){var e=n(t),r=i(e,1);return r-t>t-e?e:r}function u(e){return t(e=n(new Ds(e-1)),1),e}function i(n,e){return t(n=new Ds(+n),e),n}function o(n,r,i){var o=u(n),a=[];if(i>1)for(;r>o;)e(o)%i||a.push(new Date(+o)),t(o,1);else for(;r>o;)a.push(new Date(+o)),t(o,1);return a}function a(n,t,e){try{Ds=ao;var r=new ao;return r._=n,o(r,t,e)}finally{Ds=Date}}n.floor=n,n.round=r,n.ceil=u,n.offset=i,n.range=o;var c=n.utc=so(n);return c.floor=c,c.round=so(r),c.ceil=so(u),c.offset=so(i),c.range=a,n}function so(n){return function(t,e){try{Ds=ao;var r=new ao;return r._=t,n(r,e)._}finally{Ds=Date}}}function lo(n){function t(t){for(var r,u,i,o=[],a=-1,c=0;++a<e;)37===n.charCodeAt(a)&&(o.push(n.substring(c,a)),null!=(u=nl[r=n.charAt(++a)])&&(r=n.charAt(++a)),(i=tl[r])&&(r=i(t,null==u?"e"===r?" ":"0":u)),o.push(r),c=a+1);return o.push(n.substring(c,a)),o.join("")}var e=n.length;return t.parse=function(t){var e={y:1900,m:0,d:1,H:0,M:0,S:0,L:0,Z:null},r=fo(e,n,t,0);if(r!=t.length)return null;"p"in e&&(e.H=e.H%12+12*e.p);var u=null!=e.Z&&Ds!==ao,i=new(u?ao:Ds);return"j"in e?i.setFullYear(e.y,0,e.j):"w"in e&&("W"in e||"U"in e)?(i.setFullYear(e.y,0,1),i.setFullYear(e.y,0,"W"in e?(e.w+6)%7+7*e.W-(i.getDay()+5)%7:e.w+7*e.U-(i.getDay()+6)%7)):i.setFullYear(e.y,e.m,e.d),i.setHours(e.H+Math.floor(e.Z/100),e.M+e.Z%100,e.S,e.L),u?i._:i},t.toString=function(){return n},t}function fo(n,t,e,r){for(var u,i,o,a=0,c=t.length,s=e.length;c>a;){if(r>=s)return-1;if(u=t.charCodeAt(a++),37===u){if(o=t.charAt(a++),i=el[o in nl?t.charAt(a++):o],!i||(r=i(n,e,r))<0)return-1}else if(u!=e.charCodeAt(r++))return-1}return r}function ho(n){return new RegExp("^(?:"+n.map($o.requote).join("|")+")","i")}function go(n){for(var t=new u,e=-1,r=n.length;++e<r;)t.set(n[e].toLowerCase(),e);return t}function po(n,t,e){var r=0>n?"-":"",u=(r?-n:n)+"",i=u.length;return r+(e>i?new Array(e-i+1).join(t)+u:u)}function vo(n,t,e){$s.lastIndex=0;var r=$s.exec(t.substring(e));return r?(n.w=Bs.get(r[0].toLowerCase()),e+r[0].length):-1}function mo(n,t,e){Vs.lastIndex=0;var r=Vs.exec(t.substring(e));return r?(n.w=Xs.get(r[0].toLowerCase()),e+r[0].length):-1}function yo(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+1));return r?(n.w=+r[0],e+r[0].length):-1}function xo(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e));return r?(n.U=+r[0],e+r[0].length):-1}function Mo(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e));return r?(n.W=+r[0],e+r[0].length):-1}function _o(n,t,e){Gs.lastIndex=0;var r=Gs.exec(t.substring(e));return r?(n.m=Ks.get(r[0].toLowerCase()),e+r[0].length):-1}function bo(n,t,e){Ws.lastIndex=0;var r=Ws.exec(t.substring(e));return r?(n.m=Js.get(r[0].toLowerCase()),e+r[0].length):-1}function wo(n,t,e){return fo(n,tl.c.toString(),t,e)}function So(n,t,e){return fo(n,tl.x.toString(),t,e)}function ko(n,t,e){return fo(n,tl.X.toString(),t,e)}function Eo(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+4));return r?(n.y=+r[0],e+r[0].length):-1}function Ao(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+2));return r?(n.y=No(+r[0]),e+r[0].length):-1}function Co(n,t,e){return/^[+-]\d{4}$/.test(t=t.substring(e,e+5))?(n.Z=+t,e+5):-1}function No(n){return n+(n>68?1900:2e3)}function Lo(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+2));return r?(n.m=r[0]-1,e+r[0].length):-1}function To(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+2));return r?(n.d=+r[0],e+r[0].length):-1}function qo(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+3));return r?(n.j=+r[0],e+r[0].length):-1}function zo(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+2));return r?(n.H=+r[0],e+r[0].length):-1}function Ro(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+2));return r?(n.M=+r[0],e+r[0].length):-1}function Do(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+2));return r?(n.S=+r[0],e+r[0].length):-1}function Po(n,t,e){rl.lastIndex=0;var r=rl.exec(t.substring(e,e+3));return r?(n.L=+r[0],e+r[0].length):-1}function Uo(n,t,e){var r=ul.get(t.substring(e,e+=2).toLowerCase());return null==r?-1:(n.p=r,e)}function jo(n){var t=n.getTimezoneOffset(),e=t>0?"-":"+",r=~~(aa(t)/60),u=aa(t)%60;return e+po(r,"0",2)+po(u,"0",2)}function Ho(n,t,e){Qs.lastIndex=0;var r=Qs.exec(t.substring(e,e+1));return r?e+r[0].length:-1}function Fo(n){function t(n){try{Ds=ao;var t=new Ds;return t._=n,e(t)}finally{Ds=Date}}var e=lo(n);return t.parse=function(n){try{Ds=ao;var t=e.parse(n);return t&&t._}finally{Ds=Date}},t.toString=e.toString,t}function Oo(n){return n.toISOString()}function Yo(n,t,e){function r(t){return n(t)}function u(n,e){var r=n[1]-n[0],u=r/e,i=$o.bisect(ol,u);return i==ol.length?[t.year,si(n.map(function(n){return n/31536e6}),e)[2]]:i?t[u/ol[i-1]<ol[i]/u?i-1:i]:[ll,si(n,e)[2]]}return r.invert=function(t){return Io(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain(t),r):n.domain().map(Io)},r.nice=function(n,t){function e(e){return!isNaN(e)&&!n.range(e,Io(+e+1),t).length}var i=r.domain(),o=ni(i),a=null==n?u(o,10):"number"==typeof n&&u(o,n);return a&&(n=a[0],t=a[1]),r.domain(ri(i,t>1?{floor:function(t){for(;e(t=n.floor(t));)t=Io(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=Io(+t+1);return t}}:n))},r.ticks=function(n,t){var e=ni(r.domain()),i=null==n?u(e,10):"number"==typeof n?u(e,n):!n.range&&[{range:n},t];return i&&(n=i[0],t=i[1]),n.range(e[0],Io(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return Yo(n.copy(),t,e)},ai(r,n)}function Io(n){return new Date(n)}function Zo(n){return function(t){for(var e=n.length-1,r=n[e];!r[1](t);)r=n[--e];return r[0](t)}}function Vo(n){return JSON.parse(n.responseText)}function Xo(n){var t=Jo.createRange();return t.selectNode(Jo.body),t.createContextualFragment(n.responseText)}var $o={version:"3.3.9"};Date.now||(Date.now=function(){return+new Date});var Bo=[].slice,Wo=function(n){return Bo.call(n)},Jo=document,Go=Jo.documentElement,Ko=window;try{Wo(Go.childNodes)[0].nodeType}catch(Qo){Wo=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}try{Jo.createElement("div").style.setProperty("opacity",0,"")}catch(na){var ta=Ko.Element.prototype,ea=ta.setAttribute,ra=ta.setAttributeNS,ua=Ko.CSSStyleDeclaration.prototype,ia=ua.setProperty;ta.setAttribute=function(n,t){ea.call(this,n,t+"")},ta.setAttributeNS=function(n,t,e){ra.call(this,n,t,e+"")},ua.setProperty=function(n,t,e){ia.call(this,n,t+"",e)}}$o.ascending=function(n,t){return t>n?-1:n>t?1:n>=t?0:0/0},$o.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:0/0},$o.min=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u<i&&!(null!=(e=n[u])&&e>=e);)e=void 0;for(;++u<i;)null!=(r=n[u])&&e>r&&(e=r)}else{for(;++u<i&&!(null!=(e=t.call(n,n[u],u))&&e>=e);)e=void 0;for(;++u<i;)null!=(r=t.call(n,n[u],u))&&e>r&&(e=r)}return e},$o.max=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u<i&&!(null!=(e=n[u])&&e>=e);)e=void 0;for(;++u<i;)null!=(r=n[u])&&r>e&&(e=r)}else{for(;++u<i&&!(null!=(e=t.call(n,n[u],u))&&e>=e);)e=void 0;for(;++u<i;)null!=(r=t.call(n,n[u],u))&&r>e&&(e=r)}return e},$o.extent=function(n,t){var e,r,u,i=-1,o=n.length;if(1===arguments.length){for(;++i<o&&!(null!=(e=u=n[i])&&e>=e);)e=u=void 0;for(;++i<o;)null!=(r=n[i])&&(e>r&&(e=r),r>u&&(u=r))}else{for(;++i<o&&!(null!=(e=u=t.call(n,n[i],i))&&e>=e);)e=void 0;for(;++i<o;)null!=(r=t.call(n,n[i],i))&&(e>r&&(e=r),r>u&&(u=r))}return[e,u]},$o.sum=function(n,t){var e,r=0,u=n.length,i=-1;if(1===arguments.length)for(;++i<u;)isNaN(e=+n[i])||(r+=e);else for(;++i<u;)isNaN(e=+t.call(n,n[i],i))||(r+=e);return r},$o.mean=function(t,e){var r,u=t.length,i=0,o=-1,a=0;if(1===arguments.length)for(;++o<u;)n(r=t[o])&&(i+=(r-i)/++a);else for(;++o<u;)n(r=e.call(t,t[o],o))&&(i+=(r-i)/++a);return a?i:void 0},$o.quantile=function(n,t){var e=(n.length-1)*t+1,r=Math.floor(e),u=+n[r-1],i=e-r; -return i?u+i*(n[r]-u):u},$o.median=function(t,e){return arguments.length>1&&(t=t.map(e)),t=t.filter(n),t.length?$o.quantile(t.sort($o.ascending),.5):void 0},$o.bisector=function(n){return{left:function(t,e,r,u){for(arguments.length<3&&(r=0),arguments.length<4&&(u=t.length);u>r;){var i=r+u>>>1;n.call(t,t[i],i)<e?r=i+1:u=i}return r},right:function(t,e,r,u){for(arguments.length<3&&(r=0),arguments.length<4&&(u=t.length);u>r;){var i=r+u>>>1;e<n.call(t,t[i],i)?u=i:r=i+1}return r}}};var oa=$o.bisector(function(n){return n});$o.bisectLeft=oa.left,$o.bisect=$o.bisectRight=oa.right,$o.shuffle=function(n){for(var t,e,r=n.length;r;)e=0|Math.random()*r--,t=n[r],n[r]=n[e],n[e]=t;return n},$o.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},$o.pairs=function(n){for(var t,e=0,r=n.length-1,u=n[0],i=new Array(0>r?0:r);r>e;)i[e]=[t=u,u=n[++e]];return i},$o.zip=function(){if(!(u=arguments.length))return[];for(var n=-1,e=$o.min(arguments,t),r=new Array(e);++n<e;)for(var u,i=-1,o=r[n]=new Array(u);++i<u;)o[i]=arguments[i][n];return r},$o.transpose=function(n){return $o.zip.apply($o,n)},$o.keys=function(n){var t=[];for(var e in n)t.push(e);return t},$o.values=function(n){var t=[];for(var e in n)t.push(n[e]);return t},$o.entries=function(n){var t=[];for(var e in n)t.push({key:e,value:n[e]});return t},$o.merge=function(n){for(var t,e,r,u=n.length,i=-1,o=0;++i<u;)o+=n[i].length;for(e=new Array(o);--u>=0;)for(r=n[u],t=r.length;--t>=0;)e[--o]=r[t];return e};var aa=Math.abs;$o.range=function(n,t,r){if(arguments.length<3&&(r=1,arguments.length<2&&(t=n,n=0)),1/0===(t-n)/r)throw new Error("infinite range");var u,i=[],o=e(aa(r)),a=-1;if(n*=o,t*=o,r*=o,0>r)for(;(u=n+r*++a)>t;)i.push(u/o);else for(;(u=n+r*++a)<t;)i.push(u/o);return i},$o.map=function(n){var t=new u;if(n instanceof u)n.forEach(function(n,e){t.set(n,e)});else for(var e in n)t.set(e,n[e]);return t},r(u,{has:function(n){return ca+n in this},get:function(n){return this[ca+n]},set:function(n,t){return this[ca+n]=t},remove:function(n){return n=ca+n,n in this&&delete this[n]},keys:function(){var n=[];return this.forEach(function(t){n.push(t)}),n},values:function(){var n=[];return this.forEach(function(t,e){n.push(e)}),n},entries:function(){var n=[];return this.forEach(function(t,e){n.push({key:t,value:e})}),n},forEach:function(n){for(var t in this)t.charCodeAt(0)===sa&&n.call(this,t.substring(1),this[t])}});var ca="\x00",sa=ca.charCodeAt(0);$o.nest=function(){function n(t,a,c){if(c>=o.length)return r?r.call(i,a):e?a.sort(e):a;for(var s,l,f,h,g=-1,p=a.length,v=o[c++],d=new u;++g<p;)(h=d.get(s=v(l=a[g])))?h.push(l):d.set(s,[l]);return t?(l=t(),f=function(e,r){l.set(e,n(t,r,c))}):(l={},f=function(e,r){l[e]=n(t,r,c)}),d.forEach(f),l}function t(n,e){if(e>=o.length)return n;var r=[],u=a[e++];return n.forEach(function(n,u){r.push({key:n,values:t(u,e)})}),u?r.sort(function(n,t){return u(n.key,t.key)}):r}var e,r,i={},o=[],a=[];return i.map=function(t,e){return n(e,t,0)},i.entries=function(e){return t(n($o.map,e,0),0)},i.key=function(n){return o.push(n),i},i.sortKeys=function(n){return a[o.length-1]=n,i},i.sortValues=function(n){return e=n,i},i.rollup=function(n){return r=n,i},i},$o.set=function(n){var t=new i;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},r(i,{has:function(n){return ca+n in this},add:function(n){return this[ca+n]=!0,n},remove:function(n){return n=ca+n,n in this&&delete this[n]},values:function(){var n=[];return this.forEach(function(t){n.push(t)}),n},forEach:function(n){for(var t in this)t.charCodeAt(0)===sa&&n.call(this,t.substring(1))}}),$o.behavior={},$o.rebind=function(n,t){for(var e,r=1,u=arguments.length;++r<u;)n[e=arguments[r]]=o(n,t,t[e]);return n};var la=["webkit","ms","moz","Moz","o","O"];$o.dispatch=function(){for(var n=new s,t=-1,e=arguments.length;++t<e;)n[arguments[t]]=l(n);return n},s.prototype.on=function(n,t){var e=n.indexOf("."),r="";if(e>=0&&(r=n.substring(e+1),n=n.substring(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},$o.event=null,$o.requote=function(n){return n.replace(fa,"\\$&")};var fa=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,ha={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},ga=function(n,t){return t.querySelector(n)},pa=function(n,t){return t.querySelectorAll(n)},va=Go[a(Go,"matchesSelector")],da=function(n,t){return va.call(n,t)};"function"==typeof Sizzle&&(ga=function(n,t){return Sizzle(n,t)[0]||null},pa=function(n,t){return Sizzle.uniqueSort(Sizzle(n,t))},da=Sizzle.matchesSelector),$o.selection=function(){return Ma};var ma=$o.selection.prototype=[];ma.select=function(n){var t,e,r,u,i=[];n=v(n);for(var o=-1,a=this.length;++o<a;){i.push(t=[]),t.parentNode=(r=this[o]).parentNode;for(var c=-1,s=r.length;++c<s;)(u=r[c])?(t.push(e=n.call(u,u.__data__,c,o)),e&&"__data__"in u&&(e.__data__=u.__data__)):t.push(null)}return p(i)},ma.selectAll=function(n){var t,e,r=[];n=d(n);for(var u=-1,i=this.length;++u<i;)for(var o=this[u],a=-1,c=o.length;++a<c;)(e=o[a])&&(r.push(t=Wo(n.call(e,e.__data__,a,u))),t.parentNode=e);return p(r)};var ya={svg:"http://www.w3.org/2000/svg",xhtml:"http://www.w3.org/1999/xhtml",xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};$o.ns={prefix:ya,qualify:function(n){var t=n.indexOf(":"),e=n;return t>=0&&(e=n.substring(0,t),n=n.substring(t+1)),ya.hasOwnProperty(e)?{space:ya[e],local:n}:n}},ma.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=$o.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(m(t,n[t]));return this}return this.each(m(n,t))},ma.classed=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node(),r=(n=n.trim().split(/^|\s+/g)).length,u=-1;if(t=e.classList){for(;++u<r;)if(!t.contains(n[u]))return!1}else for(t=e.getAttribute("class");++u<r;)if(!x(n[u]).test(t))return!1;return!0}for(t in n)this.each(M(t,n[t]));return this}return this.each(M(n,t))},ma.style=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t="");for(e in n)this.each(b(e,n[e],t));return this}if(2>r)return Ko.getComputedStyle(this.node(),null).getPropertyValue(n);e=""}return this.each(b(n,t,e))},ma.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(w(t,n[t]));return this}return this.each(w(n,t))},ma.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},ma.html=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},ma.append=function(n){return n=S(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},ma.insert=function(n,t){return n=S(n),t=v(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments)||null)})},ma.remove=function(){return this.each(function(){var n=this.parentNode;n&&n.removeChild(this)})},ma.data=function(n,t){function e(n,e){var r,i,o,a=n.length,f=e.length,h=Math.min(a,f),g=new Array(f),p=new Array(f),v=new Array(a);if(t){var d,m=new u,y=new u,x=[];for(r=-1;++r<a;)d=t.call(i=n[r],i.__data__,r),m.has(d)?v[r]=i:m.set(d,i),x.push(d);for(r=-1;++r<f;)d=t.call(e,o=e[r],r),(i=m.get(d))?(g[r]=i,i.__data__=o):y.has(d)||(p[r]=k(o)),y.set(d,o),m.remove(d);for(r=-1;++r<a;)m.has(x[r])&&(v[r]=n[r])}else{for(r=-1;++r<h;)i=n[r],o=e[r],i?(i.__data__=o,g[r]=i):p[r]=k(o);for(;f>r;++r)p[r]=k(e[r]);for(;a>r;++r)v[r]=n[r]}p.update=g,p.parentNode=g.parentNode=v.parentNode=n.parentNode,c.push(p),s.push(g),l.push(v)}var r,i,o=-1,a=this.length;if(!arguments.length){for(n=new Array(a=(r=this[0]).length);++o<a;)(i=r[o])&&(n[o]=i.__data__);return n}var c=N([]),s=p([]),l=p([]);if("function"==typeof n)for(;++o<a;)e(r=this[o],n.call(r,r.parentNode.__data__,o));else for(;++o<a;)e(r=this[o],n);return s.enter=function(){return c},s.exit=function(){return l},s},ma.datum=function(n){return arguments.length?this.property("__data__",n):this.property("__data__")},ma.filter=function(n){var t,e,r,u=[];"function"!=typeof n&&(n=E(n));for(var i=0,o=this.length;o>i;i++){u.push(t=[]),t.parentNode=(e=this[i]).parentNode;for(var a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a)&&t.push(r)}return p(u)},ma.order=function(){for(var n=-1,t=this.length;++n<t;)for(var e,r=this[n],u=r.length-1,i=r[u];--u>=0;)(e=r[u])&&(i&&i!==e.nextSibling&&i.parentNode.insertBefore(e,i),i=e);return this},ma.sort=function(n){n=A.apply(this,arguments);for(var t=-1,e=this.length;++t<e;)this[t].sort(n);return this.order()},ma.each=function(n){return C(this,function(t,e,r){n.call(t,t.__data__,e,r)})},ma.call=function(n){var t=Wo(arguments);return n.apply(t[0]=this,t),this},ma.empty=function(){return!this.node()},ma.node=function(){for(var n=0,t=this.length;t>n;n++)for(var e=this[n],r=0,u=e.length;u>r;r++){var i=e[r];if(i)return i}return null},ma.size=function(){var n=0;return this.each(function(){++n}),n};var xa=[];$o.selection.enter=N,$o.selection.enter.prototype=xa,xa.append=ma.append,xa.empty=ma.empty,xa.node=ma.node,xa.call=ma.call,xa.size=ma.size,xa.select=function(n){for(var t,e,r,u,i,o=[],a=-1,c=this.length;++a<c;){r=(u=this[a]).update,o.push(t=[]),t.parentNode=u.parentNode;for(var s=-1,l=u.length;++s<l;)(i=u[s])?(t.push(r[s]=e=n.call(u.parentNode,i.__data__,s,a)),e.__data__=i.__data__):t.push(null)}return p(o)},xa.insert=function(n,t){return arguments.length<2&&(t=L(this)),ma.insert.call(this,n,t)},ma.transition=function(){for(var n,t,e=Ss||++Ns,r=[],u=ks||{time:Date.now(),ease:Hr,delay:0,duration:250},i=-1,o=this.length;++i<o;){r.push(n=[]);for(var a=this[i],c=-1,s=a.length;++c<s;)(t=a[c])&&uo(t,c,e,u),n.push(t)}return to(r,e)},ma.interrupt=function(){return this.each(T)},$o.select=function(n){var t=["string"==typeof n?ga(n,Jo):n];return t.parentNode=Go,p([t])},$o.selectAll=function(n){var t=Wo("string"==typeof n?pa(n,Jo):n);return t.parentNode=Go,p([t])};var Ma=$o.select(Go);ma.on=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(q(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(q(n,t,e))};var _a=$o.map({mouseenter:"mouseover",mouseleave:"mouseout"});_a.forEach(function(n){"on"+n in Jo&&_a.remove(n)});var ba="onselectstart"in Jo?null:a(Go.style,"userSelect"),wa=0;$o.mouse=function(n){return P(n,h())};var Sa=/WebKit/.test(Ko.navigator.userAgent)?-1:0;$o.touches=function(n,t){return arguments.length<2&&(t=h().touches),t?Wo(t).map(function(t){var e=P(n,t);return e.identifier=t.identifier,e}):[]},$o.behavior.drag=function(){function n(){this.on("mousedown.drag",o).on("touchstart.drag",a)}function t(){return $o.event.changedTouches[0].identifier}function e(n,t){return $o.touches(n).filter(function(n){return n.identifier===t})[0]}function r(n,t,e,r){return function(){function o(){var n=t(l,g),e=n[0]-v[0],r=n[1]-v[1];d|=e|r,v=n,f({type:"drag",x:n[0]+c[0],y:n[1]+c[1],dx:e,dy:r})}function a(){m.on(e+"."+p,null).on(r+"."+p,null),y(d&&$o.event.target===h),f({type:"dragend"})}var c,s=this,l=s.parentNode,f=u.of(s,arguments),h=$o.event.target,g=n(),p=null==g?"drag":"drag-"+g,v=t(l,g),d=0,m=$o.select(Ko).on(e+"."+p,o).on(r+"."+p,a),y=D();i?(c=i.apply(s,arguments),c=[c.x-v[0],c.y-v[1]]):c=[0,0],f({type:"dragstart"})}}var u=g(n,"drag","dragstart","dragend"),i=null,o=r(c,$o.mouse,"mousemove","mouseup"),a=r(t,e,"touchmove","touchend");return n.origin=function(t){return arguments.length?(i=t,n):i},$o.rebind(n,u,"on")};var ka=Math.PI,Ea=2*ka,Aa=ka/2,Ca=1e-6,Na=Ca*Ca,La=ka/180,Ta=180/ka,qa=Math.SQRT2,za=2,Ra=4;$o.interpolateZoom=function(n,t){function e(n){var t=n*y;if(m){var e=O(v),o=i/(za*h)*(e*Y(qa*t+v)-F(v));return[r+o*s,u+o*l,i*e/O(qa*t+v)]}return[r+n*s,u+n*l,i*Math.exp(qa*t)]}var r=n[0],u=n[1],i=n[2],o=t[0],a=t[1],c=t[2],s=o-r,l=a-u,f=s*s+l*l,h=Math.sqrt(f),g=(c*c-i*i+Ra*f)/(2*i*za*h),p=(c*c-i*i-Ra*f)/(2*c*za*h),v=Math.log(Math.sqrt(g*g+1)-g),d=Math.log(Math.sqrt(p*p+1)-p),m=d-v,y=(m||Math.log(c/i))/qa;return e.duration=1e3*y,e},$o.behavior.zoom=function(){function n(n){n.on(A,s).on(Ua+".zoom",h).on(C,p).on("dblclick.zoom",v).on(L,l)}function t(n){return[(n[0]-S.x)/S.k,(n[1]-S.y)/S.k]}function e(n){return[n[0]*S.k+S.x,n[1]*S.k+S.y]}function r(n){S.k=Math.max(E[0],Math.min(E[1],n))}function u(n,t){t=e(t),S.x+=n[0]-t[0],S.y+=n[1]-t[1]}function i(){_&&_.domain(M.range().map(function(n){return(n-S.x)/S.k}).map(M.invert)),w&&w.domain(b.range().map(function(n){return(n-S.y)/S.k}).map(b.invert))}function o(n){n({type:"zoomstart"})}function a(n){i(),n({type:"zoom",scale:S.k,translate:[S.x,S.y]})}function c(n){n({type:"zoomend"})}function s(){function n(){l=1,u($o.mouse(r),h),a(i)}function e(){f.on(C,Ko===r?p:null).on(N,null),g(l&&$o.event.target===s),c(i)}var r=this,i=q.of(r,arguments),s=$o.event.target,l=0,f=$o.select(Ko).on(C,n).on(N,e),h=t($o.mouse(r)),g=D();T.call(r),o(i)}function l(){function n(){var n=$o.touches(p);return g=S.k,n.forEach(function(n){n.identifier in d&&(d[n.identifier]=t(n))}),n}function e(){for(var t=$o.event.changedTouches,e=0,i=t.length;i>e;++e)d[t[e].identifier]=null;var o=n(),c=Date.now();if(1===o.length){if(500>c-x){var s=o[0],l=d[s.identifier];r(2*S.k),u(s,l),f(),a(v)}x=c}else if(o.length>1){var s=o[0],h=o[1],g=s[0]-h[0],p=s[1]-h[1];m=g*g+p*p}}function i(){for(var n,t,e,i,o=$o.touches(p),c=0,s=o.length;s>c;++c,i=null)if(e=o[c],i=d[e.identifier]){if(t)break;n=e,t=i}if(i){var l=(l=e[0]-n[0])*l+(l=e[1]-n[1])*l,f=m&&Math.sqrt(l/m);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+i[0])/2,(t[1]+i[1])/2],r(f*g)}x=null,u(n,t),a(v)}function h(){if($o.event.touches.length){for(var t=$o.event.changedTouches,e=0,r=t.length;r>e;++e)delete d[t[e].identifier];for(var u in d)return void n()}b.on(M,null).on(_,null),w.on(A,s).on(L,l),k(),c(v)}var g,p=this,v=q.of(p,arguments),d={},m=0,y=$o.event.changedTouches[0].identifier,M="touchmove.zoom-"+y,_="touchend.zoom-"+y,b=$o.select(Ko).on(M,i).on(_,h),w=$o.select(p).on(A,null).on(L,e),k=D();T.call(p),e(),o(v)}function h(){var n=q.of(this,arguments);y?clearTimeout(y):(T.call(this),o(n)),y=setTimeout(function(){y=null,c(n)},50),f();var e=m||$o.mouse(this);d||(d=t(e)),r(Math.pow(2,.002*Da())*S.k),u(e,d),a(n)}function p(){d=null}function v(){var n=q.of(this,arguments),e=$o.mouse(this),i=t(e),s=Math.log(S.k)/Math.LN2;o(n),r(Math.pow(2,$o.event.shiftKey?Math.ceil(s)-1:Math.floor(s)+1)),u(e,i),a(n),c(n)}var d,m,y,x,M,_,b,w,S={x:0,y:0,k:1},k=[960,500],E=Pa,A="mousedown.zoom",C="mousemove.zoom",N="mouseup.zoom",L="touchstart.zoom",q=g(n,"zoomstart","zoom","zoomend");return n.event=function(n){n.each(function(){var n=q.of(this,arguments),t=S;Ss?$o.select(this).transition().each("start.zoom",function(){S=this.__chart__||{x:0,y:0,k:1},o(n)}).tween("zoom:zoom",function(){var e=k[0],r=k[1],u=e/2,i=r/2,o=$o.interpolateZoom([(u-S.x)/S.k,(i-S.y)/S.k,e/S.k],[(u-t.x)/t.k,(i-t.y)/t.k,e/t.k]);return function(t){var r=o(t),c=e/r[2];this.__chart__=S={x:u-r[0]*c,y:i-r[1]*c,k:c},a(n)}}).each("end.zoom",function(){c(n)}):(this.__chart__=S,o(n),a(n),c(n))})},n.translate=function(t){return arguments.length?(S={x:+t[0],y:+t[1],k:S.k},i(),n):[S.x,S.y]},n.scale=function(t){return arguments.length?(S={x:S.x,y:S.y,k:+t},i(),n):S.k},n.scaleExtent=function(t){return arguments.length?(E=null==t?Pa:[+t[0],+t[1]],n):E},n.center=function(t){return arguments.length?(m=t&&[+t[0],+t[1]],n):m},n.size=function(t){return arguments.length?(k=t&&[+t[0],+t[1]],n):k},n.x=function(t){return arguments.length?(_=t,M=t.copy(),S={x:0,y:0,k:1},n):_},n.y=function(t){return arguments.length?(w=t,b=t.copy(),S={x:0,y:0,k:1},n):w},$o.rebind(n,q,"on")};var Da,Pa=[0,1/0],Ua="onwheel"in Jo?(Da=function(){return-$o.event.deltaY*($o.event.deltaMode?120:1)},"wheel"):"onmousewheel"in Jo?(Da=function(){return $o.event.wheelDelta},"mousewheel"):(Da=function(){return-$o.event.detail},"MozMousePixelScroll");Z.prototype.toString=function(){return this.rgb()+""},$o.hsl=function(n,t,e){return 1===arguments.length?n instanceof X?V(n.h,n.s,n.l):st(""+n,lt,V):V(+n,+t,+e)};var ja=X.prototype=new Z;ja.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),V(this.h,this.s,this.l/n)},ja.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),V(this.h,this.s,n*this.l)},ja.rgb=function(){return $(this.h,this.s,this.l)},$o.hcl=function(n,t,e){return 1===arguments.length?n instanceof W?B(n.h,n.c,n.l):n instanceof K?nt(n.l,n.a,n.b):nt((n=ft((n=$o.rgb(n)).r,n.g,n.b)).l,n.a,n.b):B(+n,+t,+e)};var Ha=W.prototype=new Z;Ha.brighter=function(n){return B(this.h,this.c,Math.min(100,this.l+Fa*(arguments.length?n:1)))},Ha.darker=function(n){return B(this.h,this.c,Math.max(0,this.l-Fa*(arguments.length?n:1)))},Ha.rgb=function(){return J(this.h,this.c,this.l).rgb()},$o.lab=function(n,t,e){return 1===arguments.length?n instanceof K?G(n.l,n.a,n.b):n instanceof W?J(n.l,n.c,n.h):ft((n=$o.rgb(n)).r,n.g,n.b):G(+n,+t,+e)};var Fa=18,Oa=.95047,Ya=1,Ia=1.08883,Za=K.prototype=new Z;Za.brighter=function(n){return G(Math.min(100,this.l+Fa*(arguments.length?n:1)),this.a,this.b)},Za.darker=function(n){return G(Math.max(0,this.l-Fa*(arguments.length?n:1)),this.a,this.b)},Za.rgb=function(){return Q(this.l,this.a,this.b)},$o.rgb=function(n,t,e){return 1===arguments.length?n instanceof at?ot(n.r,n.g,n.b):st(""+n,ot,$):ot(~~n,~~t,~~e)};var Va=at.prototype=new Z;Va.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,u=30;return t||e||r?(t&&u>t&&(t=u),e&&u>e&&(e=u),r&&u>r&&(r=u),ot(Math.min(255,~~(t/n)),Math.min(255,~~(e/n)),Math.min(255,~~(r/n)))):ot(u,u,u)},Va.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),ot(~~(n*this.r),~~(n*this.g),~~(n*this.b))},Va.hsl=function(){return lt(this.r,this.g,this.b)},Va.toString=function(){return"#"+ct(this.r)+ct(this.g)+ct(this.b)};var Xa=$o.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074});Xa.forEach(function(n,t){Xa.set(n,ut(t))}),$o.functor=pt,$o.xhr=dt(vt),$o.dsv=function(n,t){function e(n,e,i){arguments.length<3&&(i=e,e=null);var o=$o.xhr(n,t,i);return o.row=function(n){return arguments.length?o.response(null==(e=n)?r:u(n)):e},o.row(e)}function r(n){return e.parse(n.responseText)}function u(n){return function(t){return e.parse(t.responseText,n)}}function o(t){return t.map(a).join(n)}function a(n){return c.test(n)?'"'+n.replace(/\"/g,'""')+'"':n}var c=new RegExp('["'+n+"\n]"),s=n.charCodeAt(0);return e.parse=function(n,t){var r;return e.parseRows(n,function(n,e){if(r)return r(n,e-1);var u=new Function("d","return {"+n.map(function(n,t){return JSON.stringify(n)+": d["+t+"]"}).join(",")+"}");r=t?function(n,e){return t(u(n),e)}:u})},e.parseRows=function(n,t){function e(){if(l>=c)return o;if(u)return u=!1,i;var t=l;if(34===n.charCodeAt(t)){for(var e=t;e++<c;)if(34===n.charCodeAt(e)){if(34!==n.charCodeAt(e+1))break;++e}l=e+2;var r=n.charCodeAt(e+1);return 13===r?(u=!0,10===n.charCodeAt(e+2)&&++l):10===r&&(u=!0),n.substring(t+1,e).replace(/""/g,'"')}for(;c>l;){var r=n.charCodeAt(l++),a=1;if(10===r)u=!0;else if(13===r)u=!0,10===n.charCodeAt(l)&&(++l,++a);else if(r!==s)continue;return n.substring(t,l-a)}return n.substring(t)}for(var r,u,i={},o={},a=[],c=n.length,l=0,f=0;(r=e())!==o;){for(var h=[];r!==i&&r!==o;)h.push(r),r=e();(!t||(h=t(h,f++)))&&a.push(h)}return a},e.format=function(t){if(Array.isArray(t[0]))return e.formatRows(t);var r=new i,u=[];return t.forEach(function(n){for(var t in n)r.has(t)||u.push(r.add(t))}),[u.map(a).join(n)].concat(t.map(function(t){return u.map(function(n){return a(t[n])}).join(n)})).join("\n")},e.formatRows=function(n){return n.map(o).join("\n")},e},$o.csv=$o.dsv(",","text/csv"),$o.tsv=$o.dsv(" ","text/tab-separated-values");var $a,Ba,Wa,Ja,Ga,Ka=Ko[a(Ko,"requestAnimationFrame")]||function(n){setTimeout(n,17)};$o.timer=function(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var u=e+t,i={c:n,t:u,f:!1,n:null};Ba?Ba.n=i:$a=i,Ba=i,Wa||(Ja=clearTimeout(Ja),Wa=1,Ka(xt))},$o.timer.flush=function(){Mt(),_t()};var Qa=".",nc=",",tc=[3,3],ec="$",rc=["y","z","a","f","p","n","\xb5","m","","k","M","G","T","P","E","Z","Y"].map(bt);$o.formatPrefix=function(n,t){var e=0;return n&&(0>n&&(n*=-1),t&&(n=$o.round(n,wt(n,t))),e=1+Math.floor(1e-12+Math.log(n)/Math.LN10),e=Math.max(-24,Math.min(24,3*Math.floor((0>=e?e+1:e-1)/3)))),rc[8+e/3]},$o.round=function(n,t){return t?Math.round(n*(t=Math.pow(10,t)))/t:Math.round(n)},$o.format=function(n){var t=uc.exec(n),e=t[1]||" ",r=t[2]||">",u=t[3]||"",i=t[4]||"",o=t[5],a=+t[6],c=t[7],s=t[8],l=t[9],f=1,h="",g=!1;switch(s&&(s=+s.substring(1)),(o||"0"===e&&"="===r)&&(o=e="0",r="=",c&&(a-=Math.floor((a-1)/4))),l){case"n":c=!0,l="g";break;case"%":f=100,h="%",l="f";break;case"p":f=100,h="%",l="r";break;case"b":case"o":case"x":case"X":"#"===i&&(i="0"+l.toLowerCase());case"c":case"d":g=!0,s=0;break;case"s":f=-1,l="r"}"#"===i?i="":"$"===i&&(i=ec),"r"!=l||s||(l="g"),null!=s&&("g"==l?s=Math.max(1,Math.min(21,s)):("e"==l||"f"==l)&&(s=Math.max(0,Math.min(20,s)))),l=ic.get(l)||St;var p=o&&c;return function(n){if(g&&n%1)return"";var t=0>n||0===n&&0>1/n?(n=-n,"-"):u;if(0>f){var v=$o.formatPrefix(n,s);n=v.scale(n),h=v.symbol}else n*=f;n=l(n,s);var d=n.lastIndexOf("."),m=0>d?n:n.substring(0,d),y=0>d?"":Qa+n.substring(d+1);!o&&c&&(m=oc(m));var x=i.length+m.length+y.length+(p?0:t.length),M=a>x?new Array(x=a-x+1).join(e):"";return p&&(m=oc(M+m)),t+=i,n=m+y,("<"===r?t+n+M:">"===r?M+t+n:"^"===r?M.substring(0,x>>=1)+t+n+M.substring(x):t+(p?n:M+n))+h}};var uc=/(?:([^{])?([<>=^]))?([+\- ])?([$#])?(0)?(\d+)?(,)?(\.-?\d+)?([a-z%])?/i,ic=$o.map({b:function(n){return n.toString(2)},c:function(n){return String.fromCharCode(n)},o:function(n){return n.toString(8)},x:function(n){return n.toString(16)},X:function(n){return n.toString(16).toUpperCase()},g:function(n,t){return n.toPrecision(t)},e:function(n,t){return n.toExponential(t)},f:function(n,t){return n.toFixed(t)},r:function(n,t){return(n=$o.round(n,wt(n,t))).toFixed(Math.max(0,Math.min(20,wt(n*(1+1e-15),t))))}}),oc=vt;if(tc){var ac=tc.length;oc=function(n){for(var t=n.length,e=[],r=0,u=tc[0];t>0&&u>0;)e.push(n.substring(t-=u,t+u)),u=tc[r=(r+1)%ac];return e.reverse().join(nc)}}$o.geo={},kt.prototype={s:0,t:0,add:function(n){Et(n,this.t,cc),Et(cc.s,this.s,this),this.s?this.t+=cc.t:this.s=cc.t},reset:function(){this.s=this.t=0},valueOf:function(){return this.s}};var cc=new kt;$o.geo.stream=function(n,t){n&&sc.hasOwnProperty(n.type)?sc[n.type](n,t):At(n,t)};var sc={Feature:function(n,t){At(n.geometry,t)},FeatureCollection:function(n,t){for(var e=n.features,r=-1,u=e.length;++r<u;)At(e[r].geometry,t)}},lc={Sphere:function(n,t){t.sphere()},Point:function(n,t){n=n.coordinates,t.point(n[0],n[1],n[2])},MultiPoint:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)n=e[r],t.point(n[0],n[1],n[2])},LineString:function(n,t){Ct(n.coordinates,t,0)},MultiLineString:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)Ct(e[r],t,0)},Polygon:function(n,t){Nt(n.coordinates,t)},MultiPolygon:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)Nt(e[r],t)},GeometryCollection:function(n,t){for(var e=n.geometries,r=-1,u=e.length;++r<u;)At(e[r],t)}};$o.geo.area=function(n){return fc=0,$o.geo.stream(n,gc),fc};var fc,hc=new kt,gc={sphere:function(){fc+=4*ka},point:c,lineStart:c,lineEnd:c,polygonStart:function(){hc.reset(),gc.lineStart=Lt},polygonEnd:function(){var n=2*hc;fc+=0>n?4*ka+n:n,gc.lineStart=gc.lineEnd=gc.point=c}};$o.geo.bounds=function(){function n(n,t){x.push(M=[l=n,h=n]),f>t&&(f=t),t>g&&(g=t)}function t(t,e){var r=Tt([t*La,e*La]);if(m){var u=zt(m,r),i=[u[1],-u[0],0],o=zt(i,u);Pt(o),o=Ut(o);var c=t-p,s=c>0?1:-1,v=o[0]*Ta*s,d=aa(c)>180;if(d^(v>s*p&&s*t>v)){var y=o[1]*Ta;y>g&&(g=y)}else if(v=(v+360)%360-180,d^(v>s*p&&s*t>v)){var y=-o[1]*Ta;f>y&&(f=y)}else f>e&&(f=e),e>g&&(g=e);d?p>t?a(l,t)>a(l,h)&&(h=t):a(t,h)>a(l,h)&&(l=t):h>=l?(l>t&&(l=t),t>h&&(h=t)):t>p?a(l,t)>a(l,h)&&(h=t):a(t,h)>a(l,h)&&(l=t)}else n(t,e);m=r,p=t}function e(){_.point=t}function r(){M[0]=l,M[1]=h,_.point=n,m=null}function u(n,e){if(m){var r=n-p;y+=aa(r)>180?r+(r>0?360:-360):r}else v=n,d=e;gc.point(n,e),t(n,e)}function i(){gc.lineStart()}function o(){u(v,d),gc.lineEnd(),aa(y)>Ca&&(l=-(h=180)),M[0]=l,M[1]=h,m=null}function a(n,t){return(t-=n)<0?t+360:t}function c(n,t){return n[0]-t[0]}function s(n,t){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:n<t[0]||t[1]<n}var l,f,h,g,p,v,d,m,y,x,M,_={point:n,lineStart:e,lineEnd:r,polygonStart:function(){_.point=u,_.lineStart=i,_.lineEnd=o,y=0,gc.polygonStart()},polygonEnd:function(){gc.polygonEnd(),_.point=n,_.lineStart=e,_.lineEnd=r,0>hc?(l=-(h=180),f=-(g=90)):y>Ca?g=90:-Ca>y&&(f=-90),M[0]=l,M[1]=h}};return function(n){g=h=-(l=f=1/0),x=[],$o.geo.stream(n,_);var t=x.length;if(t){x.sort(c);for(var e,r=1,u=x[0],i=[u];t>r;++r)e=x[r],s(e[0],u)||s(e[1],u)?(a(u[0],e[1])>a(u[0],u[1])&&(u[1]=e[1]),a(e[0],u[1])>a(u[0],u[1])&&(u[0]=e[0])):i.push(u=e);for(var o,e,p=-1/0,t=i.length-1,r=0,u=i[t];t>=r;u=e,++r)e=i[r],(o=a(u[1],e[0]))>p&&(p=o,l=e[0],h=u[1])}return x=M=null,1/0===l||1/0===f?[[0/0,0/0],[0/0,0/0]]:[[l,f],[h,g]]}}(),$o.geo.centroid=function(n){pc=vc=dc=mc=yc=xc=Mc=_c=bc=wc=Sc=0,$o.geo.stream(n,kc);var t=bc,e=wc,r=Sc,u=t*t+e*e+r*r;return Na>u&&(t=xc,e=Mc,r=_c,Ca>vc&&(t=dc,e=mc,r=yc),u=t*t+e*e+r*r,Na>u)?[0/0,0/0]:[Math.atan2(e,t)*Ta,H(r/Math.sqrt(u))*Ta]};var pc,vc,dc,mc,yc,xc,Mc,_c,bc,wc,Sc,kc={sphere:c,point:Ht,lineStart:Ot,lineEnd:Yt,polygonStart:function(){kc.lineStart=It},polygonEnd:function(){kc.lineStart=Ot}},Ec=Bt(Zt,Qt,te,[-ka,-ka/2]),Ac=1e9;$o.geo.clipExtent=function(){var n,t,e,r,u,i,o={stream:function(n){return u&&(u.valid=!1),u=i(n),u.valid=!0,u},extent:function(a){return arguments.length?(i=ue(n=+a[0][0],t=+a[0][1],e=+a[1][0],r=+a[1][1]),u&&(u.valid=!1,u=null),o):[[n,t],[e,r]]}};return o.extent([[0,0],[960,500]])},($o.geo.conicEqualArea=function(){return oe(ae)}).raw=ae,$o.geo.albers=function(){return $o.geo.conicEqualArea().rotate([96,0]).center([-.6,38.7]).parallels([29.5,45.5]).scale(1070)},$o.geo.albersUsa=function(){function n(n){var i=n[0],o=n[1];return t=null,e(i,o),t||(r(i,o),t)||u(i,o),t}var t,e,r,u,i=$o.geo.albers(),o=$o.geo.conicEqualArea().rotate([154,0]).center([-2,58.5]).parallels([55,65]),a=$o.geo.conicEqualArea().rotate([157,0]).center([-3,19.9]).parallels([8,18]),c={point:function(n,e){t=[n,e]}};return n.invert=function(n){var t=i.scale(),e=i.translate(),r=(n[0]-e[0])/t,u=(n[1]-e[1])/t;return(u>=.12&&.234>u&&r>=-.425&&-.214>r?o:u>=.166&&.234>u&&r>=-.214&&-.115>r?a:i).invert(n)},n.stream=function(n){var t=i.stream(n),e=o.stream(n),r=a.stream(n);return{point:function(n,u){t.point(n,u),e.point(n,u),r.point(n,u)},sphere:function(){t.sphere(),e.sphere(),r.sphere()},lineStart:function(){t.lineStart(),e.lineStart(),r.lineStart()},lineEnd:function(){t.lineEnd(),e.lineEnd(),r.lineEnd()},polygonStart:function(){t.polygonStart(),e.polygonStart(),r.polygonStart()},polygonEnd:function(){t.polygonEnd(),e.polygonEnd(),r.polygonEnd()}}},n.precision=function(t){return arguments.length?(i.precision(t),o.precision(t),a.precision(t),n):i.precision()},n.scale=function(t){return arguments.length?(i.scale(t),o.scale(.35*t),a.scale(t),n.translate(i.translate())):i.scale()},n.translate=function(t){if(!arguments.length)return i.translate();var s=i.scale(),l=+t[0],f=+t[1];return e=i.translate(t).clipExtent([[l-.455*s,f-.238*s],[l+.455*s,f+.238*s]]).stream(c).point,r=o.translate([l-.307*s,f+.201*s]).clipExtent([[l-.425*s+Ca,f+.12*s+Ca],[l-.214*s-Ca,f+.234*s-Ca]]).stream(c).point,u=a.translate([l-.205*s,f+.212*s]).clipExtent([[l-.214*s+Ca,f+.166*s+Ca],[l-.115*s-Ca,f+.234*s-Ca]]).stream(c).point,n},n.scale(1070)};var Cc,Nc,Lc,Tc,qc,zc,Rc={point:c,lineStart:c,lineEnd:c,polygonStart:function(){Nc=0,Rc.lineStart=ce},polygonEnd:function(){Rc.lineStart=Rc.lineEnd=Rc.point=c,Cc+=aa(Nc/2)}},Dc={point:se,lineStart:c,lineEnd:c,polygonStart:c,polygonEnd:c},Pc={point:he,lineStart:ge,lineEnd:pe,polygonStart:function(){Pc.lineStart=ve},polygonEnd:function(){Pc.point=he,Pc.lineStart=ge,Pc.lineEnd=pe}};$o.geo.path=function(){function n(n){return n&&("function"==typeof a&&i.pointRadius(+a.apply(this,arguments)),o&&o.valid||(o=u(i)),$o.geo.stream(n,o)),i.result()}function t(){return o=null,n}var e,r,u,i,o,a=4.5;return n.area=function(n){return Cc=0,$o.geo.stream(n,u(Rc)),Cc},n.centroid=function(n){return dc=mc=yc=xc=Mc=_c=bc=wc=Sc=0,$o.geo.stream(n,u(Pc)),Sc?[bc/Sc,wc/Sc]:_c?[xc/_c,Mc/_c]:yc?[dc/yc,mc/yc]:[0/0,0/0]},n.bounds=function(n){return qc=zc=-(Lc=Tc=1/0),$o.geo.stream(n,u(Dc)),[[Lc,Tc],[qc,zc]]},n.projection=function(n){return arguments.length?(u=(e=n)?n.stream||ye(n):vt,t()):e},n.context=function(n){return arguments.length?(i=null==(r=n)?new le:new de(n),"function"!=typeof a&&i.pointRadius(a),t()):r},n.pointRadius=function(t){return arguments.length?(a="function"==typeof t?t:(i.pointRadius(+t),+t),n):a},n.projection($o.geo.albersUsa()).context(null)},$o.geo.transform=function(n){return{stream:function(t){var e=new xe(t);for(var r in n)e[r]=n[r];return e}}},xe.prototype={point:function(n,t){this.stream.point(n,t)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart() -},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}},$o.geo.projection=_e,$o.geo.projectionMutator=be,($o.geo.equirectangular=function(){return _e(Se)}).raw=Se.invert=Se,$o.geo.rotation=function(n){function t(t){return t=n(t[0]*La,t[1]*La),t[0]*=Ta,t[1]*=Ta,t}return n=Ee(n[0]%360*La,n[1]*La,n.length>2?n[2]*La:0),t.invert=function(t){return t=n.invert(t[0]*La,t[1]*La),t[0]*=Ta,t[1]*=Ta,t},t},ke.invert=Se,$o.geo.circle=function(){function n(){var n="function"==typeof r?r.apply(this,arguments):r,t=Ee(-n[0]*La,-n[1]*La,0).invert,u=[];return e(null,null,1,{point:function(n,e){u.push(n=t(n,e)),n[0]*=Ta,n[1]*=Ta}}),{type:"Polygon",coordinates:[u]}}var t,e,r=[0,0],u=6;return n.origin=function(t){return arguments.length?(r=t,n):r},n.angle=function(r){return arguments.length?(e=Le((t=+r)*La,u*La),n):t},n.precision=function(r){return arguments.length?(e=Le(t*La,(u=+r)*La),n):u},n.angle(90)},$o.geo.distance=function(n,t){var e,r=(t[0]-n[0])*La,u=n[1]*La,i=t[1]*La,o=Math.sin(r),a=Math.cos(r),c=Math.sin(u),s=Math.cos(u),l=Math.sin(i),f=Math.cos(i);return Math.atan2(Math.sqrt((e=f*o)*e+(e=s*l-c*f*a)*e),c*l+s*f*a)},$o.geo.graticule=function(){function n(){return{type:"MultiLineString",coordinates:t()}}function t(){return $o.range(Math.ceil(i/d)*d,u,d).map(h).concat($o.range(Math.ceil(s/m)*m,c,m).map(g)).concat($o.range(Math.ceil(r/p)*p,e,p).filter(function(n){return aa(n%d)>Ca}).map(l)).concat($o.range(Math.ceil(a/v)*v,o,v).filter(function(n){return aa(n%m)>Ca}).map(f))}var e,r,u,i,o,a,c,s,l,f,h,g,p=10,v=p,d=90,m=360,y=2.5;return n.lines=function(){return t().map(function(n){return{type:"LineString",coordinates:n}})},n.outline=function(){return{type:"Polygon",coordinates:[h(i).concat(g(c).slice(1),h(u).reverse().slice(1),g(s).reverse().slice(1))]}},n.extent=function(t){return arguments.length?n.majorExtent(t).minorExtent(t):n.minorExtent()},n.majorExtent=function(t){return arguments.length?(i=+t[0][0],u=+t[1][0],s=+t[0][1],c=+t[1][1],i>u&&(t=i,i=u,u=t),s>c&&(t=s,s=c,c=t),n.precision(y)):[[i,s],[u,c]]},n.minorExtent=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],a=+t[0][1],o=+t[1][1],r>e&&(t=r,r=e,e=t),a>o&&(t=a,a=o,o=t),n.precision(y)):[[r,a],[e,o]]},n.step=function(t){return arguments.length?n.majorStep(t).minorStep(t):n.minorStep()},n.majorStep=function(t){return arguments.length?(d=+t[0],m=+t[1],n):[d,m]},n.minorStep=function(t){return arguments.length?(p=+t[0],v=+t[1],n):[p,v]},n.precision=function(t){return arguments.length?(y=+t,l=qe(a,o,90),f=ze(r,e,y),h=qe(s,c,90),g=ze(i,u,y),n):y},n.majorExtent([[-180,-90+Ca],[180,90-Ca]]).minorExtent([[-180,-80-Ca],[180,80+Ca]])},$o.geo.greatArc=function(){function n(){return{type:"LineString",coordinates:[t||r.apply(this,arguments),e||u.apply(this,arguments)]}}var t,e,r=Re,u=De;return n.distance=function(){return $o.geo.distance(t||r.apply(this,arguments),e||u.apply(this,arguments))},n.source=function(e){return arguments.length?(r=e,t="function"==typeof e?null:e,n):r},n.target=function(t){return arguments.length?(u=t,e="function"==typeof t?null:t,n):u},n.precision=function(){return arguments.length?n:0},n},$o.geo.interpolate=function(n,t){return Pe(n[0]*La,n[1]*La,t[0]*La,t[1]*La)},$o.geo.length=function(n){return Uc=0,$o.geo.stream(n,jc),Uc};var Uc,jc={sphere:c,point:c,lineStart:Ue,lineEnd:c,polygonStart:c,polygonEnd:c},Hc=je(function(n){return Math.sqrt(2/(1+n))},function(n){return 2*Math.asin(n/2)});($o.geo.azimuthalEqualArea=function(){return _e(Hc)}).raw=Hc;var Fc=je(function(n){var t=Math.acos(n);return t&&t/Math.sin(t)},vt);($o.geo.azimuthalEquidistant=function(){return _e(Fc)}).raw=Fc,($o.geo.conicConformal=function(){return oe(He)}).raw=He,($o.geo.conicEquidistant=function(){return oe(Fe)}).raw=Fe;var Oc=je(function(n){return 1/n},Math.atan);($o.geo.gnomonic=function(){return _e(Oc)}).raw=Oc,Oe.invert=function(n,t){return[n,2*Math.atan(Math.exp(t))-Aa]},($o.geo.mercator=function(){return Ye(Oe)}).raw=Oe;var Yc=je(function(){return 1},Math.asin);($o.geo.orthographic=function(){return _e(Yc)}).raw=Yc;var Ic=je(function(n){return 1/(1+n)},function(n){return 2*Math.atan(n)});($o.geo.stereographic=function(){return _e(Ic)}).raw=Ic,Ie.invert=function(n,t){return[Math.atan2(F(n),Math.cos(t)),H(Math.sin(t)/O(n))]},($o.geo.transverseMercator=function(){return Ye(Ie)}).raw=Ie,$o.geom={},$o.geom.hull=function(n){function t(n){if(n.length<3)return[];var t,u,i,o,a,c,s,l,f,h,g,p,v=pt(e),d=pt(r),m=n.length,y=m-1,x=[],M=[],_=0;if(v===Ze&&r===Ve)t=n;else for(i=0,t=[];m>i;++i)t.push([+v.call(this,u=n[i],i),+d.call(this,u,i)]);for(i=1;m>i;++i)(t[i][1]<t[_][1]||t[i][1]==t[_][1]&&t[i][0]<t[_][0])&&(_=i);for(i=0;m>i;++i)i!==_&&(c=t[i][1]-t[_][1],a=t[i][0]-t[_][0],x.push({angle:Math.atan2(c,a),index:i}));for(x.sort(function(n,t){return n.angle-t.angle}),g=x[0].angle,h=x[0].index,f=0,i=1;y>i;++i){if(o=x[i].index,g==x[i].angle){if(a=t[h][0]-t[_][0],c=t[h][1]-t[_][1],s=t[o][0]-t[_][0],l=t[o][1]-t[_][1],a*a+c*c>=s*s+l*l){x[i].index=-1;continue}x[f].index=-1}g=x[i].angle,f=i,h=o}for(M.push(_),i=0,o=0;2>i;++o)x[o].index>-1&&(M.push(x[o].index),i++);for(p=M.length;y>o;++o)if(!(x[o].index<0)){for(;!Xe(M[p-2],M[p-1],x[o].index,t);)--p;M[p++]=x[o].index}var b=[];for(i=p-1;i>=0;--i)b.push(n[M[i]]);return b}var e=Ze,r=Ve;return arguments.length?t(n):(t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t)},$o.geom.polygon=function(n){return ha(n,Zc),n};var Zc=$o.geom.polygon.prototype=[];Zc.area=function(){for(var n,t=-1,e=this.length,r=this[e-1],u=0;++t<e;)n=r,r=this[t],u+=n[1]*r[0]-n[0]*r[1];return.5*u},Zc.centroid=function(n){var t,e,r=-1,u=this.length,i=0,o=0,a=this[u-1];for(arguments.length||(n=-1/(6*this.area()));++r<u;)t=a,a=this[r],e=t[0]*a[1]-a[0]*t[1],i+=(t[0]+a[0])*e,o+=(t[1]+a[1])*e;return[i*n,o*n]},Zc.clip=function(n){for(var t,e,r,u,i,o,a=We(n),c=-1,s=this.length-We(this),l=this[s-1];++c<s;){for(t=n.slice(),n.length=0,u=this[c],i=t[(r=t.length-a)-1],e=-1;++e<r;)o=t[e],$e(o,l,u)?($e(i,l,u)||n.push(Be(i,o,l,u)),n.push(o)):$e(i,l,u)&&n.push(Be(i,o,l,u)),i=o;a&&n.push(n[0]),l=u}return n};var Vc,Xc,$c,Bc,Wc,Jc=[],Gc=[];rr.prototype.prepare=function(){for(var n,t=this.edges,e=t.length;e--;)n=t[e].edge,n.b&&n.a||t.splice(e,1);return t.sort(ir),t.length},vr.prototype={start:function(){return this.edge.l===this.site?this.edge.a:this.edge.b},end:function(){return this.edge.l===this.site?this.edge.b:this.edge.a}},dr.prototype={insert:function(n,t){var e,r,u;if(n){if(t.P=n,t.N=n.N,n.N&&(n.N.P=t),n.N=t,n.R){for(n=n.R;n.L;)n=n.L;n.L=t}else n.R=t;e=n}else this._?(n=Mr(this._),t.P=null,t.N=n,n.P=n.L=t,e=n):(t.P=t.N=null,this._=t,e=null);for(t.L=t.R=null,t.U=e,t.C=!0,n=t;e&&e.C;)r=e.U,e===r.L?(u=r.R,u&&u.C?(e.C=u.C=!1,r.C=!0,n=r):(n===e.R&&(yr(this,e),n=e,e=n.U),e.C=!1,r.C=!0,xr(this,r))):(u=r.L,u&&u.C?(e.C=u.C=!1,r.C=!0,n=r):(n===e.L&&(xr(this,e),n=e,e=n.U),e.C=!1,r.C=!0,yr(this,r))),e=n.U;this._.C=!1},remove:function(n){n.N&&(n.N.P=n.P),n.P&&(n.P.N=n.N),n.N=n.P=null;var t,e,r,u=n.U,i=n.L,o=n.R;if(e=i?o?Mr(o):i:o,u?u.L===n?u.L=e:u.R=e:this._=e,i&&o?(r=e.C,e.C=n.C,e.L=i,i.U=e,e!==o?(u=e.U,e.U=n.U,n=e.R,u.L=n,e.R=o,o.U=e):(e.U=u,u=e,n=e.R)):(r=n.C,n=e),n&&(n.U=u),!r){if(n&&n.C)return n.C=!1,void 0;do{if(n===this._)break;if(n===u.L){if(t=u.R,t.C&&(t.C=!1,u.C=!0,yr(this,u),t=u.R),t.L&&t.L.C||t.R&&t.R.C){t.R&&t.R.C||(t.L.C=!1,t.C=!0,xr(this,t),t=u.R),t.C=u.C,u.C=t.R.C=!1,yr(this,u),n=this._;break}}else if(t=u.L,t.C&&(t.C=!1,u.C=!0,xr(this,u),t=u.L),t.L&&t.L.C||t.R&&t.R.C){t.L&&t.L.C||(t.R.C=!1,t.C=!0,yr(this,t),t=u.L),t.C=u.C,u.C=t.L.C=!1,xr(this,u),n=this._;break}t.C=!0,n=u,u=u.U}while(!n.C);n&&(n.C=!1)}}},$o.geom.voronoi=function(n){function t(n){var t=new Array(n.length),r=a[0][0],u=a[0][1],i=a[1][0],o=a[1][1];return _r(e(n),a).cells.forEach(function(e,a){var c=e.edges,s=e.site,l=t[a]=c.length?c.map(function(n){var t=n.start();return[t.x,t.y]}):s.x>=r&&s.x<=i&&s.y>=u&&s.y<=o?[[r,o],[i,o],[i,u],[r,u]]:[];l.point=n[a]}),t}function e(n){return n.map(function(n,t){return{x:Math.round(i(n,t)/Ca)*Ca,y:Math.round(o(n,t)/Ca)*Ca,i:t}})}var r=Ze,u=Ve,i=r,o=u,a=Kc;return n?t(n):(t.links=function(n){return _r(e(n)).edges.filter(function(n){return n.l&&n.r}).map(function(t){return{source:n[t.l.i],target:n[t.r.i]}})},t.triangles=function(n){var t=[];return _r(e(n)).cells.forEach(function(e,r){for(var u,i,o=e.site,a=e.edges.sort(ir),c=-1,s=a.length,l=a[s-1].edge,f=l.l===o?l.r:l.l;++c<s;)u=l,i=f,l=a[c].edge,f=l.l===o?l.r:l.l,r<i.i&&r<f.i&&wr(o,i,f)<0&&t.push([n[r],n[i.i],n[f.i]])}),t},t.x=function(n){return arguments.length?(i=pt(r=n),t):r},t.y=function(n){return arguments.length?(o=pt(u=n),t):u},t.clipExtent=function(n){return arguments.length?(a=null==n?Kc:n,t):a===Kc?null:a},t.size=function(n){return arguments.length?t.clipExtent(n&&[[0,0],n]):a===Kc?null:a&&a[1]},t)};var Kc=[[-1e6,-1e6],[1e6,1e6]];$o.geom.delaunay=function(n){return $o.geom.voronoi().triangles(n)},$o.geom.quadtree=function(n,t,e,r,u){function i(n){function i(n,t,e,r,u,i,o,a){if(!isNaN(e)&&!isNaN(r))if(n.leaf){var c=n.x,l=n.y;if(null!=c)if(aa(c-e)+aa(l-r)<.01)s(n,t,e,r,u,i,o,a);else{var f=n.point;n.x=n.y=n.point=null,s(n,f,c,l,u,i,o,a),s(n,t,e,r,u,i,o,a)}else n.x=e,n.y=r,n.point=t}else s(n,t,e,r,u,i,o,a)}function s(n,t,e,r,u,o,a,c){var s=.5*(u+a),l=.5*(o+c),f=e>=s,h=r>=l,g=(h<<1)+f;n.leaf=!1,n=n.nodes[g]||(n.nodes[g]=Er()),f?u=s:a=s,h?o=l:c=l,i(n,t,e,r,u,o,a,c)}var l,f,h,g,p,v,d,m,y,x=pt(a),M=pt(c);if(null!=t)v=t,d=e,m=r,y=u;else if(m=y=-(v=d=1/0),f=[],h=[],p=n.length,o)for(g=0;p>g;++g)l=n[g],l.x<v&&(v=l.x),l.y<d&&(d=l.y),l.x>m&&(m=l.x),l.y>y&&(y=l.y),f.push(l.x),h.push(l.y);else for(g=0;p>g;++g){var _=+x(l=n[g],g),b=+M(l,g);v>_&&(v=_),d>b&&(d=b),_>m&&(m=_),b>y&&(y=b),f.push(_),h.push(b)}var w=m-v,S=y-d;w>S?y=d+w:m=v+S;var k=Er();if(k.add=function(n){i(k,n,+x(n,++g),+M(n,g),v,d,m,y)},k.visit=function(n){Ar(n,k,v,d,m,y)},g=-1,null==t){for(;++g<p;)i(k,n[g],f[g],h[g],v,d,m,y);--g}else n.forEach(k.add);return f=h=n=l=null,k}var o,a=Ze,c=Ve;return(o=arguments.length)?(a=Sr,c=kr,3===o&&(u=e,r=t,e=t=0),i(n)):(i.x=function(n){return arguments.length?(a=n,i):a},i.y=function(n){return arguments.length?(c=n,i):c},i.extent=function(n){return arguments.length?(null==n?t=e=r=u=null:(t=+n[0][0],e=+n[0][1],r=+n[1][0],u=+n[1][1]),i):null==t?null:[[t,e],[r,u]]},i.size=function(n){return arguments.length?(null==n?t=e=r=u=null:(t=e=0,r=+n[0],u=+n[1]),i):null==t?null:[r-t,u-e]},i)},$o.interpolateRgb=Cr,$o.interpolateObject=Nr,$o.interpolateNumber=Lr,$o.interpolateString=Tr;var Qc=/[-+]?(?:\d+\.?\d*|\.?\d+)(?:[eE][-+]?\d+)?/g;$o.interpolate=qr,$o.interpolators=[function(n,t){var e=typeof t;return("string"===e?Xa.has(t)||/^(#|rgb\(|hsl\()/.test(t)?Cr:Tr:t instanceof Z?Cr:"object"===e?Array.isArray(t)?zr:Nr:Lr)(n,t)}],$o.interpolateArray=zr;var ns=function(){return vt},ts=$o.map({linear:ns,poly:Fr,quad:function(){return Ur},cubic:function(){return jr},sin:function(){return Or},exp:function(){return Yr},circle:function(){return Ir},elastic:Zr,back:Vr,bounce:function(){return Xr}}),es=$o.map({"in":vt,out:Dr,"in-out":Pr,"out-in":function(n){return Pr(Dr(n))}});$o.ease=function(n){var t=n.indexOf("-"),e=t>=0?n.substring(0,t):n,r=t>=0?n.substring(t+1):"in";return e=ts.get(e)||ns,r=es.get(r)||vt,Rr(r(e.apply(null,Bo.call(arguments,1))))},$o.interpolateHcl=$r,$o.interpolateHsl=Br,$o.interpolateLab=Wr,$o.interpolateRound=Jr,$o.transform=function(n){var t=Jo.createElementNS($o.ns.prefix.svg,"g");return($o.transform=function(n){if(null!=n){t.setAttribute("transform",n);var e=t.transform.baseVal.consolidate()}return new Gr(e?e.matrix:rs)})(n)},Gr.prototype.toString=function(){return"translate("+this.translate+")rotate("+this.rotate+")skewX("+this.skew+")scale("+this.scale+")"};var rs={a:1,b:0,c:0,d:1,e:0,f:0};$o.interpolateTransform=tu,$o.layout={},$o.layout.bundle=function(){return function(n){for(var t=[],e=-1,r=n.length;++e<r;)t.push(uu(n[e]));return t}},$o.layout.chord=function(){function n(){var n,s,f,h,g,p={},v=[],d=$o.range(i),m=[];for(e=[],r=[],n=0,h=-1;++h<i;){for(s=0,g=-1;++g<i;)s+=u[h][g];v.push(s),m.push($o.range(i)),n+=s}for(o&&d.sort(function(n,t){return o(v[n],v[t])}),a&&m.forEach(function(n,t){n.sort(function(n,e){return a(u[t][n],u[t][e])})}),n=(Ea-l*i)/n,s=0,h=-1;++h<i;){for(f=s,g=-1;++g<i;){var y=d[h],x=m[y][g],M=u[y][x],_=s,b=s+=M*n;p[y+"-"+x]={index:y,subindex:x,startAngle:_,endAngle:b,value:M}}r[y]={index:y,startAngle:f,endAngle:s,value:(s-f)/n},s+=l}for(h=-1;++h<i;)for(g=h-1;++g<i;){var w=p[h+"-"+g],S=p[g+"-"+h];(w.value||S.value)&&e.push(w.value<S.value?{source:S,target:w}:{source:w,target:S})}c&&t()}function t(){e.sort(function(n,t){return c((n.source.value+n.target.value)/2,(t.source.value+t.target.value)/2)})}var e,r,u,i,o,a,c,s={},l=0;return s.matrix=function(n){return arguments.length?(i=(u=n)&&u.length,e=r=null,s):u},s.padding=function(n){return arguments.length?(l=n,e=r=null,s):l},s.sortGroups=function(n){return arguments.length?(o=n,e=r=null,s):o},s.sortSubgroups=function(n){return arguments.length?(a=n,e=null,s):a},s.sortChords=function(n){return arguments.length?(c=n,e&&t(),s):c},s.chords=function(){return e||n(),e},s.groups=function(){return r||n(),r},s},$o.layout.force=function(){function n(n){return function(t,e,r,u){if(t.point!==n){var i=t.cx-n.x,o=t.cy-n.y,a=1/Math.sqrt(i*i+o*o);if(v>(u-e)*a){var c=t.charge*a*a;return n.px-=i*c,n.py-=o*c,!0}if(t.point&&isFinite(a)){var c=t.pointCharge*a*a;n.px-=i*c,n.py-=o*c}}return!t.charge}}function t(n){n.px=$o.event.x,n.py=$o.event.y,a.resume()}var e,r,u,i,o,a={},c=$o.dispatch("start","tick","end"),s=[1,1],l=.9,f=us,h=is,g=-30,p=.1,v=.8,d=[],m=[];return a.tick=function(){if((r*=.99)<.005)return c.end({type:"end",alpha:r=0}),!0;var t,e,a,f,h,v,y,x,M,_=d.length,b=m.length;for(e=0;b>e;++e)a=m[e],f=a.source,h=a.target,x=h.x-f.x,M=h.y-f.y,(v=x*x+M*M)&&(v=r*i[e]*((v=Math.sqrt(v))-u[e])/v,x*=v,M*=v,h.x-=x*(y=f.weight/(h.weight+f.weight)),h.y-=M*y,f.x+=x*(y=1-y),f.y+=M*y);if((y=r*p)&&(x=s[0]/2,M=s[1]/2,e=-1,y))for(;++e<_;)a=d[e],a.x+=(x-a.x)*y,a.y+=(M-a.y)*y;if(g)for(fu(t=$o.geom.quadtree(d),r,o),e=-1;++e<_;)(a=d[e]).fixed||t.visit(n(a));for(e=-1;++e<_;)a=d[e],a.fixed?(a.x=a.px,a.y=a.py):(a.x-=(a.px-(a.px=a.x))*l,a.y-=(a.py-(a.py=a.y))*l);c.tick({type:"tick",alpha:r})},a.nodes=function(n){return arguments.length?(d=n,a):d},a.links=function(n){return arguments.length?(m=n,a):m},a.size=function(n){return arguments.length?(s=n,a):s},a.linkDistance=function(n){return arguments.length?(f="function"==typeof n?n:+n,a):f},a.distance=a.linkDistance,a.linkStrength=function(n){return arguments.length?(h="function"==typeof n?n:+n,a):h},a.friction=function(n){return arguments.length?(l=+n,a):l},a.charge=function(n){return arguments.length?(g="function"==typeof n?n:+n,a):g},a.gravity=function(n){return arguments.length?(p=+n,a):p},a.theta=function(n){return arguments.length?(v=+n,a):v},a.alpha=function(n){return arguments.length?(n=+n,r?r=n>0?n:0:n>0&&(c.start({type:"start",alpha:r=n}),$o.timer(a.tick)),a):r},a.start=function(){function n(n,r){if(!e){for(e=new Array(c),a=0;c>a;++a)e[a]=[];for(a=0;s>a;++a){var u=m[a];e[u.source.index].push(u.target),e[u.target.index].push(u.source)}}for(var i,o=e[t],a=-1,s=o.length;++a<s;)if(!isNaN(i=o[a][n]))return i;return Math.random()*r}var t,e,r,c=d.length,l=m.length,p=s[0],v=s[1];for(t=0;c>t;++t)(r=d[t]).index=t,r.weight=0;for(t=0;l>t;++t)r=m[t],"number"==typeof r.source&&(r.source=d[r.source]),"number"==typeof r.target&&(r.target=d[r.target]),++r.source.weight,++r.target.weight;for(t=0;c>t;++t)r=d[t],isNaN(r.x)&&(r.x=n("x",p)),isNaN(r.y)&&(r.y=n("y",v)),isNaN(r.px)&&(r.px=r.x),isNaN(r.py)&&(r.py=r.y);if(u=[],"function"==typeof f)for(t=0;l>t;++t)u[t]=+f.call(this,m[t],t);else for(t=0;l>t;++t)u[t]=f;if(i=[],"function"==typeof h)for(t=0;l>t;++t)i[t]=+h.call(this,m[t],t);else for(t=0;l>t;++t)i[t]=h;if(o=[],"function"==typeof g)for(t=0;c>t;++t)o[t]=+g.call(this,d[t],t);else for(t=0;c>t;++t)o[t]=g;return a.resume()},a.resume=function(){return a.alpha(.1)},a.stop=function(){return a.alpha(0)},a.drag=function(){return e||(e=$o.behavior.drag().origin(vt).on("dragstart.force",au).on("drag.force",t).on("dragend.force",cu)),arguments.length?(this.on("mouseover.force",su).on("mouseout.force",lu).call(e),void 0):e},$o.rebind(a,c,"on")};var us=20,is=1;$o.layout.hierarchy=function(){function n(t,o,a){var c=u.call(e,t,o);if(t.depth=o,a.push(t),c&&(s=c.length)){for(var s,l,f=-1,h=t.children=new Array(s),g=0,p=o+1;++f<s;)l=h[f]=n(c[f],p,a),l.parent=t,g+=l.value;r&&h.sort(r),i&&(t.value=g)}else delete t.children,i&&(t.value=+i.call(e,t,o)||0);return t}function t(n,r){var u=n.children,o=0;if(u&&(a=u.length))for(var a,c=-1,s=r+1;++c<a;)o+=t(u[c],s);else i&&(o=+i.call(e,n,r)||0);return i&&(n.value=o),o}function e(t){var e=[];return n(t,0,e),e}var r=vu,u=gu,i=pu;return e.sort=function(n){return arguments.length?(r=n,e):r},e.children=function(n){return arguments.length?(u=n,e):u},e.value=function(n){return arguments.length?(i=n,e):i},e.revalue=function(n){return t(n,0),n},e},$o.layout.partition=function(){function n(t,e,r,u){var i=t.children;if(t.x=e,t.y=t.depth*u,t.dx=r,t.dy=u,i&&(o=i.length)){var o,a,c,s=-1;for(r=t.value?r/t.value:0;++s<o;)n(a=i[s],e,c=a.value*r,u),e+=c}}function t(n){var e=n.children,r=0;if(e&&(u=e.length))for(var u,i=-1;++i<u;)r=Math.max(r,t(e[i]));return 1+r}function e(e,i){var o=r.call(this,e,i);return n(o[0],0,u[0],u[1]/t(o[0])),o}var r=$o.layout.hierarchy(),u=[1,1];return e.size=function(n){return arguments.length?(u=n,e):u},hu(e,r)},$o.layout.pie=function(){function n(i){var o=i.map(function(e,r){return+t.call(n,e,r)}),a=+("function"==typeof r?r.apply(this,arguments):r),c=(("function"==typeof u?u.apply(this,arguments):u)-a)/$o.sum(o),s=$o.range(i.length);null!=e&&s.sort(e===os?function(n,t){return o[t]-o[n]}:function(n,t){return e(i[n],i[t])});var l=[];return s.forEach(function(n){var t;l[n]={data:i[n],value:t=o[n],startAngle:a,endAngle:a+=t*c}}),l}var t=Number,e=os,r=0,u=Ea;return n.value=function(e){return arguments.length?(t=e,n):t},n.sort=function(t){return arguments.length?(e=t,n):e},n.startAngle=function(t){return arguments.length?(r=t,n):r},n.endAngle=function(t){return arguments.length?(u=t,n):u},n};var os={};$o.layout.stack=function(){function n(a,c){var s=a.map(function(e,r){return t.call(n,e,r)}),l=s.map(function(t){return t.map(function(t,e){return[i.call(n,t,e),o.call(n,t,e)]})}),f=e.call(n,l,c);s=$o.permute(s,f),l=$o.permute(l,f);var h,g,p,v=r.call(n,l,c),d=s.length,m=s[0].length;for(g=0;m>g;++g)for(u.call(n,s[0][g],p=v[g],l[0][g][1]),h=1;d>h;++h)u.call(n,s[h][g],p+=l[h-1][g][1],l[h][g][1]);return a}var t=vt,e=Mu,r=_u,u=xu,i=mu,o=yu;return n.values=function(e){return arguments.length?(t=e,n):t},n.order=function(t){return arguments.length?(e="function"==typeof t?t:as.get(t)||Mu,n):e},n.offset=function(t){return arguments.length?(r="function"==typeof t?t:cs.get(t)||_u,n):r},n.x=function(t){return arguments.length?(i=t,n):i},n.y=function(t){return arguments.length?(o=t,n):o},n.out=function(t){return arguments.length?(u=t,n):u},n};var as=$o.map({"inside-out":function(n){var t,e,r=n.length,u=n.map(bu),i=n.map(wu),o=$o.range(r).sort(function(n,t){return u[n]-u[t]}),a=0,c=0,s=[],l=[];for(t=0;r>t;++t)e=o[t],c>a?(a+=i[e],s.push(e)):(c+=i[e],l.push(e));return l.reverse().concat(s)},reverse:function(n){return $o.range(n.length).reverse()},"default":Mu}),cs=$o.map({silhouette:function(n){var t,e,r,u=n.length,i=n[0].length,o=[],a=0,c=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];r>a&&(a=r),o.push(r)}for(e=0;i>e;++e)c[e]=(a-o[e])/2;return c},wiggle:function(n){var t,e,r,u,i,o,a,c,s,l=n.length,f=n[0],h=f.length,g=[];for(g[0]=c=s=0,e=1;h>e;++e){for(t=0,u=0;l>t;++t)u+=n[t][e][1];for(t=0,i=0,a=f[e][0]-f[e-1][0];l>t;++t){for(r=0,o=(n[t][e][1]-n[t][e-1][1])/(2*a);t>r;++r)o+=(n[r][e][1]-n[r][e-1][1])/a;i+=o*n[t][e][1]}g[e]=c-=u?i/u*a:0,s>c&&(s=c)}for(e=0;h>e;++e)g[e]-=s;return g},expand:function(n){var t,e,r,u=n.length,i=n[0].length,o=1/u,a=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];if(r)for(t=0;u>t;t++)n[t][e][1]/=r;else for(t=0;u>t;t++)n[t][e][1]=o}for(e=0;i>e;++e)a[e]=0;return a},zero:_u});$o.layout.histogram=function(){function n(n,i){for(var o,a,c=[],s=n.map(e,this),l=r.call(this,s,i),f=u.call(this,l,s,i),i=-1,h=s.length,g=f.length-1,p=t?1:1/h;++i<g;)o=c[i]=[],o.dx=f[i+1]-(o.x=f[i]),o.y=0;if(g>0)for(i=-1;++i<h;)a=s[i],a>=l[0]&&a<=l[1]&&(o=c[$o.bisect(f,a,1,g)-1],o.y+=p,o.push(n[i]));return c}var t=!0,e=Number,r=Au,u=ku;return n.value=function(t){return arguments.length?(e=t,n):e},n.range=function(t){return arguments.length?(r=pt(t),n):r},n.bins=function(t){return arguments.length?(u="number"==typeof t?function(n){return Eu(n,t)}:pt(t),n):u},n.frequency=function(e){return arguments.length?(t=!!e,n):t},n},$o.layout.tree=function(){function n(n,i){function o(n,t){var r=n.children,u=n._tree;if(r&&(i=r.length)){for(var i,a,s,l=r[0],f=l,h=-1;++h<i;)s=r[h],o(s,a),f=c(s,a,f),a=s;Pu(n);var g=.5*(l._tree.prelim+s._tree.prelim);t?(u.prelim=t._tree.prelim+e(n,t),u.mod=u.prelim-g):u.prelim=g}else t&&(u.prelim=t._tree.prelim+e(n,t))}function a(n,t){n.x=n._tree.prelim+t;var e=n.children;if(e&&(r=e.length)){var r,u=-1;for(t+=n._tree.mod;++u<r;)a(e[u],t)}}function c(n,t,r){if(t){for(var u,i=n,o=n,a=t,c=n.parent.children[0],s=i._tree.mod,l=o._tree.mod,f=a._tree.mod,h=c._tree.mod;a=Lu(a),i=Nu(i),a&&i;)c=Nu(c),o=Lu(o),o._tree.ancestor=n,u=a._tree.prelim+f-i._tree.prelim-s+e(a,i),u>0&&(Uu(ju(a,n,r),n,u),s+=u,l+=u),f+=a._tree.mod,s+=i._tree.mod,h+=c._tree.mod,l+=o._tree.mod;a&&!Lu(o)&&(o._tree.thread=a,o._tree.mod+=f-l),i&&!Nu(c)&&(c._tree.thread=i,c._tree.mod+=s-h,r=n)}return r}var s=t.call(this,n,i),l=s[0];Du(l,function(n,t){n._tree={ancestor:n,prelim:0,mod:0,change:0,shift:0,number:t?t._tree.number+1:0}}),o(l),a(l,-l._tree.prelim);var f=Tu(l,zu),h=Tu(l,qu),g=Tu(l,Ru),p=f.x-e(f,h)/2,v=h.x+e(h,f)/2,d=g.depth||1;return Du(l,u?function(n){n.x*=r[0],n.y=n.depth*r[1],delete n._tree}:function(n){n.x=(n.x-p)/(v-p)*r[0],n.y=n.depth/d*r[1],delete n._tree}),s}var t=$o.layout.hierarchy().sort(null).value(null),e=Cu,r=[1,1],u=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(u=null==(r=t),n):u?null:r},n.nodeSize=function(t){return arguments.length?(u=null!=(r=t),n):u?r:null},hu(n,t)},$o.layout.pack=function(){function n(n,i){var o=e.call(this,n,i),a=o[0],c=u[0],s=u[1],l=null==t?Math.sqrt:"function"==typeof t?t:function(){return t};if(a.x=a.y=0,Du(a,function(n){n.r=+l(n.value)}),Du(a,Iu),r){var f=r*(t?1:Math.max(2*a.r/c,2*a.r/s))/2;Du(a,function(n){n.r+=f}),Du(a,Iu),Du(a,function(n){n.r-=f})}return Xu(a,c/2,s/2,t?1:1/Math.max(2*a.r/c,2*a.r/s)),o}var t,e=$o.layout.hierarchy().sort(Hu),r=0,u=[1,1];return n.size=function(t){return arguments.length?(u=t,n):u},n.radius=function(e){return arguments.length?(t=null==e||"function"==typeof e?e:+e,n):t},n.padding=function(t){return arguments.length?(r=+t,n):r},hu(n,e)},$o.layout.cluster=function(){function n(n,i){var o,a=t.call(this,n,i),c=a[0],s=0;Du(c,function(n){var t=n.children;t&&t.length?(n.x=Wu(t),n.y=Bu(t)):(n.x=o?s+=e(n,o):0,n.y=0,o=n)});var l=Ju(c),f=Gu(c),h=l.x-e(l,f)/2,g=f.x+e(f,l)/2;return Du(c,u?function(n){n.x=(n.x-c.x)*r[0],n.y=(c.y-n.y)*r[1]}:function(n){n.x=(n.x-h)/(g-h)*r[0],n.y=(1-(c.y?n.y/c.y:1))*r[1]}),a}var t=$o.layout.hierarchy().sort(null).value(null),e=Cu,r=[1,1],u=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(u=null==(r=t),n):u?null:r},n.nodeSize=function(t){return arguments.length?(u=null!=(r=t),n):u?r:null},hu(n,t)},$o.layout.treemap=function(){function n(n,t){for(var e,r,u=-1,i=n.length;++u<i;)r=(e=n[u]).value*(0>t?0:t),e.area=isNaN(r)||0>=r?0:r}function t(e){var i=e.children;if(i&&i.length){var o,a,c,s=f(e),l=[],h=i.slice(),p=1/0,v="slice"===g?s.dx:"dice"===g?s.dy:"slice-dice"===g?1&e.depth?s.dy:s.dx:Math.min(s.dx,s.dy);for(n(h,s.dx*s.dy/e.value),l.area=0;(c=h.length)>0;)l.push(o=h[c-1]),l.area+=o.area,"squarify"!==g||(a=r(l,v))<=p?(h.pop(),p=a):(l.area-=l.pop().area,u(l,v,s,!1),v=Math.min(s.dx,s.dy),l.length=l.area=0,p=1/0);l.length&&(u(l,v,s,!0),l.length=l.area=0),i.forEach(t)}}function e(t){var r=t.children;if(r&&r.length){var i,o=f(t),a=r.slice(),c=[];for(n(a,o.dx*o.dy/t.value),c.area=0;i=a.pop();)c.push(i),c.area+=i.area,null!=i.z&&(u(c,i.z?o.dx:o.dy,o,!a.length),c.length=c.area=0);r.forEach(e)}}function r(n,t){for(var e,r=n.area,u=0,i=1/0,o=-1,a=n.length;++o<a;)(e=n[o].area)&&(i>e&&(i=e),e>u&&(u=e));return r*=r,t*=t,r?Math.max(t*u*p/r,r/(t*i*p)):1/0}function u(n,t,e,r){var u,i=-1,o=n.length,a=e.x,s=e.y,l=t?c(n.area/t):0;if(t==e.dx){for((r||l>e.dy)&&(l=e.dy);++i<o;)u=n[i],u.x=a,u.y=s,u.dy=l,a+=u.dx=Math.min(e.x+e.dx-a,l?c(u.area/l):0);u.z=!0,u.dx+=e.x+e.dx-a,e.y+=l,e.dy-=l}else{for((r||l>e.dx)&&(l=e.dx);++i<o;)u=n[i],u.x=a,u.y=s,u.dx=l,s+=u.dy=Math.min(e.y+e.dy-s,l?c(u.area/l):0);u.z=!1,u.dy+=e.y+e.dy-s,e.x+=l,e.dx-=l}}function i(r){var u=o||a(r),i=u[0];return i.x=0,i.y=0,i.dx=s[0],i.dy=s[1],o&&a.revalue(i),n([i],i.dx*i.dy/i.value),(o?e:t)(i),h&&(o=u),u}var o,a=$o.layout.hierarchy(),c=Math.round,s=[1,1],l=null,f=Ku,h=!1,g="squarify",p=.5*(1+Math.sqrt(5));return i.size=function(n){return arguments.length?(s=n,i):s},i.padding=function(n){function t(t){var e=n.call(i,t,t.depth);return null==e?Ku(t):Qu(t,"number"==typeof e?[e,e,e,e]:e)}function e(t){return Qu(t,n)}if(!arguments.length)return l;var r;return f=null==(l=n)?Ku:"function"==(r=typeof n)?t:"number"===r?(n=[n,n,n,n],e):e,i},i.round=function(n){return arguments.length?(c=n?Math.round:Number,i):c!=Number},i.sticky=function(n){return arguments.length?(h=n,o=null,i):h},i.ratio=function(n){return arguments.length?(p=n,i):p},i.mode=function(n){return arguments.length?(g=n+"",i):g},hu(i,a)},$o.random={normal:function(n,t){var e=arguments.length;return 2>e&&(t=1),1>e&&(n=0),function(){var e,r,u;do e=2*Math.random()-1,r=2*Math.random()-1,u=e*e+r*r;while(!u||u>1);return n+t*e*Math.sqrt(-2*Math.log(u)/u)}},logNormal:function(){var n=$o.random.normal.apply($o,arguments);return function(){return Math.exp(n())}},irwinHall:function(n){return function(){for(var t=0,e=0;n>e;e++)t+=Math.random();return t/n}}},$o.scale={};var ss={floor:vt,ceil:vt};$o.scale.linear=function(){return oi([0,1],[0,1],qr,!1)};var ls={s:1,g:1,p:1,r:1,e:1};$o.scale.log=function(){return pi($o.scale.linear().domain([0,1]),10,!0,[1,10])};var fs=$o.format(".0e"),hs={floor:function(n){return-Math.ceil(-n)},ceil:function(n){return-Math.floor(-n)}};$o.scale.pow=function(){return vi($o.scale.linear(),1,[0,1])},$o.scale.sqrt=function(){return $o.scale.pow().exponent(.5)},$o.scale.ordinal=function(){return mi([],{t:"range",a:[[]]})},$o.scale.category10=function(){return $o.scale.ordinal().range(gs)},$o.scale.category20=function(){return $o.scale.ordinal().range(ps)},$o.scale.category20b=function(){return $o.scale.ordinal().range(vs)},$o.scale.category20c=function(){return $o.scale.ordinal().range(ds)};var gs=[2062260,16744206,2924588,14034728,9725885,9197131,14907330,8355711,12369186,1556175].map(it),ps=[2062260,11454440,16744206,16759672,2924588,10018698,14034728,16750742,9725885,12955861,9197131,12885140,14907330,16234194,8355711,13092807,12369186,14408589,1556175,10410725].map(it),vs=[3750777,5395619,7040719,10264286,6519097,9216594,11915115,13556636,9202993,12426809,15186514,15190932,8666169,11356490,14049643,15177372,8077683,10834324,13528509,14589654].map(it),ds=[3244733,7057110,10406625,13032431,15095053,16616764,16625259,16634018,3253076,7652470,10607003,13101504,7695281,10394312,12369372,14342891,6513507,9868950,12434877,14277081].map(it);$o.scale.quantile=function(){return yi([],[])},$o.scale.quantize=function(){return xi(0,1,[0,1])},$o.scale.threshold=function(){return Mi([.5],[0,1])},$o.scale.identity=function(){return _i([0,1])},$o.svg={},$o.svg.arc=function(){function n(){var n=t.apply(this,arguments),i=e.apply(this,arguments),o=r.apply(this,arguments)+ms,a=u.apply(this,arguments)+ms,c=(o>a&&(c=o,o=a,a=c),a-o),s=ka>c?"0":"1",l=Math.cos(o),f=Math.sin(o),h=Math.cos(a),g=Math.sin(a);return c>=ys?n?"M0,"+i+"A"+i+","+i+" 0 1,1 0,"+-i+"A"+i+","+i+" 0 1,1 0,"+i+"M0,"+n+"A"+n+","+n+" 0 1,0 0,"+-n+"A"+n+","+n+" 0 1,0 0,"+n+"Z":"M0,"+i+"A"+i+","+i+" 0 1,1 0,"+-i+"A"+i+","+i+" 0 1,1 0,"+i+"Z":n?"M"+i*l+","+i*f+"A"+i+","+i+" 0 "+s+",1 "+i*h+","+i*g+"L"+n*h+","+n*g+"A"+n+","+n+" 0 "+s+",0 "+n*l+","+n*f+"Z":"M"+i*l+","+i*f+"A"+i+","+i+" 0 "+s+",1 "+i*h+","+i*g+"L0,0"+"Z"}var t=bi,e=wi,r=Si,u=ki;return n.innerRadius=function(e){return arguments.length?(t=pt(e),n):t},n.outerRadius=function(t){return arguments.length?(e=pt(t),n):e},n.startAngle=function(t){return arguments.length?(r=pt(t),n):r},n.endAngle=function(t){return arguments.length?(u=pt(t),n):u},n.centroid=function(){var n=(t.apply(this,arguments)+e.apply(this,arguments))/2,i=(r.apply(this,arguments)+u.apply(this,arguments))/2+ms;return[Math.cos(i)*n,Math.sin(i)*n]},n};var ms=-Aa,ys=Ea-Ca;$o.svg.line=function(){return Ei(vt)};var xs=$o.map({linear:Ai,"linear-closed":Ci,step:Ni,"step-before":Li,"step-after":Ti,basis:Ui,"basis-open":ji,"basis-closed":Hi,bundle:Fi,cardinal:Ri,"cardinal-open":qi,"cardinal-closed":zi,monotone:Xi});xs.forEach(function(n,t){t.key=n,t.closed=/-closed$/.test(n)});var Ms=[0,2/3,1/3,0],_s=[0,1/3,2/3,0],bs=[0,1/6,2/3,1/6];$o.svg.line.radial=function(){var n=Ei($i);return n.radius=n.x,delete n.x,n.angle=n.y,delete n.y,n},Li.reverse=Ti,Ti.reverse=Li,$o.svg.area=function(){return Bi(vt)},$o.svg.area.radial=function(){var n=Bi($i);return n.radius=n.x,delete n.x,n.innerRadius=n.x0,delete n.x0,n.outerRadius=n.x1,delete n.x1,n.angle=n.y,delete n.y,n.startAngle=n.y0,delete n.y0,n.endAngle=n.y1,delete n.y1,n},$o.svg.chord=function(){function n(n,a){var c=t(this,i,n,a),s=t(this,o,n,a);return"M"+c.p0+r(c.r,c.p1,c.a1-c.a0)+(e(c,s)?u(c.r,c.p1,c.r,c.p0):u(c.r,c.p1,s.r,s.p0)+r(s.r,s.p1,s.a1-s.a0)+u(s.r,s.p1,c.r,c.p0))+"Z"}function t(n,t,e,r){var u=t.call(n,e,r),i=a.call(n,u,r),o=c.call(n,u,r)+ms,l=s.call(n,u,r)+ms;return{r:i,a0:o,a1:l,p0:[i*Math.cos(o),i*Math.sin(o)],p1:[i*Math.cos(l),i*Math.sin(l)]}}function e(n,t){return n.a0==t.a0&&n.a1==t.a1}function r(n,t,e){return"A"+n+","+n+" 0 "+ +(e>ka)+",1 "+t}function u(n,t,e,r){return"Q 0,0 "+r}var i=Re,o=De,a=Wi,c=Si,s=ki;return n.radius=function(t){return arguments.length?(a=pt(t),n):a},n.source=function(t){return arguments.length?(i=pt(t),n):i},n.target=function(t){return arguments.length?(o=pt(t),n):o},n.startAngle=function(t){return arguments.length?(c=pt(t),n):c},n.endAngle=function(t){return arguments.length?(s=pt(t),n):s},n},$o.svg.diagonal=function(){function n(n,u){var i=t.call(this,n,u),o=e.call(this,n,u),a=(i.y+o.y)/2,c=[i,{x:i.x,y:a},{x:o.x,y:a},o];return c=c.map(r),"M"+c[0]+"C"+c[1]+" "+c[2]+" "+c[3]}var t=Re,e=De,r=Ji;return n.source=function(e){return arguments.length?(t=pt(e),n):t},n.target=function(t){return arguments.length?(e=pt(t),n):e},n.projection=function(t){return arguments.length?(r=t,n):r},n},$o.svg.diagonal.radial=function(){var n=$o.svg.diagonal(),t=Ji,e=n.projection;return n.projection=function(n){return arguments.length?e(Gi(t=n)):t},n},$o.svg.symbol=function(){function n(n,r){return(ws.get(t.call(this,n,r))||no)(e.call(this,n,r))}var t=Qi,e=Ki;return n.type=function(e){return arguments.length?(t=pt(e),n):t},n.size=function(t){return arguments.length?(e=pt(t),n):e},n};var ws=$o.map({circle:no,cross:function(n){var t=Math.sqrt(n/5)/2;return"M"+-3*t+","+-t+"H"+-t+"V"+-3*t+"H"+t+"V"+-t+"H"+3*t+"V"+t+"H"+t+"V"+3*t+"H"+-t+"V"+t+"H"+-3*t+"Z"},diamond:function(n){var t=Math.sqrt(n/(2*As)),e=t*As;return"M0,"+-t+"L"+e+",0"+" 0,"+t+" "+-e+",0"+"Z"},square:function(n){var t=Math.sqrt(n)/2;return"M"+-t+","+-t+"L"+t+","+-t+" "+t+","+t+" "+-t+","+t+"Z"},"triangle-down":function(n){var t=Math.sqrt(n/Es),e=t*Es/2;return"M0,"+e+"L"+t+","+-e+" "+-t+","+-e+"Z"},"triangle-up":function(n){var t=Math.sqrt(n/Es),e=t*Es/2;return"M0,"+-e+"L"+t+","+e+" "+-t+","+e+"Z"}});$o.svg.symbolTypes=ws.keys();var Ss,ks,Es=Math.sqrt(3),As=Math.tan(30*La),Cs=[],Ns=0; -Cs.call=ma.call,Cs.empty=ma.empty,Cs.node=ma.node,Cs.size=ma.size,$o.transition=function(n){return arguments.length?Ss?n.transition():n:Ma.transition()},$o.transition.prototype=Cs,Cs.select=function(n){var t,e,r,u=this.id,i=[];n=v(n);for(var o=-1,a=this.length;++o<a;){i.push(t=[]);for(var c=this[o],s=-1,l=c.length;++s<l;)(r=c[s])&&(e=n.call(r,r.__data__,s,o))?("__data__"in r&&(e.__data__=r.__data__),uo(e,s,u,r.__transition__[u]),t.push(e)):t.push(null)}return to(i,u)},Cs.selectAll=function(n){var t,e,r,u,i,o=this.id,a=[];n=d(n);for(var c=-1,s=this.length;++c<s;)for(var l=this[c],f=-1,h=l.length;++f<h;)if(r=l[f]){i=r.__transition__[o],e=n.call(r,r.__data__,f,c),a.push(t=[]);for(var g=-1,p=e.length;++g<p;)(u=e[g])&&uo(u,g,o,i),t.push(u)}return to(a,o)},Cs.filter=function(n){var t,e,r,u=[];"function"!=typeof n&&(n=E(n));for(var i=0,o=this.length;o>i;i++){u.push(t=[]);for(var e=this[i],a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a)&&t.push(r)}return to(u,this.id)},Cs.tween=function(n,t){var e=this.id;return arguments.length<2?this.node().__transition__[e].tween.get(n):C(this,null==t?function(t){t.__transition__[e].tween.remove(n)}:function(r){r.__transition__[e].tween.set(n,t)})},Cs.attr=function(n,t){function e(){this.removeAttribute(a)}function r(){this.removeAttributeNS(a.space,a.local)}function u(n){return null==n?e:(n+="",function(){var t,e=this.getAttribute(a);return e!==n&&(t=o(e,n),function(n){this.setAttribute(a,t(n))})})}function i(n){return null==n?r:(n+="",function(){var t,e=this.getAttributeNS(a.space,a.local);return e!==n&&(t=o(e,n),function(n){this.setAttributeNS(a.space,a.local,t(n))})})}if(arguments.length<2){for(t in n)this.attr(t,n[t]);return this}var o="transform"==n?tu:qr,a=$o.ns.qualify(n);return eo(this,"attr."+n,t,a.local?i:u)},Cs.attrTween=function(n,t){function e(n,e){var r=t.call(this,n,e,this.getAttribute(u));return r&&function(n){this.setAttribute(u,r(n))}}function r(n,e){var r=t.call(this,n,e,this.getAttributeNS(u.space,u.local));return r&&function(n){this.setAttributeNS(u.space,u.local,r(n))}}var u=$o.ns.qualify(n);return this.tween("attr."+n,u.local?r:e)},Cs.style=function(n,t,e){function r(){this.style.removeProperty(n)}function u(t){return null==t?r:(t+="",function(){var r,u=Ko.getComputedStyle(this,null).getPropertyValue(n);return u!==t&&(r=qr(u,t),function(t){this.style.setProperty(n,r(t),e)})})}var i=arguments.length;if(3>i){if("string"!=typeof n){2>i&&(t="");for(e in n)this.style(e,n[e],t);return this}e=""}return eo(this,"style."+n,t,u)},Cs.styleTween=function(n,t,e){function r(r,u){var i=t.call(this,r,u,Ko.getComputedStyle(this,null).getPropertyValue(n));return i&&function(t){this.style.setProperty(n,i(t),e)}}return arguments.length<3&&(e=""),this.tween("style."+n,r)},Cs.text=function(n){return eo(this,"text",n,ro)},Cs.remove=function(){return this.each("end.transition",function(){var n;this.__transition__.count<2&&(n=this.parentNode)&&n.removeChild(this)})},Cs.ease=function(n){var t=this.id;return arguments.length<1?this.node().__transition__[t].ease:("function"!=typeof n&&(n=$o.ease.apply($o,arguments)),C(this,function(e){e.__transition__[t].ease=n}))},Cs.delay=function(n){var t=this.id;return C(this,"function"==typeof n?function(e,r,u){e.__transition__[t].delay=+n.call(e,e.__data__,r,u)}:(n=+n,function(e){e.__transition__[t].delay=n}))},Cs.duration=function(n){var t=this.id;return C(this,"function"==typeof n?function(e,r,u){e.__transition__[t].duration=Math.max(1,n.call(e,e.__data__,r,u))}:(n=Math.max(1,n),function(e){e.__transition__[t].duration=n}))},Cs.each=function(n,t){var e=this.id;if(arguments.length<2){var r=ks,u=Ss;Ss=e,C(this,function(t,r,u){ks=t.__transition__[e],n.call(t,t.__data__,r,u)}),ks=r,Ss=u}else C(this,function(r){var u=r.__transition__[e];(u.event||(u.event=$o.dispatch("start","end"))).on(n,t)});return this},Cs.transition=function(){for(var n,t,e,r,u=this.id,i=++Ns,o=[],a=0,c=this.length;c>a;a++){o.push(n=[]);for(var t=this[a],s=0,l=t.length;l>s;s++)(e=t[s])&&(r=Object.create(e.__transition__[u]),r.delay+=r.duration,uo(e,s,i,r)),n.push(e)}return to(o,i)},$o.svg.axis=function(){function n(n){n.each(function(){var n,s=$o.select(this),l=this.__chart__||e,f=this.__chart__=e.copy(),h=null==c?f.ticks?f.ticks.apply(f,a):f.domain():c,g=null==t?f.tickFormat?f.tickFormat.apply(f,a):vt:t,p=s.selectAll(".tick").data(h,f),v=p.enter().insert("g",".domain").attr("class","tick").style("opacity",Ca),d=$o.transition(p.exit()).style("opacity",Ca).remove(),m=$o.transition(p).style("opacity",1),y=ti(f),x=s.selectAll(".domain").data([0]),M=(x.enter().append("path").attr("class","domain"),$o.transition(x));v.append("line"),v.append("text");var _=v.select("line"),b=m.select("line"),w=p.select("text").text(g),S=v.select("text"),k=m.select("text");switch(r){case"bottom":n=io,_.attr("y2",u),S.attr("y",Math.max(u,0)+o),b.attr("x2",0).attr("y2",u),k.attr("x",0).attr("y",Math.max(u,0)+o),w.attr("dy",".71em").style("text-anchor","middle"),M.attr("d","M"+y[0]+","+i+"V0H"+y[1]+"V"+i);break;case"top":n=io,_.attr("y2",-u),S.attr("y",-(Math.max(u,0)+o)),b.attr("x2",0).attr("y2",-u),k.attr("x",0).attr("y",-(Math.max(u,0)+o)),w.attr("dy","0em").style("text-anchor","middle"),M.attr("d","M"+y[0]+","+-i+"V0H"+y[1]+"V"+-i);break;case"left":n=oo,_.attr("x2",-u),S.attr("x",-(Math.max(u,0)+o)),b.attr("x2",-u).attr("y2",0),k.attr("x",-(Math.max(u,0)+o)).attr("y",0),w.attr("dy",".32em").style("text-anchor","end"),M.attr("d","M"+-i+","+y[0]+"H0V"+y[1]+"H"+-i);break;case"right":n=oo,_.attr("x2",u),S.attr("x",Math.max(u,0)+o),b.attr("x2",u).attr("y2",0),k.attr("x",Math.max(u,0)+o).attr("y",0),w.attr("dy",".32em").style("text-anchor","start"),M.attr("d","M"+i+","+y[0]+"H0V"+y[1]+"H"+i)}if(f.rangeBand){var E=f.rangeBand()/2,A=function(n){return f(n)+E};v.call(n,A),m.call(n,A)}else v.call(n,l),m.call(n,f),d.call(n,f)})}var t,e=$o.scale.linear(),r=Ls,u=6,i=6,o=3,a=[10],c=null;return n.scale=function(t){return arguments.length?(e=t,n):e},n.orient=function(t){return arguments.length?(r=t in Ts?t+"":Ls,n):r},n.ticks=function(){return arguments.length?(a=arguments,n):a},n.tickValues=function(t){return arguments.length?(c=t,n):c},n.tickFormat=function(e){return arguments.length?(t=e,n):t},n.tickSize=function(t){var e=arguments.length;return e?(u=+t,i=+arguments[e-1],n):u},n.innerTickSize=function(t){return arguments.length?(u=+t,n):u},n.outerTickSize=function(t){return arguments.length?(i=+t,n):i},n.tickPadding=function(t){return arguments.length?(o=+t,n):o},n.tickSubdivide=function(){return arguments.length&&n},n};var Ls="bottom",Ts={top:1,right:1,bottom:1,left:1};$o.svg.brush=function(){function n(i){i.each(function(){var i=$o.select(this).style("pointer-events","all").style("-webkit-tap-highlight-color","rgba(0,0,0,0)").on("mousedown.brush",u).on("touchstart.brush",u),o=i.selectAll(".background").data([0]);o.enter().append("rect").attr("class","background").style("visibility","hidden").style("cursor","crosshair"),i.selectAll(".extent").data([0]).enter().append("rect").attr("class","extent").style("cursor","move");var a=i.selectAll(".resize").data(d,vt);a.exit().remove(),a.enter().append("g").attr("class",function(n){return"resize "+n}).style("cursor",function(n){return qs[n]}).append("rect").attr("x",function(n){return/[ew]$/.test(n)?-3:null}).attr("y",function(n){return/^[ns]/.test(n)?-3:null}).attr("width",6).attr("height",6).style("visibility","hidden"),a.style("display",n.empty()?"none":null);var l,f=$o.transition(i),h=$o.transition(o);c&&(l=ti(c),h.attr("x",l[0]).attr("width",l[1]-l[0]),e(f)),s&&(l=ti(s),h.attr("y",l[0]).attr("height",l[1]-l[0]),r(f)),t(f)})}function t(n){n.selectAll(".resize").attr("transform",function(n){return"translate("+l[+/e$/.test(n)]+","+h[+/^s/.test(n)]+")"})}function e(n){n.select(".extent").attr("x",l[0]),n.selectAll(".extent,.n>rect,.s>rect").attr("width",l[1]-l[0])}function r(n){n.select(".extent").attr("y",h[0]),n.selectAll(".extent,.e>rect,.w>rect").attr("height",h[1]-h[0])}function u(){function u(){32==$o.event.keyCode&&(C||(x=null,L[0]-=l[1],L[1]-=h[1],C=2),f())}function g(){32==$o.event.keyCode&&2==C&&(L[0]+=l[1],L[1]+=h[1],C=0,f())}function d(){var n=$o.mouse(_),u=!1;M&&(n[0]+=M[0],n[1]+=M[1]),C||($o.event.altKey?(x||(x=[(l[0]+l[1])/2,(h[0]+h[1])/2]),L[0]=l[+(n[0]<x[0])],L[1]=h[+(n[1]<x[1])]):x=null),E&&m(n,c,0)&&(e(S),u=!0),A&&m(n,s,1)&&(r(S),u=!0),u&&(t(S),w({type:"brush",mode:C?"move":"resize"}))}function m(n,t,e){var r,u,a=ti(t),c=a[0],s=a[1],f=L[e],g=e?h:l,d=g[1]-g[0];return C&&(c-=f,s-=d+f),r=(e?v:p)?Math.max(c,Math.min(s,n[e])):n[e],C?u=(r+=f)+d:(x&&(f=Math.max(c,Math.min(s,2*x[e]-r))),r>f?(u=r,r=f):u=f),g[0]!=r||g[1]!=u?(e?o=null:i=null,g[0]=r,g[1]=u,!0):void 0}function y(){d(),S.style("pointer-events","all").selectAll(".resize").style("display",n.empty()?"none":null),$o.select("body").style("cursor",null),T.on("mousemove.brush",null).on("mouseup.brush",null).on("touchmove.brush",null).on("touchend.brush",null).on("keydown.brush",null).on("keyup.brush",null),N(),w({type:"brushend"})}var x,M,_=this,b=$o.select($o.event.target),w=a.of(_,arguments),S=$o.select(_),k=b.datum(),E=!/^(n|s)$/.test(k)&&c,A=!/^(e|w)$/.test(k)&&s,C=b.classed("extent"),N=D(),L=$o.mouse(_),T=$o.select(Ko).on("keydown.brush",u).on("keyup.brush",g);if($o.event.changedTouches?T.on("touchmove.brush",d).on("touchend.brush",y):T.on("mousemove.brush",d).on("mouseup.brush",y),S.interrupt().selectAll("*").interrupt(),C)L[0]=l[0]-L[0],L[1]=h[0]-L[1];else if(k){var q=+/w$/.test(k),z=+/^n/.test(k);M=[l[1-q]-L[0],h[1-z]-L[1]],L[0]=l[q],L[1]=h[z]}else $o.event.altKey&&(x=L.slice());S.style("pointer-events","none").selectAll(".resize").style("display",null),$o.select("body").style("cursor",b.style("cursor")),w({type:"brushstart"}),d()}var i,o,a=g(n,"brushstart","brush","brushend"),c=null,s=null,l=[0,0],h=[0,0],p=!0,v=!0,d=zs[0];return n.event=function(n){n.each(function(){var n=a.of(this,arguments),t={x:l,y:h,i:i,j:o},e=this.__chart__||t;this.__chart__=t,Ss?$o.select(this).transition().each("start.brush",function(){i=e.i,o=e.j,l=e.x,h=e.y,n({type:"brushstart"})}).tween("brush:brush",function(){var e=zr(l,t.x),r=zr(h,t.y);return i=o=null,function(u){l=t.x=e(u),h=t.y=r(u),n({type:"brush",mode:"resize"})}}).each("end.brush",function(){i=t.i,o=t.j,n({type:"brush",mode:"resize"}),n({type:"brushend"})}):(n({type:"brushstart"}),n({type:"brush",mode:"resize"}),n({type:"brushend"}))})},n.x=function(t){return arguments.length?(c=t,d=zs[!c<<1|!s],n):c},n.y=function(t){return arguments.length?(s=t,d=zs[!c<<1|!s],n):s},n.clamp=function(t){return arguments.length?(c&&s?(p=!!t[0],v=!!t[1]):c?p=!!t:s&&(v=!!t),n):c&&s?[p,v]:c?p:s?v:null},n.extent=function(t){var e,r,u,a,f;return arguments.length?(c&&(e=t[0],r=t[1],s&&(e=e[0],r=r[0]),i=[e,r],c.invert&&(e=c(e),r=c(r)),e>r&&(f=e,e=r,r=f),(e!=l[0]||r!=l[1])&&(l=[e,r])),s&&(u=t[0],a=t[1],c&&(u=u[1],a=a[1]),o=[u,a],s.invert&&(u=s(u),a=s(a)),u>a&&(f=u,u=a,a=f),(u!=h[0]||a!=h[1])&&(h=[u,a])),n):(c&&(i?(e=i[0],r=i[1]):(e=l[0],r=l[1],c.invert&&(e=c.invert(e),r=c.invert(r)),e>r&&(f=e,e=r,r=f))),s&&(o?(u=o[0],a=o[1]):(u=h[0],a=h[1],s.invert&&(u=s.invert(u),a=s.invert(a)),u>a&&(f=u,u=a,a=f))),c&&s?[[e,u],[r,a]]:c?[e,r]:s&&[u,a])},n.clear=function(){return n.empty()||(l=[0,0],h=[0,0],i=o=null),n},n.empty=function(){return!!c&&l[0]==l[1]||!!s&&h[0]==h[1]},$o.rebind(n,a,"on")};var qs={n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},zs=[["n","e","s","w","nw","ne","se","sw"],["e","w"],["n","s"],[]],Rs=$o.time={},Ds=Date,Ps=["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"];ao.prototype={getDate:function(){return this._.getUTCDate()},getDay:function(){return this._.getUTCDay()},getFullYear:function(){return this._.getUTCFullYear()},getHours:function(){return this._.getUTCHours()},getMilliseconds:function(){return this._.getUTCMilliseconds()},getMinutes:function(){return this._.getUTCMinutes()},getMonth:function(){return this._.getUTCMonth()},getSeconds:function(){return this._.getUTCSeconds()},getTime:function(){return this._.getTime()},getTimezoneOffset:function(){return 0},valueOf:function(){return this._.valueOf()},setDate:function(){Us.setUTCDate.apply(this._,arguments)},setDay:function(){Us.setUTCDay.apply(this._,arguments)},setFullYear:function(){Us.setUTCFullYear.apply(this._,arguments)},setHours:function(){Us.setUTCHours.apply(this._,arguments)},setMilliseconds:function(){Us.setUTCMilliseconds.apply(this._,arguments)},setMinutes:function(){Us.setUTCMinutes.apply(this._,arguments)},setMonth:function(){Us.setUTCMonth.apply(this._,arguments)},setSeconds:function(){Us.setUTCSeconds.apply(this._,arguments)},setTime:function(){Us.setTime.apply(this._,arguments)}};var Us=Date.prototype,js="%a %b %e %X %Y",Hs="%m/%d/%Y",Fs="%H:%M:%S",Os=["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],Ys=["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],Is=["January","February","March","April","May","June","July","August","September","October","November","December"],Zs=["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"];Rs.year=co(function(n){return n=Rs.day(n),n.setMonth(0,1),n},function(n,t){n.setFullYear(n.getFullYear()+t)},function(n){return n.getFullYear()}),Rs.years=Rs.year.range,Rs.years.utc=Rs.year.utc.range,Rs.day=co(function(n){var t=new Ds(2e3,0);return t.setFullYear(n.getFullYear(),n.getMonth(),n.getDate()),t},function(n,t){n.setDate(n.getDate()+t)},function(n){return n.getDate()-1}),Rs.days=Rs.day.range,Rs.days.utc=Rs.day.utc.range,Rs.dayOfYear=function(n){var t=Rs.year(n);return Math.floor((n-t-6e4*(n.getTimezoneOffset()-t.getTimezoneOffset()))/864e5)},Ps.forEach(function(n,t){n=n.toLowerCase(),t=7-t;var e=Rs[n]=co(function(n){return(n=Rs.day(n)).setDate(n.getDate()-(n.getDay()+t)%7),n},function(n,t){n.setDate(n.getDate()+7*Math.floor(t))},function(n){var e=Rs.year(n).getDay();return Math.floor((Rs.dayOfYear(n)+(e+t)%7)/7)-(e!==t)});Rs[n+"s"]=e.range,Rs[n+"s"].utc=e.utc.range,Rs[n+"OfYear"]=function(n){var e=Rs.year(n).getDay();return Math.floor((Rs.dayOfYear(n)+(e+t)%7)/7)}}),Rs.week=Rs.sunday,Rs.weeks=Rs.sunday.range,Rs.weeks.utc=Rs.sunday.utc.range,Rs.weekOfYear=Rs.sundayOfYear,Rs.format=lo;var Vs=ho(Os),Xs=go(Os),$s=ho(Ys),Bs=go(Ys),Ws=ho(Is),Js=go(Is),Gs=ho(Zs),Ks=go(Zs),Qs=/^%/,nl={"-":"",_:" ",0:"0"},tl={a:function(n){return Ys[n.getDay()]},A:function(n){return Os[n.getDay()]},b:function(n){return Zs[n.getMonth()]},B:function(n){return Is[n.getMonth()]},c:lo(js),d:function(n,t){return po(n.getDate(),t,2)},e:function(n,t){return po(n.getDate(),t,2)},H:function(n,t){return po(n.getHours(),t,2)},I:function(n,t){return po(n.getHours()%12||12,t,2)},j:function(n,t){return po(1+Rs.dayOfYear(n),t,3)},L:function(n,t){return po(n.getMilliseconds(),t,3)},m:function(n,t){return po(n.getMonth()+1,t,2)},M:function(n,t){return po(n.getMinutes(),t,2)},p:function(n){return n.getHours()>=12?"PM":"AM"},S:function(n,t){return po(n.getSeconds(),t,2)},U:function(n,t){return po(Rs.sundayOfYear(n),t,2)},w:function(n){return n.getDay()},W:function(n,t){return po(Rs.mondayOfYear(n),t,2)},x:lo(Hs),X:lo(Fs),y:function(n,t){return po(n.getFullYear()%100,t,2)},Y:function(n,t){return po(n.getFullYear()%1e4,t,4)},Z:jo,"%":function(){return"%"}},el={a:vo,A:mo,b:_o,B:bo,c:wo,d:To,e:To,H:zo,I:zo,j:qo,L:Po,m:Lo,M:Ro,p:Uo,S:Do,U:xo,w:yo,W:Mo,x:So,X:ko,y:Ao,Y:Eo,Z:Co,"%":Ho},rl=/^\s*\d+/,ul=$o.map({am:0,pm:1});lo.utc=Fo;var il=Fo("%Y-%m-%dT%H:%M:%S.%LZ");lo.iso=Date.prototype.toISOString&&+new Date("2000-01-01T00:00:00.000Z")?Oo:il,Oo.parse=function(n){var t=new Date(n);return isNaN(t)?null:t},Oo.toString=il.toString,Rs.second=co(function(n){return new Ds(1e3*Math.floor(n/1e3))},function(n,t){n.setTime(n.getTime()+1e3*Math.floor(t))},function(n){return n.getSeconds()}),Rs.seconds=Rs.second.range,Rs.seconds.utc=Rs.second.utc.range,Rs.minute=co(function(n){return new Ds(6e4*Math.floor(n/6e4))},function(n,t){n.setTime(n.getTime()+6e4*Math.floor(t))},function(n){return n.getMinutes()}),Rs.minutes=Rs.minute.range,Rs.minutes.utc=Rs.minute.utc.range,Rs.hour=co(function(n){var t=n.getTimezoneOffset()/60;return new Ds(36e5*(Math.floor(n/36e5-t)+t))},function(n,t){n.setTime(n.getTime()+36e5*Math.floor(t))},function(n){return n.getHours()}),Rs.hours=Rs.hour.range,Rs.hours.utc=Rs.hour.utc.range,Rs.month=co(function(n){return n=Rs.day(n),n.setDate(1),n},function(n,t){n.setMonth(n.getMonth()+t)},function(n){return n.getMonth()}),Rs.months=Rs.month.range,Rs.months.utc=Rs.month.utc.range;var ol=[1e3,5e3,15e3,3e4,6e4,3e5,9e5,18e5,36e5,108e5,216e5,432e5,864e5,1728e5,6048e5,2592e6,7776e6,31536e6],al=[[Rs.second,1],[Rs.second,5],[Rs.second,15],[Rs.second,30],[Rs.minute,1],[Rs.minute,5],[Rs.minute,15],[Rs.minute,30],[Rs.hour,1],[Rs.hour,3],[Rs.hour,6],[Rs.hour,12],[Rs.day,1],[Rs.day,2],[Rs.week,1],[Rs.month,1],[Rs.month,3],[Rs.year,1]],cl=[[lo("%Y"),Zt],[lo("%B"),function(n){return n.getMonth()}],[lo("%b %d"),function(n){return 1!=n.getDate()}],[lo("%a %d"),function(n){return n.getDay()&&1!=n.getDate()}],[lo("%I %p"),function(n){return n.getHours()}],[lo("%I:%M"),function(n){return n.getMinutes()}],[lo(":%S"),function(n){return n.getSeconds()}],[lo(".%L"),function(n){return n.getMilliseconds()}]],sl=Zo(cl);al.year=Rs.year,Rs.scale=function(){return Yo($o.scale.linear(),al,sl)};var ll={range:function(n,t,e){return $o.range(+n,+t,e).map(Io)}},fl=al.map(function(n){return[n[0].utc,n[1]]}),hl=[[Fo("%Y"),Zt],[Fo("%B"),function(n){return n.getUTCMonth()}],[Fo("%b %d"),function(n){return 1!=n.getUTCDate()}],[Fo("%a %d"),function(n){return n.getUTCDay()&&1!=n.getUTCDate()}],[Fo("%I %p"),function(n){return n.getUTCHours()}],[Fo("%I:%M"),function(n){return n.getUTCMinutes()}],[Fo(":%S"),function(n){return n.getUTCSeconds()}],[Fo(".%L"),function(n){return n.getUTCMilliseconds()}]],gl=Zo(hl);return fl.year=Rs.year.utc,Rs.scale.utc=function(){return Yo($o.scale.linear(),fl,gl)},$o.text=dt(function(n){return n.responseText}),$o.json=function(n,t){return mt(n,"application/json",Vo,t)},$o.html=function(n,t){return mt(n,"text/html",Xo,t)},$o.xml=dt(function(n){return n.responseXML}),$o}();</script> -<script type="text/javascript">var RadarChart = { - draw: function(id, d, options){ - var cfg = { - radius: 5, - w: 600, - h: 600, - factor: .95, - factorLegend: 1, - levels: 3, - maxValue: 0, - radians: 2 * Math.PI, - opacityArea: 0.5, - color: d3.scale.category10(), - fontSize: 10 - }; - if('undefined' !== typeof options){ - for(var i in options){ - if('undefined' !== typeof options[i]){ - cfg[i] = options[i]; - } - } - } - cfg.maxValue = Math.max(cfg.maxValue, d3.max(d, function(i){return d3.max(i.map(function(o){return o.value;}))})); - var allAxis = (d[0].map(function(i, j){return i.axis})); - var total = allAxis.length; - var radius = cfg.factor*Math.min(cfg.w/2, cfg.h/2); - d3.select(id).select("svg").remove(); - var g = d3.select(id).append("svg").attr("width", cfg.w).attr("height", 0.75 * cfg.h).append("g"); - - var tooltip; - function getPosition(i, range, factor, func){ - factor = typeof factor !== 'undefined' ? factor : 1; - return range * (1 - factor * func(i * cfg.radians / total)); - } - function getHorizontalPosition(i, range, factor){ - return getPosition(i, range, factor, Math.sin); - } - function getVerticalPosition(i, range, factor){ - return getPosition(i, range, factor, Math.cos); - } - - for(var j=0; j<cfg.levels; j++){ - var levelFactor = radius*((j+1)/cfg.levels); - g.selectAll(".levels").data(allAxis).enter().append("svg:line") - .attr("x1", function(d, i){return getHorizontalPosition(i, levelFactor);}) - .attr("y1", function(d, i){return getVerticalPosition(i, levelFactor);}) - .attr("x2", function(d, i){return getHorizontalPosition(i+1, levelFactor);}) - .attr("y2", function(d, i){return getVerticalPosition(i+1, levelFactor);}) - .attr("class", "line").style("stroke", "grey").style("stroke-width", "0.5px").attr("transform", "translate(" + (cfg.w/2-levelFactor) + ", " + (cfg.h/2-levelFactor) + ")"); - - } - - series = 0; - - var axis = g.selectAll(".axis").data(allAxis).enter().append("g").attr("class", "axis"); - - axis.append("line") - .attr("x1", cfg.w/2) - .attr("y1", cfg.h/2) - .attr("x2", function(j, i){return getHorizontalPosition(i, cfg.w/2, cfg.factor);}) - .attr("y2", function(j, i){return getVerticalPosition(i, cfg.h/2, cfg.factor);}) - .attr("class", "line").style("stroke", "grey").style("stroke-width", "1px"); - - axis.append("text").attr("class", function(d){ return "legend_" + d.replace("/", "") }) - .text(function(d){return d}) - .style("font-size", cfg.fontSize + "px") - .style("text-anchor", function(d, i){ - var p = getHorizontalPosition(i, 0.5); - return (p < 0.4) ? "start" : ((p > 0.6) ? "end" : "middle"); - }) - .attr("transform", function(d, i){ - var p = getVerticalPosition(i, cfg.h / 2); - return p < cfg.fontSize ? "translate(0, " + (cfg.fontSize - p) + ")" : ""; - }) - .attr("x", function(d, i){return getHorizontalPosition(i, cfg.w / 2, cfg.factorLegend);}) - .attr("y", function(d, i){return getVerticalPosition(i, cfg.h / 2, cfg.factorLegend);}); - - d.forEach(function(y, x){ - dataValues = []; - g.selectAll(".nodes") - .data(y, function(j, i){ - dataValues.push([ - getHorizontalPosition(i, cfg.w/2, (parseFloat(Math.max(j.value, 0))/cfg.maxValue)*cfg.factor), - getVerticalPosition(i, cfg.h/2, (parseFloat(Math.max(j.value, 0))/cfg.maxValue)*cfg.factor) - ]); - }); - dataValues.push(dataValues[0]); - g.selectAll(".area") - .data([dataValues]) - .enter() - .append("polygon") - .attr("class", "radar-chart-serie"+series) - .style("stroke-width", "2px") - .style("stroke", cfg.color(series)) - .attr("points",function(d) { - var str=""; - for(var pti=0;pti<d.length;pti++){ - str=str+d[pti][0]+","+d[pti][1]+" "; - } - return str; - }) - .style("fill", function(j, i){return cfg.color(series)}) - .style("fill-opacity", cfg.opacityArea) - .on('mouseover', function (d){ - z = "polygon."+d3.select(this).attr("class"); - g.selectAll("polygon").transition(200).style("fill-opacity", 0.1); - g.selectAll(z).transition(200).style("fill-opacity", .7); - }) - .on('mouseout', function(){ - g.selectAll("polygon").transition(200).style("fill-opacity", cfg.opacityArea); - }); - series++; - }); - series=0; - - - d.forEach(function(y, x){ - g.selectAll(".nodes") - .data(y).enter() - .append("svg:circle").attr("class", "radar-chart-serie"+series) - .attr('r', cfg.radius) - .attr("alt", function(j){return Math.max(j.value, 0)}) - .attr("cx", function(j, i){ - dataValues.push([ - getHorizontalPosition(i, cfg.w/2, (parseFloat(Math.max(j.value, 0))/cfg.maxValue)*cfg.factor), - getVerticalPosition(i, cfg.h/2, (parseFloat(Math.max(j.value, 0))/cfg.maxValue)*cfg.factor) - ]); - return getHorizontalPosition(i, cfg.w/2, (Math.max(j.value, 0)/cfg.maxValue)*cfg.factor); - }) - .attr("cy", function(j, i){ - return getVerticalPosition(i, cfg.h/2, (Math.max(j.value, 0)/cfg.maxValue)*cfg.factor); - }) - .attr("data-id", function(j){return j.axis}) - .style("fill", cfg.color(series)).style("fill-opacity", .9) - .on('mouseover', function (d){ - newX = parseFloat(d3.select(this).attr('cx')) - 10; - newY = parseFloat(d3.select(this).attr('cy')) - 5; - tooltip.attr('x', newX).attr('y', newY).text(d.value).transition(200).style('opacity', 1); - z = "polygon."+d3.select(this).attr("class"); - g.selectAll("polygon").transition(200).style("fill-opacity", 0.1); - g.selectAll(z).transition(200).style("fill-opacity", .7); - }) - .on('mouseout', function(){ - tooltip.transition(200).style('opacity', 0); - g.selectAll("polygon").transition(200).style("fill-opacity", cfg.opacityArea); - }) - .append("svg:title") - .text(function(j){return Math.max(j.value, 0)}); - - series++; - }); - //Tooltip - tooltip = g.append('text').style('opacity', 0).style('font-family', 'sans-serif').style('font-size', '13px'); - } -} - -</script> -<script type="text/javascript">window.onload = function() { - var cpu = 88.6; - var mpi = 11.4; - var io = 0.0; - - // Draw radar chart, choose its color based on application classification - var radar_data = [[ { axis: "CPU", value: cpu }, - { axis: "MPI", value: mpi }, - { axis: "I/O", value: io } ]]; - var radar_options = { w: 200, h: 200, factor: 0.7, fontSize: 16, radius: 0, - opacityArea: 0.64, maxValue: 100, - color: function() { - if ("cpu" == "io") return "#ed8140"; - if ("cpu" == "cpu") return "#4fd32e"; - if ("cpu" == "mpi") return "#409ded"; - else return "#bb58d6"; - } - }; - RadarChart.draw("#time_radar", radar_data, radar_options); - - // Wrap the exe path on slashes if necessary - var exe_path = document.getElementById("exe_path"); - exe_path.innerHTML = exe_path.innerHTML.replace(/\//g, '/​'); // insert a zero width space after slashes as a word wrap hint - - // Format our byte value numbers to show at most 2 decimal places, but only if required to attain 3 digits of precision - // 1234.56 -> "1234" - // 12.3456 -> "12.3" - // 1.23456 -> "1.23" - // 0.00000 -> "0.00" - var formatNumber = function(num) { - if (num > 100) return new Number(num).toFixed(0); // 1234.56 -> 1234 - if (num > 10) return new Number(num).toFixed(1); // 12.3456 -> 12.3 - else return new Number(num).toFixed(2); // 1.23456 -> 1.23 and 0 -> 0.00 - } - - // Parse a number, but return 0 for invalid numbers, not NaN - var toNumber = function(num_str) { - var num = Number(num_str) - return isNaN(num) ? 0 : num; - } - - // Scale byte values to bytes, kB, MB or GB as appropriate - var fillBytes = function(base_name, bytes_str, per_second) { - var units = "bytes"; - var scale = 1.0; - var bytes = Number(bytes_str) - if (isNaN(bytes) ) { units = "" ; } - else if (bytes > 1000000000) { units = "GB"; scale = 1*1000*1000*1000; } - else if (bytes > 1000000 ) { units = "MB"; scale = 1*1000*1000 } - else if (bytes > 1000 ) { units = "kB"; scale = 1*1000 } - var num = isNaN(bytes) ? bytes_str : formatNumber(bytes / scale); - var elementNum = document.getElementById(base_name + "_num"); - elementNum.innerHTML = num; - var elementUnits = document.getElementById(base_name + "_units"); - elementUnits.innerHTML = units + (per_second && !isNaN(bytes) ? "/s" : ""); - }; - - // Fill all the byte numbers and their units with the scaled values - fillBytes("mpi_colrate", "1.65e+02", true); - fillBytes("mpi_p2prate", "0.00e+00", true); - fillBytes("io_readrate", "0.00e+00", true); - fillBytes("io_writerate", "0.00e+00", true); - fillBytes("ram_mean", "2.33e+07", false); - fillBytes("ram_peak", "2.35e+07", false); - - // Set widths for all graphs - var bar = function(name, width) { - var rounded = Math.round(width); - if (!isFinite(rounded) || rounded < 1) - document.getElementById(name).style.width = "1px" - else - document.getElementById(name).style.width = rounded + "px"; - }; - bar("cpu_bar", cpu * 2); - bar("mpi_bar", mpi * 2); - bar("io_bar", io * 2); - - bar("cpu_num_bar", toNumber("50.0") / 2); - bar("cpu_vec_bar", toNumber("50.0") / 2); - bar("cpu_mem_bar", toNumber("0.0") / 2); - bar("cpu_other_bar", toNumber("0.0") / 2); - - bar("mpi_col_bar", toNumber("100.0") / 2); - bar("mpi_p2p_bar", toNumber("0.0") / 2); - var mpi_scale = Math.max(toNumber("1.65e+02"), toNumber("0.00e+00")); - bar("mpi_colrate_bar", 50 * toNumber("1.65e+02") / mpi_scale); - bar("mpi_p2prate_bar", 50 * toNumber("0.00e+00") / mpi_scale); - - bar("io_read_bar", toNumber("0.0") / 2); - bar("io_write_bar", toNumber("0.0") / 2); - var io_scale = Math.max(toNumber("0.00e+00"), toNumber("0.00e+00")); - bar("io_readrate_bar", 50 * toNumber("0.00e+00") / io_scale); - bar("io_writerate_bar", 50 * toNumber("0.00e+00") / io_scale); - - bar("ram_mean_bar", 50 * toNumber("2.33e+07") / toNumber("2.35e+07")); - bar("ram_peak_bar", 50); - bar("ram_node_bar", toNumber("2.8") / 2); - - // Formatting replacement helper function - var replaceIn = function(elementId, re, new_text) { - element = document.getElementById(elementId); - element.innerHTML = element.innerHTML.replace(re, new_text); - }; - - // Add formatting for the overview advice section - replaceIn('overview_advice', /CPU/g, '<span class="cpu_span">CPU</span>'); - replaceIn('overview_advice', /MPI/g, '<span class="mpi_span">MPI</span>'); - replaceIn('overview_advice', /I\/O/g, '<span class="io_span">I\/O</span>'); - - // Add formatting for the CPU advice section - if ( 88.6 < 0.05 ) - replaceIn('cpu_explanation', /application code/g, '<span class="cpu_span">application code</span>'); - replaceIn('cpu_explanation', /vectorized instructions/g, '<span class="cpu_vec_span">vectorized instructions</span>'); - replaceIn('cpu_explanation', /memory accesses/g, '<span class="cpu_mem_span">memory accesses</span>'); - replaceIn('cpu_explanation', /memory-bound/g, '<span class="cpu_mem_span">memory-bound</span>'); - replaceIn('cpu_explanation', /arithmetic-bound/g, '<span class="cpu_num_span">arithmetic-bound</span>'); - replaceIn('cpu_explanation', /numerical computation/g, '<span class="cpu_num_span">numerical computation</span>'); - - // Add formatting for the MPI advice section - if ( 11.4 < 0.05 ) - replaceIn('mpi_explanation', /MPI/g, '<span class="mpi_span">MPI</span>'); - replaceIn('mpi_explanation', /collective calls/g, '<span class="mpi_col_span">collective calls</span>'); - replaceIn('mpi_explanation', /point-to-point calls/g, '<span class="mpi_p2p_span">point-to-point calls</span>'); - replaceIn('mpi_explanation', / very low/g, ' <span class="mpi_colrate_span">very low</span>'); - replaceIn('mpi_explanation', / low/g, ' <span class="mpi_colrate_span">low</span>'); - replaceIn('mpi_explanation', / average/g, ' <span class="mpi_colrate_span">average</span>'); - replaceIn('mpi_explanation', / very high/g, ' <span class="mpi_colrate_span">very high</span>'); - replaceIn('mpi_explanation', / high/g, ' <span class="mpi_colrate_span">high</span>'); - - // Add formatting for the I/O advice section - if ( 0.0 < 0.05 ) - replaceIn('io_explanation', /I\/O/g, '<span class="io_span">I/O</span>'); - replaceIn('io_explanation', /read operations/g, '<span class="io_read_span">read operations</span>'); - replaceIn('io_explanation', /write operations/g, '<span class="io_write_span">write operations</span>'); - replaceIn('io_explanation', / very low/g, ' <span class="io_readrate_span">very low</span>'); - replaceIn('io_explanation', / low/g, ' <span class="io_readrate_span">low</span>'); - replaceIn('io_explanation', / average/g, ' <span class="io_readrate_span">average</span>'); - replaceIn('io_explanation', / very high/g, ' <span class="io_readrate_span">very high</span>'); - replaceIn('io_explanation', / high/g, ' <span class="io_readrate_span">high</span>'); - - // Add formatting for the RAM advice section - replaceIn('ram_explanation', /high/g, ' <span class="ram_node_span">high</span>'); - replaceIn('ram_explanation', /well-balanced/g, '<span class="ram_peak_span">well-balanced</span>'); - replaceIn('ram_explanation', /significant variation/g, '<span class="ram_peak_span">significant variation</span>'); - replaceIn('ram_explanation', /peak node memory usage/g, '<span class="ram_node_span">peak node memory usage</span>'); - - // Hide the error warning if the script got this far - document.getElementById('error').style.display = 'none'; -}; -</script> -</head> - -<body> -<div id="content"> - -<div class="header"> - <div class="logo"><img src="http://content.allinea.com/downloads/performance-report-logo.png" alt="Allinea Performance Reports" /></div> - <div class="header_left"> - <div class="application_details"> - <table> - <tr><td class="details_key">Executable:</td><td id="exe_name">mympiprog.x</td></tr> - <tr><td class="details_key">Resources:</td><td id="num_procs">32 processes, 2 nodes</td></tr> - <tr><td class="details_key">Machine:</td><td id="machine_name">cn182</td></tr> - <tr><td class="details_key">Start time:</td><td id="start_date">Wed Oct 15 16:56:23 2014</td></tr> - <tr><td class="details_key">Total time:</td><td id="time_string">7 seconds (0 minutes)</td></tr> - <tr><td class="details_key">Full path:</td><td id="exe_path">/home/user</td></tr> - <tr><td class="details_key">Notes:</td><td id="notes"></td></tr> - </table> - </div> - </div> - <div id="time_radar"></div> - <div class="clear"></div> -</div> -<hr /> -<div id="error"> -<p><strong>Error: javascript is not running</strong></p> -<p>The graphs in this Performance Report require <strong>javascript</strong>, which is disabled or not working.</p> -<p>Check whether your javascript support is enabled or try another browser.<p> -<p>Remember, you can always contact <a href="mailto:support@allinea.com">support@allinea.com</a>, we're very nice!</p> -</div> -<div class="summary"> - <div class="heading">Summary: mympiprog.x is <span class="cpu_span">CPU-bound</span> in this configuration</div> - <div>The total wallclock time was spent as follows:</div> - <table class="summary_table"> - <tr><td class="heading_cpu">CPU</td><td class="percent">88.6%</td><td class="bar_graph"><div id="cpu_bar" /></td> - <td class="details"><p>Time spent running application code. High values are usually good.</p><p>This is <span id="summary_cpu_class">high</span>; check the CPU performance section for optimization advice.</p></td></tr> - <tr><td class="heading_mpi">MPI</td><td class="percent">11.4%</td><td class="bar_graph"><div id="mpi_bar" /></td> - <td class="details"><p>Time spent in MPI calls. High values are usually bad.</p><p>This is <span id="summary_mpi_class">very low</span>; this code may benefit from increasing the process count.</p></td></tr> - <tr><td class="heading_io">I/O</td><td class="percent">0.0%</td><td class="bar_graph"><div id="io_bar" /></td> - <td class="details"><p>Time spent in filesystem I/O. High values are usually bad.</p><p>This is <span id="summary_io_class">negligible</span>; there's no need to investigate I/O performance.</p></td></tr> - </table> - <div class="overview_general_advice"><p>This application run was <span class="cpu_span">CPU-bound</span>. A breakdown of this time and advice for investigating further is in the <span class="cpu_span">CPU</span> section below.</p><p id="overview_advice">As very little time is spent in MPI calls, this code may also benefit from running at larger scales.</p></div> -</div> -<hr /> -<div class="subsections"> - <div class="ltcol"> - <div class="heading_cpu">CPU</div> - <div>A breakdown of how the <span class="cpu_span">88.6</span>% total CPU time was spent:</div> - <table id="cpu_chart"> - <tr><td>Scalar numeric ops</td><td class="right_cell"><span class="cpu_num_span">50.0</span>%</td><td class="mini_bar_graph"><div id="cpu_num_bar" /></td></tr> - <tr><td>Vector numeric ops</td><td class="right_cell"><span class="cpu_vec_span">50.0</span>%</td><td class="mini_bar_graph"><div id="cpu_vec_bar" /></td></tr> - <tr><td>Memory accesses</td><td class="right_cell"><span class="cpu_mem_span">0.0</span>%</td><td class="mini_bar_graph"><div id="cpu_mem_bar" /></td></tr> - <tr><td>Other</td><td class="right_cell"><span class="cpu_other_span">0.0</span>%</td><td class="mini_bar_graph"><div id="cpu_other_bar" /></td></tr> - </table> - <div id="cpu_explanation"> - <div class="explanation">The per-core performance is arithmetic-bound. Try to increase the amount of time spent in vectorized instructions by analyzing the compiler's vectorization reports.</div> - <div class="explanation"></div> - </div> - </div> - <div class="rtcol"> - <div class="heading_mpi">MPI</div> - <div>Of the <span class="mpi_span">11.4</span>% total time spent in MPI calls:</div> - <table id="mpi_chart"> - <tr><td>Time in collective calls</td><td class="right_cell"><span class="mpi_col_span">100.0</span>%</td><td class="mini_bar_graph"><div id="mpi_col_bar" /></td></tr> - <tr><td>Time in point-to-point calls</td><td class="right_cell"><span class="mpi_p2p_span">0.0</span>%</td><td class="mini_bar_graph"><div id="mpi_p2p_bar" /></td></tr> - <tr><td>Effective process collective rate</td><td class="right_cell"><span class="mpi_colrate_span"><span id="mpi_colrate_num">1.65e+02</span></span> <span id="mpi_colrate_units"></span></td><td class="mini_bar_graph"><div id="mpi_colrate_bar" /></td></tr> - <tr><td>Effective process point-to-point rate</td><td class="right_cell"><span class="mpi_p2prate_span"><span id="mpi_p2prate_num">0.00e+00</span></span> <span id="mpi_p2prate_units"></span></td><td class="mini_bar_graph"><div id="mpi_p2prate_bar" /></td></tr> - </table> - <div id="mpi_explanation"> - <div class="explanation">Most of the time is spent in collective calls with a very low transfer rate. This suggests load imbalance is causing synchonization overhead; use an MPI profiler to investigate further.</div> - <div class="explanation"></div> - </div> - </div> - <div class="clear"></div> -</div> -<div class="subsections"> - <div class="ltcol"> - <div class="heading_io">I/O</div> - <div>A breakdown of how the <span class="io_span">0.0</span>% total I/O time was spent:</div> - <table id="io_chart"> - <tr><td>Time in reads</td><td class="right_cell"><span class="io_read_span">0.0</span>%</td><td class="mini_bar_graph"><div id="io_read_bar" /></td></tr> - <tr><td>Time in writes</td><td class="right_cell"><span class="io_write_span">0.0</span>%</td><td class="mini_bar_graph"><div id="io_write_bar" /></td></tr> - <tr><td>Effective process read rate</td><td class="right_cell"><span class="io_readrate_span"><span id="io_readrate_num">0.00e+00</span></span> <span id="io_readrate_units"></span></td><td class="mini_bar_graph"><div id="io_readrate_bar" /></td></tr> - <tr><td>Effective process write rate</td><td class="right_cell"><span class="io_writerate_span"><span id="io_writerate_num">0.00e+00</span></span> <span id="io_writerate_units"></span></td><td class="mini_bar_graph"><div id="io_writerate_bar" /></td></tr> - </table> - <div id="io_explanation"> - <div class="explanation">No time is spent in I/O operations. There's nothing to optimize here!</div> - <div class="explanation"></div> - </div> - </div> - <div class="rtcol"> - <div class="heading_ram">Memory</div> - <div>Per-process memory usage may also affect scaling:</div> - <table id="ram_chart"> - <tr><td>Mean process memory usage</td><td class="right_cell"><span class="ram_mean_span"><span id="ram_mean_num">2.33e+07</span></span> <span id="ram_mean_units"></span></td><td class="mini_bar_graph"><div id="ram_mean_bar" /></td></tr> - <tr><td>Peak process memory usage</td><td class="right_cell"><span class="ram_peak_span"><span id="ram_peak_num">2.35e+07</span></span> <span id="ram_peak_units"></span></td><td class="mini_bar_graph"><div id="ram_peak_bar" /></td></tr> - <tr><td>Peak node memory usage</td><td class="right_cell"><span class="ram_node_span">2.8</span>%</td><td class="mini_bar_graph"><div id="ram_node_bar" /></td></tr> - </table> - <div id="ram_explanation"> - <div class="explanation">The peak node memory usage is very low. You may be able to reduce the amount of allocation time used by running with fewer MPI processes and more data on each process.</div> - <div class="explanation"></div> - </div> - </div> - <div class="clear"></div> -</div> -</div> <!-- content --> -</body> -</html> diff --git a/docs.it4i/salomon/software/debuggers/mympiprog_32p_2014-10-15_16-56.txt b/docs.it4i/salomon/software/debuggers/mympiprog_32p_2014-10-15_16-56.txt deleted file mode 100644 index de8449179..000000000 --- a/docs.it4i/salomon/software/debuggers/mympiprog_32p_2014-10-15_16-56.txt +++ /dev/null @@ -1,50 +0,0 @@ -Executable: mympiprog.x -Resources: 32 processes, 2 nodes -Machine: cn182 -Started on: Wed Oct 15 16:56:23 2014 -Total time: 7 seconds (0 minutes) -Full path: /home/user -Notes: - -Summary: mympiprog.x is CPU-bound in this configuration -CPU: 88.6% |========| -MPI: 11.4% || -I/O: 0.0% | -This application run was CPU-bound. A breakdown of this time and advice for investigating further is found in the CPU section below. -As very little time is spent in MPI calls, this code may also benefit from running at larger scales. - -CPU: -A breakdown of how the 88.6% total CPU time was spent: -Scalar numeric ops: 50.0% |====| -Vector numeric ops: 50.0% |====| -Memory accesses: 0.0% | -Other: 0.0% | -The per-core performance is arithmetic-bound. Try to increase the amount of time spent in vectorized instructions by analyzing the compiler's vectorization reports. - - -MPI: -A breakdown of how the 11.4% total MPI time was spent: -Time in collective calls: 100.0% |=========| -Time in point-to-point calls: 0.0% | -Effective collective rate: 1.65e+02 bytes/s -Effective point-to-point rate: 0.00e+00 bytes/s -Most of the time is spent in collective calls with a very low transfer rate. This suggests load imbalance is causing synchonization overhead; use an MPI profiler to investigate further. - - -I/O: -A breakdown of how the 0.0% total I/O time was spent: -Time in reads: 0.0% | -Time in writes: 0.0% | -Effective read rate: 0.00e+00 bytes/s -Effective write rate: 0.00e+00 bytes/s -No time is spent in I/O operations. There's nothing to optimize here! - - -Memory: -Per-process memory usage may also affect scaling: -Mean process memory usage: 2.33e+07 bytes -Peak process memory usage: 2.35e+07 bytes -Peak node memory usage: 2.8% | -The peak node memory usage is very low. You may be able to reduce the amount of allocation time used by running with fewer MPI processes and more data on each process. - - diff --git a/docs.it4i/salomon/software/debuggers/total-view.md b/docs.it4i/salomon/software/debuggers/total-view.md deleted file mode 100644 index 172836a05..000000000 --- a/docs.it4i/salomon/software/debuggers/total-view.md +++ /dev/null @@ -1,150 +0,0 @@ -# Total View - -TotalView is a GUI-based source code multi-process, multi-thread debugger. - -## License and Limitations for Cluster Users - -On the cluster users can debug OpenMP or MPI code that runs up to 64 parallel processes. These limitation means that: - -```console - 1 user can debug up 64 processes, or - 32 users can debug 2 processes, etc. -``` - -Debugging of GPU accelerated codes is also supported. - -You can check the status of the licenses [here](https://extranet.it4i.cz/rsweb/anselm/license/Totalview). - -## Compiling Code to Run With TotalView - -### Modules - -Load all necessary modules to compile the code. For example: - -```console - ml intel -``` - -Load the TotalView module: - -```console - ml TotalView/8.15.4-6-linux-x86-64 -``` - -Compile the code: - -```console - mpicc -g -O0 -o test_debug test.c - mpif90 -g -O0 -o test_debug test.f -``` - -### Compiler Flags - -Before debugging, you need to compile your code with theses flags: - -!!! note - **-g** : Generates extra debugging information usable by GDB. -g3 includes even more debugging information. This option is available for GNU and INTEL C/C++ and Fortran compilers. - - **-O0** : Suppress all optimizations. - -## Starting a Job With TotalView - -Be sure to log in with an X window forwarding enabled. This could mean using the -X in the ssh: - -```console -ssh -X username@salomon.it4i.cz -``` - -Other options is to access login node using VNC. Please see the detailed information on how to use graphic user interface on Anselm. - -From the login node an interactive session with X windows forwarding (-X option) can be started by following command: - -```console -$ qsub -I -X -A NONE-0-0 -q qexp -lselect=1:ncpus=24:mpiprocs=24,walltime=01:00:00 -``` - -Then launch the debugger with the totalview command followed by the name of the executable to debug. - -### Debugging a Serial Code - -To debug a serial code use: - -```console -totalview test_debug -``` - -### Debugging a Parallel Code - Option 1 - -To debug a parallel code compiled with **OpenMPI** you need to setup your TotalView environment: - -!!! hint - To be able to run parallel debugging procedure from the command line without stopping the debugger in the mpiexec source code you have to add the following function to your **~/.tvdrc** file. - -```console - proc mpi_auto_run_starter {loaded_id} { - set starter_programs {mpirun mpiexec orterun} - set executable_name [TV::symbol get $loaded_id full_pathname] - set file_component [file tail $executable_name] - - if {[lsearch -exact $starter_programs $file_component] != -1} { - puts "*************************************" - puts "Automatically starting $file_component" - puts "*************************************" - dgo - } - } - - # Append this function to TotalView's image load callbacks so that - # TotalView run this program automatically. - - dlappend TV::image_load_callbacks mpi_auto_run_starter -``` - -The source code of this function can be also found in - -```console -$ /apps/all/OpenMPI/1.10.1-GNU-4.9.3-2.25/etc/openmpi-totalview.tcl -``` - -You can also add only following line to you ~/.tvdrc file instead of -the entire function: - -```console -$ source /apps/all/OpenMPI/1.10.1-GNU-4.9.3-2.25/etc/openmpi-totalview.tcl -``` - -You need to do this step only once. See also [OpenMPI FAQ entry](https://www.open-mpi.org/faq/?category=running#run-with-tv) - -Now you can run the parallel debugger using: - -```console -$ mpirun -tv -n 5 ./test_debug -``` - -When following dialog appears click on "Yes" - - - -At this point the main TotalView GUI window will appear and you can insert the breakpoints and start debugging: - - - -### Debugging a Parallel Code - Option 2 - -Other option to start new parallel debugging session from a command line is to let TotalView to execute mpirun by itself. In this case user has to specify a MPI implementation used to compile the source code. - -The following example shows how to start debugging session with Intel MPI: - -```console -$ ml intel -$ ml TotalView/8.15.4-6-linux-x86-64 -$ totalview -mpi "Intel MPI-Hydra" -np 8 ./hello_debug_impi -``` - -After running previous command you will see the same window as shown in the screenshot above. - -More information regarding the command line parameters of the TotalView can be found TotalView Reference Guide, Chapter 7: TotalView Command Syntax. - -## Documentation - -[1] The [TotalView documentation](http://www.roguewave.com/support/product-documentation/totalview-family.aspx#totalview) web page is a good resource for learning more about some of the advanced TotalView features. diff --git a/docs.it4i/salomon/software/debuggers/valgrind.md b/docs.it4i/salomon/software/debuggers/valgrind.md deleted file mode 100644 index 6a897e39f..000000000 --- a/docs.it4i/salomon/software/debuggers/valgrind.md +++ /dev/null @@ -1,265 +0,0 @@ -# Valgrind - -## About Valgrind - -Valgrind is an open-source tool, used mainly for debuggig memory-related problems, such as memory leaks, use of uninitalized memory etc. in C/C++ applications. The toolchain was however extended over time with more functionality, such as debugging of threaded applications, cache profiling, not limited only to C/C++. - -Valgind is an extremely useful tool for debugging memory errors such as [off-by-one](http://en.wikipedia.org/wiki/Off-by-one_error). Valgrind uses a virtual machine and dynamic recompilation of binary code, because of that, you can expect that programs being debugged by Valgrind run 5-100 times slower. - -The main tools available in Valgrind are : - -* **Memcheck**, the original, must used and default tool. Verifies memory access in you program and can detect use of unitialized memory, out of bounds memory access, memory leaks, double free, etc. -* **Massif**, a heap profiler. -* **Hellgrind** and **DRD** can detect race conditions in multi-threaded applications. -* **Cachegrind**, a cache profiler. -* **Callgrind**, a callgraph analyzer. -* For a full list and detailed documentation, please refer to the [official Valgrind documentation](http://valgrind.org/docs/). - -## Installed Versions - -There are two versions of Valgrind available on the cluster. - -* Version 3.8.1, installed by operating system vendor in /usr/bin/valgrind. This version is available by default, without the need to load any module. This version however does not provide additional MPI support. Also, it does not support AVX2 instructions, debugging of an AVX2-enabled executable with this version will fail -* Version 3.11.0 built by ICC with support for Intel MPI, available in module Valgrind/3.11.0-intel-2015b. After loading the module, this version replaces the default valgrind. -* Version 3.11.0 built by GCC with support for Open MPI, module Valgrind/3.11.0-foss-2015b - -## Usage - -Compile the application which you want to debug as usual. It is advisable to add compilation flags -g (to add debugging information to the binary so that you will see original source code lines in the output) and -O0 (to disable compiler optimizations). - -For example, lets look at this C code, which has two problems: - -```cpp -#include <stdlib.h> - -void f(void) -{ - int* x = malloc(10 * sizeof(int)); - x[10] = 0; // problem 1: heap block overrun -} // problem 2: memory leak -- x not freed - -int main(void) -{ - f(); - return 0; -} -``` - -Now, compile it with Intel compiler: - -```console -$ module add intel -$ icc -g valgrind-example.c -o valgrind-example -``` - -Now, lets run it with Valgrind. The syntax is: - -valgrind [valgrind options] < your program binary > [your program options] - -If no Valgrind options are specified, Valgrind defaults to running Memcheck tool. Please refer to the Valgrind documentation for a full description of command line options. - -```console -$ valgrind ./valgrind-example - ==12652== Memcheck, a memory error detector - ==12652== Copyright (C) 2002-2013, and GNU GPL'd, by Julian Seward et al. - ==12652== Using Valgrind-3.9.0 and LibVEX; rerun with -h for copyright info - ==12652== Command: ./valgrind-example - ==12652== - ==12652== Invalid write of size 4 - ==12652== at 0x40053E: f (valgrind-example.c:6) - ==12652== by 0x40054E: main (valgrind-example.c:11) - ==12652== Address 0x5861068 is 0 bytes after a block of size 40 alloc'd - ==12652== at 0x4C27AAA: malloc (vg_replace_malloc.c:291) - ==12652== by 0x400528: f (valgrind-example.c:5) - ==12652== by 0x40054E: main (valgrind-example.c:11) - ==12652== - ==12652== - ==12652== HEAP SUMMARY: - ==12652== in use at exit: 40 bytes in 1 blocks - ==12652== total heap usage: 1 allocs, 0 frees, 40 bytes allocated - ==12652== - ==12652== LEAK SUMMARY: - ==12652== definitely lost: 40 bytes in 1 blocks - ==12652== indirectly lost: 0 bytes in 0 blocks - ==12652== possibly lost: 0 bytes in 0 blocks - ==12652== still reachable: 0 bytes in 0 blocks - ==12652== suppressed: 0 bytes in 0 blocks - ==12652== Rerun with --leak-check=full to see details of leaked memory - ==12652== - ==12652== For counts of detected and suppressed errors, rerun with: -v - ==12652== ERROR SUMMARY: 1 errors from 1 contexts (suppressed: 6 from 6) -``` - -In the output we can see that Valgrind has detected both errors - the off-by-one memory access at line 5 and a memory leak of 40 bytes. If we want a detailed analysis of the memory leak, we need to run Valgrind with --leak-check=full option: - -```console -$ valgrind --leak-check=full ./valgrind-example - ==23856== Memcheck, a memory error detector - ==23856== Copyright (C) 2002-2010, and GNU GPL'd, by Julian Seward et al. - ==23856== Using Valgrind-3.6.0 and LibVEX; rerun with -h for copyright info - ==23856== Command: ./valgrind-example - ==23856== - ==23856== Invalid write of size 4 - ==23856== at 0x40067E: f (valgrind-example.c:6) - ==23856== by 0x40068E: main (valgrind-example.c:11) - ==23856== Address 0x66e7068 is 0 bytes after a block of size 40 alloc'd - ==23856== at 0x4C26FDE: malloc (vg_replace_malloc.c:236) - ==23856== by 0x400668: f (valgrind-example.c:5) - ==23856== by 0x40068E: main (valgrind-example.c:11) - ==23856== - ==23856== - ==23856== HEAP SUMMARY: - ==23856== in use at exit: 40 bytes in 1 blocks - ==23856== total heap usage: 1 allocs, 0 frees, 40 bytes allocated - ==23856== - ==23856== 40 bytes in 1 blocks are definitely lost in loss record 1 of 1 - ==23856== at 0x4C26FDE: malloc (vg_replace_malloc.c:236) - ==23856== by 0x400668: f (valgrind-example.c:5) - ==23856== by 0x40068E: main (valgrind-example.c:11) - ==23856== - ==23856== LEAK SUMMARY: - ==23856== definitely lost: 40 bytes in 1 blocks - ==23856== indirectly lost: 0 bytes in 0 blocks - ==23856== possibly lost: 0 bytes in 0 blocks - ==23856== still reachable: 0 bytes in 0 blocks - ==23856== suppressed: 0 bytes in 0 blocks - ==23856== - ==23856== For counts of detected and suppressed errors, rerun with: -v - ==23856== ERROR SUMMARY: 2 errors from 2 contexts (suppressed: 6 from 6) -``` - -Now we can see that the memory leak is due to the malloc() at line 6. - -## Usage With MPI - -Although Valgrind is not primarily a parallel debugger, it can be used to debug parallel applications as well. When launching your parallel applications, prepend the valgrind command. For example: - -```console -$ mpirun -np 4 valgrind myapplication -``` - -The default version without MPI support will however report a large number of false errors in the MPI library, such as: - -```console - ==30166== Conditional jump or move depends on uninitialised value(s) - ==30166== at 0x4C287E8: strlen (mc_replace_strmem.c:282) - ==30166== by 0x55443BD: I_MPI_Processor_model_number (init_interface.c:427) - ==30166== by 0x55439E0: I_MPI_Processor_arch_code (init_interface.c:171) - ==30166== by 0x558D5AE: MPID_nem_impi_init_shm_configuration (mpid_nem_impi_extensions.c:1091) - ==30166== by 0x5598F4C: MPID_nem_init_ckpt (mpid_nem_init.c:566) - ==30166== by 0x5598B65: MPID_nem_init (mpid_nem_init.c:489) - ==30166== by 0x539BD75: MPIDI_CH3_Init (ch3_init.c:64) - ==30166== by 0x5578743: MPID_Init (mpid_init.c:193) - ==30166== by 0x554650A: MPIR_Init_thread (initthread.c:539) - ==30166== by 0x553369F: PMPI_Init (init.c:195) - ==30166== by 0x4008BD: main (valgrind-example-mpi.c:18) -``` - -so it is better to use the MPI-enabled valgrind from module. The MPI versions requires library: - -$EBROOTVALGRIND/lib/valgrind/libmpiwrap-amd64-linux.so - -which must be included in the LD_PRELOAD environment variable. - -Lets look at this MPI example: - -```cpp -#include <stdlib.h> -#include <mpi.h> - -int main(int argc, char *argv[]) -{ - int *data = malloc(sizeof(int)*99); - - MPI_Init(&argc, &argv); - MPI_Bcast(data, 100, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Finalize(); - - return 0; -} -``` - -There are two errors - use of uninitialized memory and invalid length of the buffer. Lets debug it with valgrind : - -```console -$ module add intel impi -$ mpiicc -g valgrind-example-mpi.c -o valgrind-example-mpi -$ module add Valgrind/3.11.0-intel-2015b -$ mpirun -np 2 -env LD_PRELOAD $EBROOTVALGRIND/lib/valgrind/libmpiwrap-amd64-linux.so valgrind ./valgrind-example-mpi -``` - -Prints this output : (note that there is output printed for every launched MPI process) - -```console - ==31318== Memcheck, a memory error detector - ==31318== Copyright (C) 2002-2013, and GNU GPL'd, by Julian Seward et al. - ==31318== Using Valgrind-3.9.0 and LibVEX; rerun with -h for copyright info - ==31318== Command: ./valgrind-example-mpi - ==31318== - ==31319== Memcheck, a memory error detector - ==31319== Copyright (C) 2002-2013, and GNU GPL'd, by Julian Seward et al. - ==31319== Using Valgrind-3.9.0 and LibVEX; rerun with -h for copyright info - ==31319== Command: ./valgrind-example-mpi - ==31319== - valgrind MPI wrappers 31319: Active for pid 31319 - valgrind MPI wrappers 31319: Try MPIWRAP_DEBUG=help for possible options - valgrind MPI wrappers 31318: Active for pid 31318 - valgrind MPI wrappers 31318: Try MPIWRAP_DEBUG=help for possible options - ==31319== Unaddressable byte(s) found during client check request - ==31319== at 0x4E35974: check_mem_is_addressable_untyped (libmpiwrap.c:960) - ==31319== by 0x4E5D0FE: PMPI_Bcast (libmpiwrap.c:908) - ==31319== by 0x400911: main (valgrind-example-mpi.c:20) - ==31319== Address 0x69291cc is 0 bytes after a block of size 396 alloc'd - ==31319== at 0x4C27AAA: malloc (vg_replace_malloc.c:291) - ==31319== by 0x4007BC: main (valgrind-example-mpi.c:8) - ==31319== - ==31318== Uninitialised byte(s) found during client check request - ==31318== at 0x4E3591D: check_mem_is_defined_untyped (libmpiwrap.c:952) - ==31318== by 0x4E5D06D: PMPI_Bcast (libmpiwrap.c:908) - ==31318== by 0x400911: main (valgrind-example-mpi.c:20) - ==31318== Address 0x6929040 is 0 bytes inside a block of size 396 alloc'd - ==31318== at 0x4C27AAA: malloc (vg_replace_malloc.c:291) - ==31318== by 0x4007BC: main (valgrind-example-mpi.c:8) - ==31318== - ==31318== Unaddressable byte(s) found during client check request - ==31318== at 0x4E3591D: check_mem_is_defined_untyped (libmpiwrap.c:952) - ==31318== by 0x4E5D06D: PMPI_Bcast (libmpiwrap.c:908) - ==31318== by 0x400911: main (valgrind-example-mpi.c:20) - ==31318== Address 0x69291cc is 0 bytes after a block of size 396 alloc'd - ==31318== at 0x4C27AAA: malloc (vg_replace_malloc.c:291) - ==31318== by 0x4007BC: main (valgrind-example-mpi.c:8) - ==31318== - ==31318== - ==31318== HEAP SUMMARY: - ==31318== in use at exit: 3,172 bytes in 67 blocks - ==31318== total heap usage: 191 allocs, 124 frees, 81,203 bytes allocated - ==31318== - ==31319== - ==31319== HEAP SUMMARY: - ==31319== in use at exit: 3,172 bytes in 67 blocks - ==31319== total heap usage: 175 allocs, 108 frees, 48,435 bytes allocated - ==31319== - ==31318== LEAK SUMMARY: - ==31318== definitely lost: 408 bytes in 3 blocks - ==31318== indirectly lost: 256 bytes in 1 blocks - ==31318== possibly lost: 0 bytes in 0 blocks - ==31318== still reachable: 2,508 bytes in 63 blocks - ==31318== suppressed: 0 bytes in 0 blocks - ==31318== Rerun with --leak-check=full to see details of leaked memory - ==31318== - ==31318== For counts of detected and suppressed errors, rerun with: -v - ==31318== Use --track-origins=yes to see where uninitialised values come from - ==31318== ERROR SUMMARY: 2 errors from 2 contexts (suppressed: 4 from 4) - ==31319== LEAK SUMMARY: - ==31319== definitely lost: 408 bytes in 3 blocks - ==31319== indirectly lost: 256 bytes in 1 blocks - ==31319== possibly lost: 0 bytes in 0 blocks - ==31319== still reachable: 2,508 bytes in 63 blocks - ==31319== suppressed: 0 bytes in 0 blocks - ==31319== Rerun with --leak-check=full to see details of leaked memory - ==31319== - ==31319== For counts of detected and suppressed errors, rerun with: -v - ==31319== ERROR SUMMARY: 1 errors from 1 contexts (suppressed: 4 from 4) -``` - -We can see that Valgrind has reported use of unitialised memory on the master process (which reads the array to be broadcasted) and use of unaddresable memory on both processes. diff --git a/docs.it4i/salomon/software/debuggers/vampir.md b/docs.it4i/salomon/software/debuggers/vampir.md deleted file mode 100644 index 852374d22..000000000 --- a/docs.it4i/salomon/software/debuggers/vampir.md +++ /dev/null @@ -1,24 +0,0 @@ -# Vampir - -Vampir is a commercial trace analysis and visualisation tool. It can work with traces in OTF and OTF2 formats. It does not have the functionality to collect traces, you need to use a trace collection tool (such as [Score-P](score-p/)) first to collect the traces. - - - -## Installed Versions - -```console -$ ml av Vampir -``` - -```console -$ ml Vampir -$ vampir & -``` - -## User Manual - -You can find the detailed user manual in PDF format in $EBROOTVAMPIR/doc/vampir-manual.pdf - -## References - -1. <https://www.vampir.eu> diff --git a/docs.it4i/salomon/software/intel-suite/intel-advisor.md b/docs.it4i/salomon/software/intel-suite/intel-advisor.md deleted file mode 100644 index 688deda17..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-advisor.md +++ /dev/null @@ -1,31 +0,0 @@ -# Intel Advisor - -is tool aiming to assist you in vectorization and threading of your code. You can use it to profile your application and identify loops, that could benefit from vectorization and/or threading parallelism. - -## Installed Versions - -The following versions are currently available on Salomon as modules: - -2016 Update 2 - Advisor/2016_update2 - -## Usage - -Your program should be compiled with -g switch to include symbol names. You should compile with -O2 or higher to see code that is already vectorized by the compiler. - -Profiling is possible either directly from the GUI, or from command line. - -To profile from GUI, launch Advisor: - -```console -$ advixe-gui -``` - -Then select menu File -> New -> Project. Choose a directory to save project data to. After clicking OK, Project properties window will appear, where you can configure path to your binary, launch arguments, working directory etc. After clicking OK, the project is ready. - -In the left pane, you can switch between Vectorization and Threading workflows. Each has several possible steps which you can execute by clicking Collect button. Alternatively, you can click on Command Line, to see the command line required to run the analysis directly from command line. - -## References - -1. [Intel® Advisor 2015 Tutorial: Find Where to Add Parallelism - C++ Sample](https://software.intel.com/en-us/intel-advisor-tutorial-vectorization-windows-cplusplus) -1. [Product page](https://software.intel.com/en-us/intel-advisor-xe) -1. [Documentation](https://software.intel.com/en-us/intel-advisor-2016-user-guide-linux) diff --git a/docs.it4i/salomon/software/intel-suite/intel-compilers.md b/docs.it4i/salomon/software/intel-suite/intel-compilers.md deleted file mode 100644 index 8e2ee714f..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-compilers.md +++ /dev/null @@ -1,36 +0,0 @@ -# Intel Compilers - -The Intel compilers in multiple versions are available, via module intel. The compilers include the icc C and C++ compiler and the ifort fortran 77/90/95 compiler. - -```console -$ ml intel -$ icc -v -$ ifort -v -``` - -The intel compilers provide for vectorization of the code, via the AVX2 instructions and support threading parallelization via OpenMP - -For maximum performance on the Salomon cluster compute nodes, compile your programs using the AVX2 instructions, with reporting where the vectorization was used. We recommend following compilation options for high performance - -```console -$ icc -ipo -O3 -xCORE-AVX2 -qopt-report1 -qopt-report-phase=vec myprog.c mysubroutines.c -o myprog.x -$ ifort -ipo -O3 -xCORE-AVX2 -qopt-report1 -qopt-report-phase=vec myprog.f mysubroutines.f -o myprog.x -``` - -In this example, we compile the program enabling interprocedural optimizations between source files (-ipo), aggresive loop optimizations (-O3) and vectorization (-xCORE-AVX2) - -The compiler recognizes the omp, simd, vector and ivdep pragmas for OpenMP parallelization and AVX2 vectorization. Enable the OpenMP parallelization by the **-openmp** compiler switch. - -```console -$ icc -ipo -O3 -xCORE-AVX2 -qopt-report1 -qopt-report-phase=vec -openmp myprog.c mysubroutines.c -o myprog.x -$ ifort -ipo -O3 -xCORE-AVX2 -qopt-report1 -qopt-report-phase=vec -openmp myprog.f mysubroutines.f -o myprog.x -``` - -Read more at <https://software.intel.com/en-us/intel-cplusplus-compiler-16.0-user-and-reference-guide> - -## Sandy Bridge/Ivy Bridge/Haswell Binary Compatibility - - Anselm nodes are currently equipped with Sandy Bridge CPUs, while Salomon compute nodes are equipped with Haswell based architecture. The UV1 SMP compute server has Ivy Bridge CPUs, which are equivalent to Sandy Bridge (only smaller manufacturing technology). The new processors are backward compatible with the Sandy Bridge nodes, so all programs that ran on the Sandy Bridge processors, should also run on the new Haswell nodes. To get optimal performance out of the Haswell processors a program should make use of the special AVX2 instructions for this processor. One can do this by recompiling codes with the compiler flags designated to invoke these instructions. For the Intel compiler suite, there are two ways of doing this: - -* Using compiler flag (both for Fortran and C): -xCORE-AVX2. This will create a binary with AVX2 instructions, specifically for the Haswell processors. Note that the executable will not run on Sandy Bridge/Ivy Bridge nodes. -* Using compiler flags (both for Fortran and C): -xAVX -axCORE-AVX2. This will generate multiple, feature specific auto-dispatch code paths for Intel® processors, if there is a performance benefit. So this binary will run both on Sandy Bridge/Ivy Bridge and Haswell processors. During runtime it will be decided which path to follow, dependent on which processor you are running on. In general this will result in larger binaries. diff --git a/docs.it4i/salomon/software/intel-suite/intel-debugger.md b/docs.it4i/salomon/software/intel-suite/intel-debugger.md deleted file mode 100644 index 15788c798..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-debugger.md +++ /dev/null @@ -1,73 +0,0 @@ -# Intel Debugger - -IDB is no longer available since Intel Parallel Studio 2015 - -## Debugging Serial Applications - -The intel debugger version 13.0 is available, via module intel. The debugger works for applications compiled with C and C++ compiler and the ifort fortran 77/90/95 compiler. The debugger provides java GUI environment. Use [X display](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/) for running the GUI. - -```console -$ ml intel -$ ml Java -$ idb -``` - -The debugger may run in text mode. To debug in text mode, use - -```console -$ idbc -``` - -To debug on the compute nodes, module intel must be loaded. The GUI on compute nodes may be accessed using the same way as in [the GUI section](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/) - -Example: - -```console -$ qsub -q qexp -l select=1:ncpus=24 -X -I - qsub: waiting for job 19654.srv11 to start - qsub: job 19654.srv11 ready -$ ml intel -$ ml Java -$ icc -O0 -g myprog.c -o myprog.x -$ idb ./myprog.x -``` - -In this example, we allocate 1 full compute node, compile program myprog.c with debugging options -O0 -g and run the idb debugger interactively on the myprog.x executable. The GUI access is via X11 port forwarding provided by the PBS workload manager. - -## Debugging Parallel Applications - - Intel debugger is capable of debugging multithreaded and MPI parallel programs as well. - -### Small Number of MPI Ranks - -For debugging small number of MPI ranks, you may execute and debug each rank in separate xterm terminal (do not forget the [X display](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/)). Using Intel MPI, this may be done in following way: - -```console -$ qsub -q qexp -l select=2:ncpus=24 -X -I - qsub: waiting for job 19654.srv11 to start - qsub: job 19655.srv11 ready -$ ml intel -$ mpirun -ppn 1 -hostfile $PBS_NODEFILE --enable-x xterm -e idbc ./mympiprog.x -``` - -In this example, we allocate 2 full compute node, run xterm on each node and start idb debugger in command line mode, debugging two ranks of mympiprog.x application. The xterm will pop up for each rank, with idb prompt ready. The example is not limited to use of Intel MPI - -### Large Number of MPI Ranks - -Run the idb debugger from within the MPI debug option. This will cause the debugger to bind to all ranks and provide aggregated outputs across the ranks, pausing execution automatically just after startup. You may then set break points and step the execution manually. Using Intel MPI: - -```console -$ qsub -q qexp -l select=2:ncpus=24 -X -I - qsub: waiting for job 19654.srv11 to start - qsub: job 19655.srv11 ready -$ ml intel -$ mpirun -n 48 -idb ./mympiprog.x -``` - -### Debugging Multithreaded Application - -Run the idb debugger in GUI mode. The menu Parallel contains number of tools for debugging multiple threads. One of the most useful tools is the **Serialize Execution** tool, which serializes execution of concurrent threads for easy orientation and identification of concurrency related bugs. - -## Further Information - -Exhaustive manual on idb features and usage is published at Intel website, <https://software.intel.com/sites/products/documentation/doclib/iss/2013/compiler/cpp-lin/> diff --git a/docs.it4i/salomon/software/intel-suite/intel-inspector.md b/docs.it4i/salomon/software/intel-suite/intel-inspector.md deleted file mode 100644 index bd2989238..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-inspector.md +++ /dev/null @@ -1,39 +0,0 @@ -# Intel Inspector - -Intel Inspector is a dynamic memory and threading error checking tool for C/C++/Fortran applications. It can detect issues such as memory leaks, invalid memory references, uninitalized variables, race conditions, deadlocks etc. - -## Installed Versions - -The following versions are currently available on Salomon as modules: - -2016 Update 1 - Inspector/2016_update1 - -## Usage - -Your program should be compiled with -g switch to include symbol names. Optimizations can be turned on. - -Debugging is possible either directly from the GUI, or from command line. - -### GUI Mode - -To debug from GUI, launch Inspector: - -```console -$ inspxe-gui & -``` - -Then select menu File -> New -> Project. Choose a directory to save project data to. After clicking OK, Project properties window will appear, where you can configure path to your binary, launch arguments, working directory etc. After clicking OK, the project is ready. - -In the main pane, you can start a predefined analysis type or define your own. Click Start to start the analysis. Alternatively, you can click on Command Line, to see the command line required to run the analysis directly from command line. - -### Batch Mode - -Analysis can be also run from command line in batch mode. Batch mode analysis is run with command inspxe-cl. To obtain the required parameters, either consult the documentation or you can configure the analysis in the GUI and then click "Command Line" button in the lower right corner to the respective command line. - -Results obtained from batch mode can be then viewed in the GUI by selecting File -> Open -> Result... - -## References - -1. [Product page](https://software.intel.com/en-us/intel-inspector-xe) -1. [Documentation and Release Notes](https://software.intel.com/en-us/intel-inspector-xe-support/documentation) -1. [Tutorials](https://software.intel.com/en-us/articles/inspectorxe-tutorials) diff --git a/docs.it4i/salomon/software/intel-suite/intel-integrated-performance-primitives.md b/docs.it4i/salomon/software/intel-suite/intel-integrated-performance-primitives.md deleted file mode 100644 index 60628eed0..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-integrated-performance-primitives.md +++ /dev/null @@ -1,78 +0,0 @@ -# Intel IPP - -## Intel Integrated Performance Primitives - -Intel Integrated Performance Primitives, version 9.0.1, compiled for AVX2 vector instructions is available, via module ipp. The IPP is a very rich library of highly optimized algorithmic building blocks for media and data applications. This includes signal, image and frame processing algorithms, such as FFT, FIR, Convolution, Optical Flow, Hough transform, Sum, MinMax, as well as cryptographic functions, linear algebra functions and many more. - -Check out IPP before implementing own math functions for data processing, it is likely already there. - -```console -$ ml ipp -``` - -The module sets up environment variables, required for linking and running ipp enabled applications. - -## IPP Example - -```cpp - #include "ipp.h" - #include <stdio.h> - int main(int argc, char* argv[]) - { - const IppLibraryVersion *lib; - Ipp64u fm; - IppStatus status; - - status= ippInit(); //IPP initialization with the best optimization layer - if( status != ippStsNoErr ) { - printf("IppInit() Error:n"); - printf("%sn", ippGetStatusString(status) ); - return -1; - } - - //Get version info - lib = ippiGetLibVersion(); - printf("%s %sn", lib->Name, lib->Version); - - //Get CPU features enabled with selected library level - fm=ippGetEnabledCpuFeatures(); - printf("SSE :%cn",(fm>1)&1?'Y':'N'); - printf("SSE2 :%cn",(fm>2)&1?'Y':'N'); - printf("SSE3 :%cn",(fm>3)&1?'Y':'N'); - printf("SSSE3 :%cn",(fm>4)&1?'Y':'N'); - printf("SSE41 :%cn",(fm>6)&1?'Y':'N'); - printf("SSE42 :%cn",(fm>7)&1?'Y':'N'); - printf("AVX :%cn",(fm>8)&1 ?'Y':'N'); - printf("AVX2 :%cn", (fm>15)&1 ?'Y':'N' ); - printf("----------n"); - printf("OS Enabled AVX :%cn", (fm>9)&1 ?'Y':'N'); - printf("AES :%cn", (fm>10)&1?'Y':'N'); - printf("CLMUL :%cn", (fm>11)&1?'Y':'N'); - printf("RDRAND :%cn", (fm>13)&1?'Y':'N'); - printf("F16C :%cn", (fm>14)&1?'Y':'N'); - - return 0; - } -``` - -Compile above example, using any compiler and the ipp module. - -```console -$ ml intel -$ ml ipp -$ icc testipp.c -o testipp.x -lippi -lipps -lippcore -``` - -You will need the ipp module loaded to run the ipp enabled executable. This may be avoided, by compiling library search paths into the executable - -```console -$ ml intel -$ ml ipp -$ icc testipp.c -o testipp.x -Wl,-rpath=$LIBRARY_PATH -lippi -lipps -lippcore -``` - -## Code Samples and Documentation - -Intel provides number of [Code Samples for IPP](https://software.intel.com/en-us/articles/code-samples-for-intel-integrated-performance-primitives-library), illustrating use of IPP. - -Read full documentation on IPP [on Intel website,](http://software.intel.com/sites/products/search/search.php?q=&x=15&y=6&product=ipp&version=7.1&docos=lin) in particular the [IPP Reference manual.](http://software.intel.com/sites/products/documentation/doclib/ipp_sa/71/ipp_manual/index.htm) diff --git a/docs.it4i/salomon/software/intel-suite/intel-mkl.md b/docs.it4i/salomon/software/intel-suite/intel-mkl.md deleted file mode 100644 index 6b54e0890..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-mkl.md +++ /dev/null @@ -1,119 +0,0 @@ -# Intel MKL - -## Intel Math Kernel Library - -Intel Math Kernel Library (Intel MKL) is a library of math kernel subroutines, extensively threaded and optimized for maximum performance. Intel MKL provides these basic math kernels: - -* BLAS (level 1, 2, and 3) and LAPACK linear algebra routines, offering vector, vector-matrix, and matrix-matrix operations. -* The PARDISO direct sparse solver, an iterative sparse solver, and supporting sparse BLAS (level 1, 2, and 3) routines for solving sparse systems of equations. -* ScaLAPACK distributed processing linear algebra routines for Linux and Windows operating systems, as well as the Basic Linear Algebra Communications Subprograms (BLACS) and the Parallel Basic Linear Algebra Subprograms (PBLAS). -* Fast Fourier transform (FFT) functions in one, two, or three dimensions with support for mixed radices (not limited to sizes that are powers of 2), as well as distributed versions of these functions. -* Vector Math Library (VML) routines for optimized mathematical operations on vectors. -* Vector Statistical Library (VSL) routines, which offer high-performance vectorized random number generators (RNG) for several probability distributions, convolution and correlation routines, and summary statistics functions. -* Data Fitting Library, which provides capabilities for spline-based approximation of functions, derivatives and integrals of functions, and search. -* Extended Eigensolver, a shared memory version of an eigensolver based on the Feast Eigenvalue Solver. - -For details see the [Intel MKL Reference Manual](http://software.intel.com/sites/products/documentation/doclib/mkl_sa/11/mklman/index.htm). - -Intel MKL version 11.2.3.187 is available on the cluster - -```console -$ ml imkl -``` - -The module sets up environment variables, required for linking and running mkl enabled applications. The most important variables are the $MKLROOT, $CPATH, $LD_LIBRARY_PATH and $MKL_EXAMPLES - -Intel MKL library may be linked using any compiler. With intel compiler use -mkl option to link default threaded MKL. - -### Interfaces - -Intel MKL library provides number of interfaces. The fundamental once are the LP64 and ILP64. The Intel MKL ILP64 libraries use the 64-bit integer type (necessary for indexing large arrays, with more than 231^-1 elements), whereas the LP64 libraries index arrays with the 32-bit integer type. - -| Interface | Integer type | -| --------- | -------------------------------------------- | -| LP64 | 32-bit, int, integer(kind=4), MPI_INT | -| ILP64 | 64-bit, long int, integer(kind=8), MPI_INT64 | - -### Linking - -Linking Intel MKL libraries may be complex. Intel [mkl link line advisor](http://software.intel.com/en-us/articles/intel-mkl-link-line-advisor) helps. See also [examples](intel-mkl/#examples) below. - -You will need the mkl module loaded to run the mkl enabled executable. This may be avoided, by compiling library search paths into the executable. Include rpath on the compile line: - -```console -$ icc .... -Wl,-rpath=$LIBRARY_PATH ... -``` - -### Threading - -Advantage in using Intel MKL library is that it brings threaded parallelization to applications that are otherwise not parallel. - -For this to work, the application must link the threaded MKL library (default). Number and behaviour of MKL threads may be controlled via the OpenMP environment variables, such as OMP_NUM_THREADS and KMP_AFFINITY. MKL_NUM_THREADS takes precedence over OMP_NUM_THREADS - -```console -$ export OMP_NUM_THREADS=24 -$ export KMP_AFFINITY=granularity=fine,compact,1,0 -``` - -The application will run with 24 threads with affinity optimized for fine grain parallelization. - -## Examples - -Number of examples, demonstrating use of the Intel MKL library and its linking is available on clusters, in the $MKL_EXAMPLES directory. In the examples below, we demonstrate linking Intel MKL to Intel and GNU compiled program for multi-threaded matrix multiplication. - -### Working With Examples - -```console -$ ml intel -$ ml imkl -$ cp -a $MKL_EXAMPLES/cblas /tmp/ -$ cd /tmp/cblas -$ make sointel64 function=cblas_dgemm -``` - -In this example, we compile, link and run the cblas_dgemm example, demonstrating use of MKL example suite installed on clusters. - -### Example: MKL and Intel Compiler - -```console -$ ml intel -$ ml imkl -$ cp -a $MKL_EXAMPLES/cblas /tmp/ -$ cd /tmp/cblas -$ -$ icc -w source/cblas_dgemmx.c source/common_func.c -mkl -o cblas_dgemmx.x -$ ./cblas_dgemmx.x data/cblas_dgemmx.d -``` - -In this example, we compile, link and run the cblas_dgemm example, demonstrating use of MKL with icc -mkl option. Using the -mkl option is equivalent to: - -```console -$ icc -w source/cblas_dgemmx.c source/common_func.c -o cblas_dgemmx.x -I$MKL_INC_DIR -L$MKL_LIB_DIR -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -liomp5 -``` - -In this example, we compile and link the cblas_dgemm example, using LP64 interface to threaded MKL and Intel OMP threads implementation. - -### Example: Intel MKL and GNU Compiler - -```console -$ ml GCC -$ ml imkl -$ cp -a $MKL_EXAMPLES/cblas /tmp/ -$ cd /tmp/cblas -$ gcc -w source/cblas_dgemmx.c source/common_func.c -o cblas_dgemmx.x -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lm -$ ./cblas_dgemmx.x data/cblas_dgemmx.d -``` - -In this example, we compile, link and run the cblas_dgemm example, using LP64 interface to threaded MKL and gnu OMP threads implementation. - -## MKL and MIC Accelerators - -The Intel MKL is capable to automatically offload the computations o the MIC accelerator. See section [Intel Xeon Phi](../intel-xeon-phi/) for details. - -## LAPACKE C Interface - -MKL includes LAPACKE C Interface to LAPACK. For some reason, although Intel is the author of LAPACKE, the LAPACKE header files are not present in MKL. For this reason, we have prepared LAPACKE module, which includes Intel's LAPACKE headers from official LAPACK, which you can use to compile code using LAPACKE interface against MKL. - -## Further Reading - -Read more on [Intel website](http://software.intel.com/en-us/intel-mkl), in particular the [MKL users guide](https://software.intel.com/en-us/intel-mkl/documentation/linux). diff --git a/docs.it4i/salomon/software/intel-suite/intel-parallel-studio-introduction.md b/docs.it4i/salomon/software/intel-suite/intel-parallel-studio-introduction.md deleted file mode 100644 index b22274a0e..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-parallel-studio-introduction.md +++ /dev/null @@ -1,69 +0,0 @@ -# Intel Parallel Studio - -The Salomon cluster provides following elements of the Intel Parallel Studio XE - -Intel Parallel Studio XE - -* Intel Compilers -* Intel Debugger -* Intel MKL Library -* Intel Integrated Performance Primitives Library -* Intel Threading Building Blocks Library -* Intel Trace Analyzer and Collector -* Intel Advisor -* Intel Inspector - -## Intel Compilers - -The Intel compilers version 131.3 are available, via module iccifort/2013.5.192-GCC-4.8.3. The compilers include the icc C and C++ compiler and the ifort fortran 77/90/95 compiler. - -```console -$ ml intel -$ icc -v -$ ifort -v -``` - -Read more at the [Intel Compilers](intel-compilers/) page. - -## Intel Debugger - -IDB is no longer available since Parallel Studio 2015. - -The intel debugger version 13.0 is available, via module intel. The debugger works for applications compiled with C and C++ compiler and the ifort fortran 77/90/95 compiler. The debugger provides java GUI environment. - -```console -$ ml intel -$ idb -``` - -Read more at the [Intel Debugger](intel-debugger/) page. - -## Intel Math Kernel Library - -Intel Math Kernel Library (Intel MKL) is a library of math kernel subroutines, extensively threaded and optimized for maximum performance. Intel MKL unites and provides these basic components: BLAS, LAPACK, ScaLapack, PARDISO, FFT, VML, VSL, Data fitting, Feast Eigensolver and many more. - -```console -$ ml imkl -``` - -Read more at the [Intel MKL](intel-mkl/) page. - -## Intel Integrated Performance Primitives - -Intel Integrated Performance Primitives, version 7.1.1, compiled for AVX is available, via module ipp. The IPP is a library of highly optimized algorithmic building blocks for media and data applications. This includes signal, image and frame processing algorithms, such as FFT, FIR, Convolution, Optical Flow, Hough transform, Sum, MinMax and many more. - -```console -$ ml ipp -``` - -Read more at the [Intel IPP](intel-integrated-performance-primitives/) page. - -## Intel Threading Building Blocks - -Intel Threading Building Blocks (Intel TBB) is a library that supports scalable parallel programming using standard ISO C++ code. It does not require special languages or compilers. It is designed to promote scalable data parallel programming. Additionally, it fully supports nested parallelism, so you can build larger parallel components from smaller parallel components. To use the library, you specify tasks, not threads, and let the library map tasks onto threads in an efficient manner. - -```console -$ ml tbb -``` - -Read more at the [Intel TBB](intel-tbb/) page. diff --git a/docs.it4i/salomon/software/intel-suite/intel-tbb.md b/docs.it4i/salomon/software/intel-suite/intel-tbb.md deleted file mode 100644 index 59976aa7e..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-tbb.md +++ /dev/null @@ -1,40 +0,0 @@ -# Intel TBB - -## Intel Threading Building Blocks - -Intel Threading Building Blocks (Intel TBB) is a library that supports scalable parallel programming using standard ISO C++ code. It does not require special languages or compilers. To use the library, you specify tasks, not threads, and let the library map tasks onto threads in an efficient manner. The tasks are executed by a runtime scheduler and may be offloaded to [MIC accelerator](../intel-xeon-phi/). - -Intel is available on the cluster. - -```console -$ ml av tbb -``` - -The module sets up environment variables, required for linking and running tbb enabled applications. - -Link the tbb library, using -ltbb - -## Examples - -Number of examples, demonstrating use of TBB and its built-in scheduler is available on Anselm, in the $TBB_EXAMPLES directory. - -```console -$ ml intel -$ ml tbb -$ cp -a $TBB_EXAMPLES/common $TBB_EXAMPLES/parallel_reduce /tmp/ -$ cd /tmp/parallel_reduce/primes -$ icc -O2 -DNDEBUG -o primes.x main.cpp primes.cpp -ltbb -$ ./primes.x -``` - -In this example, we compile, link and run the primes example, demonstrating use of parallel task-based reduce in computation of prime numbers. - -You will need the tbb module loaded to run the tbb enabled executable. This may be avoided, by compiling library search paths into the executable. - -```console -$ icc -O2 -o primes.x main.cpp primes.cpp -Wl,-rpath=$LIBRARY_PATH -ltbb -``` - -## Further Reading - -Read more on Intel website, <http://software.intel.com/sites/products/documentation/doclib/tbb_sa/help/index.htm> diff --git a/docs.it4i/salomon/software/intel-suite/intel-trace-analyzer-and-collector.md b/docs.it4i/salomon/software/intel-suite/intel-trace-analyzer-and-collector.md deleted file mode 100644 index 9cae361ca..000000000 --- a/docs.it4i/salomon/software/intel-suite/intel-trace-analyzer-and-collector.md +++ /dev/null @@ -1,40 +0,0 @@ -# Intel Trace Analyzer and Collector - -Intel Trace Analyzer and Collector (ITAC) is a tool to collect and graphicaly analyze behaviour of MPI applications. It helps you to analyze communication patterns of your application, identify hotspots, perform correctnes checking (identify deadlocks, data corruption etc), simulate how your application would run on a different interconnect. - -ITAC is a offline analysis tool - first you run your application to collect a trace file, then you can open the trace in a GUI analyzer to view it. - -## Installed Version - -Currently on Salomon is version 9.1.2.024 available as module itac/9.1.2.024 - -## Collecting Traces - -ITAC can collect traces from applications that are using Intel MPI. To generate a trace, simply add -trace option to your mpirun command : - -```console -$ ml itac/9.1.2.024 -$ mpirun -trace myapp -``` - -The trace will be saved in file myapp.stf in the current directory. - -## Viewing Traces - -To view and analyze the trace, open ITAC GUI in a [graphical environment](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/): - -```console -$ ml itac/9.1.2.024 -$ traceanalyzer -``` - -The GUI will launch and you can open the produced `*`.stf file. - - - -Please refer to Intel documenation about usage of the GUI tool. - -## References - -1. [Getting Started with Intel® Trace Analyzer and Collector](https://software.intel.com/en-us/get-started-with-itac-for-linux) -1. [Intel® Trace Analyzer and Collector - Documentation](https://software.intel.com/en-us/intel-trace-analyzer) diff --git a/docs.it4i/salomon/software/java.md b/docs.it4i/salomon/software/java.md deleted file mode 100644 index 83c3738c0..000000000 --- a/docs.it4i/salomon/software/java.md +++ /dev/null @@ -1,25 +0,0 @@ -# Java - -Java is available on the cluster. Activate java by loading the Java module - -```console -$ ml Java -``` - -Note that the Java module must be loaded on the compute nodes as well, in order to run java on compute nodes. - -Check for java version and path - -```console -$ java -version -$ which java -``` - -With the module loaded, not only the runtime environment (JRE), but also the development environment (JDK) with the compiler is available. - -```console -$ javac -version -$ which javac -``` - -Java applications may use MPI for inter-process communication, in conjunction with Open MPI. Read more on <http://www.open-mpi.org/faq/?category=java>. This functionality is currently not supported on Anselm cluster. In case you require the java interface to MPI, please contact [cluster support](https://support.it4i.cz/rt/). diff --git a/docs.it4i/salomon/software/machine-learning/introduction.md b/docs.it4i/salomon/software/machine-learning/introduction.md deleted file mode 100644 index 4c5aba660..000000000 --- a/docs.it4i/salomon/software/machine-learning/introduction.md +++ /dev/null @@ -1,19 +0,0 @@ -# Machine Learning - -This section overviews machine learning frameworks and libraries available on the Salomon cluster. - -## TensorFlow - -Load TensorFlow module: - -```console -$ ml Tensorflow -``` - -Test module: - -```console -$ ml Tensorflow -``` - -Read more about available versions at the [TensorFlow page](tensorflow). diff --git a/docs.it4i/salomon/software/machine-learning/tensorflow.md b/docs.it4i/salomon/software/machine-learning/tensorflow.md deleted file mode 100644 index cad52480f..000000000 --- a/docs.it4i/salomon/software/machine-learning/tensorflow.md +++ /dev/null @@ -1,68 +0,0 @@ -# TensorFlow - -TensorFlow is an open-source software library for machine intelligence. - -## TensorFlow modules - -Salomon provides three different TensorFlow modules: - -* Tensorflow/1.1.0 -* Tensorflow/1.2.0-GCC-7.1.0-2.28 -* Tensorflow/1.2.0-intel-2017.05-mkl - -### Tensorflow/1.1.0 (not recommended) - -TensorFlow 1.1 build. - -```console - -$ ml Tensorflow/1.1.0 -``` - -This module was built with: - -* GCC/4.9.3 -* Python/3.6.1 - -### Tensorflow/1.2.0-GCC-7.1.0-2.28 (default, recommended) - -TensorFlow 1.2 with SIMD support. TensorFlow build taking advantage of the Salomon CPU architecture. - -```console - -$ ml Tensorflow/1.2.0-GCC-7.1.0-2.28 -``` - -This module was built with: - -* GCC/7.1.0-2.28 -* Python/3.6.1 -* protobuf/3.2.0-GCC-7.1.0-2.28-Python-3.6.1 - -### Tensorflow/1.2.0-intel-2017.05-mkl - -TensorFlow 1.2 with MKL support. - -```console - -$ ml Tensorflow/1.2.0-intel-2017.05-mkl -``` - -This module was built with: - -* icc/2017.4.196-GCC-7.1.0-2.28 -* Python/3.6.1 -* protobuf/3.2.0-GCC-7.1.0-2.28-Python-3.6.1 - -## TensorFlow application example - -After loading one of the available TensorFlow modules, you can check the functionality running the following python script. - -```python - -import tensorflow as tf - -c = tf.constant('Hello World!') -sess = tf.Session() -print(sess.run(c)) -``` diff --git a/docs.it4i/salomon/software/mpi/Running_OpenMPI.md b/docs.it4i/salomon/software/mpi/Running_OpenMPI.md deleted file mode 100644 index e2633236a..000000000 --- a/docs.it4i/salomon/software/mpi/Running_OpenMPI.md +++ /dev/null @@ -1,203 +0,0 @@ -# Running OpenMPI - -## OpenMPI Program Execution - -The OpenMPI programs may be executed only via the PBS Workload manager, by entering an appropriate queue. On the cluster, the **OpenMPI 1.8.6** is OpenMPI based MPI implementation. - -### Basic Usage - -Use the mpiexec to run the OpenMPI code. - -Example: - -```console -$ qsub -q qexp -l select=4:ncpus=24 -I - qsub: waiting for job 15210.isrv5 to start - qsub: job 15210.isrv5 ready -$ pwd - /home/username -$ ml OpenMPI -$ mpiexec -pernode ./helloworld_mpi.x - Hello world! from rank 0 of 4 on host r1i0n17 - Hello world! from rank 1 of 4 on host r1i0n5 - Hello world! from rank 2 of 4 on host r1i0n6 - Hello world! from rank 3 of 4 on host r1i0n7 -``` - -Please be aware, that in this example, the directive **-pernode** is used to run only **one task per node**, which is normally an unwanted behaviour (unless you want to run hybrid code with just one MPI and 24 OpenMP tasks per node). In normal MPI programs **omit the -pernode directive** to run up to 24 MPI tasks per each node. - -In this example, we allocate 4 nodes via the express queue interactively. We set up the openmpi environment and interactively run the helloworld_mpi.x program. -Note that the executable helloworld_mpi.x must be available within the same path on all nodes. This is automatically fulfilled on the /home and /scratch filesystem. - -You need to preload the executable, if running on the local ramdisk /tmp filesystem - -```console -$ pwd - /tmp/pbs.15210.isrv5 -$ mpiexec -pernode --preload-binary ./helloworld_mpi.x - Hello world! from rank 0 of 4 on host r1i0n17 - Hello world! from rank 1 of 4 on host r1i0n5 - Hello world! from rank 2 of 4 on host r1i0n6 - Hello world! from rank 3 of 4 on host r1i0n7 -``` - -In this example, we assume the executable helloworld_mpi.x is present on compute node r1i0n17 on ramdisk. We call the mpiexec whith the **--preload-binary** argument (valid for openmpi). The mpiexec will copy the executable from r1i0n17 to the /tmp/pbs.15210.isrv5 directory on r1i0n5, r1i0n6 and r1i0n7 and execute the program. - -MPI process mapping may be controlled by PBS parameters. - -The mpiprocs and ompthreads parameters allow for selection of number of running MPI processes per node as well as number of OpenMP threads per MPI process. - -### One MPI Process Per Node - -Follow this example to run one MPI process per node, 24 threads per process. - -```console -$ qsub -q qexp -l select=4:ncpus=24:mpiprocs=1:ompthreads=24 -I -$ ml OpenMPI -$ mpiexec --bind-to-none ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 1 MPI processes per node and 24 threads per socket, on 4 nodes. - -### Two MPI Processes Per Node - -Follow this example to run two MPI processes per node, 8 threads per process. Note the options to mpiexec. - -```console -$ qsub -q qexp -l select=4:ncpus=24:mpiprocs=2:ompthreads=12 -I -$ ml OpenMPI -$ mpiexec -bysocket -bind-to-socket ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 2 MPI processes per node and 12 threads per socket, each process and its threads bound to a separate processor socket of the node, on 4 nodes - -### 24 MPI Processes Per Node - -Follow this example to run 24 MPI processes per node, 1 thread per process. Note the options to mpiexec. - -```console -$ qsub -q qexp -l select=4:ncpus=24:mpiprocs=24:ompthreads=1 -I -$ ml OpenMPI -$ mpiexec -bycore -bind-to-core ./helloworld_mpi.x -``` - -In this example, we demonstrate recommended way to run an MPI application, using 24 MPI processes per node, single threaded. Each process is bound to separate processor core, on 4 nodes. - -### OpenMP Thread Affinity - -!!! note - Important! Bind every OpenMP thread to a core! - -In the previous two examples with one or two MPI processes per node, the operating system might still migrate OpenMP threads between cores. You might want to avoid this by setting these environment variable for GCC OpenMP: - -```console -$ export GOMP_CPU_AFFINITY="0-23" -``` - -or this one for Intel OpenMP: - -```console -$ export KMP_AFFINITY=granularity=fine,compact,1,0 -``` - -As of OpenMP 4.0 (supported by GCC 4.9 and later and Intel 14.0 and later) the following variables may be used for Intel or GCC: - -```console -$ export OMP_PROC_BIND=true -$ export OMP_PLACES=cores -``` - -## OpenMPI Process Mapping and Binding - -The mpiexec allows for precise selection of how the MPI processes will be mapped to the computational nodes and how these processes will bind to particular processor sockets and cores. - -MPI process mapping may be specified by a hostfile or rankfile input to the mpiexec program. Altough all implementations of MPI provide means for process mapping and binding, following examples are valid for the openmpi only. - -### Hostfile - -Example hostfile - -```console - r1i0n17.smc.salomon.it4i.cz - r1i0n5.smc.salomon.it4i.cz - r1i0n6.smc.salomon.it4i.cz - r1i0n7.smc.salomon.it4i.cz -``` - -Use the hostfile to control process placement - -```console -$ mpiexec -hostfile hostfile ./helloworld_mpi.x - Hello world! from rank 0 of 4 on host r1i0n17 - Hello world! from rank 1 of 4 on host r1i0n5 - Hello world! from rank 2 of 4 on host r1i0n6 - Hello world! from rank 3 of 4 on host r1i0n7 -``` - -In this example, we see that ranks have been mapped on nodes according to the order in which nodes show in the hostfile - -### Rankfile - -Exact control of MPI process placement and resource binding is provided by specifying a rankfile - -Appropriate binding may boost performance of your application. - -Example rankfile - -```console - rank 0=r1i0n7.smc.salomon.it4i.cz slot=1:0,1 - rank 1=r1i0n6.smc.salomon.it4i.cz slot=0:* - rank 2=r1i0n5.smc.salomon.it4i.cz slot=1:1-2 - rank 3=r1i0n17.smc.salomon slot=0:1,1:0-2 - rank 4=r1i0n6.smc.salomon.it4i.cz slot=0:*,1:* -``` - -This rankfile assumes 5 ranks will be running on 4 nodes and provides exact mapping and binding of the processes to the processor sockets and cores - -Explanation: -rank 0 will be bounded to r1i0n7, socket1 core0 and core1 -rank 1 will be bounded to r1i0n6, socket0, all cores -rank 2 will be bounded to r1i0n5, socket1, core1 and core2 -rank 3 will be bounded to r1i0n17, socket0 core1, socket1 core0, core1, core2 -rank 4 will be bounded to r1i0n6, all cores on both sockets - -```console - $ mpiexec -n 5 -rf rankfile --report-bindings ./helloworld_mpi.x - [r1i0n17:11180] MCW rank 3 bound to socket 0[core 1] socket 1[core 0-2]: [. B . . . . . . . . . .][B B B . . . . . . . . .] (slot list 0:1,1:0-2) - [r1i0n7:09928] MCW rank 0 bound to socket 1[core 0-1]: [. . . . . . . . . . . .][B B . . . . . . . . . .] (slot list 1:0,1) - [r1i0n6:10395] MCW rank 1 bound to socket 0[core 0-7]: [B B B B B B B B B B B B][. . . . . . . . . . . .] (slot list 0:*) - [r1i0n5:10406] MCW rank 2 bound to socket 1[core 1-2]: [. . . . . . . . . . . .][. B B . . . . . . . . .] (slot list 1:1-2) - [r1i0n6:10406] MCW rank 4 bound to socket 0[core 0-7] socket 1[core 0-7]: [B B B B B B B B B B B B][B B B B B B B B B B B B] (slot list 0:*,1:*) - Hello world! from rank 3 of 5 on host r1i0n17 - Hello world! from rank 1 of 5 on host r1i0n6 - Hello world! from rank 0 of 5 on host r1i0n7 - Hello world! from rank 4 of 5 on host r1i0n6 - Hello world! from rank 2 of 5 on host r1i0n5 -``` - -In this example we run 5 MPI processes (5 ranks) on four nodes. The rankfile defines how the processes will be mapped on the nodes, sockets and cores. The **--report-bindings** option was used to print out the actual process location and bindings. Note that ranks 1 and 4 run on the same node and their core binding overlaps. - -It is users responsibility to provide correct number of ranks, sockets and cores. - -### Bindings Verification - -In all cases, binding and threading may be verified by executing for example: - -```console -$ mpiexec -bysocket -bind-to-socket --report-bindings echo -$ mpiexec -bysocket -bind-to-socket numactl --show -$ mpiexec -bysocket -bind-to-socket echo $OMP_NUM_THREADS -``` - -## Changes in OpenMPI 1.8 - -Some options have changed in OpenMPI version 1.8. - -| version 1.6.5 | version 1.8.1 | -| ---------------- | ------------------- | -| --bind-to-none | --bind-to none | -| --bind-to-core | --bind-to core | -| --bind-to-socket | --bind-to socket | -| -bysocket | --map-by socket | -| -bycore | --map-by core | -| -pernode | --map-by ppr:1:node | diff --git a/docs.it4i/salomon/software/mpi/mpi.md b/docs.it4i/salomon/software/mpi/mpi.md deleted file mode 100644 index 536335b3d..000000000 --- a/docs.it4i/salomon/software/mpi/mpi.md +++ /dev/null @@ -1,142 +0,0 @@ -# MPI - -## Setting Up MPI Environment - -The Salomon cluster provides several implementations of the MPI library: - -| MPI Library | Thread support | -| ----------------- | ------------------------------------------------------------------------------------------------------ | -| **Intel MPI 4.1** | Full thread support up to, MPI_THREAD_MULTIPLE | -| **Intel MPI 5.0** | Full thread support up to, MPI_THREAD_MULTIPLE | -| OpenMPI 1.8.6 | Full thread support up to, MPI_THREAD_MULTIPLE, MPI-3.0, support | -| SGI MPT 2.12 | | - -MPI libraries are activated via the environment modules. - -Look up section modulefiles/mpi in module avail - -```console -$ ml av - ------------------------------ /apps/modules/mpi ------------------------------- - impi/4.1.1.036-iccifort-2013.5.192 - impi/4.1.1.036-iccifort-2013.5.192-GCC-4.8.3 - impi/5.0.3.048-iccifort-2015.3.187 - impi/5.0.3.048-iccifort-2015.3.187-GNU-5.1.0-2.25 - MPT/2.12 - OpenMPI/1.8.6-GNU-5.1.0-2.25 -``` - -There are default compilers associated with any particular MPI implementation. The defaults may be changed, the MPI libraries may be used in conjunction with any compiler. The defaults are selected via the modules in following way - -| Module | MPI | Compiler suite | -| ---------------------------------------- | ---------- | -------------- | -| impi-5.0.3.048-iccifort- Intel MPI 5.0.3 | 2015.3.187 | | -| OpenMP-1.8.6-GNU-5.1.0-2 OpenMPI 1.8.6 | .25 | | - -Examples: - -```console -$ ml gompi/2015b -``` - -In this example, we activate the latest OpenMPI with latest GNU compilers (OpenMPI 1.8.6 and GCC 5.1). Please see more information about toolchains in section [Environment and Modules](../../environment-and-modules/) . - -To use OpenMPI with the intel compiler suite, use - -```console -$ ml iompi/2015.03 -``` - -In this example, the openmpi 1.8.6 using intel compilers is activated. It's used "iompi" toolchain. - -## Compiling MPI Programs - -After setting up your MPI environment, compile your program using one of the mpi wrappers - -```console -$ mpicc -v -$ mpif77 -v -$ mpif90 -v -``` - -When using Intel MPI, use the following MPI wrappers: - -```console -$ mpicc -$ mpiifort -``` - -Wrappers mpif90, mpif77 that are provided by Intel MPI are designed for gcc and gfortran. You might be able to compile MPI code by them even with Intel compilers, but you might run into problems (for example, native MIC compilation with -mmic does not work with mpif90). - -Example program: - -```cpp -// helloworld_mpi.c -#include <stdio.h> - -#include<mpi.h> - -int main(int argc, char **argv) { - -int len; -int rank, size; -char node[MPI_MAX_PROCESSOR_NAME]; - -// Initiate MPI -MPI_Init(&argc, &argv); -MPI_Comm_rank(MPI_COMM_WORLD,&rank); -MPI_Comm_size(MPI_COMM_WORLD,&size); - -// Get hostame and print -MPI_Get_processor_name(node,&len); -printf("Hello world! from rank %d of %d on host %sn",rank,size,node); - -// Finalize and exit -MPI_Finalize(); - -return 0; -} -``` - -Compile the above example with - -```console -$ mpicc helloworld_mpi.c -o helloworld_mpi.x -``` - -## Running MPI Programs - -The MPI program executable must be compatible with the loaded MPI module. -Always compile and execute using the very same MPI module. - -It is strongly discouraged to mix mpi implementations. Linking an application with one MPI implementation and running mpirun/mpiexec form other implementation may result in unexpected errors. - -The MPI program executable must be available within the same path on all nodes. This is automatically fulfilled on the /home and /scratch filesystem. You need to preload the executable, if running on the local scratch /lscratch filesystem. - -### Ways to Run MPI Programs - -Optimal way to run an MPI program depends on its memory requirements, memory access pattern and communication pattern. - -!!! note - Consider these ways to run an MPI program: - - 1. One MPI process per node, 24 threads per process - 2. Two MPI processes per node, 12 threads per process - 3. 24 MPI processes per node, 1 thread per process. - -**One MPI** process per node, using 24 threads, is most useful for memory demanding applications, that make good use of processor cache memory and are not memory bound. This is also a preferred way for communication intensive applications as one process per node enjoys full bandwidth access to the network interface. - -**Two MPI** processes per node, using 12 threads each, bound to processor socket is most useful for memory bandwidth bound applications such as BLAS1 or FFT, with scalable memory demand. However, note that the two processes will share access to the network interface. The 12 threads and socket binding should ensure maximum memory access bandwidth and minimize communication, migration and numa effect overheads. - -!!! note - Important! Bind every OpenMP thread to a core! - -In the previous two cases with one or two MPI processes per node, the operating system might still migrate OpenMP threads between cores. You want to avoid this by setting the KMP_AFFINITY or GOMP_CPU_AFFINITY environment variables. - -**24 MPI** processes per node, using 1 thread each bound to processor core is most suitable for highly scalable applications with low communication demand. - -### Running OpenMPI - -The [**OpenMPI 1.8.6**](http://www.open-mpi.org/) is based on OpenMPI. Read more on [how to run OpenMPI](Running_OpenMPI/) based MPI. - -The Intel MPI may run on the[Intel Xeon Ph](../intel-xeon-phi/)i accelerators as well. Read more on [how to run Intel MPI on accelerators](../intel-xeon-phi/). diff --git a/docs.it4i/salomon/software/mpi/mpi4py-mpi-for-python.md b/docs.it4i/salomon/software/mpi/mpi4py-mpi-for-python.md deleted file mode 100644 index cdad4f556..000000000 --- a/docs.it4i/salomon/software/mpi/mpi4py-mpi-for-python.md +++ /dev/null @@ -1,110 +0,0 @@ -# MPI4Py (MPI for Python) - -OpenMPI interface to Python - -## Introduction - -MPI for Python provides bindings of the Message Passing Interface (MPI) standard for the Python programming language, allowing any Python program to exploit multiple processors. - -This package is constructed on top of the MPI-1/2 specifications and provides an object oriented interface which closely follows MPI-2 C++ bindings. It supports point-to-point (sends, receives) and collective (broadcasts, scatters, gathers) communications of any picklable Python object, as well as optimized communications of Python object exposing the single-segment buffer interface (NumPy arrays, builtin bytes/string/array objects). - -On Salomon MPI4Py is available in standard Python modules. - -## Modules - -MPI4Py is build for OpenMPI. Before you start with MPI4Py you need to load Python and OpenMPI modules. You can use toolchain, that loads Python and OpenMPI at once. - -```console -$ ml av Python/ ---------------------------------------- /apps/modules/lang ------------------------- - Python/2.7.8-intel-2015b Python/2.7.11-intel-2016a Python/3.5.1-intel-2017.00 - Python/2.7.11-intel-2017a Python/2.7.9-foss-2015b Python/2.7.9-intel-2015b - Python/2.7.11-foss-2016a Python/3.5.2-foss-2016a Python/3.5.1 - Python/2.7.9-foss-2015g Python/3.4.3-intel-2015b Python/2.7.9 - Python/2.7.11-intel-2015b Python/3.5.2 - -$ ml av OpenMPI/ ---------------------------------------- /apps/modules/mpi -------------------------- -OpenMPI/1.8.6-GCC-4.4.7-system OpenMPI/1.8.8-GNU-4.9.3-2.25 OpenMPI/1.10.1-GCC-4.9.3-2.25 -OpenMPI/1.8.6-GNU-5.1.0-2.25 OpenMPI/1.8.8-GNU-5.1.0-2.25 OpenMPI/1.10.1-GNU-4.9.3-2.25 - OpenMPI/1.8.8-iccifort-2015.3.187-GNU-4.9.3-2.25 OpenMPI/2.0.2-GCC-6.3.0-2.27 -``` - -!!! Warning "" - - * modules Python/x.x.x-intel... - intel MPI - * modules Python/x.x.x-foss... - OpenMPI - * modules Python/x.x.x - without MPI - -## Execution - -You need to import MPI to your python program. Include the following line to the python script: - -```python -from mpi4py import MPI -``` - -The MPI4Py enabled python programs [execute as any other OpenMPI](Running_OpenMPI/) code.The simpliest way is to run - -```console -$ mpiexec python <script>.py -``` - -For example - -```console -$ mpiexec python hello_world.py -``` - -## Examples - -### Hello World! - -```python -from mpi4py import MPI - -comm = MPI.COMM_WORLD - -print "Hello! I'm rank %d from %d running in total..." % (comm.rank, comm.size) - -comm.Barrier() # wait for everybody to synchronize -``` - -### Collective Communication With NumPy Arrays - -```python -from __future__ import division -from mpi4py import MPI -import numpy as np - -comm = MPI.COMM_WORLD - -print("-"*78) -print(" Running on %d cores" % comm.size) -print("-"*78) - -comm.Barrier() - -# Prepare a vector of N=5 elements to be broadcasted... -N = 5 -if comm.rank == 0: - A = np.arange(N, dtype=np.float64) # rank 0 has proper data -else: - A = np.empty(N, dtype=np.float64) # all other just an empty array - -# Broadcast A from rank 0 to everybody -comm.Bcast( [A, MPI.DOUBLE] ) - -# Everybody should now have the same... -print "[%02d] %s" % (comm.rank, A) -``` - -Execute the above code as: - -```console -$ qsub -q qexp -l select=4:ncpus=24:mpiprocs=24:ompthreads=1 -I -$ ml Python/2.7.9-foss-2015g -$ mpiexec --map-by core --bind-to core python hello_world.py -``` - -In this example, we run MPI4Py enabled code on 4 nodes, 24 cores per node (total of 96 processes), each python process is bound to a different core. More examples and documentation can be found on [MPI for Python webpage](https://pypi.python.org/pypi/mpi4py). diff --git a/docs.it4i/salomon/software/numerical-languages/introduction.md b/docs.it4i/salomon/software/numerical-languages/introduction.md deleted file mode 100644 index 13ba67071..000000000 --- a/docs.it4i/salomon/software/numerical-languages/introduction.md +++ /dev/null @@ -1,40 +0,0 @@ -# Numerical languages - -Interpreted languages for numerical computations and analysis - -## Introduction - -This section contains a collection of high-level interpreted languages, primarily intended for numerical computations. - -## Matlab - -MATLAB®^ is a high-level language and interactive environment for numerical computation, visualization, and programming. - -```console -$ ml MATLAB -$ matlab -``` - -Read more at the [Matlab page](matlab/). - -## Octave - -GNU Octave is a high-level interpreted language, primarily intended for numerical computations. The Octave language is quite similar to Matlab so that most programs are easily portable. - -```console -$ ml Octave -$ octave -``` - -Read more at the [Octave page](octave/). - -## R - -The R is an interpreted language and environment for statistical computing and graphics. - -```console -$ ml R -$ R -``` - -Read more at the [R page](r/). diff --git a/docs.it4i/salomon/software/numerical-languages/matlab.md b/docs.it4i/salomon/software/numerical-languages/matlab.md deleted file mode 100644 index 5a64b1ff1..000000000 --- a/docs.it4i/salomon/software/numerical-languages/matlab.md +++ /dev/null @@ -1,278 +0,0 @@ -# Matlab - -## Introduction - -Matlab is available in versions R2015a and R2015b. There are always two variants of the release: - -* Non commercial or so called EDU variant, which can be used for common research and educational purposes. -* Commercial or so called COM variant, which can used also for commercial activities. The licenses for commercial variant are much more expensive, so usually the commercial variant has only subset of features compared to the EDU available. - -To load the latest version of Matlab load the module - -```console -$ ml MATLAB -``` - -By default the EDU variant is marked as default. If you need other version or variant, load the particular version. To obtain the list of available versions use - -```console -$ module avail MATLAB -``` - -If you need to use the Matlab GUI to prepare your Matlab programs, you can use Matlab directly on the login nodes. But for all computations use Matlab on the compute nodes via PBS Pro scheduler. - -If you require the Matlab GUI, please follow the general information about [running graphical applications](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/). - -Matlab GUI is quite slow using the X forwarding built in the PBS (qsub -X), so using X11 display redirection either via SSH or directly by xauth (please see the "GUI Applications on Compute Nodes over VNC" part [here](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/)) is recommended. - -To run Matlab with GUI, use - -```console -$ matlab -``` - -To run Matlab in text mode, without the Matlab Desktop GUI environment, use - -```console -$ matlab -nodesktop -nosplash -``` - -plots, images, etc... will be still available. - -## Running Parallel Matlab Using Distributed Computing Toolbox / Engine - -Distributed toolbox is available only for the EDU variant - -The MPIEXEC mode available in previous versions is no longer available in MATLAB 2015. Also, the programming interface has changed. Refer to [Release Notes](http://www.mathworks.com/help/distcomp/release-notes.html#buanp9e-1). - -Delete previously used file mpiLibConf.m, we have observed crashes when using Intel MPI. - -To use Distributed Computing, you first need to setup a parallel profile. We have provided the profile for you, you can either import it in MATLAB command line: - -```console -> parallel.importProfile('/apps/all/MATLAB/2015b-EDU/SalomonPBSPro.settings') - -ans = - -SalomonPBSPro -``` - -Or in the GUI, go to tab HOME -> Parallel -> Manage Cluster Profiles..., click Import and navigate to : - -/apps/all/MATLAB/2015b-EDU/SalomonPBSPro.settings - -With the new mode, MATLAB itself launches the workers via PBS, so you can either use interactive mode or a batch mode on one node, but the actual parallel processing will be done in a separate job started by MATLAB itself. Alternatively, you can use "local" mode to run parallel code on just a single node. - -### Parallel Matlab Interactive Session - -Following example shows how to start interactive session with support for Matlab GUI. For more information about GUI based applications on Anselm see [this page](../../../general/accessing-the-clusters/graphical-user-interface/x-window-system/). - -```console -$ xhost + -$ qsub -I -v DISPLAY=$(uname -n):$(echo $DISPLAY | cut -d ':' -f 2) -A NONE-0-0 -q qexp -l select=1 -l walltime=00:30:00 -l feature__matlab__MATLAB=1 -``` - -This qsub command example shows how to run Matlab on a single node. - -The second part of the command shows how to request all necessary licenses. In this case 1 Matlab-EDU license and 48 Distributed Computing Engines licenses. - -Once the access to compute nodes is granted by PBS, user can load following modules and start Matlab: - -```console -r1i0n17$ ml MATLAB/2015a-EDU -r1i0n17$ matlab & -``` - -### Parallel Matlab Batch Job in Local Mode - -To run matlab in batch mode, write an matlab script, then write a bash jobscript and execute via the qsub command. By default, matlab will execute one matlab worker instance per allocated core. - -```bash -#!/bin/bash -#PBS -A PROJECT ID -#PBS -q qprod -#PBS -l select=1:ncpus=24:mpiprocs=24:ompthreads=1 - -# change to shared scratch directory -SCR=/scratch/work/user/$USER/$PBS_JOBID -mkdir -p $SCR ; cd $SCR || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/matlabcode.m . - -# load modules -module load MATLAB/2015a-EDU - -# execute the calculation -matlab -nodisplay -r matlabcode > output.out - -# copy output file to home -cp output.out $PBS_O_WORKDIR/. -``` - -This script may be submitted directly to the PBS workload manager via the qsub command. The inputs and matlab script are in matlabcode.m file, outputs in output.out file. Note the missing .m extension in the matlab -r matlabcodefile call, **the .m must not be included**. Note that the **shared /scratch must be used**. Further, it is **important to include quit** statement at the end of the matlabcode.m script. - -Submit the jobscript using qsub - -```console -$ qsub ./jobscript -``` - -### Parallel Matlab Local Mode Program Example - -The last part of the configuration is done directly in the user Matlab script before Distributed Computing Toolbox is started. - -```console -cluster = parcluster('local') -``` - -This script creates scheduler object "cluster" of type "local" that starts workers locally. - -!!! hint - Every Matlab script that needs to initialize/use matlabpool has to contain these three lines prior to calling parpool(sched, ...) function. - -The last step is to start matlabpool with "cluster" object and correct number of workers. We have 24 cores per node, so we start 24 workers. - -```console -parpool(cluster,24); - - -... parallel code ... - - -parpool close -``` - -The complete example showing how to use Distributed Computing Toolbox in local mode is shown here. - -```console -cluster = parcluster('local'); -cluster - -parpool(cluster,24); - -n=2000; - -W = rand(n,n); -W = distributed(W); -x = (1:n)'; -x = distributed(x); -spmd -[~, name] = system('hostname') - - T = W*x; % Calculation performed on labs, in parallel. - % T and W are both codistributed arrays here. -end -T; -whos % T and W are both distributed arrays here. - -parpool close -quit -``` - -You can copy and paste the example in a .m file and execute. Note that the parpool size should correspond to **total number of cores** available on allocated nodes. - -### Parallel Matlab Batch Job Using PBS Mode (Workers Spawned in a Separate Job) - -This mode uses PBS scheduler to launch the parallel pool. It uses the SalomonPBSPro profile that needs to be imported to Cluster Manager, as mentioned before. This methodod uses MATLAB's PBS Scheduler interface - it spawns the workers in a separate job submitted by MATLAB using qsub. - -This is an example of m-script using PBS mode: - -```console -cluster = parcluster('SalomonPBSPro'); -set(cluster, 'SubmitArguments', '-A OPEN-0-0'); -set(cluster, 'ResourceTemplate', '-q qprod -l select=10:ncpus=24'); -set(cluster, 'NumWorkers', 240); - -pool = parpool(cluster,240); - -n=2000; - -W = rand(n,n); -W = distributed(W); -x = (1:n)'; -x = distributed(x); -spmd -[~, name] = system('hostname') - - T = W*x; % Calculation performed on labs, in parallel. - % T and W are both codistributed arrays here. -end -whos % T and W are both distributed arrays here. - -% shut down parallel pool -delete(pool) -``` - -Note that we first construct a cluster object using the imported profile, then set some important options, namely : SubmitArguments, where you need to specify accounting id, and ResourceTemplate, where you need to specify number of nodes to run the job. - -You can start this script using batch mode the same way as in Local mode example. - -### Parallel Matlab Batch With Direct Launch (Workers Spawned Within the Existing Job) - -This method is a "hack" invented by us to emulate the mpiexec functionality found in previous MATLAB versions. We leverage the MATLAB Generic Scheduler interface, but instead of submitting the workers to PBS, we launch the workers directly within the running job, thus we avoid the issues with master script and workers running in separate jobs (issues with license not available, waiting for the worker's job to spawn etc.) - -!!! warning - This method is experimental. - -For this method, you need to use SalomonDirect profile, import it using [the same way as SalomonPBSPro](matlab.md#running-parallel-matlab-using-distributed-computing-toolbox---engine) - -This is an example of m-script using direct mode: - -```console -parallel.importProfile('/apps/all/MATLAB/2015b-EDU/SalomonDirect.settings') -cluster = parcluster('SalomonDirect'); -set(cluster, 'NumWorkers', 48); - -pool = parpool(cluster, 48); - -n=2000; - -W = rand(n,n); -W = distributed(W); -x = (1:n)'; -x = distributed(x); -spmd -[~, name] = system('hostname') - - T = W*x; % Calculation performed on labs, in parallel. - % T and W are both codistributed arrays here. -end -whos % T and W are both distributed arrays here. - -% shut down parallel pool -delete(pool) -``` - -### Non-Interactive Session and Licenses - -If you want to run batch jobs with Matlab, be sure to request appropriate license features with the PBS Pro scheduler, at least the `-l __feature__matlab__MATLAB=1` for EDU variant of Matlab. More information about how to check the license features states and how to request them with PBS Pro, please [look here](../../../anselm/software/isv_licenses/). - -The licensing feature of PBS is currently disabled. - -In case of non-interactive session please read the [following information](../../../anselm/software/isv_licenses/) on how to modify the qsub command to test for available licenses prior getting the resource allocation. - -### Matlab Distributed Computing Engines Start Up Time - -Starting Matlab workers is an expensive process that requires certain amount of time. For your information please see the following table: - -| compute nodes | number of workers | start-up time[s] | -| ------------- | ----------------- | ---------------- | -| 16 | 384 | 831 | -| 8 | 192 | 807 | -| 4 | 96 | 483 | -| 2 | 48 | 16 | - -## MATLAB on UV2000 - -UV2000 machine available in queue "qfat" can be used for MATLAB computations. This is a SMP NUMA machine with large amount of RAM, which can be beneficial for certain types of MATLAB jobs. CPU cores are allocated in chunks of 8 for this machine. - -You can use MATLAB on UV2000 in two parallel modes: - -### Threaded Mode - -Since this is a SMP machine, you can completely avoid using Parallel Toolbox and use only MATLAB's threading. MATLAB will automatically detect the number of cores you have allocated and will set maxNumCompThreads accordingly and certain operations, such as fft, , eig, svd, etc. will be automatically run in threads. The advantage of this mode is that you don't need to modify your existing sequential codes. - -### Local Cluster Mode - -You can also use Parallel Toolbox on UV2000. Use l[ocal cluster mode](matlab/#parallel-matlab-batch-job-in-local-mode), "SalomonPBSPro" profile will not work. diff --git a/docs.it4i/salomon/software/numerical-languages/octave.md b/docs.it4i/salomon/software/numerical-languages/octave.md deleted file mode 100644 index ebb451f2c..000000000 --- a/docs.it4i/salomon/software/numerical-languages/octave.md +++ /dev/null @@ -1,56 +0,0 @@ -# Octave - -GNU Octave is a high-level interpreted language, primarily intended for numerical computations. It provides capabilities for the numerical solution of linear and nonlinear problems, and for performing other numerical experiments. It also provides extensive graphics capabilities for data visualization and manipulation. Octave is normally used through its interactive command line interface, but it can also be used to write non-interactive programs. The Octave language is quite similar to Matlab so that most programs are easily portable. Read more on <http://www.gnu.org/software/octave/> - -Two versions of octave are available on the cluster, via module - -| Status | Version | module | -| ---------- | ------------ | ------ | -| **Stable** | Octave 3.8.2 | Octave | - -```console -$ ml Octave -``` - -The octave on the cluster is linked to highly optimized MKL mathematical library. This provides threaded parallelization to many octave kernels, notably the linear algebra subroutines. Octave runs these heavy calculation kernels without any penalty. By default, octave would parallelize to 24 threads. You may control the threads by setting the OMP_NUM_THREADS environment variable. - -To run octave interactively, log in with ssh -X parameter for X11 forwarding. Run octave: - -```console -$ octave -``` - -To run octave in batch mode, write an octave script, then write a bash jobscript and execute via the qsub command. By default, octave will use 16 threads when running MKL kernels. - -```bash -#!/bin/bash - -# change to local scratch directory -mkdir -p /scratch/work/user/$USER/$PBS_JOBID -cd /scratch/work/user/$USER/$PBS_JOBID || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/octcode.m . - -# load octave module -module load Octave - -# execute the calculation -octave -q --eval octcode > output.out - -# copy output file to home -cp output.out $PBS_O_WORKDIR/. - -#exit -exit -``` - -This script may be submitted directly to the PBS workload manager via the qsub command. The inputs are in octcode.m file, outputs in output.out file. See the single node jobscript example in the [Job execution section](../../). - -The octave c compiler mkoctfile calls the GNU gcc 4.8.1 for compiling native c code. This is very useful for running native c subroutines in octave environment. - -```console -$ mkoctfile -v -``` - -Octave may use MPI for interprocess communication This functionality is currently not supported on the cluster cluster. In case you require the octave interface to MPI, please contact our [cluster support](https://support.it4i.cz/rt/). diff --git a/docs.it4i/salomon/software/numerical-languages/opencoarrays.md b/docs.it4i/salomon/software/numerical-languages/opencoarrays.md deleted file mode 100644 index ada43753a..000000000 --- a/docs.it4i/salomon/software/numerical-languages/opencoarrays.md +++ /dev/null @@ -1,127 +0,0 @@ -# OpenCoarrays - -## Introduction - -Coarray Fortran (CAF) is an extension of Fortran language and offers a simple interface for parallel processing and memory sharing. -The advantage is that only small changes are required to convert existing Fortran code to support a robust and potentially efficient parallelism. - -A CAF program is interpreted as if it was replicated a number of times and all copies were executed asynchronously. -The number of copies is decided at execution time. Each copy (called *image*) has its own private variables. -The variable syntax of Fortran language is extended with indexes in square brackets (called *co-dimension*) representing a reference to data distributed across images. - -By default, the CAF is using Message Passing Interface (MPI) for lower-level communication, so there are some similarities with MPI. - -Read more on <http://www.opencoarrays.org/> - -## Coarray Basics - -### Indexing of Coarray Images - -Indexing of individual images can be shown on the simple *Hello World* program: - -```fortran -program hello_world - implicit none - print *, 'Hello world from image ', this_image() , 'of', num_images() -end program hello_world -``` - -* num_images() - returns the number of all images -* this_image() - returns the image index - numbered from 1 to num_images() - -### Co-dimension Variables Declaration - -Coarray variables can be declared with the **codimension[*]** attribute or by adding trailing index **[*]** after the variable name. -Notice, the ***** character always has to be in the square brackets. - -```fortran -integer, codimension[*] :: scalar -integer :: scalar[*] -real, dimension(64), codimension[*] :: vector -real :: vector(64)[*] -``` - -### Images Synchronization - -Because each image is running on its own, the image synchronization is needed to ensure, that all altered data are distributed to all images. -Synchronization can be done across all images or only between selected images. Be aware, that selective synchronization can lead to the race condition problems like deadlock. - -Example program: - -```fortran -program synchronization_test - implicit none - integer :: i ! Local variable - integer :: numbers[*] ! Scalar coarray - - ! Genereate random number on image 1 - if (this_image() == 1) then - numbers = floor(rand(1) * 1000) - ! Distribute information to other images - do i = 2, num_images() - numbers[i] = numbers - end do - end if - - sync all ! Barrier to synchronize all images - - print *, 'The random number is', numbers -end program synchronization_test -``` - -* sync all - Synchronize all images between each other -* sync images(*) - Synchronize this image to all other -* sync images(*index*) - Synchronize this image to image with *index* - -!!! note - **number** is the local variable while **number[*index*]** accesses the variable in a specific image. - **number[this_image()]** is the same as **number**. - -## Compile and Run - -Currently, version 1.8.10 compiled with OpenMPI 1.10.7 library is installed on Cluster. The OpenCoarrays module can be load as follows: - -```console -$ ml OpenCoarrays/1.8.10-GCC-6.3.0-2.27 -``` - -### Compile CAF Program - -The preferred method for compiling a CAF program is by invoking the *caf* compiler wrapper. -The above mentioned *Hello World* program can be compiled as follows: - -```console -$ caf hello_world.f90 -o hello_world.x -``` - -!!! warning - The input file extension **.f90** or **.F90** are to be interpreted as *Fortran 90*. - If the input file extension is **.f** or **.F** the source code will be interpreted as *Fortran 77*. - -Another method for compiling is by invoking the *mpif90* compiler wrapper directly: - -```console -$ mpif90 hello_world.f90 -o hello_world.x -fcoarray=lib -lcaf_mpi -``` - -### Run CAF Program - -A CAF program can be run by invoking the *cafrun* wrapper or directly by the *mpiexec*: - -```console -$ cafrun -np 4 ./hello_world.x - Hello world from image 1 of 4 - Hello world from image 2 of 4 - Hello world from image 3 of 4 - Hello world from image 4 of 4 - -$ mpiexec -np 4 ./synchronization_test.x - The random number is 242 - The random number is 242 - The random number is 242 - The random number is 242 -``` - -**-np 4** is number of images to run. The parameters of **cafrun** and **mpiexec** are the same. - -For more information about running CAF program please follow [Running OpenMPI](../mpi/Running_OpenMPI.md) diff --git a/docs.it4i/salomon/software/numerical-languages/r.md b/docs.it4i/salomon/software/numerical-languages/r.md deleted file mode 100644 index 3af38b318..000000000 --- a/docs.it4i/salomon/software/numerical-languages/r.md +++ /dev/null @@ -1,405 +0,0 @@ -# R - -## Introduction - -The R is a language and environment for statistical computing and graphics. R provides a wide variety of statistical (linear and nonlinear modelling, classical statistical tests, time-series analysis, classification, clustering, ...) and graphical techniques, and is highly extensible. - -One of R's strengths is the ease with which well-designed publication-quality plots can be produced, including mathematical symbols and formulae where needed. Great care has been taken over the defaults for the minor design choices in graphics, but the user retains full control. - -Another convenience is the ease with which the C code or third party libraries may be integrated within R. - -Extensive support for parallel computing is available within R. - -Read more on <http://www.r-project.org/>, <http://cran.r-project.org/doc/manuals/r-release/R-lang.html> - -## Modules - -The R version 3.1.1 is available on the cluster, along with GUI interface Rstudio - -| Application | Version | module | -| ----------- | ----------------- | ------------------- | -| **R** | R 3.1.1 | R/3.1.1-intel-2015b | -| **Rstudio** | Rstudio 0.98.1103 | Rstudio | - -```console -$ ml R -``` - -## Execution - -The R on Anselm is linked to highly optimized MKL mathematical library. This provides threaded parallelization to many R kernels, notably the linear algebra subroutines. The R runs these heavy calculation kernels without any penalty. By default, the R would parallelize to 24 threads. You may control the threads by setting the OMP_NUM_THREADS environment variable. - -### Interactive Execution - -To run R interactively, using Rstudio GUI, log in with ssh -X parameter for X11 forwarding. Run rstudio: - -```console -$ ml Rstudio -$ rstudio -``` - -### Batch Execution - -To run R in batch mode, write an R script, then write a bash jobscript and execute via the qsub command. By default, R will use 24 threads when running MKL kernels. - -Example jobscript: - -```bash -#!/bin/bash - -# change to local scratch directory -cd /lscratch/$PBS_JOBID || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/rscript.R . - -# load R module -module load R - -# execute the calculation -R CMD BATCH rscript.R routput.out - -# copy output file to home -cp routput.out $PBS_O_WORKDIR/. - -#exit -exit -``` - -This script may be submitted directly to the PBS workload manager via the qsub command. The inputs are in rscript.R file, outputs in routput.out file. See the single node jobscript example in the [Job execution section](../../job-submission-and-execution/). - -## Parallel R - -Parallel execution of R may be achieved in many ways. One approach is the implied parallelization due to linked libraries or specially enabled functions, as [described above](r/#interactive-execution). In the following sections, we focus on explicit parallelization, where parallel constructs are directly stated within the R script. - -## Package Parallel - -The package parallel provides support for parallel computation, including by forking (taken from package multicore), by sockets (taken from package snow) and random-number generation. - -The package is activated this way: - -```console -$ R -> library(parallel) -``` - -More information and examples may be obtained directly by reading the documentation available in R - -```r -> ?parallel -> library(help = "parallel") -> vignette("parallel") -``` - -Download the package [parallell](package-parallel-vignette.pdf) vignette. - -The forking is the most simple to use. Forking family of functions provide parallelized, drop in replacement for the serial apply() family of functions. - -!!! warning - Forking via package parallel provides functionality similar to OpenMP construct omp parallel for - - Only cores of single node can be utilized this way! - -Forking example: - -```r -library(parallel) - -#integrand function -f <- function(i,h) { -x <- h*(i-0.5) -return (4/(1 + x*x)) -} - -#initialize -size <- detectCores() - -while (TRUE) -{ - #read number of intervals - cat("Enter the number of intervals: (0 quits) ") - fp<-file("stdin"); n<-scan(fp,nmax=1); close(fp) - - if(n<=0) break - - #run the calculation - n <- max(n,size) - h <- 1.0/n - - i <- seq(1,n); - pi3 <- h*sum(simplify2array(mclapply(i,f,h,mc.cores=size))); - - #print results - cat(sprintf("Value of PI %16.14f, diff= %16.14fn",pi3,pi3-pi)) -} -``` - -The above example is the classic parallel example for calculating the number Ď€. Note the **detectCores()** and **mclapply()** functions. Execute the example as: - -```console -$ R --slave --no-save --no-restore -f pi3p.R -``` - -Every evaluation of the integrad function runs in parallel on different process. - -## Package Rmpi - -package Rmpi provides an interface (wrapper) to MPI APIs. - -It also provides interactive R slave environment. On the cluster, Rmpi provides interface to the [OpenMPI](../mpi/Running_OpenMPI/). - -Read more on Rmpi at <http://cran.r-project.org/web/packages/Rmpi/>, reference manual is available at <http://cran.r-project.org/web/packages/Rmpi/Rmpi.pdf> - -When using package Rmpi, both openmpi and R modules must be loaded - -```console -$ ml OpenMPI -$ ml R -``` - -Rmpi may be used in three basic ways. The static approach is identical to executing any other MPI programm. In addition, there is Rslaves dynamic MPI approach and the mpi.apply approach. In the following section, we will use the number Ď€ integration example, to illustrate all these concepts. - -### Static Rmpi - -Static Rmpi programs are executed via mpiexec, as any other MPI programs. Number of processes is static - given at the launch time. - -Static Rmpi example: - -```r -library(Rmpi) - -#integrand function -f <- function(i,h) { -x <- h*(i-0.5) -return (4/(1 + x*x)) -} - -#initialize -invisible(mpi.comm.dup(0,1)) -rank <- mpi.comm.rank() -size <- mpi.comm.size() -n<-0 - -while (TRUE) -{ - #read number of intervals - if (rank==0) { - cat("Enter the number of intervals: (0 quits) ") - fp<-file("stdin"); n<-scan(fp,nmax=1); close(fp) - } - - #broadcat the intervals - n <- mpi.bcast(as.integer(n),type=1) - - if(n<=0) break - - #run the calculation - n <- max(n,size) - h <- 1.0/n - - i <- seq(rank+1,n,size); - mypi <- h*sum(sapply(i,f,h)); - - pi3 <- mpi.reduce(mypi) - - #print results - if (rank==0) cat(sprintf("Value of PI %16.14f, diff= %16.14fn",pi3,pi3-pi)) -} - -mpi.quit() -``` - -The above is the static MPI example for calculating the number Ď€. Note the **library(Rmpi)** and **mpi.comm.dup()** function calls. Execute the example as: - -```console -$ mpirun R --slave --no-save --no-restore -f pi3.R -``` - -### Dynamic Rmpi - -Dynamic Rmpi programs are executed by calling the R directly. OpenMPI module must be still loaded. The R slave processes will be spawned by a function call within the Rmpi program. - -Dynamic Rmpi example: - -```r -#integrand function -f <- function(i,h) { -x <- h*(i-0.5) -return (4/(1 + x*x)) -} - -#the worker function -workerpi <- function() -{ -#initialize -rank <- mpi.comm.rank() -size <- mpi.comm.size() -n<-0 - -while (TRUE) -{ - #read number of intervals - if (rank==0) { - cat("Enter the number of intervals: (0 quits) ") - fp<-file("stdin"); n<-scan(fp,nmax=1); close(fp) - } - - #broadcat the intervals - n <- mpi.bcast(as.integer(n),type=1) - - if(n<=0) break - - #run the calculation - n <- max(n,size) - h <- 1.0/n - - i <- seq(rank+1,n,size); - mypi <- h*sum(sapply(i,f,h)); - - pi3 <- mpi.reduce(mypi) - - #print results - if (rank==0) cat(sprintf("Value of PI %16.14f, diff= %16.14fn",pi3,pi3-pi)) -} -} - -#main -library(Rmpi) - -cat("Enter the number of slaves: ") -fp<-file("stdin"); ns<-scan(fp,nmax=1); close(fp) - -mpi.spawn.Rslaves(nslaves=ns) -mpi.bcast.Robj2slave(f) -mpi.bcast.Robj2slave(workerpi) - -mpi.bcast.cmd(workerpi()) -workerpi() - -mpi.quit() -``` - -The above example is the dynamic MPI example for calculating the number Ď€. Both master and slave processes carry out the calculation. Note the mpi.spawn.Rslaves(), mpi.bcast.Robj2slave()** and the mpi.bcast.cmd()** function calls. - -Execute the example as: - -```console -$ mpirun -np 1 R --slave --no-save --no-restore -f pi3Rslaves.R -``` - -Note that this method uses MPI_Comm_spawn (Dynamic process feature of MPI-2) to start the slave processes - the master process needs to be launched with MPI. In general, Dynamic processes are not well supported among MPI implementations, some issues might arise. Also, environment variables are not propagated to spawned processes, so they will not see paths from modules. - -### mpi.apply Rmpi - -mpi.apply is a specific way of executing Dynamic Rmpi programs. - -mpi.apply() family of functions provide MPI parallelized, drop in replacement for the serial apply() family of functions. - -Execution is identical to other dynamic Rmpi programs. - -mpi.apply Rmpi example: - -```r -#integrand function -f <- function(i,h) { -x <- h*(i-0.5) -return (4/(1 + x*x)) -} - -#the worker function -workerpi <- function(rank,size,n) -{ - #run the calculation - n <- max(n,size) - h <- 1.0/n - - i <- seq(rank,n,size); - mypi <- h*sum(sapply(i,f,h)); - - return(mypi) -} - -#main -library(Rmpi) - -cat("Enter the number of slaves: ") -fp<-file("stdin"); ns<-scan(fp,nmax=1); close(fp) - -mpi.spawn.Rslaves(nslaves=ns) -mpi.bcast.Robj2slave(f) -mpi.bcast.Robj2slave(workerpi) - -while (TRUE) -{ - #read number of intervals - cat("Enter the number of intervals: (0 quits) ") - fp<-file("stdin"); n<-scan(fp,nmax=1); close(fp) - if(n<=0) break - - #run workerpi - i=seq(1,2*ns) - pi3=sum(mpi.parSapply(i,workerpi,2*ns,n)) - - #print results - cat(sprintf("Value of PI %16.14f, diff= %16.14fn",pi3,pi3-pi)) -} - -mpi.quit() -``` - -The above is the mpi.apply MPI example for calculating the number Ď€. Only the slave processes carry out the calculation. Note the **mpi.parSapply()**, function call. The package parallel [example](r/#package-parallel) [above](r/#package-parallel) may be trivially adapted (for much better performance) to this structure using the mclapply() in place of mpi.parSapply(). - -Execute the example as: - -```console -$ mpirun -np 1 R --slave --no-save --no-restore -f pi3parSapply.R -``` - -## Combining Parallel and Rmpi - -Currently, the two packages can not be combined for hybrid calculations. - -## Parallel Execution - -The R parallel jobs are executed via the PBS queue system exactly as any other parallel jobs. User must create an appropriate jobscript and submit via the **qsub** - -Example jobscript for [static Rmpi](r/#static-rmpi) parallel R execution, running 1 process per core: - -```bash -#!/bin/bash -#PBS -q qprod -#PBS -N Rjob -#PBS -l select=100:ncpus=24:mpiprocs=24:ompthreads=1 - -# change to scratch directory -SCRDIR=/scratch/work/user/$USER/myjob -cd $SCRDIR || exit - -# copy input file to scratch -cp $PBS_O_WORKDIR/rscript.R . - -# load R and openmpi module -module load R -module load OpenMPI - -# execute the calculation -mpiexec -bycore -bind-to-core R --slave --no-save --no-restore -f rscript.R - -# copy output file to home -cp routput.out $PBS_O_WORKDIR/. - -#exit -exit -``` - -For more information about jobscripts and MPI execution refer to the [Job submission](../../job-submission-and-execution/) and general [MPI](../mpi/mpi/) sections. - -## Xeon Phi Offload - -By leveraging MKL, R can accelerate certain computations, most notably linear algebra operations on the Xeon Phi accelerator by using Automated Offload. To use MKL Automated Offload, you need to first set this environment variable before R execution: - -```console -$ export MKL_MIC_ENABLE=1 -``` - -[Read more about automatic offload](../intel-xeon-phi/) diff --git a/docs.it4i/salomon/software/paraview.md b/docs.it4i/salomon/software/paraview.md deleted file mode 100644 index 0255dfc0a..000000000 --- a/docs.it4i/salomon/software/paraview.md +++ /dev/null @@ -1,81 +0,0 @@ -# ParaView - -Open-Source, Multi-Platform Data Analysis and Visualization Application - -## Introduction - -**ParaView** is an open-source, multi-platform data analysis and visualization application. ParaView users can quickly build visualizations to analyze their data using qualitative and quantitative techniques. The data exploration can be done interactively in 3D or programmatically using ParaView's batch processing capabilities. - -ParaView was developed to analyze extremely large datasets using distributed memory computing resources. It can be run on supercomputers to analyze datasets of exascale size as well as on laptops for smaller data. - -Homepage : <http://www.paraview.org/> - -## Installed Version - -Currently, version 5.1.2 compiled with intel/2017a against intel MPI library and OSMesa 12.0.2 is installed on Salomon. - -## Usage - -On Salomon, ParaView is to be used in client-server mode. A parallel ParaView server is launched on compute nodes by the user, and client is launched on your desktop PC to control and view the visualization. Download ParaView client application for your OS here: <http://paraview.org/paraview/resources/software.php>. - -!!!Warning - Your version must match the version number installed on Salomon. - -### Launching Server - -To launch the server, you must first allocate compute nodes, for example - -```console -$ qsub -I -q qprod -A OPEN-0-0 -l select=2 -``` - -to launch an interactive session on 2 nodes. Refer to [Resource Allocation and Job Execution](../job-submission-and-execution/) for details. - -After the interactive session is opened, load the ParaView module : - -```console -$ ml ParaView/5.1.2-intel-2017a-mpi -``` - -Now launch the parallel server, with number of nodes times 24 processes: - -```console -$ mpirun -np 48 pvserver --use-offscreen-rendering - Waiting for client... - Connection URL: cs://r37u29n1006:11111 - Accepting connection(s): r37u29n1006:11111 -``` - -Note the that the server is listening on compute node r37u29n1006 in this case, we shall use this information later. - -### Client Connection - -Because a direct connection is not allowed to compute nodes on Salomon, you must establish a SSH tunnel to connect to the server. Choose a port number on your PC to be forwarded to ParaView server, for example 12345. If your PC is running Linux, use this command to establish a SSH tunnel: - -```console -$ ssh -TN -L 12345:r37u29n1006:11111 username@salomon.it4i.cz -``` - -replace username with your login and r37u29n1006 with the name of compute node your ParaView server is running on (see previous step). - -If you use PuTTY on Windows, load Salomon connection configuration, then go to *Connection* -> *SSH* -> *Tunnels* to set up the port forwarding. - -Fill the Source port and Destination fields. **Do not forget to click the Add button.** - - - -Now launch ParaView client installed on your desktop PC. Select *File* -> *Connect...* and fill in the following : - - - -The configuration is now saved for later use. Now click Connect to connect to the ParaView server. In your terminal where you have interactive session with ParaView server launched, you should see: - -```console -Client connected. -``` - -You can now use Parallel ParaView. - -### Close Server - -Remember to close the interactive session after you finish working with ParaView server, as it will remain launched even after your client is disconnected and will continue to consume resources. diff --git a/docs.it4i/software/debuggers/Introduction.md b/docs.it4i/software/debuggers/Introduction.md index 1e8157c46..4ce2fc77b 100644 --- a/docs.it4i/software/debuggers/Introduction.md +++ b/docs.it4i/software/debuggers/Introduction.md @@ -15,7 +15,7 @@ $ ml intel $ idb ``` -Read more at the [Intel Debugger](../../anselm/software/intel-suite/intel-debugger/) page. +Read more at the [Intel Debugger](../intel-suite/intel-debugger/) page. ## Allinea Forge (DDT/MAP) diff --git a/docs.it4i/software/debuggers/score-p.md b/docs.it4i/software/debuggers/score-p.md index a30df38e7..186762617 100644 --- a/docs.it4i/software/debuggers/score-p.md +++ b/docs.it4i/software/debuggers/score-p.md @@ -10,8 +10,8 @@ Score-P can be used as an instrumentation tool for [Scalasca](scalasca/). There are currently two versions of Score-P version 1.2.6 [modules](../../modules-matrix/) installed on Anselm : -* scorep/1.2.3-gcc-openmpi, for usage with [GNU Compiler](../../anselm/software/compilers/) and [OpenMPI](../mpi/Running_OpenMPI/) -* scorep/1.2.3-icc-impi, for usage with [Intel Compiler](../../anselm/software/compilers/)> and [Intel MPI](../mpi/running-mpich2/)>. +* scorep/1.2.3-gcc-openmpi, for usage with [GNU Compiler](../compilers/) and [OpenMPI](../mpi/Running_OpenMPI/) +* scorep/1.2.3-icc-impi, for usage with [Intel Compiler](../compilers/)> and [Intel MPI](../mpi/running-mpich2/)>. ## Instrumentation diff --git a/docs.it4i/software/numerical-languages/opencoarrays.md b/docs.it4i/software/numerical-languages/opencoarrays.md index 5949756f0..55ca845b5 100644 --- a/docs.it4i/software/numerical-languages/opencoarrays.md +++ b/docs.it4i/software/numerical-languages/opencoarrays.md @@ -124,4 +124,4 @@ $ mpiexec -np 4 ./synchronization_test.x **-np 4** is number of images to run. The parameters of **cafrun** and **mpiexec** are the same. -For more information about running CAF program please follow [Running OpenMPI - Salomon](../../salomon/software/mpi/Running_OpenMPI.md) +For more information about running CAF program please follow [Running OpenMPI - Salomon](../mpi/Running_OpenMPI.md) diff --git a/docs.it4i/software/numerical-languages/r.md b/docs.it4i/software/numerical-languages/r.md index 75ec9aaa8..771c3efd5 100644 --- a/docs.it4i/software/numerical-languages/r.md +++ b/docs.it4i/software/numerical-languages/r.md @@ -146,7 +146,7 @@ Every evaluation of the integrad function runs in parallel on different process. package Rmpi provides an interface (wrapper) to MPI APIs. -It also provides interactive R slave environment. On the cluster, Rmpi provides interface to the [OpenMPI](../../anselm/software/mpi/Running_OpenMPI/). +It also provides interactive R slave environment. On the cluster, Rmpi provides interface to the [OpenMPI](../mpi/Running_OpenMPI/). Read more on Rmpi at <http://cran.r-project.org/web/packages/Rmpi/>, reference manual is available at <http://cran.r-project.org/web/packages/Rmpi/Rmpi.pdf> @@ -392,7 +392,7 @@ cp routput.out $PBS_O_WORKDIR/. exit ``` -For more information about jobscripts and MPI execution refer to the [Job submission](../../anselm/job-submission-and-execution/) and general [MPI](../../anselm/software/mpi/mpi/) sections. +For more information about jobscripts and MPI execution refer to the [Job submission](../../anselm/job-submission-and-execution/) and general [MPI](../mpi/mpi/) sections. ## Xeon Phi Offload diff --git a/docs.it4i/software/omics-master/diagnostic-component-team.md b/docs.it4i/software/omics-master/diagnostic-component-team.md index d8d0c4fc4..24dc71778 100644 --- a/docs.it4i/software/omics-master/diagnostic-component-team.md +++ b/docs.it4i/software/omics-master/diagnostic-component-team.md @@ -13,6 +13,6 @@ VCF files are scanned by this diagnostic tool for known diagnostic disease-assoc TEAM (27) is an intuitive and easy-to-use web tool that fills the gap between the predicted mutations and the final diagnostic in targeted enrichment sequencing analysis. The tool searches for known diagnostic mutations, corresponding to a disease panel, among the predicted patient’s variants. Diagnostic variants for the disease are taken from four databases of disease-related variants (HGMD-public, HUMSAVAR , ClinVar and COSMIC) If no primary diagnostic variant is found, then a list of secondary findings that can help to establish a diagnostic is produced. TEAM also provides with an interface for the definition of and customization of panels, by means of which, genes and mutations can be added or discarded to adjust panel definitions. - + ** Figure 5. **Interface of the application. Panels for defining targeted regions of interest can be set up by just drag and drop known disease genes or disease definitions from the lists. Thus, virtual panels can be interactively improved as the knowledge of the disease increases. diff --git a/docs.it4i/software/omics-master/overview.md b/docs.it4i/software/omics-master/overview.md index 8038a2907..e29f1daec 100644 --- a/docs.it4i/software/omics-master/overview.md +++ b/docs.it4i/software/omics-master/overview.md @@ -9,7 +9,7 @@ The scope of this OMICS MASTER solution is restricted to human genomics research The pipeline inputs the raw data produced by the sequencing machines and undergoes a processing procedure that consists on a quality control, the mapping and variant calling steps that result in a file containing the set of variants in the sample. From this point, the prioritization component or the diagnostic component can be launched.  +them, depending of the experimental design carried out.](../../img/fig1.png) Figure 1. OMICS MASTER solution overview. Data is produced in the external labs and comes to IT4I (represented by the blue dashed line). The data pre-processor converts raw data into a list of variants and annotations for each sequenced patient. These lists files together with primary and secondary (alignment) data files are stored in IT4I sequence DB and uploaded to the discovery (candidate prioritization) or diagnostic component where they can be analyzed directly by the user that produced them, depending of the experimental design carried out. @@ -41,7 +41,7 @@ Output: FASTQ file plus an HTML file containing statistics on the data. FASTQ format It represents the nucleotide sequence and its corresponding quality scores. - + Figure 2.FASTQ file. #### Mapping @@ -81,7 +81,7 @@ corresponding information is unavailable. The standard CIGAR description of pairwise alignment defines three operations: â€M’ for match/mismatch, â€I’ for insertion compared with the reference and â€D’ for deletion. The extended CIGAR proposed in SAM added four more operations: â€N’ for skipped bases on the reference, â€S’ for soft clipping, â€H’ for hard clipping and â€P’ for padding. These support splicing, clipping, multi-part and padded alignments. Figure 3 shows examples of CIGAR strings for different types of alignments. - + Figure 3 . SAM format file. The â€@SQ’ line in the header section gives the order of reference sequences. Notably, r001 is the name of a read pair. According to FLAG 163 (=1+2+32+128), the read mapped to position 7 is the second read in the pair (128) and regarded as properly paired (1 + 2); its mate is mapped to 37 on the reverse strand (32). Read r002 has three soft-clipped (unaligned) bases. The coordinate shown in SAM is the position of the first aligned base. The CIGAR string for this alignment contains a P (padding) operation which correctly aligns the inserted sequences. Padding operations can be absent when an aligner does not support multiple sequence alignment. The last six bases of read r003 map to position 9, and the first five to position 29 on the reverse strand. The hard clipping operation H indicates that the clipped sequence is not present in the sequence field. The NM tag gives the number of mismatches. Read r004 is aligned across an intron, indicated by the N operation. @@ -125,7 +125,7 @@ A VCF file consists of a header section and a data section. The header contains  +two bases by another base (SAMPLE2); the second line shows a SNP and an insertion; the third a SNP; the fourth a large structural variant described by the annotation in the INFO column, the coordinate is that of the base before the variant. (b–f ) Alignments and VCF representations of different sequence variants: SNP, insertion, deletion, replacement, and a large deletion. The REF columns shows the reference bases replaced by the haplotype in the ALT column. The coordinate refers to the first reference base. (g) Users are advised to use simplest representation possible and lowest coordinate in cases where the position is ambiguous.](../../img/fig4.png) Figure 4 . (a) Example of valid VCF. The header lines ##fileformat and #CHROM are mandatory, the rest is optional but strongly recommended. Each line of the body describes variants present in the sampled population at one genomic position or region. All alternate alleles are listed in the ALT column and referenced from the genotype fields as 1-based indexes to this list; the reference haplotype is designated as 0. For multiploid data, the separator indicates whether the data are phased (|) or unphased (/). Thus, the two alleles C and G at the positions 2 and 5 in this figure occur on the same chromosome in SAMPLE1. The first data line shows an example of a deletion (present in SAMPLE1) and a replacement of two bases by another base (SAMPLE2); the second line shows a SNP and an insertion; the third a SNP; the fourth a large structural variant described by the annotation in the INFO column, the coordinate is that of the base before the variant. (b–f ) Alignments and VCF representations of different sequence variants: SNP, insertion, deletion, replacement, and a large deletion. The REF columns shows the reference bases replaced by the haplotype in the ALT column. The coordinate refers to the first reference base. (g) Users are advised to use simplest representation possible and lowest coordinate in cases where the position is ambiguous. @@ -231,7 +231,7 @@ second one. --project>. Project ID of your supercomputer allocation. - --queue. [Queue](../../resources-allocation-policy/) to run the jobs in. + --queue. [Queue](../../salomon/resources-allocation-policy/) to run the jobs in. ``` Input, output and ped arguments are mandatory. If the output folder does not exist, the pipeline will create it. @@ -264,7 +264,7 @@ The ped file ( file.ped) contains the following info: FAM sample_B 0 0 2 2 ``` -Now, lets load the NGSPipeline module and copy the sample data to a [scratch directory](../../storage/): +Now, lets load the NGSPipeline module and copy the sample data to a [scratch directory](../../salomon/storage/): ```console $ ml ngsPipeline @@ -278,7 +278,7 @@ Now, we can launch the pipeline (replace OPEN-0-0 with your Project ID): $ ngsPipeline -i /scratch/$USER/omics/sample_data/data -o /scratch/$USER/omics/results -p /scratch/$USER/omics/sample_data/data/file.ped --project OPEN-0-0 --queue qprod ``` -This command submits the processing [jobs to the queue](../../job-submission-and-execution/). +This command submits the processing [jobs to the queue](../../salomon/job-submission-and-execution/). If we want to re-launch the pipeline from stage 4 until stage 20 we should use the next command: @@ -336,25 +336,25 @@ This listing show which tools are used in each step of the pipeline The output folder contains all the subfolders with the intermediate data. This folder contains the final VCF with all the variants. This file can be uploaded into [TEAM](diagnostic-component-team/) by using the VCF file button. It is important to note here that the entire management of the VCF file is local: no patient’s sequence data is sent over the Internet thus avoiding any problem of data privacy or confidentiality. -![TEAM upload panel. Once the file has been uploaded, a panel must be chosen from the Panel list. Then, pressing the Run button the diagnostic process starts.]\((../../../img/fig7.png) +![TEAM upload panel. Once the file has been uploaded, a panel must be chosen from the Panel list. Then, pressing the Run button the diagnostic process starts.]\((../../img/fig7.png) Figure 7. _TEAM upload panel._ _Once the file has been uploaded, a panel must be chosen from the Panel_ list. Then, pressing the Run button the diagnostic process starts. Once the file has been uploaded, a panel must be chosen from the Panel list. Then, pressing the Run button the diagnostic process starts. TEAM searches first for known diagnostic mutation(s) taken from four databases: HGMD-public (20), [HUMSAVAR](http://www.uniprot.org/docs/humsavar), ClinVar (29) and COSMIC (23). - + Figure 7. The panel manager. The elements used to define a panel are ( A ) disease terms, ( B ) diagnostic mutations and ( C ) genes. Arrows represent actions that can be taken in the panel manager. Panels can be defined by using the known mutations and genes of a particular disease. This can be done by dragging them to the Primary Diagnostic box (action D ). This action, in addition to defining the diseases in the Primary Diagnostic box, automatically adds the corresponding genes to the Genes box. The panels can be customized by adding new genes (action F ) or removing undesired genes (action G). New disease mutations can be added independently or associated to an already existing disease term (action E ). Disease terms can be removed by simply dragging them back (action H ). For variant discovering/filtering we should upload the VCF file into BierApp by using the following form: -\\ +\\ Figure 8 . \BierApp VCF upload panel. It is recommended to choose a name for the job as well as a description \\. Each prioritization (â€job’) has three associated screens that facilitate the filtering steps. The first one, the â€Summary’ tab, displays a statistic of the data set analyzed, containing the samples analyzed, the number and types of variants found and its distribution according to consequence types. The second screen, in the â€Variants and effect’ tab, is the actual filtering tool, and the third one, the â€Genome view’ tab, offers a representation of the selected variants within the genomic context provided by an embedded version of the Genome Maps Tool (30). - + Figure 9 . This picture shows all the information associated to the variants. If a variant has an associated phenotype we could see it in the last column. In this case, the variant 7:132481242 CT is associated to the phenotype: large intestine tumor. diff --git a/docs.it4i/software/omics-master/priorization-component-bierapp.md b/docs.it4i/software/omics-master/priorization-component-bierapp.md index 6f88fecc7..07c763fb6 100644 --- a/docs.it4i/software/omics-master/priorization-component-bierapp.md +++ b/docs.it4i/software/omics-master/priorization-component-bierapp.md @@ -13,7 +13,7 @@ BiERapp is available at the [following address](http://omics.it4i.cz/bierapp/) BiERapp (28) efficiently helps in the identification of causative variants in family and sporadic genetic diseases. The program reads lists of predicted variants (nucleotide substitutions and indels) in affected individuals or tumor samples and controls. In family studies, different modes of inheritance can easily be defined to filter out variants that do not segregate with the disease along the family. Moreover, BiERapp integrates additional information such as allelic frequencies in the general population and the most popular damaging scores to further narrow down the number of putative variants in successive filtering steps. BiERapp provides an interactive and user-friendly interface that implements the filtering strategy used in the context of a large-scale genomic project carried out by the Spanish Network for Research, in Rare Diseases (CIBERER) and the Medical Genome Project. in which more than 800 exomes have been analyzed. - + ** Figure 6 **. Web interface to the prioritization tool. This figure shows the interface of the web tool for candidate gene prioritization with the filters available. The tool includes a genomic viewer (Genome Maps 30) that enables the representation of the variants in the corresponding genomic coordinates. -- GitLab