sun.chpc.ac.za¶

Here you find information about the system http://www.chpc.ac.za/sun/.

The installation of user’s packages on SUSE 10 login01, 64-bit described below uses modules, and assumes sh shell:

packages are installed under ~/CAMd:
```
mkdir ~/CAMd
cd ~/CAMd
```
module files are located under ~/CAMd/modulefiles:
```
mkdir ~/CAMd/modulefiles
```

download the customize_sun_chpc_SUSE10.py file:

compiler = 'gcc'
mpicompiler = '/opt/SUNWhpc/HPC8.2/gnu/bin/mpicc'
mpilinker = 'MPICH_CC=gcc mpicc -Xlinker --export-dynamic'

extra_compile_args = ['-O3', '-std=c99', '-fpic']

scalapack = False

mkl_dir = '/opt/gridware/intel/Compiler/11.1/056/mkl/lib/em64t/'

library_dirs += [mkl_dir]
libraries = ['mkl_intel_lp64' ,'mkl_sequential' ,'mkl_core',
             'mkl_lapack',
             ]

extra_link_args = ['-Wl,-rpath=' + mkl_dir]

download packages with download_sun_chpc.sh:

export APPS="~/CAMd"
export MODULEFILES="${APPS}/modulefiles"

cd ${APPS}
# download packages
nose_version=1.1.2
http://pypi.python.org/packages/source/n/nose/nose-${nose_version}.tar.gz
numpy_version=1.5.1
wget http://downloads.sourceforge.net/numpy/numpy-${numpy_version}.tar.gz
ase_version=3.6.0.2515
wget https://wiki.fysik.dtu.dk/ase-files/python-ase-${ase_version}.tar.gz
gpaw_version=0.9.0.8965
wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-${gpaw_version}.tar.gz
gpaw_setups_version=0.8.7929
wget http://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-${gpaw_setups_version}.tar.gz

install packages, deploy modules and test with install_sun_chpc_SUSE10.sh:

export APPS=`echo ~/CAMd`
export MODULEFILES="${APPS}/modulefiles"

# build packages

python_version=2.7.3
wget http://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz
tar zxf Python-${python_version}.tgz
cd Python-${python_version}
./configure --prefix=${APPS}/Python-${python_version}-1
make 2>&1 | tee make.log
make install 2>&1 | tee make_install.log
cd ..

mkdir -p ${MODULEFILES}/python
cat <<EOF > ${MODULEFILES}/python/${python_version}-1
#%Module1.0
set apps_path ${APPS}
prepend-path    PATH                \$apps_path/Python-${python_version}-1/bin
unset apps_path
EOF

module use --append ${MODULEFILES}
module load python

export GPAW_PLATFORM=`python -c "from distutils import util, sysconfig; print util.get_platform()+'-'+sysconfig.get_python_version()"`
export PYTHONVERSION=`python -c "from distutils import sysconfig; print sysconfig.get_python_version()"`

nose_version=1.1.2
tar zxf nose-${nose_version}.tar.gz
cd nose-${nose_version}
python setup.py install --root=${APPS}/nose-${nose_version}-1
cd ..

numpy_version=1.6.1
tar zxf numpy-${numpy_version}.tar.gz
cd  numpy-${numpy_version}
# Use /usr/bin/gfortran and numpy's internal blas/lapack
sed -i "s/_lib_names = \['blas'\]/_lib_names = ['']/g"  numpy/distutils/system_info.py
sed -i "s/_lib_names = \['lapack'\]/_lib_names = ['']/g"  numpy/distutils/system_info.py
# avoid "Both g77 and gfortran runtimes linked in lapack_lite !" setting --fcompiler=gnu95
# note that this forces /usr/bin/gfortran to be used
python setup.py build --fcompiler=gnu95 2>&1 | tee build.log
python setup.py install --root=${APPS}/numpy-${numpy_version}-1 2>&1 | tee install.log
cd ..

ase_version=3.6.0.2515
tar zxf python-ase-${ase_version}.tar.gz

gpaw_version=0.9.0.8965
tar zxf gpaw-${gpaw_version}.tar.gz

gpaw_setups_version=0.8.7929
tar zxf gpaw-setups-${gpaw_setups_version}.tar.gz

mkdir -p ${MODULEFILES}/nose
cat <<EOF > ${MODULEFILES}/nose/${nose_version}-1
#%Module1.0
set apps_path ${APPS}
prereq python
prepend-path    PATH                \$apps_path/nose-${nose_version}-1${APPS}/Python-${python_version}-1/bin
prepend-path    PYTHONPATH          \$apps_path/nose-${nose_version}-1${APPS}/Python-${python_version}-1/lib/python${PYTHONVERSION}/site-packages/
unset apps_path
EOF

mkdir -p ${MODULEFILES}/numpy
cat <<EOF > ${MODULEFILES}/numpy/${numpy_version}-1
#%Module1.0
set apps_path ${APPS}
prereq nose
prepend-path    PATH                \$apps_path/numpy-${numpy_version}-1${APPS}/Python-${python_version}-1/bin
prepend-path    PYTHONPATH          \$apps_path/numpy-${numpy_version}-1${APPS}/Python-${python_version}-1/lib/python${PYTHONVERSION}/site-packages/
unset apps_path
EOF

mkdir -p ${MODULEFILES}/python-ase
cat <<EOF > ${MODULEFILES}/python-ase/${ase_version}-1
#%Module1.0
set apps_path ${APPS}
prereq numpy
prepend-path    PATH                \$apps_path/python-ase-${ase_version}/tools
prepend-path    PYTHONPATH          \$apps_path/python-ase-${ase_version}/
unset apps_path
EOF

mkdir -p ${MODULEFILES}/gpaw-setups
cat <<EOF > ${MODULEFILES}/gpaw-setups/${gpaw_setups_version}-1
#%Module1.0
set apps_path ${APPS}
prepend-path    GPAW_SETUP_PATH     \$apps_path/gpaw-setups-${gpaw_setups_version}
unset apps_path
EOF

mkdir -p ${MODULEFILES}/gpaw
cat <<EOF > ${MODULEFILES}/gpaw/${gpaw_version}-1
#%Module1.0
set apps_path ${APPS}
prereq python-ase
prereq gpaw-setups
prepend-path    PATH                \$apps_path/gpaw-${gpaw_version}/tools
prepend-path    PATH                \$apps_path/gpaw-${gpaw_version}/build/bin.${GPAW_PLATFORM}
prepend-path    PYTHONPATH          \$apps_path/gpaw-${gpaw_version}/
prepend-path    PYTHONPATH          \$apps_path/gpaw-${gpaw_version}/build/lib.${GPAW_PLATFORM}
setenv OMP_NUM_THREADS 1
unset apps_path
EOF

module load nose
module load numpy
# test numpy
python -c "import numpy; numpy.test()"

module load python-ase
# test ase
mkdir -p testase
cd testase
testase.py --no-display 2>&1 | tee testase.log
cd ..
# build gpaw
cd gpaw-${gpaw_version}
# disable fftw (something is wrong with /usr/local/lib/libfftw3.a)
sed -i 's/libfftw3\.so/libfftw3_not.so/' gpaw/fftw.py
python setup.py build_ext --customize=../customize_sun_chpc_SUSE10.py --remove-default-flags 2>&1 | tee build_ext.log
cd ..
module load gpaw-setups
module load gpaw
mkdir -p testgpaw
cd testgpaw
mpiexec -np 4 gpaw-python `which gpaw-test` 2>&1 | tee testgpaw.log

Note that every time you wish to install a new version of a package, and deploy new module file, better keep the old module file.

submit the test job (jobs must be submitted from under ~/scratch):

mqsub msub_sun_chpc.sh

using the following msub_sun_chpc.sh:

#!/bin/sh
###These lines are for Moab
#MSUB -l nodes=1:ppn=8
#MSUB -l walltime=02:00:00
#MSUB -l partition=nehalem|harpertown|westmere

. /opt/gridware/modules-3.2.7/modules.sh
export APPS=`echo ~/CAMd`
export MODULEFILES="${APPS}/modulefiles"
module use --append ${MODULEFILES}
module load python
module load nose
module load numpy
module load python-ase
module load gpaw-setups
module load gpaw/0.9.0.8965-1

export OMP_NUM_THREADS=1

##### Running commands
cat $PBS_NODEFILE
NP=`cat $PBS_NODEFILE | wc -l`
mpirun -x PYTHONPATH -x GPAW_SETUP_PATH -x PATH -np $NP -machinefile $PBS_NODEFILE gpaw-python `which gpaw-test`