sun.chpc.ac.za

Here you find information about the the system http://www.chpc.ac.za/sun/.

The installation of user’s packages on SUSE 10 login01, 64-bit described below uses modules, and assumes sh shell:

  • packages are installed under ~/CAMd:

    mkdir ~/CAMd
    cd ~/CAMd
    
  • module files are located under ~/CAMd/modulefiles:

    mkdir ~/CAMd/modulefiles
    
  • download the customize_sun_chpc_SUSE10.py file:

    compiler = 'gcc'
    mpicompiler = '/opt/SUNWhpc/HPC8.2/gnu/bin/mpicc'
    mpilinker = 'MPICH_CC=gcc mpicc -Xlinker --export-dynamic'
    
    extra_compile_args = ['-O3', '-std=c99', '-fpic']
    
    scalapack = False
    
    mkl_dir = '/opt/gridware/intel/Compiler/11.1/056/mkl/lib/em64t/'
    
    library_dirs += [mkl_dir]
    libraries = ['mkl_intel_lp64' ,'mkl_sequential' ,'mkl_core',
                 'mkl_lapack',
                 ]
    
    extra_link_args = ['-Wl,-rpath=' + mkl_dir]
    
    
  • download packages with download_sun_chpc.sh:

    export APPS="~/CAMd"
    export MODULEFILES="${APPS}/modulefiles"
    
    cd ${APPS}
    # download packages
    nose_version=1.1.2
    http://pypi.python.org/packages/source/n/nose/nose-${nose_version}.tar.gz
    numpy_version=1.5.1
    wget http://downloads.sourceforge.net/numpy/numpy-${numpy_version}.tar.gz
    ase_version=3.6.0.2515
    wget https://wiki.fysik.dtu.dk/ase-files/python-ase-${ase_version}.tar.gz
    gpaw_version=0.9.0.8965
    wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-${gpaw_version}.tar.gz
    gpaw_setups_version=0.8.7929
    wget http://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-${gpaw_setups_version}.tar.gz
    
  • install packages, deploy modules and test with install_sun_chpc_SUSE10.sh:

    export APPS=`echo ~/CAMd`
    export MODULEFILES="${APPS}/modulefiles"
    
    # build packages
    
    python_version=2.7.3
    wget http://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz
    tar zxf Python-${python_version}.tgz
    cd Python-${python_version}
    ./configure --prefix=${APPS}/Python-${python_version}-1
    make 2>&1 | tee make.log
    make install 2>&1 | tee make_install.log
    cd ..
    
    mkdir -p ${MODULEFILES}/python
    cat <<EOF > ${MODULEFILES}/python/${python_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prepend-path    PATH                \$apps_path/Python-${python_version}-1/bin
    unset apps_path
    EOF
    
    module use --append ${MODULEFILES}
    module load python
    
    export GPAW_PLATFORM=`python -c "from distutils import util, sysconfig; print util.get_platform()+'-'+sysconfig.get_python_version()"`
    export PYTHONVERSION=`python -c "from distutils import sysconfig; print sysconfig.get_python_version()"`
    
    nose_version=1.1.2
    tar zxf nose-${nose_version}.tar.gz
    cd nose-${nose_version}
    python setup.py install --root=${APPS}/nose-${nose_version}-1
    cd ..
    
    numpy_version=1.6.1
    tar zxf numpy-${numpy_version}.tar.gz
    cd  numpy-${numpy_version}
    # Use /usr/bin/gfortran and numpy's internal blas/lapack
    sed -i "s/_lib_names = \['blas'\]/_lib_names = ['']/g"  numpy/distutils/system_info.py
    sed -i "s/_lib_names = \['lapack'\]/_lib_names = ['']/g"  numpy/distutils/system_info.py
    # avoid "Both g77 and gfortran runtimes linked in lapack_lite !" setting --fcompiler=gnu95
    # note that this forces /usr/bin/gfortran to be used
    python setup.py build --fcompiler=gnu95 2>&1 | tee build.log
    python setup.py install --root=${APPS}/numpy-${numpy_version}-1 2>&1 | tee install.log
    cd ..
    
    ase_version=3.6.0.2515
    tar zxf python-ase-${ase_version}.tar.gz
    
    gpaw_version=0.9.0.8965
    tar zxf gpaw-${gpaw_version}.tar.gz
    
    gpaw_setups_version=0.8.7929
    tar zxf gpaw-setups-${gpaw_setups_version}.tar.gz
    
    mkdir -p ${MODULEFILES}/nose
    cat <<EOF > ${MODULEFILES}/nose/${nose_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prereq python
    prepend-path    PATH                \$apps_path/nose-${nose_version}-1${APPS}/Python-${python_version}-1/bin
    prepend-path    PYTHONPATH          \$apps_path/nose-${nose_version}-1${APPS}/Python-${python_version}-1/lib/python${PYTHONVERSION}/site-packages/
    unset apps_path
    EOF
    
    mkdir -p ${MODULEFILES}/numpy
    cat <<EOF > ${MODULEFILES}/numpy/${numpy_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prereq nose
    prepend-path    PATH                \$apps_path/numpy-${numpy_version}-1${APPS}/Python-${python_version}-1/bin
    prepend-path    PYTHONPATH          \$apps_path/numpy-${numpy_version}-1${APPS}/Python-${python_version}-1/lib/python${PYTHONVERSION}/site-packages/
    unset apps_path
    EOF
    
    mkdir -p ${MODULEFILES}/python-ase
    cat <<EOF > ${MODULEFILES}/python-ase/${ase_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prereq numpy
    prepend-path    PATH                \$apps_path/python-ase-${ase_version}/tools
    prepend-path    PYTHONPATH          \$apps_path/python-ase-${ase_version}/
    unset apps_path
    EOF
    
    mkdir -p ${MODULEFILES}/gpaw-setups
    cat <<EOF > ${MODULEFILES}/gpaw-setups/${gpaw_setups_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prepend-path    GPAW_SETUP_PATH     \$apps_path/gpaw-setups-${gpaw_setups_version}
    unset apps_path
    EOF
    
    mkdir -p ${MODULEFILES}/gpaw
    cat <<EOF > ${MODULEFILES}/gpaw/${gpaw_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prereq python-ase
    prereq gpaw-setups
    prepend-path    PATH                \$apps_path/gpaw-${gpaw_version}/tools
    prepend-path    PATH                \$apps_path/gpaw-${gpaw_version}/build/bin.${GPAW_PLATFORM}
    prepend-path    PYTHONPATH          \$apps_path/gpaw-${gpaw_version}/
    prepend-path    PYTHONPATH          \$apps_path/gpaw-${gpaw_version}/build/lib.${GPAW_PLATFORM}
    setenv OMP_NUM_THREADS 1
    unset apps_path
    EOF
    
    module load nose
    module load numpy
    # test numpy
    python -c "import numpy; numpy.test()"
    
    module load python-ase
    # test ase
    mkdir -p testase
    cd testase
    testase.py --no-display 2>&1 | tee testase.log
    cd ..
    # build gpaw
    cd gpaw-${gpaw_version}
    # disable fftw (something is wrong with /usr/local/lib/libfftw3.a)
    sed -i 's/libfftw3\.so/libfftw3_not.so/' gpaw/fftw.py
    python setup.py build_ext --customize=../customize_sun_chpc_SUSE10.py --remove-default-flags 2>&1 | tee build_ext.log
    cd ..
    module load gpaw-setups
    module load gpaw
    mkdir -p testgpaw
    cd testgpaw
    mpiexec -np 4 gpaw-python `which gpaw-test` 2>&1 | tee testgpaw.log
    

    Note that every time you wish to install a new version of a package, and deploy new module file, better keep the old module file.

  • submit the test job (jobs must be submitted from under ~/scratch):

    mqsub msub_sun_chpc.sh
    

    using the following msub_sun_chpc.sh:

    #!/bin/sh
    ###These lines are for Moab
    #MSUB -l nodes=1:ppn=8
    #MSUB -l walltime=02:00:00
    #MSUB -l partition=nehalem|harpertown|westmere
    
    . /opt/gridware/modules-3.2.7/modules.sh
    export APPS=`echo ~/CAMd`
    export MODULEFILES="${APPS}/modulefiles"
    module use --append ${MODULEFILES}
    module load python
    module load nose
    module load numpy
    module load python-ase
    module load gpaw-setups
    module load gpaw/0.9.0.8965-1
    
    export OMP_NUM_THREADS=1
    
    ##### Running commands
    cat $PBS_NODEFILE
    NP=`cat $PBS_NODEFILE | wc -l`
    mpirun -x PYTHONPATH -x GPAW_SETUP_PATH -x PATH -np $NP -machinefile $PBS_NODEFILE gpaw-python `which gpaw-test`