Running OLAM at HCC

OLAM compilation

pgi/11 compilation with mpi and openmp enabled
  1. Load modules: 
        module load compiler/pgi/11 openmpi/1.6 szip/2.1 zlib/1.2 NCL/6.1dist
  2. Edit the include.mk file.
include.mk
#-----------------  LINUX Intel Fortran ifort/gcc ---------------
F_COMP=mpif90
# If the compiler supports (and the user wants to use)
# the module IEEE_ARITHMETIC, uncomment below
IEEE_ARITHMETIC=yes
# If using MPI libraries:
OLAM_MPI=yes
# If parallel hdf5 is supported, uncomment the next line
OLAM_PARALLEL_HDF5=yes
# If you use the ED2 model, uncomment the next line
#USE_ED2=yes
MPI_PATH=/util/opt/openmpi/1.6/pgi/11
PAR_INCS=-I$(MPI_PATH)/include:$(MPI_PATH)/lib
PAR_LIBS=-L$(MPI_PATH)/lib -lmpi
# OPTIMIZED:
F_OPTS=-O3 -traceback -mp
#F_OPTS=-xHost -O3 -fno-alias -ip -openmp -traceback
#F_OPTS=-g -O3 -xHost -traceback
# DEBUG:
#F_OPTS=-g -fp-model precise -check bounds -traceback \
#        -debug extended -check uninit -ftrapuv
# FORTRAN FLAGS FOR BIG FILES WHICH WOULD HAVE EXCESSIVE COMPILATION TIME
#SLOW_FFLAGS=-O1 -g -no-ip -traceback
C_COMP=mpicc
#C_COMP=mpicc
C_OPTS=-DUNDERSCORE -DLITTLE
NCARG_DIR=/util/src/ncl_ncarg/ncl_ncarg-6.1.2/lib
LIBNCARG=-L$(NCARG_DIR) -lncarg -lncarg_gks -lncarg_c \
          -L/usr/lib64 -lX11 -ldl -lpthread -lgfortran -lcairo
HDF5_LIBS=-L/util/opt/hdf5/1.8.13/openmpi/1.6/pgi/11/lib -lhdf5_fortran -lhdf5 -lz -lm
HDF5_INCS=-I/util/opt/hdf5/1.8.13/openmpi/1.6/pgi/11/include
NETCDF_LIBS=-L/util/opt/netcdf/4.2/pgi/11/lib -lnetcdf
NETCDF_INCS=-I/util/opt/netcdf/4.2/pgi/11/include

LOADER=$(F_COMP)
LOADER_OPTS=-mp
#LOADER_OPTS=-static-intel $(F_OPTS)
# For Apple OSX: the stack size needs to be increased at link time
# LOADER_OPTS=-static-intel $(F_OPTS) -Wl,-stack_size -Wl,0x10000000
# to allow ifort compiler to link with pg-compiled ncar graphics:
# LIBS=-z muldefs -L/opt/pgi/linux86-64/5.2/lib -lpgftnrtl -lpgc
## IMPORTANT:  Need to specify this flag in ED2
#USE_HDF5=1

  1. Command: make clean

  2. Command: make -j 8

intel/12 compilation with mpi and openmp enabled
  1. Load modules: 
        module load compiler/intel/12 openmpi/1.6 szip/2.1 zlib/1.2

  2. Edit the include.mk file.

    include.mk
    #-----------------  LINUX Intel Fortran ifort/gcc ---------------
F_COMP=mpif90
# If the compiler supports (and the user wants to use)
# the module IEEE_ARITHMETIC, uncomment below
IEEE_ARITHMETIC=yes
# If using MPI libraries:
OLAM_MPI=yes
# If parallel hdf5 is supported, uncomment the next line
OLAM_PARALLEL_HDF5=yes
# If you use the ED2 model, uncomment the next line
#USE_ED2=yes
MPI_PATH=/util/opt/openmpi/1.6/intel/12
PAR_INCS=-I$(MPI_PATH)/include:$(MPI_PATH)/lib
PAR_LIBS=-L$(MPI_PATH)/lib -lmpi
# OPTIMIZED:
F_OPTS=-O3 -traceback -openmp
#F_OPTS=-xHost -O3 -fno-alias -ip -openmp -traceback
#F_OPTS=-g -O3 -xHost -traceback
# DEBUG:
#F_OPTS=-g -fp-model precise -check bounds -traceback \
#        -debug extended -check uninit -ftrapuv
# FORTRAN FLAGS FOR BIG FILES WHICH WOULD HAVE EXCESSIVE COMPILATION TIME
#SLOW_FFLAGS=-O1 -g -no-ip -traceback
C_COMP=mpicc
#C_COMP=mpicc
C_OPTS=-DUNDERSCORE -DLITTLE
NCARG_DIR=/util/src/ncl_ncarg/ncl_ncarg-6.1.2/lib
LIBNCARG=-L$(NCARG_DIR) -lncarg -lncarg_gks -lncarg_c \
          -L/usr/lib64 -lX11 -ldl -lpthread -lgfortran -lcairo
HDF5_LIBS=-L/util/opt/hdf5/1.8.13/openmpi/1.6/intel/12/lib -lhdf5_fortran -lhdf5 -lz -lm
HDF5_INCS=-I/util/opt/hdf5/1.8.13/openmpi/1.6/intel/12/include
NETCDF_LIBS=-L/util/opt/netcdf/4.2/intel/12/lib -lnetcdf
NETCDF_INCS=-I/util/opt/netcdf/4.2/intel/12/include
LOADER=$(F_COMP)
LOADER_OPTS=-openmp
#LOADER_OPTS=-static-intel $(F_OPTS)
# For Apple OSX: the stack size needs to be increased at link time
# LOADER_OPTS=-static-intel $(F_OPTS) -Wl,-stack_size -Wl,0x10000000
# to allow ifort compiler to link with pg-compiled ncar graphics:
# LIBS=-z muldefs -L/opt/pgi/linux86-64/5.2/lib -lpgftnrtl -lpgc
## IMPORTANT:  Need to specify this flag in ED2
#USE_HDF5=1

  3. Command: make clean

  4. Command: make -j 8

 

OLAM compilation on Crane

Intel/15 compiler with OpenMPI/1.10
  1. Load modules: 
        module load compiler/intel/15 openmpi/1.10 NCL/6.1 netcdf/4.4 phdf5/1.8 szip/2.1 zlib/1.2

  2. Edit the include.mk file:

    include.mk
    #-----------------  LINUX Intel Fortran ifort/gcc ---------------
F_COMP=/util/opt/hdf5/1.8/openmpi/1.10/intel/15/bin/h5pfc

# If the compiler supports (and the user wants to use)
# the module IEEE_ARITHMETIC, uncomment below
IEEE_ARITHMETIC=yes

# If using MPI libraries:
OLAM_MPI=yes

# If parallel hdf5 is supported, uncomment the next line
OLAM_PARALLEL_HDF5=yes

# If you use the ED2 model, uncomment the next line
#USE_ED2=yes

#MPI_PATH=/usr/local/mpich
PAR_INCS=-I/util/opt/openmpi/1.10/intel/15/include
PAR_LIBS=-L/util/opt/openmpi/1.10/intel/15/lib

# OPTIMIZED:
F_OPTS=-xHost -O3 -fno-alias -ip -openmp -traceback
#F_OPTS=-g -O3 -xHost -traceback

# DEBUG:
#F_OPTS=-g -fp-model precise -check bounds -traceback \
#        -debug extended -check uninit -ftrapuv

# EXTRA OPTIONS FOR FIXED-SOURCE CODE
FIXED_SRC_FLAGS=-fixed -132

# FORTRAN FLAGS FOR BIG FILES WHICH WOULD HAVE EXCESSIVE COMPILATION TIME
SLOW_FFLAGS=-O1 -g -no-ip -traceback

#C_COMP=icc
C_COMP=mpicc
C_OPTS=-O3 -DUNDERSCORE -DLITTLE

NCARG_DIR=/util/opt/NCL/6.1/lib
LIBNCARG=-L$(NCARG_DIR) -lncarg -lncarg_gks -lncarg_c \
-L/usr/lib64 -lX11 -ldl -lpng -lpthread -lgfortran -lcairo

HDF5_LIBS=-L/util/opt/hdf5/1.8/openmpi/1.10/intel/15/lib
HDF5_INCS=-I/util/opt/hdf5/1.8/openmpi/1.10/intel/15/include

NETCDF_LIBS=-L/util/opt/netcdf/4.4/intel/15/lib -lnetcdf
NETCDF_INCS=-I/util/opt/netcdf/4.4/intel/15/include

LOADER=$(F_COMP)
LOADER_OPTS=-static-intel $(F_OPTS)

# For Apple OSX: the stack size needs to be increased at link time
# LOADER_OPTS=-static-intel $(F_OPTS) -Wl,-stack_size -Wl,0x10000000

# to allow ifort compiler to link with pg-compiled ncar graphics:
# LIBS=-z muldefs -L/opt/pgi/linux86-64/5.2/lib -lpgftnrtl -lpgc

## IMPORTANT:  Need to specify this flag in ED2
USE_HDF5=1

  3. Command: make clean

  4. Command: make -j 8

 

Sample SLURM submit scripts

PGI compiler:
Sample submit script for PGI compiler
#!/bin/bash
#SBATCH --ntasks=8                                          # 8 cores
#SBATCH --mem-per-cpu=1024                                  # Minimum memory required per CPU (in megabytes)
#SBATCH --time=03:15:00                                     # Run time in hh:mm:ss
#SBATCH --error=/work/[groupname]/[username]/job.%J.err
#SBATCH --output=/work/[groupname]/[username]/job.%J.out
 
module load compiler/pgi/11 openmpi/1.6 szip/2.1 zlib/1.2
 
mpirun /path/to/olam-4.2c-mpi
Intel compiler:
Sample submit script for Intel compiler
#!/bin/bash
#SBATCH --ntasks=8                                          # 8 cores
#SBATCH --mem-per-cpu=1024                                  # Minimum memory required per CPU (in megabytes)
#SBATCH --time=03:15:00                                     # Run time in hh:mm:ss
#SBATCH --error=/work/[groupname]/[username]/job.%J.err
#SBATCH --output=/work/[groupname]/[username]/job.%J.out
 
module load compiler/intel/12 openmpi/1.6 szip/2.1 zlib/1.2
 
mpirun /path/to/olam-4.2c-mpi