#!/bin/bash
#  MASTER - run the data assimilation scheme with 6-hour forecasts
#  (or whatever other time interval is specified).
#  Execute line: [-s starttime] [-C config] endtime
#  where both dates have the form yyyymmddhh.  By default, the
#  starttime is 2000010100; this can be changed with the -s option.
#  Most configuration parameters are stored in a file called "config"
#  in the current directory by default.  A configuration file with
#  a different name or location may be specified as the argument of
#  the -C option.
#
#  This version is tailored to the Karman Beowulf cluster with one master
#  node and other nodes named nodeN, n=2,..,25.
#  08/24/05 - added support for either 2001 or 2004 model, according to the
#     MODEL variable in the config file.
#  08/10/05 - moved forecast code to scriptdir/forecast.2001; changed
#     STARTDATE to STARTTIME.
#  06/17/05 - added -f option to skip the assimilation step and run
#             a forecast when the starting time is 0 (=starttime)
#  06/14/05 - restored capability to take initial conditions from physical
#             grid perturbation files and take one assimilation step before
#             forecasts (added the startphys variable in config file to
#             handle this).
#  06/04/05 - bug fixes to handle nature run surface files
#  02/04/05 - added AHOUR and ADATE variables to complement ATIME
#  01/28/05 - added checks for zrad.dat file, for existence of result
#             directory, changed permissions of starting sigma files;
#             changed references from MRF to GFS.
#  12/15/04 - deleted special initialization case; we always read sigma files
#             to define the grid.
#  12/14/04 - moved final grib postprocessing to beo_assim
#  11/29/04 - syntax corrections for improved portability from bash to ksh
#  07/27/04 - fixed bug in which output surface analysis files for all
#      ensemble members were copied to the _same_ output file.  Also
#      better supports the config file members, removing hardwired names
#      like pnew$ens and so on.  Eliminates hardwired 6-hour forecast
#      interval.
#  01/23/04 - added support for taking surface boundary conditions
#      from the 6-hr forecast instead of the nature run.
#
version="08/24/05"
# ---------------------------------------
function usage {
   echo "$0: version $version" 1>&2
   echo "$0: usage: [-s starttime] [-C config] endtime" 1>&2 
   echo "$0: starttime and endtime are specified as yyyymmddhh" 1>&2
}
# ---------------------------------------
#  EXPAND - expand a quoted value using current variables, especially ATIME
#  and its derivatives.
#  Optional arguments allow for temporary overrides of variables named
#  in the first argument.

function expand {  
   local v=$1
   local prev=""
   local var

   shift 1
   for var in $*
   do
      local $var
   done
   local ADATE=$(echo $ATIME | cut -c 1-8)
   local AHOUR=$(echo $ATIME | cut -c 9-10)

   while [ "$prev" != "$v" ]
   do
      prev=$v
      v=$(eval echo $v)
   done
   echo $v
   return
}  #expand
# ---------------------------------------
function initialize {
   set -u  # uninitialized variables are errors
   CL_CONFIG=config  # configuration file
   CL_STARTDATE=""   # if specified, overrides STARTDATE in config file
   RUN_FORECAST_FIRST=false  # by default, run an assimilation step at T=T0
   while getopts C:fhs: opt
   do
       case $opt in
       C)  CL_CONFIG=$OPTARG;;
       f)  RUN_FORECAST_FIRST=true;;
       h)  usage; exit 0;;
       s)  CL_STARTDATE=$OPTARG;;
       \?) echo "$0: unknown option" 1>&2; usage; exit 1;;
       esac
   done

   # The ending time is a required argument that comes last on the command line
   if [ $OPTIND -eq 1 ]  # no options
   then
      endtime=$1
   elif [ $OPTIND -gt $# ]
   then
      usage
      exit 1
   else
      shift $(expr $OPTIND - 1)
      endtime=$1
   fi
   #  Source the config file
   if [ -n "$CL_CONFIG" ]
   then
      CONFIG=$CL_CONFIG
   fi
   if [ ! -f $CONFIG ]
   then
      echo $0: cannot read configuration file \"$CONFIG\" 1>&2
      exit 1
   elif ! . $CONFIG
   then
      echo $0: error parsing config file 1>&2
      exit 1
   fi

   #  Check for valid ending time
   if ! $scriptdir/ndate 0 $endtime > /dev/null
   then
      echo "$0: invalid format for endtime; must be yyyymmddhh"  1>&2
      exit 1
   fi

   #  Check for valid forecast model
   if [ -z "$MODEL" ]
   then
      echo "$0: the variable MODEL (either 2001 or 2004) must be specified in the config file" 1>&2
      exit 1
   elif [ $MODEL -ne 2001 -a $MODEL -ne 2004 ]
   then
      echo "$0: MODEL must be either 2001 or 2004" 1>&2
      exit 1
   fi

   # If the parent of the result directory does not exist, then create it
   PARENT=$(dirname $RESULTDIR)
   if [ -z "$PARENT" ]
   then
      echo $0: No result directory specified 1>&2
      exit 1
   elif [ ! -d "$PARENT" ]
   then
      echo $0: creating $PARENT 1>&2
      if ! mkdir -p $PARENT
      then
         echo $0: error creating $PARENT 1>&2
         exit 1
      fi
   fi

   # Put the vertical localization data file in the right place
   if [ -n "$ZRAD" -a ! -f "$PARENT/$ZRAD" ]
   then
      if ! cp $ZRAD $PARENT
      then
         echo $0: Cannot locate $ZRAD for vertical localization 1>&2
         exit 1
      else
         echo $0: Warning: using $ZRAD for vertical localization 1>&2
      fi
   fi

   # Although the master script itself references all the executables
   # that it needs by full path names, some of the scripts that it calls,
   # such as run_gfs, need to know the PATH variable.
   export PATH=${exedir}:${scriptdir}:$PATH

   # Ensure that we have enough processors - PROCLIST is defined in config file
   declare -a machlist=($(nodelist -p $NODE $PROCLIST))  # bash only
   nnodes=$(nodelist -n $PROCLIST)
   let "ncpu=nnodes*procpernode"   # procpernode from config file

   # Check for sane starting and ending times
   ATIME=${CL_STARTDATE:-$starttime}  # use starttime if CL_STARTDATE is null
   if ! $scriptdir/ndate 0 $ATIME > /dev/null
   then
      echo $0: invalid format for ATIME 1>&2
      exit 1
   fi
   if [ $ATIME -gt $endtime ]
   then
      echo $0: starting time is after ending time 1>&2
      exit 1
   fi
   return
}  #initialize
# ------------------------------------------------------------------------
#  Main code here.

initialize $*

if [ $VERBLEV -gt 1 ]  # make beo_assim more verbose
then 
   beo_v="-v"
else
   beo_v=""
fi

echo "$0: master script (version ${version}) for 2001 and 2004 versions of the GFS"
echo $0: starting simulation at $(date)
echo $0: using config file $CONFIG

#  Do an assimilation step at time 0 before running a forecast unless
#  RUN_FORECAST_FIRST is "true".

if [ $ATIME -eq $starttime ]
then
   ADATE=$(echo $ATIME | cut -c 1-8)  # these can be used in filenames
   AHOUR=$(echo $ATIME | cut -c 9-10)
   if [ -n "$startsigma" ]   # initial conditions are sigma files in RESULTDIR
   then
      echo "$0: using initial sigma files for $ATIME"
      startsigma=$(expand $startsigma)
      startphys=""
   elif [ -n "$startphys" ]  # initial conditions are physical grid
   then
      echo "$0: using initial physical perturbation grids for $ATIME"
      startphys=$(expand $startphys)
   else
      echo $0: one of startsigma or startphys must be \
         specified in config file 1>&2
      exit 1
   fi

   if [ "$RUN_FORECAST_FIRST" = true ]
   then
      echo $0: skipping initial assimilation step
      if [ -n "$startphys" ]
      then
         if ! $scriptdir/beo_assim -post $beo_v -C $CONFIG $ATIME
         then
            echo $0: error converting physical grids to sigma files
            exit 1
         fi
      fi
   else
      echo $0: $ATIME: assimilating obs into starting grids
      if ! $scriptdir/beo_assim $beo_v -C $CONFIG $ATIME
      then
         echo $0: error running data assimilation
         exit 1
      fi
   fi
fi

# Main loop: run a forecast, then assimilate observations

while [ $ATIME -lt $endtime ]
do
   echo $ATIME: running ${fcsttime}-hr forecasts
   if ! $scriptdir/forecast.$MODEL -C $CONFIG $ATIME
   then
      echo $0: error running forecasts
      exit 1
   fi
   ATIME=$($scriptdir/ndate $fcsttime $ATIME)
   echo $0: $ATIME: assimilating data
   beo_assim $beo_v -C $CONFIG $ATIME
   errcode=$?
   if [ $errcode -ne 0 ]
   then
      echo $0: data assimilation failed at $(date)
      exit $errcode
   fi
done
echo "$0: simulation ended at" $(date)
exit 0
