sentinel2_pipeline/bin/s2_pipeline

#!/bin/bash

set -e

aoi=''
algorithm=''
startdate='NOW-5DAYS'
enddate='NOW'
cloud='20'
outdir="$PWD"
cleanup=''
export S2_USER
export S2_PASS

# handle flags
while test $# -gt 0; do
  case "$1" in
    -h|--help)
      echo "s2_pipeline - sentintel 2 change detection processing pipeline"
      echo " "
      echo "s2_pipeline [options]"
      echo " "
      echo "options:"
      echo "-h, --help                show brief help."
      echo "-a, --aoi                 area of interest as geojson polygon geometry json file. Required."
      echo "--algorithm               select single algorithm. Possible is 'dnbr' or 'bais2'. Defaults to both."
      echo "-s, --start               start date 'YYYYMMDD' or 'NOW-XDAYS'. Default 'NOW-5DAYS'."
      echo "-e, --end                 end date 'YYYYMMDD' or 'NOW-XDAYS'. Default 'NOW'."
      echo "-o, --outdir              path to output directory. Defaults to directory where the script was executed."
      echo "-c, --cloudcoverage       maximum percentage of allowed cloud coverage. Default '20'."
      echo "    --cleanup             remove intermediate results after processing"
      exit 0
      ;;
    -a|--aoi)
      shift
      aoi=$1
      aoistring=$(cat $aoi)
      shift
      ;;
    --algorithm)
      shift
      algorithm=$1
      shift
      ;;
    -s|--start)
      shift
      startdate=$1
      shift
      ;;
    -e|--end)
      shift
      enddate=$1
      shift
      ;;
    -o|--outdir)
      shift
      outdir=$1
      shift
      ;;
    -c|--cloudcoverage)
      shift
      cloud=$1
      shift
      ;;
    --cleanup)
      cleanup=true
      shift
      ;;
    *)
      break
      ;;
  esac
done

if [ -z $aoi ]; then
  echo "flag --aoi is required!"
  exit 1
fi

function parallelism {
  # checks the available memory, and returns the number of instances that can be
  # launched in parallel for a given amount of required RAM (maximum: no. cpu cores)
  # only works on linux!
  mb_required=$1
  mem_instances=`awk '/MemAvailable/ { print int($2 / 1024 / '$mb_required') }' /proc/meminfo`
  cpu_instances=`< /proc/cpuinfo awk '/cpu cores/ { print $4 }' | uniq`
  echo -e "$cpu_instances\n$mem_instances" | sort -n | head -n1 # minimum of cores & ram
}

function filesexist {
  if [ ! -z "$(find $outdir/data/ -maxdepth 1 -name $1)" ]; then
    return 0
  else
    return 1
  fi
}

sen2cor_instances=`parallelism 2900`

if [ "$sen2cor_instances" -eq "0" ]; then
  echo "not enough memory to do sen2cor preprocessing!";
  exit 1;
fi

SCRIPTDIR=${BASH_SOURCE%/*}
outdir=${outdir%/}
mkdir -p $outdir/data

# using bash pipes and xargs gives us smart parallelism, all without Apache Storm, Spark,
# Hadoop or whatever! (who needs replayability, just start the pipe again, completed work is cached)

###########################
#
# Download & Preprocessing
#
###########################
echo -e "====> pipeline 1/4: download & preprocessing\n"
# first pipeline:
# - download two in parallel (API limitation)
# - filter L1C tiles to apply correction to
# - apply preprocessing (athmospheric correction with sen2cor) to L1C tiles

$SCRIPTDIR/s2_query --aoi "$aoistring" --startdate "$startdate" --enddate "$enddate" --cloud "$cloud" --processed-dir "$outdir/data" |
 xargs -r -L1 -P2 $SCRIPTDIR/s2_download $outdir/data |
 grep 'MSIL1C' | # only L1C products need to be processed. TODO: also skip for already existing L2A
 xargs -r -L1 -P$sen2cor_instances $SCRIPTDIR/s2_preprocess

# wait for all previous steps to complete, as we need /all/ images for merging, then start second pipeline:

###########################
#
# Orbit tile stitching and clipping
#
###########################
echo -e "\n====> pipeline 2/4: orbit tile stitching and clipping\n"
# second pipeline:
# - combine bands of each tile into .vrt files
# - join/stitch images of the same orbit
# - clip to AoI

tiffs=`$SCRIPTDIR/s2_grouporbit --input $outdir/data/ |
  xargs -r -L1 -P2 $SCRIPTDIR/s2_clip --aoi "$aoi" --outputdir "$outdir/data" --input`

# wait for completion again, as we need to compare two adjacent images each.

###########################
#
# Change detection
#
###########################
echo -e "\n====> pipeline 3/4: change detection\n"
# calling change detection script(s)
# add --algorithm flag if it was defined

if [ ! -z $algorithm ]
  then
    $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" --process $algorithm
  else
    $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data"
fi

###########################
#
# Web Visualization
#
###########################
echo -e "\n====> pipeline 4/4: web visualization\n"
# create web visualization at last
# always checks if files with matching suffix exist
htmls=''
if filesexist '*_bais2.tif'; then
  $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_bais2\.tif$" --outfile "viz_bais2.html" &
  htmls="${htmls}bais2 "
fi

if filesexist '*_dnbr.tif'; then
  $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_dnbr\.tif$" --outfile "viz_dnbr.html" &
  htmls="${htmls}dnbr "
fi

if filesexist '*_l2a.tif'; then
  $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_l2a\.tif$" --outfile "viz_l2a.html"
  htmls="${htmls}l2a "
fi

if [ ! -z "$htmls" ]; then
  $SCRIPTDIR/s2_visindex "$outdir/data" --htmls $htmls
fi

wait # until s2_visualize background processes finished

###########################
#
# Cleanup
#
###########################
# clean up intermediate files
if [ ! -z $cleanup ]; then
  echo -e "\n====> cleanup of intermediate files"
  rm -rf $outdir/data/{*.SAFE,*.zip}
fi

echo -e "\n====> done!"