#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Main driver for the CSPP VIIRS Flood package Copyright (C) 2017-2018 University of Wisconsin Space Science and Engineering Center This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . """ import os import sys from tempfile import mkdtemp import re import subprocess import shutil import argparse import glob import h5py import numpy as np from nagg_wrapper import nagg import logging LOG = logging.getLogger(__name__) """ # 1. RUN SWATH PROJECTION # The parameters to run VIIRS Swath Projection include: # -i: [Necessary], VIIRS sdr data path, for example, /home/VIIRS/sdr/ # -a: [Necessary], path of users’ AOI definition file path, for example, /home/ # VIIRS /prj/. If -n is set as 0, it is an optional parameter. Otherwise, it is a necessary parameter. # -u: filename of users’ AOI definition file, for example, User_AOI_Definition.txt. If -n is set as 0, it is an optional parameter. Otherwise, it is a necessary parameter. # -g: [Necessary], VIIRS SDR375-m geo-location filename, for example, GITCO_npp_d20130525_t1900226_e1901468_b08163_c20130526011532558954_noaa_ops.h5 # 10 # -o: [Necessary], file path for projected SDR data, for example, /home/VIIRS/prj/ # -l: [Necessary], log file path, for example, /home/VIIRS/logfile/ # -n: [Necessary], switch of projection range between projecting whole granule and projecting users’ subsets: 0 is to project the whole granule, and 1 is to project subsets according to users’ AOIs which are listed in users’ AOI definition txt file. # To run the executive module: VIIRS_Swath_Projection in subsets with VIIRS SDR data: GITCO_npp_d20130525_t1900226_e1901468_b08163_c20130526011532558954_noaa_ops.h5, the test script is written as: # ./ VIIRS_Swath_Projection -i /home/VIIRS/sdr/ -a /home/ VIIRS /prj/ -u User_AOI_Definition.txt -g GITCO_npp_d20130525_t1900226_e1901468_b08163_c20130526011532558954_no aa_ops.h5 -o /home/VIIRS/prj/ -n 1 # To run the executive module: VIIRS_Swath_Projection in whole granule, the test script is written as: # ./ VIIRS_Swath_Projection -i /home/VIIRS/sdr/ -a /home/ VIIRS /prj/ -u User_AOI_Definition.txt -g GITCO_npp_d20130525_t1900226_e1901468_b08163_c20130526011532558954_no aa_ops.h5 -o /home/VIIRS/prj/ -n 0 # Or: # ./ VIIRS_Swath_Projection -i /home/VIIRS/sdr/ -g GITCO_npp_d20130525_t1900226_e1901468_b08163_c20130526011532558954_no aa_ops.h5 -o /home/VIIRS/prj/ -n 0 export CSPP_FLOOD_HOME=/workspace/cspp-flood-dev/ export PATH=${CSPP_FLOOD_HOME}/bin:${CSPP_FLOOD_HOME}/gdl:${PATH} export GDL_PATH=${CSPP_FLOOD_HOME}/gdl:${GDL_PATH}:${PATH} export SHELL=/bin/bash #${CSPP_FLOOD_HOME}/bin/VIIRS_Swath_Projection \ ${CSPP_FLOOD_HOME}/bin/CSPP_VIIRS_Swath_Projection \ -i ${INPUT_DIR}/ \ -g GITCO_npp_d20170326_t1950501_e1952143_b28042_c20170326200446192609_cspp_dev.h5 \ -o ${PRJ_DIR}/ \ -l ${LOG_DIR}/ \ -n 0 """ def run_swath_projection(sdr_path, gitco_filename, prj_dir, log_dir, aoi_definition_file=None): cmd = ["VIIRS_Swath_Projection", "-i", sdr_path, "-g", gitco_filename, "-o", prj_dir, "-l", log_dir,] if aoi_definition_file is not None: aoi_definition_file = os.path.abspath(aoi_definition_file) # to avoid aoi_dir being null aoi_dir, aoi_filename = os.path.split(aoi_definition_file) cmd.append('-a') cmd.append(aoi_dir + "/") # requires a trailing slash or the swath projection binary segfaults cmd.append('-u') cmd.append(aoi_filename) cmd.append('-n') cmd.append('1') else: cmd.append('-n') cmd.append('0') LOG.debug(" ".join(cmd)) console_log = open('swath_projection.log', 'w') subprocess.call(cmd, stdout=console_log, stderr=console_log) console_log.close() return """ # 2. RUN FLOOD DETECTION # There are five parameters (each one with an index name such as -h): -h: [Necessary], VIIRS projected hdf5 file path, for example, # /home/VIIRS/prj # 11 # -a: [Necessary], ancillary data file path, for example, /home/VIIRS/assdata # -v: [Necessary], VIIRS sdr projected 375-m geo-location filename, for example, # GITCO_Prj_SVI_npp_d20130525_t2105329_e2106571_b08165_cspp_dev.h5 -c: [Optional], VIIRS projected cloud mask file path, for example, # /home/VIIRS/prj # -l: [Necessary], log file path, for example, /home/VIIRS/logfile # To run the executive module: VIIRS_Flood_Detection with projected data: GITCO_Prj_SVI_npp_d20130525_t2105329_e2106571_b08165_cspp_dev.h, the test script is written as: # ./ VIIRS_Flood_Detection -h /home/VIIRS/prj -a /home/VIIRS/assdata -v GITCO_Prj_SVI_npp_d20130525_t2105329_e2106571_b08165_cspp_dev.h5 -c /home/VIIRS/prj -l /home/VIIRS/logfile # FIXME: THESE CAN'T BE HARDCODED PRJ_FILENAME=GITCO_Prj_SVI_npp_d20170326_t1950501_e1952143_b28042_cspp_dev.h5 CMASK_FILENAME=GMTCO_IICMO_Prj_SVI_npp_d20170326_t1950501_e1952143_b28042_cspp_dev.h5 ASS_DIR=${CSPP_FLOOD_HOME}/assdata ${CSPP_FLOOD_HOME}/bin/VIIRS_Flood_Detection_Ver01 \ -h ${PRJ_DIR} \ -a ${ASS_DIR} \ -v ${PRJ_FILENAME} \ -c ${CMASK_FILENAME} \ # FIXME: THIS SHOULD'VE BEEN PATH -l ${LOG_DIR} """ def run_flood_detection(anc_dir, prj_dir, gitco_filename, log_dir): cmd = ["VIIRS_Flood_Detection", "-h", prj_dir, "-v", gitco_filename, "-a", anc_dir, # "-c", prj_dir, "-o", ".", "-l", log_dir,] LOG.debug(" ".join(cmd)) console_log = open('flood_detection.log', 'w') subprocess.call(cmd, stdout=console_log, stderr=console_log) console_log.close() return def is_day(gitco_filename, sz_threshold=85): """ Simple check to see whether it's a daytime or nighttime swath. """ gitco = h5py.File(gitco_filename, 'r') sz = gitco['All_Data/VIIRS-IMG-GEO-TC_All/SCSolarZenithAngle'][:] nday = ((sz < sz_threshold) & (sz > -1)).sum() LOG.debug("is_day sz check: nday == %d, if > 0 we continue" % nday) return nday > 0 def is_polar(gitco_filename, lat_threshold=85): gitco = h5py.File(gitco_filename, 'r') lat = gitco['All_Data/VIIRS-IMG-GEO-TC_All/Latitude'][:] abslat = np.absolute(lat) polars = np.where( np.logical_and(abslat <= 90, abslat >= lat_threshold) ) return len(polars[0]) > 0 def _setup_argparse(): parser = argparse.ArgumentParser( description="CSPP VIIRS Flood Detection software.") # Required arguments: parser.add_argument('input_dir', help='A directory with a single pass of VIIRS SDR data. Should include GITCO, SVI, and optionally IICMO/GMTCO.') # Optional arguments: parser.add_argument('-o', '--output', dest='output_file', metavar='FILE', action='store', default=None, help="write output to FILE instead of the default.") parser.add_argument('-a', '--aoi', dest='aoi_file', action="store", default=None, help="Provide an AOI definition file for subsetting.") parser.add_argument('-d', '--debug', dest='debug', action="store_true", default=False, help="Enable debug mode, which leaves the working directory around after we're done.") parser.add_argument('-v', '--verbose', dest='verbosity', action="count", default=0, help='Each occurrence increases verbosity through ERROR, WARNING, INFO, DEBUG.') return parser def main(): # Get input args parser = _setup_argparse() args = parser.parse_args() # Setup logging levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] verbosity = min(args.verbosity, len(levels) - 1) logging.basicConfig(level=verbosity) # it's import this happens before we change directories if args.output_file: output_file = os.path.abspath(args.output_file) # create & move to working dir input_dir = os.path.abspath(args.input_dir) # don't use args.input_dir after this, in case it's relative start_dir = os.getcwd() tmp_dir = prj_dir = log_dir = mkdtemp(prefix='cspp-floods-') + "/" os.chdir(tmp_dir) # aggregate all the datasets we care about from input_dir LOG.info("input_dir: %s" % (input_dir)) NAGG_DATASETS = ['GITCO', 'GMTCO', 'IICMO', 'SVI01', 'SVI02', 'SVI03', 'SVI05'] REQUIRED_DATASETS = ['GITCO', 'SVI01', 'SVI02', 'SVI03', 'SVI05'] polar_granules = 0 for gitco_file in glob.glob(os.path.join(input_dir, 'GITCO*.h5')): if is_polar(gitco_file): LOG.error("{} crosses a pole and cannot be processed".format(gitco_file)) polar_granules = polar_granules + 1 if polar_granules > 0: raise RuntimeError("Cannot process due to polar granules in the input, see previous error messages for details") nagg(input_dir, NAGG_DATASETS, REQUIRED_DATASETS) gitco_file = glob.glob('GITCO*.h5')[0] # FIXME: dangerous gitco_path, gitco_file = os.path.split(os.path.abspath(gitco_file)) # FIXME: is there a better way to do this? gitco_path = os.path.abspath(gitco_path) + "/" if not is_day(os.path.join(gitco_path, gitco_file)): LOG.error(gitco_path + "is a nighttime granule. VIIRS flood detection requires a daytime scene.") exit(1) anc_dir = os.getenv("CSPP_FLOOD_ANCILLARY") LOG.info("input: %s %s" % (gitco_path, gitco_file)) # FIXME: not a fan of splitting this up, but currently uses relative paths on one end and abs on the other run_swath_projection(gitco_path, gitco_file, prj_dir, log_dir, args.aoi_file) # making prj_gitco_file name # GITCO_npp_d20170326_t1950501_e1952143_b28042_c20170326200446192609_cspp_dev.h5 # becomes # GITCO_Prj_SVI_npp_d20170326_t1950501_e1952143_b28042_cspp_dev.h5 # or # GITCO_Prj_SVI_npp_d20170326_t1950501_e1952143_b28042_cspp_dev_001.h5 # GITCO_Prj_SVI_npp_d20170326_t1950501_e1952143_b28042_cspp_dev_002.h5 # if an AOI file is used # prj_gitco_file = gitco_file.replace('GITCO_npp', 'GITCO_Prj_SVI_npp') # prj_gitco_file = re.sub('_c\d*_cspp', '_cspp', prj_gitco_file) # # if not os.path.exists(prj_gitco_file): # LOG.error("Could not find expected swath projection output: %s" % (prj_gitco_file)) # raise RuntimeError("Swath Projection failed. See %s for more details." % (os.path.join(log_dir, "VIIRS_Swath_Projection_log.txt"))) prj_gitco_files = glob.glob('GITCO_Prj_SVI_*.h5') if len(prj_gitco_files) == 0: LOG.error("Could not find expected swath projection output.") raise RuntimeError("Swath Projection failed. See %s for more details." % (os.path.join(log_dir, "VIIRS_Swath_Projection_log.txt"))) for n_prj_gitco_file, prj_gitco_file in enumerate(prj_gitco_files): LOG.debug(os.path.join(prj_dir, prj_gitco_file)) run_flood_detection(anc_dir, prj_dir, prj_gitco_file, log_dir) # grab the water file and move it into place as output # GITCO_Prj_SVI_npp_d20170326_t1950501_e1952143_b28042_cspp_dev.h5 water_glob = prj_gitco_file.replace('GITCO_', 'WATER_VIIRS_') # water_glob = prj_gitco_file.replace('GMTCO_IICMO_', 'WATER_VIIRS_') water_glob = water_glob.replace('.h5', '*.hdf') # need to remove .h5 and glob the rest to get the water file; it's hdf4 water_files = glob.glob(water_glob) if len(water_files) != 1: LOG.error("Found %d flood detection output files." % (len(water_files))) LOG.error("While looking for: %s" % water_glob) # raise RuntimeError("Flood Detection failed. See %s for more details." % (os.path.join(log_dir, "VIIRS_Flood_Detection_Log.txt"))) water_file = water_files[0] if args.output_file: # don't use args.output_file below here; directories have changed since initial invocation if args.aoi_file: # we want this to be (water_file)_001.hdf out_base, out_ext = os.path.splitext(output_file) dest_file = out_base + "_%03d" % (n_prj_gitco_file) + out_ext # don't assign to output_file or we mess up the next loop else: dest_file = output_file else: dest_file = os.path.join(start_dir, water_file) LOG.debug("Moving %s -> %s" % (water_file, dest_file)) shutil.move(water_file, dest_file) # delete tmp dir if not args.debug: shutil.rmtree(tmp_dir) if __name__ == "__main__": main()