Commit a90ed4bf authored by Germain Salgues's avatar Germain Salgues

ENH: improve snow_annual_map.py and provide a hpc script

parent 2982b12a
#!/bin/bash
#PBS -N TheiaNeige
#PBS -j oe
#PBS -l select=1:ncpus=8:mem=20000mb
#PBS -l walltime=24:00:00
# run LIS for one Sentinel-2 Level-2A tile and one date (walltime is higher)
# specify the path to the tile folder, the path the DEM and the template configuration file (.json)
# First argument is the tile name (nnccc): qsub -v config="path/to/config/json" run_snow_annual_map.sh
if [ -z $config ]; then
echo "config is not set, exit"
exit
fi
echo $config
#config_list=`ls /work/OT/siaa/Theia/Neige/Snow_Annual_Maps/*/*.json`
#Load LIS modules
#module load lis/develop
source /home/qt/salguesg/load_lis.sh
#configure gdal_cachemax to speedup gdal polygonize and gdal rasterize (half of requested RAM)
export GDAL_CACHEMAX=2048
echo $GDAL_CACHEMAX
export PATH=/home/qt/salguesg/local/bin:/home/qt/salguesg/local/bin:$PATH
echo $PATH
# run the snow detection
date ; echo "START run_snow_annual_map.py $config"
run_snow_annual_map.py $config
date ; echo "END run_snow_annual_map.py"
......@@ -358,6 +358,8 @@ def get_app_output(app, out_key, mode="RUNTIME"):
- or a filename corresponding to the output written on the disk
"""
app_output = app.GetParameterString(out_key)
if "?" in app_output:
app_output = app_output.split("?")[0]
if mode == "RUNTIME":
app.Execute()
......
......@@ -113,12 +113,15 @@ class snow_annual_map():
self.output_dates_filename = params.get("output_dates_filename", None)
self.mode = params.get("mode", "RUNTIME")
self.processing_id = str(self.tile_id + "_" + \
datetime_to_str(self.date_start) + "_" + \
datetime_to_str(self.date_stop))
self.input_dir = str(op.join(params.get("input_dir"), self.tile_id))
self.path_tmp = str(params.get("path_tmp", os.environ.get('TMPDIR')))
self.path_out = op.join(str(params.get("path_out")),
self.tile_id + "_" + datetime_to_str(self.date_start)
+ "-" + datetime_to_str(self.date_stop))
#self.path_tmp = str(params.get("path_tmp", os.environ.get('TMPDIR')))
self.path_tmp = str(os.environ.get('TMPDIR'))
self.path_out = op.join(str(params.get("path_out")), self.processing_id)
if not os.path.exists(self.path_out):
os.mkdir(self.path_out)
......@@ -143,9 +146,9 @@ class snow_annual_map():
self.output_dates_filename = op.join(self.path_tmp, "output_dates.txt")
self.multitemp_snow_vrt = op.join(self.path_tmp, "multitemp_snow_mask.vrt")
self.multitemp_cloud_vrt = op.join(self.path_tmp, "multitemp_cloud_mask.vrt")
self.gapfilled_timeserie = op.join(self.path_tmp, "gap_filled_snow_mask.tif")
self.annual_snow_map = op.join(self.path_tmp, "gap_filled_snow_mask_annual_snow.tif")
self.cloud_occurence_img = op.join(self.path_tmp, "annual_cloud_occurence.tif")
self.gapfilled_timeserie = op.join(self.path_tmp, "DAILY_SNOW_MASKS_" + self.processing_id + ".tif")
self.annual_snow_map = op.join(self.path_tmp, "SNOW_OCCURENCE_" + self.processing_id + ".tif")
self.cloud_occurence_img = op.join(self.path_tmp, "CLOUD_OCCURENCE_" + self.processing_id +".tif")
def run(self):
logging.info("Run snow_annual_map")
......@@ -215,7 +218,7 @@ class snow_annual_map():
output_dates = read_list_from_file(self.output_dates_filename)
else:
tmp_date = self.date_start
while tmp_date < self.date_stop:
while tmp_date <= self.date_stop:
output_dates.append(datetime_to_str(tmp_date))
tmp_date += timedelta(days=1)
write_list_to_file(self.output_dates_filename, output_dates)
......@@ -303,7 +306,7 @@ class snow_annual_map():
logging.info("Copying outputs from tmp to output folder")
shutil.copy2(self.annual_snow_map, self.path_out)
logging.info("End snow_annual_map")
logging.info("End of snow_annual_map")
if self.mode == "DEBUG":
dest_debug_dir = op.join(self.path_out, "tmpdir")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment