From 7871d68bc388470cc4081cdac1fd591b56f27784 Mon Sep 17 00:00:00 2001
From: Valentin Genin <valentin.genin@cnes.fr>
Date: Mon, 16 Mar 2020 08:43:25 +0000
Subject: [PATCH] Add new schema json for SAR_MultiSlc_IW.py
---
json_schemas/schema_MultiSlc_IW.json | 107 +++++++++++++++++++
python_src/SAR_MultiSlc.py | 2 +-
python_src/SAR_MultiSlc_IW.py | 148 ++++++++++++---------------
python_src/utils/func_utils.py | 12 +--
4 files changed, 181 insertions(+), 88 deletions(-)
create mode 100644 json_schemas/schema_MultiSlc_IW.json
diff --git a/json_schemas/schema_MultiSlc_IW.json b/json_schemas/schema_MultiSlc_IW.json
new file mode 100644
index 0000000..04fdd7d
--- /dev/null
+++ b/json_schemas/schema_MultiSlc_IW.json
@@ -0,0 +1,107 @@
+{
+ "$schema": "http://json-schema.org/schema#",
+ "title": "JSON SCHEMA for MultiSlc",
+ "description": "JSON organization for the script SAR_MultiSlc_IW.py",
+
+ "type": "object",
+
+ "allOf": [{"required": ["Global", "Pre_Processing", "DIn_SAR"]}],
+
+ "properties":
+ {
+ "Global":
+ {
+ "type": "object",
+ "properties":
+ {
+ "in":
+ {
+ "type": "object",
+ "required": ["SRTM_Shapefile", "SRTM_Path", "Geoid", "Master_Image", "Start_Date",
+ "End_Date", "Input_Path"],
+ "additionalProperties": false,
+ "properties": {"SRTM_Shapefile": {"type": "string"},
+ "SRTM_Path": {"type": "string"},
+ "Geoid": {"type": "string"},
+ "Master_Image": {"type": "string"},
+ "Start_Date": {"type": "string"},
+ "End_Date": {"type": "string"},
+ "Exclude": {"type": "string", "default": "-9999"},
+ "Input_Path": {"type": "string"}}
+ },
+ "out":
+ {
+ "type": "object",
+ "required": ["Output_Path"],
+ "additionalProperties": false,
+ "properties": {"Output_Path": {"type": "string"}}
+ },
+ "parameter":
+ {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {"clean": {"type": "boolean", "default": true},
+ "burst_index": {"type": "string", "default": "0-8"},
+ "optram": {"type": "number", "default": 4000}}
+ }
+ },
+ "additionalProperties": false,
+ "required": ["in", "out"]
+ },
+
+ "Pre_Processing":
+ {
+ "type": "object",
+ "properties":
+ {
+ "out":
+ {
+ "type": "object",
+ "required": ["doppler_file"],
+ "additionalProperties": false,
+ "properties": {"doppler_file": {"type": "string"}}
+ },
+ "parameter":
+ {
+ "type": "object",
+ "required": ["ML_gain"],
+ "additionalProperties": false,
+ "properties": {"ML_ran": {"type": "number", "default" : 3},
+ "ML_azi": {"type": "number", "default" : 3},
+ "ML_gain": {"type": "number"}}
+
+ }
+ },
+ "additionalProperties": false,
+ "required": ["out", "parameter"]
+ },
+
+ "DIn_SAR":
+ {
+ "type": "object",
+ "properties":
+ {
+ "parameter":
+ {
+ "type": "object",
+ "required": ["GridStep_range", "GridStep_azimut", "Grid_Threshold", "Grid_Gap",
+ "Interferogram_gain"],
+ "additionalProperties": false,
+ "properties": {"GridStep_range": {"type": "number"},
+ "GridStep_azimut": {"type": "number"},
+ "Grid_Threshold": {"type": "number"},
+ "Grid_Gap": {"type": "number"},
+ "Interferogram_gain": {"type": "number"},
+ "Activate_Interferogram": {"type": "string", "default": "yes"},
+ "Activate_Ortho": {"type": "string"},
+ "Spacingxy": {"type": "number", "default": 0.0001},
+ "roi": {"type": "string"},
+ "ESD_iter" : {"type": ["integer","string"]}
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": ["parameter"]
+ }
+ }
+}
diff --git a/python_src/SAR_MultiSlc.py b/python_src/SAR_MultiSlc.py
index a428d41..aeabfc0 100644
--- a/python_src/SAR_MultiSlc.py
+++ b/python_src/SAR_MultiSlc.py
@@ -114,7 +114,7 @@ if __name__ == "__main__":
ram = 4000
if 'parameter' in dict_Global:
if 'optram' in dict_Global['parameter']:
- ram = dict_Global['parameter']['ram']
+ ram = dict_Global['parameter']['optram']
if 'clean' in dict_Global['parameter']:
light_version = dict_Global['parameter']['clean']
diff --git a/python_src/SAR_MultiSlc_IW.py b/python_src/SAR_MultiSlc_IW.py
index 9fc4e2e..50ca997 100644
--- a/python_src/SAR_MultiSlc_IW.py
+++ b/python_src/SAR_MultiSlc_IW.py
@@ -69,84 +69,14 @@ if __name__ == "__main__":
parser.add_argument("configfile",
help="""input conguration file for the
application DiapOTB""")
- parser.add_argument("-d", "--start_date", dest="start_date",
- action="store", help="start date, fmt('20151222')",
- default=None)
- parser.add_argument("-f", "--end_date", dest="end_date", action="store",
- help="end date, fmt('20151223')", default=None)
- parser.add_argument("-m", "--master", dest="master", action="store",
- help="master image, fmt (.tiff or .h5)", default=None)
- parser.add_argument("-e", "--exclude", dest="exclude", nargs="+",
- action="store", help=""""excluded date(s) from the time
- serie, exemple('20151221 20160114')""", default="-9999")
- parser.add_argument("-b", "--burst", dest="burst", action="store",
- help="Select the bursts you want to work on, fmt(0-4)",
- default="0-8")
- parser.add_argument("-roi", "--roi", dest="roi", nargs="+", action="store",
- help=""""Define the lng lat ROI coordinates,
- fmt('ulx uly lrx lry'), ex: -roi 2.44115 48.96126
- 2.44176 48.95927""", default=None)
- parser.add_argument("-i", "--interferogram", dest="interferogram",
- action="store", help=""" Simply write ' -i no ' to
- deactivate interferogram output. Activated by default.
- Returns: Interferogram.tif""", default="yes")
- parser.add_argument("-o", "--ortho", dest="ortho",
- action="store", help=""" Simply write ' -o yes ' to
- activate Orthorectified interferogram output. If activated,
- returns: Ortho_Interferogram.tif""", default=None)
- parser.add_argument("-l", "--light", dest="light", action="store",
- help="""" -l no' to deactivate. If activated, returns a
- light version: Interferogram.tif + Coregistrated.tif +
- multilook.tif""", default="yes")
- parser.add_argument("-ram", "--optram", dest="optram",
- action="store", help="""Available RAM (mb),
- by default value is 4OOO""",
- default="4000")
- parser.add_argument("-spacingxy", "--spacingxy", dest="spacingxy",
- action="store", help="""Set the spatial resolution
- for OrthoRectification in degrees,
- Default value is 0.0001""",
- default="0.0001")
- parser.add_argument("-ml", "--multilook", dest="multi", nargs="+", action="store",
- help=""""Set the range and azimuth (in this order) you want
- for the multilook. Default is 3 3""", default='3' '3')
args = parser.parse_args()
print(args.configfile)
func_utils.init_logger()
- dataConfig = func_utils.load_configfile(args.configfile, "S1_IW")
+ dataConfig = func_utils.load_configfile(args.configfile, "multi_SW")
- # =================================
- # Get elements from user's arguments
- # =================================
-
- # ====== Set the variables names
- iso_start, iso_end = func_utils.argDates_to_isoDates(args.start_date, args.end_date)
- start_time = int(args.start_date)
- end_time = int(args.end_date)
- master_Image_base = args.master
- master_Image = func_utils.get_imgFromSAFE(args.master)
- master_date = master_Image_base.split("-")[4].split("t")[0]
- pol = master_Image_base.split("-")[3]
- iw = master_Image_base.split("-")[1]
- burst_index = args.burst
- print(burst_index)###############################################################################
- exclude = args.exclude
- version_interferogram = args.interferogram
- light_version = args.light
- manifest = master_Image.split("measurement")[0]+"/manifest.safe"
- relative_orbit = func_utils.get_relative_orbit(manifest)
- ortho_interferogram = args.ortho
- roi = args.roi
- if roi:
- ortho_interferogram = "yes"
- print("ortho_interferogram", ortho_interferogram)
- ram = args.optram
- spacingxy = args.spacingxy
- rng,azi = args.multi
-
# ====================================
# Get elements from configuration file
# ====================================
@@ -156,15 +86,54 @@ if __name__ == "__main__":
dict_PreProcessing = dataConfig['Pre_Processing']
dict_Ground = dataConfig['Ground']
dict_DInSAR = dataConfig['DIn_SAR']
-
# ====== Get elements from dictionaries
# ====== Global
- srtm_shapefile = dict_Global['in']['SRTMShapefile']
- hgts_path = dict_Global['in']['Datalake']
+ srtm_shapefile = dict_Global['in']['SRTM_Shapefile']
+ hgts_path = dict_Global['in']['SRTM_Path']
geoid_path = dict_Global['in']['Geoid']
+ output_dir = dict_Global['out']['Output_Path']
+
+ if not os.path.exists(output_dir):
+ print("The output directory does not exist and will be created")
+ os.makedirs(output_dir)
+ else :
+ print("The output directory exists. Some files can be overwritten")
+
+ # ====== Set the variables names
+ iso_start, iso_end = func_utils.argDates_to_isoDates(dict_Global['in']['Start_Date'], dict_Global['in']['End_Date'])
+ start_time = int(dict_Global['in']['Start_Date'])
+ end_time = int(dict_Global['in']['End_Date'])
+ master_Image_base = dict_Global['in']['Master_Image']
+ master_Image = func_utils.get_imgFromSAFE(dict_Global['in']['Master_Image'], dict_Global['in']['Input_Path'])
+ master_date = master_Image_base.split("-")[4].split("t")[0]
+ pol = master_Image_base.split("-")[3]
+ iw = master_Image_base.split("-")[1]
+ burst_index = "0-8"
+ ram = 4000
+ light_version = True
+ if 'parameter' in dict_Global:
+ if 'optram' in dict_Global['parameter']:
+ ram = dict_Global['parameter']['optram']
+ if 'clean' in dict_Global['parameter']:
+ light_version = dict_Global['parameter']['clean']
+ if 'burst_index' in dict_Global['parameter']:
+ burst_index = dict_Global['parameter']['burst_index']
+ print(burst_index)###############################################################################
+ exclude = "-9999"
+ if 'Exclude' in dict_Global['in']:
+ exclude = dict_Global['in']['Exclude']
+ manifest = master_Image.split("measurement")[0]+"/manifest.safe"
+ relative_orbit = func_utils.get_relative_orbit(manifest)
+
# ====== Pre_Processing
+ rng = 3
+ azi = 3
+ if "ML_ran" in dict_PreProcessing['parameter']:
+ rng = dict_PreProcessing['parameter'].get('ML_ran')
+ if "ML_azi" in dict_PreProcessing['parameter']:
+ azi = dict_PreProcessing['parameter'].get('ML_azi')
ml_range = int(rng)
ml_azimut = int(azi)
ml_gain = dict_PreProcessing['parameter']['ML_gain']
@@ -173,6 +142,21 @@ if __name__ == "__main__":
# ====== Ground
# ====== DIn_SAR
+ version_interferogram = "yes"
+ if "Activate_Interferogram" in dict_DInSAR['parameter']:
+ version_interferogram = dict_DInSAR['parameter']['Activate_Interferogram']
+ ortho_interferogram = None
+ if 'Activate_Ortho' in dict_DInSAR['parameter']:
+ ortho_interferogram = dict_DInSAR['parameter']['Activate_Ortho']
+ roi = None
+ if 'roi' in dict_DInSAR['parameter']:
+ roi = dict_DInSAR['parameter']['roi']
+ if roi:
+ ortho_interferogram = "yes"
+ print("ortho_interferogram", ortho_interferogram)
+ spacingxy = 0.0001
+ if "Spacingxy" in dict_DInSAR['parameter']:
+ spacingxy = dict_DInSAR['parameter']['Spacingxy']
geoGrid_gridstep_range = dict_DInSAR['parameter']['GridStep_range']
geoGrid_gridstep_azimut = dict_DInSAR['parameter']['GridStep_azimut']
geoGrid_threshold = dict_DInSAR['parameter']['Grid_Threshold']
@@ -180,6 +164,10 @@ if __name__ == "__main__":
ml_geoGrid_range = int(rng)
ml_geoGrid_azimut = int(azi)
gain_interfero = dict_DInSAR['parameter']['Interferogram_gain']
+ esd_NbIter = 0
+ esd_AutoMode = "none"
+ if 'ESD_iter' in dict_DInSAR['parameter']:
+ esd_NbIter, esd_AutoMode = dict_DInSAR['parameter']['ESD_iter']
# ====== Check Threshold
if (geoGrid_threshold < 0) or (geoGrid_threshold > 1) :
@@ -196,8 +184,8 @@ if __name__ == "__main__":
print("from check regex",firstBurst, lastBurst)###########################################################
# ====== Create global folder with starting and ending dates + master date
- output_glob = "./output_{}_to_{}_m_{}".format(start_time, end_time,
- master_date)
+ output_glob = "{}/output_{}_to_{}_m_{}".format(output_dir, start_time,
+ end_time, master_date)
if os.path.exists(output_glob):
shutil.rmtree(output_glob)
if not os.path.exists(output_glob):
@@ -243,14 +231,14 @@ if __name__ == "__main__":
# Get the elements from os.dirs
# ============================
- tiff_list = func_utils.get_AllTiff(pol, iw)
+ tiff_list = func_utils.get_AllTiff(pol, iw, searchDir=dict_Global["in"]["Input_Path"])
tiff_dates = func_utils.get_Tiff_WithDates(start_time, end_time, exclude, tiff_list)
counter = 0
for i in (i for i in tiff_dates if i != master_Image_base):
total_slaves = len(tiff_dates)-1
slave_Image_base = i
- slave_Image = func_utils.get_imgFromSAFE(slave_Image_base)
+ slave_Image = func_utils.get_imgFromSAFE(slave_Image_base, searchDir=dict_Global["in"]["Input_Path"])
slave_date = func_utils.get_Date(i)
counter += 1
output_dir = output_glob + "/{}_m_{}_s".format(master_date, slave_date)
@@ -447,8 +435,8 @@ if __name__ == "__main__":
param['doppler0'] = dop0Slave
param['gain_interfero'] = gain_interfero
param['advantage'] = "projection" # By default projection
- param['esd_NbIter'] = 0
- param['esd_AutoMode'] = "none"
+ param['esd_NbIter'] = esd_NbIter
+ param['esd_AutoMode'] = esd_AutoMode
param['with_interferogram'] = version_interferogram
param['with_concatenate'] = "no"
diff --git a/python_src/utils/func_utils.py b/python_src/utils/func_utils.py
index 75859f8..4792c38 100644
--- a/python_src/utils/func_utils.py
+++ b/python_src/utils/func_utils.py
@@ -158,10 +158,12 @@ def load_configfile(configfile, mode="Others"):
schema_json = "schema_S1IW.json"
elif mode == "multi_S1":
schema_json = "schema_MultiSlc.json"
+ elif mode == "multi_SW":
+ schema_json = "schema_MultiSlc_IW.json"
# Load schema (into DiapOTB install)
diapOTB_install = os.getenv('DIAPOTB_INSTALL')
- print(diapOTB_install)
+
if diapOTB_install is not None and os.path.exists(diapOTB_install):
schemas_path = os.path.join(diapOTB_install, "json_schemas")
if os.path.exists(schemas_path):
@@ -231,9 +233,7 @@ def get_imgFromSAFE(arg, searchDir="."):
img = []
for root, dirs, files in os.walk(searchDir):
for i in (i for i in files if i == arg):
- # img.append(os.path.join(os.getcwd()+root.split(".")[1] +
- # ".SAFE/measurement/"+(i)))
- img.append(os.path.join(root+"/measurement/", i))
+ img.append(os.path.join(root, i))
img = str("".join(img))
return img
@@ -244,8 +244,6 @@ def get_imgFromDir(arg, searchDir="."):
img = []
for root, dirs, files in os.walk(searchDir):
for i in (i for i in files if i == arg):
- # img.append(os.path.join(os.getcwd()+root.split(".")[1] +
- # "/"+(i)))
img.append(os.path.join(root, i))
img = str("".join(img))
return img
@@ -315,7 +313,7 @@ def get_AllTiff(pol, iw="", ext="", searchDir="."):
for i in (i for i in files): # if i != inTIF):
if i.endswith(".tiff") and pol == i.split("-")[3]:
TiffList.append(i)
- print(TiffList)
+
return TiffList
def get_AllFilesWithExt(searchDir, ext) :
--
GitLab