Commit b40b65b1 authored by Manuel Grizonnet's avatar Manuel Grizonnet

Merge branch 'release/1.2'

parents e154be98 038f3910
......@@ -11,4 +11,5 @@ doc/tex/*aux
doc/tex/*log
doc/tex/*out
doc/tex/*toc
doc/tex/_minted*
# Change Log
All notable changes to LIS will be documented in this file.
## [1.2] - 2017-06-04
- add json schema to ATBD to document all parameters
- Add version of lis in atbd
- Document how to build the documentation in doc/tex directory
- Compact histogram files and copy it in LIS_PRODUCTS
- Apply autopep8 to all Python scripts to imprve code quality
- Add a changelog
## [1.1.1] - 2016-11-28
- minor update in build scripts
- change ctest launcher location
## [1.1.0] - 2016-11-28
- Change license from GPL to A-GPL
- Improvments in cmake configuration
- launch tests in separate directories
## [1.0.0] - 2016-07-06
- First released version of LIS with support with last MUSCATE format
- Support for image splitted in multiple files
- Use high clouds mask in cloud refine
- Prototype for multi-T synthesis
......@@ -11,32 +11,27 @@ To read more about the "Centre d'Expertise Scientifique surface enneigée" (in F
* [Bulletin THEIA](https://www.theia-land.fr/sites/default/files/imce/BulletinTHEIA3.pdf#page=10)
The input files are Sentinel-2 or Landsat-8 level-2A products from the [Theai Land Data Centre](https://theia.cnes.fr/) or [SPOT-4/5 Take 5 level-2A products](https://spot-take5.org) and the SRTM digital elevation model.
The input files are Sentinel-2 or Landsat-8 level-2A products from the [Theai Land Data Centre](https://theia.cnes.fr/) or [SPOT-4/5 Take 5 level-2A products](https://spot-take5.org) and the SRTM digital elevation model reprojected at the same resolution as the input image.
## Code Example
##Usage
To build DEM data download the SRTM files corresponding to the study area and build the .vrt using gdalbuildvrt. Edit config.json file to activate preprocessing : Set "preprocessing" to true and set the vrt path.
Run the python script run_snow_detector.py with a json configuration file as unique argument:
The snow detection is performed in the Python script app/run_snow_detector.py.
Configure PYTHONPATH environment
```bash
export PYTHONPATH=${lis-build-dir}/app/:$PYTHONPATH
python run_snow_detector.py param.json
```
Run the main python script:
The snow detection is performed in the Python script [run_snow_detector.py](app/run_snow_detector.py).
```bash
run_snow_detector.py param.json
```
All the parameters of the algorithm, paths to input and output data are stored in the json file. See the provided example [param_test_s2_template.json](tes/param_test_s2_template.json) file for an example.
Moreover The JSON schema is available in the [Algorithm theoritical basis documentation](doc/tex/ATBD_CES-Neige.tex) and gives more information about the roles of these parameters.
NB: To build DEM data download the SRTM files corresponding to the study area and build the .vrt using gdalbuildvrt. Edit config.json file to activate preprocessing : Set "preprocessing" to true and set the vrt path.
There is a Bash script in app directory which allows to set the env variable and run the script:
```bash
runLis.sh param.json
```
## Products format
* COMPO: RGB composition with snow mask
* COMPO: Raster image showing the outlines of the cloud (including cloud shadow) and snow masks drawn on the RGB composition of the L2A image (bands SWIR/Red/Green).
* SNOW_ALL: Binary mask of snow and clouds.
* 1st bit: Snow mask after pass1
* 2nd bit: Snow mask after pass2
......
#!/usr/bin/env python
import sys
import os.path as op
import json
import argparse
def show_help():
"""Show help of the build_json script for theia N2A products"""
print "This script is used to build json configuration file use then to compute snow mask using OTB applications on Spot/LandSat/Sentinel-2 products from theia platform"
print "Usage: python build_theia_json -s [landsat|s2|take5] -d image_directory -e srtm_tile -o file.json"
print "python run_snow_detector.py help to show help"
#----------------- MAIN ---------------------------------------------------
def main():
""" Script to build json from theia N2A product"""
parser = argparse.ArgumentParser(description='Build json from THEIA product')
parser.add_argument("-s", help="select input sensors")
parser.add_argument("-d", help="input dir")
parser.add_argument("-o", help="input dir")
parser.add_argument("-do", help="input dir")
args = parser.parse_args()
#print(args.accumulate(args.integers))
#Parse sensor
if (args.s == 's2'):
multi=10
#Build json file
data = {}
data["general"]={
"pout":args.do,
"nodata":-10000,
"ram":1024,
"nb_threads":1,
"generate_vector":"false",
"preprocessing":"false",
"log":"true",
"multi":10
}
data["cloud"]={
"shadow_mask":32,
"all_cloud_mask":1,
"high_cloud_mask":128,
"rf":12,
"red_darkcloud":500,
"red_backtocloud":100
}
data["snow"]={
"dz":100,
"ndsi_pass1":0.4,
"red_pass1":200,
"ndsi_pass2":0.15,
"red_pass2":120,
"fsnow_lim":0.1,
"fsnow_total_lim":0.001
}
fp = open(args.o, 'w')
fp.write(json.dumps(data,indent=4, sort_keys=True))
fp.close()
if __name__ == "__main__":
main()
......@@ -5,7 +5,8 @@ import os.path as op
import json
from s2snow import snow_detector
VERSION="1.1.1"
VERSION = "1.2"
def show_help():
"""Show help of the run_snow_detector script"""
......@@ -14,34 +15,35 @@ def show_help():
print "python run_snow_detector.py version to show version"
print "python run_snow_detector.py help to show help"
def show_version():
print VERSION
#----------------- MAIN ---------------------------------------------------
# ----------------- MAIN ---------------------------------------------------
def main(argv):
""" main script of snow extraction procedure"""
json_file=argv[1]
json_file = argv[1]
#load json_file from json files
# Load json_file from json files
with open(json_file) as json_data_file:
data = json.load(json_data_file)
data = json.load(json_data_file)
general = data["general"]
pout = general.get("pout")
log = general.get("log", True)
if log:
sys.stdout = open(op.join(pout, "stdout.log"), 'w')
sys.stderr = open(op.join(pout, "stderr.log"), 'w')
sd = snow_detector.snow_detector(data)
sd.detect_snow(2)
if __name__ == "__main__":
if len(sys.argv) != 2 :
if len(sys.argv) != 2:
show_help()
else:
if sys.argv[1] == "version":
......
......@@ -29,7 +29,7 @@
"shadow_mask":64,
"all_cloud_mask":0,
"high_cloud_mask":32,
"rf":8,
"rf":12,
"red_darkcloud":650,
"red_backtocloud":100
},
......
......@@ -30,7 +30,8 @@ pdftitle=Algorithm theoretical basis documentation for an operational snow cover
\usepackage[]{algorithm2e}
\usepackage{fancyhdr}
\usepackage{tabularx}
% \usepackage{listings}
%\usepackage{listings}
\usepackage{minted}
% \lstset{language=Matlab}
\usepackage{tikz}
......@@ -227,7 +228,8 @@ The snow detection (Sect.~\ref{par:snowdetec}) is performed a first time using t
The L2A cloud mask is conservative because it is computed at a coarser resolution and also because it is developed for a large range of applications. However, the detection of the snow cover is robust to a thin, transparent, cloud cover. More importantly, the L2A cloud mask tends to falsely classify the edges of the snow cover as cloud. Hence, it is possible to recover many pixels from the L2A cloud mask and reclassify them as snow or no-snow. This step is important because it substantially increases the number of observations. A pixel from the L2A cloud mask cannot be reclassified as snow or no-snow if:
\begin{itemize}
\item it is coded as ``cloud shadow'' in L2A cloud mask;
\item it is coded as ``cloud shadow'' in L2A cloud mask. Note that it can be
cloud shadows matched with a cloud or cloud shadows in the zone where clouds could be outside the image ;
\item or: it is coded as ``high altitude cloud'' (or ``cirrus'') in the L2A cloud mask;
\item or: it is not a ``dark'' cloud (see below).
\end{itemize}
......@@ -256,7 +258,7 @@ After passing the pass 1 and 2 snow tests, some pixels that were originally mark
minimum height=2em]
\tikzstyle{bigbox}=[inner sep=20pt]
\begin{figure}
\begin{figure}[H]
\begin{tikzpicture}[node distance = 3.5cm, auto]
% Place nodes
......@@ -314,9 +316,13 @@ After passing the pass 1 and 2 snow tests, some pixels that were originally mark
\caption{Flowchart of the snow detection algorithm}
\end{figure}\label{fig:flowchart}
\subsection{Parameter values}\label{par:param}
\subsection{Parameters description}\label{par:param}
\begin{table}
\subsubsection{Main algorithm parameters}\label{par:sciparam}
The table below gives the description of the main parameters of the algorithm:
\begin{table}[!htbp]
\begin{center}
\begin{tabularx}{\textwidth}{|l X l l|}
\hline
......@@ -338,7 +344,23 @@ Parameter & Description & Name in the configuration file & Default value\\
\caption{LIS algorithm parameters description and default values.}
\end{table}\label{tab:param}
Above default values related to reflectance are given as float values between 0
and 1. Threshold related to reflectance values follow the convention of
considering milli-reflectance as input (values between 0 and 1000) in the json
file. Some products can encode reflectance with other convention (floating
values between 0 and 1 or reflectance between 0 and 10000), to handle those
cases, there is a parameter 'multi' in the json configuration file which allows
to scale reflectance parameters. For instance, for products with reflectance
between 0 and 10000 you can use
\subsubsection{JSON schema of configuration file}\label{par:jsonparam}
The JSON Schema here describes the parameter file format and provide a clear, human-
and machine-readable documentation of all the algorithm parameters. JSON schema
was generated on \href{https://jsonschema.net} with the following options (with
metada and relative id).
\inputminted[tabsize=2, fontsize=\tiny]{js}{schema.json}
\section{Validation}\label{par:validation}
......
# Let-it-snow documentation
## Build instructions
```bash
texi2pdf --shell-escape ATBD_CES-Neige.tex
```
The "--shell-escape" option is required by the minted package
......@@ -14,7 +14,7 @@
{ \huge \bfseries Algorithm theoritical basis documentation for an operational snow cover extent product from Sentinel-2 and Landsat-8 data (Let-it-snow)\\}
\rule{\linewidth}{0.5mm}
{ \large \bfseries Simon Gascoin (CNRS/CESBIO), Manuel Grizonnet (CNES/CT/DSI), Tristan Klempka (CNES/CT/DSI)\\ }
{ \large \bfseries V1.0 - \today \\ }
{ \large \bfseries V1.0 (Updated for LIS 1.2) - \today \\ }
% \vspace{3cm}
% \includegraphics[width=1\textwidth]{./Images/Theia_en.png}
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"id": "http://tully.ups-tlse.fr/grizonnet/let-it-snow/blob/master/test/param_test_s2_template.json",
"properties": {
"cloud": {
"id": "cloud",
"properties": {
"all_cloud_mask": {
"default": 1,
"description": "Threshold apply to Theia cloud mask to retrieve a strict cloud mask (greater than all_cloud_mask).",
"id": "all_cloud_mask",
"title": "The All_cloud_mask schema.",
"type": "integer"
},
"high_cloud_mask": {
"default": 128,
"description": "bitmask apply to retrieve high clouds for input mask",
"id": "high_cloud_mask",
"title": "The High_cloud_mask schema.",
"type": "integer"
},
"red_backtocloud": {
"default": 100,
"description": "Minimum value of the red band reflectance to return a non-snow pixel to the cloud mask.",
"id": "red_backtocloud",
"title": "The Red_backtocloud schema.",
"type": "integer"
},
"red_darkcloud": {
"default": 500,
"description": "Maximum value of the down-sampled red band reflectance to define a dark cloud pixel.",
"id": "red_darkcloud",
"title": "The Red_darkcloud schema.",
"type": "integer"
},
"rf": {
"default": 12,
"description": "Resize factor to produce the down-sampled red band (use for cloud refinement).",
"id": "rf",
"title": "The Rf schema.",
"type": "integer"
},
"shadow_in_mask": {
"default": 32,
"description": "bitmask apply to retrieve cloud shadows (for cloud inside the image).",
"id": "shadow_in_mask",
"title": "The Shadow_in_mask schema.",
"type": "integer"
},
"shadow_out_mask": {
"default": 64,
"description": "bitmask apply to retrieve cloud shadows (for cloud outside the image).",
"id": "shadow_out_mask",
"title": "The Shadow_out_mask schema.",
"type": "integer"
}
},
"type": "object"
},
"general": {
"id": "general",
"properties": {
"generate_vector": {
"default": false,
"description": "Generate vector masks of the detection (pass 1 and 2 and final result).",
"id": "generate_vector",
"title": "The Generate_vector schema.",
"type": "boolean"
},
"log": {
"default": true,
"description": "Log output and error to files (std***.log).",
"id": "log",
"title": "The Log schema.",
"type": "boolean"
},
"multi": {
"default": 10,
"description": "Scale input paramters to map reflectance interval (parameters are provided in milli-reflectance but L2A S2 Theia product are between 0 and 10000).",
"id": "multi",
"title": "The Multi schema.",
"type": "integer"
},
"nb_threads": {
"default": 1,
"description": "Maximum number of threads use by the program.",
"id": "nb_threads",
"title": "The Nb_threads schema.",
"type": "integer"
},
"nodata": {
"default": -10000,
"description": "No-data value in the input L2A product.",
"id": "nodata",
"title": "The Nodata schema.",
"type": "integer"
},
"pout": {
"description": "Path to output directory.",
"id": "pout",
"title": "The Pout schema.",
"type": "string"
},
"preprocessing": {
"default": false,
"description": "Activate the extraction and resampling of the DEM.",
"id": "preprocessing",
"title": "The Preprocessing schema.",
"type": "boolean"
},
"ram": {
"default": 1024,
"description": "Maximum number of RAM memory used by the program.",
"id": "ram",
"title": "The Ram schema.",
"type": "integer"
}
},
"type": "object"
},
"inputs": {
"id": "inputs",
"properties": {
"cloud_mask": {
"description": "Input mask image (in MASK directory for Theia product).",
"id": "cloud_mask",
"title": "The Cloud_mask schema.",
"type": "string"
},
"dem": {
"description": "Input DEM with the same resolution and extent as the input image if preprocessing is deactivated.",
"id": "dem",
"title": "The Dem schema.",
"type": "string"
},
"green_band": {
"id": "green_band",
"properties": {
"noBand": {
"default": 1,
"description": "Green band number.",
"id": "noBand",
"title": "The Noband schema.",
"type": "integer"
},
"path": {
"description": "Path to input L2A image or L2A green band.",
"id": "path",
"title": "The Path schema.",
"type": "string"
}
},
"type": "object"
},
"red_band": {
"id": "red_band",
"properties": {
"noBand": {
"default": 1,
"description": "Red band number.",
"id": "noBand",
"title": "The Noband schema.",
"type": "integer"
},
"path": {
"description": "Path to input L2A image or L2A red band.",
"id": "path",
"title": "The Path schema.",
"type": "string"
}
},
"type": "object"
},
"swir_band": {
"id": "swir_band",
"properties": {
"noBand": {
"default": 1,
"description": "SWIR band number.",
"id": "noBand",
"title": "The Noband schema.",
"type": "integer"
},
"path": {
"description": "Path to input L2A image or L2A swir band.",
"id": "path",
"title": "The Path schema.",
"type": "string"
}
},
"type": "object"
}
},
"type": "object"
},
"snow": {
"id": "snow",
"properties": {
"dz": {
"default": 100,
"description": "Minimum snow fraction in an elevation band to define zs.",
"id": "dz",
"title": "The Dz schema.",
"type": "integer"
},
"fsnow_lim": {
"default": 0.1,
"description": "Minimum snow fraction in an elevation band to define zs.",
"id": "fsnow_lim",
"title": "The Fsnow_lim schema.",
"type": "number"
},
"fsnow_total_lim": {
"default": 0.001,
"description": "Minimum snow fraction in the image to activate the pass 2 snow test.",
"id": "fsnow_total_lim",
"title": "The Fsnow_total_lim schema.",
"type": "number"
},
"ndsi_pass1": {
"default": 0.4,
"description": "Minimum value of the NDSI for the pass 1 snow test.",
"id": "ndsi_pass1",
"title": "The Ndsi_pass1 schema.",
"type": "number"
},
"ndsi_pass2": {
"default": 0.15,
"description": "Minimum value of the NDSI for the pass 2 snow test.",
"id": "ndsi_pass2",
"title": "The Ndsi_pass2 schema.",
"type": "number"
},
"red_pass1": {
"default": 200,
"description": "Minimum value of the red band reflectance the pass 1 snow test.",
"id": "red_pass1",
"title": "The Red_pass1 schema.",
"type": "integer"
},
"red_pass2": {
"default": 120,
"description": "Minimum value of the red band reflectance the pass 2 snow test.",
"id": "red_pass2",
"title": "The Red_pass2 schema.",
"type": "integer"
}
},
"type": "object"
}
},
"type": "object"
}
import sys, os, numpy, random
import sys
import os
import numpy
import random
import os.path as op
import gdal, gdalconst
import gdal
import gdalconst
from subprocess import call
def show_help():
print "This script is used to create clouds on data"
print "Usage: cloud_builder.py mode plaincloudthreshold randomcloudthreshold inputpath outputplaincloudpath ouputrandomcloudpath"
print "Mode : 0 %plain cloud image, 1 %random cloud image, 2 both"
print "This script is used to create clouds on data"
print "Usage: cloud_builder.py mode plaincloudthreshold randomcloudthreshold inputpath outputplaincloudpath ouputrandomcloudpath"
print "Mode : 0 %plain cloud image, 1 %random cloud image, 2 both"
def main(argv):
mode = argv[1]
mode = int(mode)
......@@ -17,35 +24,38 @@ def main(argv):
input_path = argv[4]
output_plain_cloud_path = argv[5]
output_random_cloud_path = argv[6]
dataset = gdal.Open(input_path, gdalconst.GA_ReadOnly)
wide = dataset.RasterXSize
high = dataset.RasterYSize
if(mode == 0 or mode == 2):
#build half cloud image
str_exp = "idxX>"+str(int(wide*plain_cloud_threshold))+"?im1b1:205"
call(["otbcli_BandMathX", "-il", input_path, "-out", output_plain_cloud_path, "-exp", str_exp])
# build half cloud image
str_exp = "idxX>" + \
str(int(wide * plain_cloud_threshold)) + "?im1b1:205"
call(["otbcli_BandMathX", "-il", input_path, "-out",
output_plain_cloud_path, "-exp", str_exp])
if(mode == 1 or mode == 2):
#build random cloud image
# build random cloud image
band = dataset.GetRasterBand(1)
array = band.ReadAsArray(0, 0, wide, high)
for i in range(0, wide):
for j in range(0, high):
if(random.randint(0, 100) < random_cloud_threshold):
array[i,j] = 205
output_random_cloud_raster = gdal.GetDriverByName('GTiff').Create(output_random_cloud_path, wide, high, 1 ,gdal.GDT_Byte)
output_random_cloud_raster.GetRasterBand(1).WriteArray(array)
array[i, j] = 205
output_random_cloud_raster = gdal.GetDriverByName('GTiff').Create(
output_random_cloud_path, wide, high, 1, gdal.GDT_Byte)
output_random_cloud_raster.GetRasterBand(1).WriteArray(array)
output_random_cloud_raster.FlushCache()
# georeference the image and set the projection
output_random_cloud_raster.SetGeoTransform(dataset.GetGeoTransform())
output_random_cloud_raster.SetProjection(dataset.GetProjection())
output_random_cloud_raster.SetProjection(dataset.GetProjection())
if __name__ == "__main__":
if len(sys.argv) != 7:
show_help()
else:
main(sys.argv)
This diff is collapsed.
......@@ -5,25 +5,30 @@ import sys
import subprocess
import ast
def show_help():
print "This script is used to compute srtm mask from a vrt file to a region extent"
print "Usage: preprocessing.py srtm.vrt img.tif output.tif"