Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • s1-tiling/s1tiling
  • abricier/s1tiling
  • gbonnefille/s1tiling
3 results
Show changes
Commits on Source (1)
Hide whitespace changes
Inline Side-by-side
s1tiling/libs/S1FileManager.py
View file @ 67b768f2
......@@ -47,7 +47,7 @@ import tempfile
from eodag.api.core import EODataAccessGateway
from eodag.utils.logging import setup_logging
from eodag.utils.exceptions import NotAvailableError
from eodag.utils.exceptions import NotAvailableError, DownloadError
from eodag.utils import get_geometry_from_various
try:
from shapely.errors import TopologicalError
......@@ -56,15 +56,15 @@ except ImportError:
import numpy as np
from s1tiling.libs import exits
from .Utils import (get_shape, list_dirs, Layer, extract_product_start_time, get_orbit_direction, get_relative_orbit, find_dem_intersecting_poly)
from s1tiling.libs import exceptions
from .Utils import (get_shape, list_dirs, Layer, extract_product_start_time, get_orbit_direction, get_relative_orbit, find_dem_intersecting_poly, get_platform_from_s1_raster)
from .S1DateAcquisition import S1DateAcquisition
from .otbpipeline import mp_worker_config
from .outcome import Outcome
setup_logging(verbose=1)
logger = logging.getLogger('s1tiling')
logger = logging.getLogger('s1tiling.filemanager')
class WorkspaceKinds(Enum):
......@@ -142,8 +142,8 @@ def product_cover(product, geometry):
# pass
def does_final_product_need_to_be_generated_for(product, tile_name,
polarizations, cfg, s2images):
def does_final_product_need_to_be_generated_for(
product, tile_name, polarizations, cfg, s2images):
"""
Tells whether finals products associated to a tile needs to be generated.
......@@ -155,7 +155,8 @@ def does_final_product_need_to_be_generated_for(product, tile_name,
Searchs in `s2images` whether all the expected product filenames for the given S2 tile name
and the requested polarizations exists.
"""
logger.debug('Searching whether %s final product has already been generated in %s', product, s2images)
logger.debug('> Searching whether %s final products have already been generated (in polarizations: %s)',
product, polarizations)
if len(s2images) == 0:
return True
# e.g. id=S1A_IW_GRDH_1SDV_20200108T044150_20200108T044215_030704_038506_C7F5,
......@@ -176,12 +177,16 @@ def does_final_product_need_to_be_generated_for(product, tile_name,
# We should use the `Processing.fname_fmt.concatenation` option
pat = fname_fmt_concatenation.format(**keys, polarisation=polarisation)
pat_filtered = fname_fmt_filtered.format(**keys, polarisation=polarisation)
# pat = f'{sat.lower()}_{tile_name}_{polarisation}_*_{start}t??????.tif'
# pat_filtered = f'{sat.lower()}_{tile_name}_{polarisation}_*_{start}t??????_filtered.tif'
found = fnmatch.filter(s2images, pat) or fnmatch.filter(s2images, pat_filtered)
logger.debug('searching w/ %s and %s ==> Found: %s', pat, pat_filtered, found)
found_s2 = fnmatch.filter(s2images, pat)
found_filt = fnmatch.filter(s2images, pat_filtered)
found = found_s2 or found_filt
logger.debug(' searching w/ %s and %s ==> Found: %s', pat, pat_filtered, found)
if not found:
return True
# FIXME:
# - if found_s2 and not found_filt => we have everything that is needed
# - if found_filt and not found_s2 => we have prevent the required S1 products from being downloaded
# if the S2 product is required
return False
......@@ -204,8 +209,8 @@ def filter_images_or_ortho(kind, all_images):
return images
def _filter_images_providing_enough_cover_by_pair(products, target_cover,
ident, get_cover, get_orbit):
def _filter_images_providing_enough_cover_by_pair(
products, target_cover, ident, get_cover, get_orbit):
"""
Associate products of the same date and orbit into pairs (at most),
to compute the total coverage of the target zone.
......@@ -315,7 +320,7 @@ def _discard_small_redundant(products, ident=None):
def _keep_requested_orbits(content_info, rq_orbit_direction, rq_relative_orbit_list):
"""
Takes care of discarding product that don't match the request orbit
Takes care of discarding products that don't match the requested orbit
specification.
Note: Beware that specifications could be contradictory and end up
......@@ -326,8 +331,6 @@ def _keep_requested_orbits(content_info, rq_orbit_direction, rq_relative_orbit_l
kept_products = []
for ci in content_info:
p = ci['product']
# safe_dir = ci['safe_dir']
# manifest = ci['manifest']
direction = ci['orbit_direction']
orbit = ci['relative_orbit']
# logger.debug('CHECK orbit: %s / %s / %s', p, safe_dir, manifest)
......@@ -345,6 +348,28 @@ def _keep_requested_orbits(content_info, rq_orbit_direction, rq_relative_orbit_l
kept_products.append(ci)
return kept_products
def _keep_requested_platforms(content_info, rq_platform_list):
"""
Takes care of discarding products that don't match the requested platform specification.
Note: Beware that specifications could be contradictory and end up discarding everything.
"""
if not rq_platform_list:
return content_info
kept_products = []
for ci in content_info:
p = ci['product']
platform = ci['platform']
logger.debug('CHECK platform: %s / %s', p, platform)
if rq_platform_list:
if platform not in rq_platform_list:
logger.debug('Discard %s as its platform (%s) differs from the requested ones %s',
p.name, platform, rq_platform_list)
continue
kept_products.append(ci)
return kept_products
def _download_and_extract_one_product(dag, raw_directory, dl_wait, dl_timeout, product):
"""
......@@ -355,7 +380,8 @@ def _download_and_extract_one_product(dag, raw_directory, dl_wait, dl_timeout, p
"""
logging.info("Starting download of %s...", product)
ok_msg = f"Successful download (and extraction) of {product}" # because eodag'll clear product
file = os.path.join(raw_directory, product.as_dict()['id']) + '.zip'
prod_id = product.as_dict()['id']
zip_file = os.path.join(raw_directory, prod_id) + '.zip'
try:
path = Outcome(dag.download(
product, # EODAG will clear this variable
......@@ -364,12 +390,20 @@ def _download_and_extract_one_product(dag, raw_directory, dl_wait, dl_timeout, p
timeout=dl_timeout # Maximum time in mins before stop retrying to download (default=20’)
))
logging.debug(ok_msg)
if os.path.exists(file) :
if os.path.exists(zip_file) :
try:
logger.debug('Removing downloaded ZIP: %s', file)
os.remove(file)
logger.debug('Removing downloaded ZIP: %s', zip_file)
os.remove(zip_file)
except OSError:
pass
# eodag may say the product is correctly downloaded while it failed to do so
# => let's do a quick sanity check
manifest = os.path.join(raw_directory, prod_id, prod_id+'.SAFE', 'manifest.safe')
if not os.path.exists(manifest):
logger.error('Actually download of %s failed, the expected manifest could not be found (%s)', prod_id, manifest)
e = exceptions.CorruptedDataSAFEError(manifest)
path = Outcome(e)
path.add_related_filename(product)
except BaseException as e: # pylint: disable=broad-except
logger.warning('%s', e) # EODAG error message is good and precise enough, just use it!
# logger.error('Product is %s', product_property(product, 'storageStatus', 'online?'))
......@@ -467,8 +501,8 @@ class S1FileManager:
self.last_date = cfg.last_date
self._refresh_s1_product_list()
if self.cfg.download:
logger.debug('Using %s EODAG configuration file', self.cfg.eodagConfig or 'user default')
self._dag = EODataAccessGateway(self.cfg.eodagConfig)
logger.debug('Using %s EODAG configuration file', self.cfg.eodag_config or 'user default')
self._dag = EODataAccessGateway(self.cfg.eodag_config)
# TODO: update once eodag directly offers "DL directory setting" feature v1.7? +?
dest_dir = os.path.abspath(self.cfg.raw_directory)
logger.debug('Override EODAG output directory to %s', dest_dir)
......@@ -562,7 +596,7 @@ class S1FileManager:
for dem_tile, dem_tile_info in dem_tile_infos.items():
dem_file = dem_filename_format.format_map(dem_tile_info)
dem_tile_filepath=Path(self.cfg.dem, dem_file)
dem_tile_filelink=Path(self.__tmpdemdir.name, dem_file)
dem_tile_filelink=Path(self.__tmpdemdir.name, os.path.basename(dem_file))
if self.__caching_option == 'symlink':
logger.debug('- ln -s %s <-- %s', dem_tile_filepath, dem_tile_filelink)
dem_tile_filelink.symlink_to(dem_tile_filepath)
......@@ -589,7 +623,7 @@ class S1FileManager:
def _search_products(self, dag: EODataAccessGateway,
extent, first_date, last_date,
orbit_direction, relative_orbit_list, polarization,
platform_list, orbit_direction, relative_orbit_list, polarization,
searched_items_per_page,dryrun):
"""
Process with the call to eodag search.
......@@ -602,9 +636,10 @@ class S1FileManager:
assert polarization in ['VV VH', 'VV', 'VH', 'HH HV', 'HH', 'HV']
# In case only 'VV' or 'VH' is requested, we still need to
# request 'VV VH' to the data provider through eodag.
dag_polarization_param = 'VV VH' if polarization in ['VV VH', 'VV', 'VH'] else 'HH HV'
dag_orbit_dir_param = k_dir_assoc.get(orbit_direction, None) # None => all
dag_orbit_list_param = relative_orbit_list[0] if len(relative_orbit_list) == 1 else None
dag_polarization_param = 'VV VH' if polarization in ['VV VH', 'VV', 'VH'] else 'HH HV'
dag_orbit_dir_param = k_dir_assoc.get(orbit_direction, None) # None => all
dag_orbit_list_param = relative_orbit_list[0] if len(relative_orbit_list) == 1 else None
dag_platform_list_param = platform_list[0] if len(platform_list) == 1 else None
while True:
page_products, _ = dag.search(
page=page, items_per_page=searched_items_per_page,
......@@ -616,6 +651,7 @@ class S1FileManager:
sensorMode="IW",
orbitDirection=dag_orbit_dir_param, # None => all
relativeOrbitNumber=dag_orbit_list_param, # List doesn't work. Single number yes!
platformSerialIdentifier=dag_platform_list_param,
)
logger.info("%s remote S1 products returned in page %s: %s", len(page_products), page, page_products)
products += page_products
......@@ -633,19 +669,32 @@ class S1FileManager:
filtered_products.extend(products.filter_property(relativeOrbitNumber=rel_orbit))
products = filtered_products
# Filter platform -- if it could not be done earlier in the search() request.
if len(platform_list) > 1:
filtered_products = []
for platform in platform_list:
filtered_products.extend(products.filter_property(platformSerialIdentifier=platform))
products = filtered_products
# Final log
extra_filter_log1 = ''
orbit_filter_log1 = ''
if dag_orbit_dir_param:
extra_filter_log1 = f'{dag_orbit_dir_param} '
extra_filter_log2 = ''
orbit_filter_log1 = f'{dag_orbit_dir_param} '
orbit_filter_log2 = ''
if len(relative_orbit_list) > 0:
if len(relative_orbit_list) > 1:
extra_filter_log2 = 's'
extra_filter_log2 += ' ' + ', '.join([str(i) for i in relative_orbit_list])
extra_filter_log = ''
if extra_filter_log1 or extra_filter_log2:
extra_filter_log = f' && {extra_filter_log1}orbit{extra_filter_log2}'
logger.info("%s remote S1 product(s) found and filtered (IW && %s%s): %s", len(products), polarization, extra_filter_log, products)
orbit_filter_log2 = 's'
orbit_filter_log2 += ' ' + ', '.join([str(i) for i in relative_orbit_list])
orbit_filter_log = ''
if orbit_filter_log1 or orbit_filter_log2:
orbit_filter_log = f'{orbit_filter_log1}orbit{orbit_filter_log2}'
extra_filters = ['IW', polarization]
if platform_list:
extra_filters.append('|'.join(platform_list))
if orbit_filter_log:
extra_filters.append(orbit_filter_log)
logger.info("%s remote S1 product(s) found and filtered (%s): %s", len(products),
" && ".join(extra_filters), products)
return products
......@@ -682,7 +731,7 @@ class S1FileManager:
# And let's suppose nobody deletd files
# manually!
products = [p for p in products
if not p.as_dict()['id'] in self._product_list.keys()
if p.as_dict()['id'] not in self._product_list.keys()
]
# logger.debug('Products cache: %s', self._product_list.keys())
logger.debug("%s remote S1 product(s) are not yet in the cache: %s", len(products), products)
......@@ -695,11 +744,12 @@ class S1FileManager:
# generator in order to download what is stricly necessary and nothing more
polarizations = polarization.lower().split(' ')
s2images_pat = f's1?_{tile_name}_*.tif'
logger.debug('Search %s for %s on disk in %s', s2images_pat, polarizations, tile_out_dir)
logger.debug('Search %s for %s on disk in %s(/filtered)/%s', s2images_pat, polarizations, tile_out_dir, tile_name)
def glob1(pat, *paths):
pathname = glob.escape(os.path.join(*paths))
return [os.path.basename(p) for p in glob.glob(os.path.join(pathname, pat))]
s2images = glob1(s2images_pat, tile_out_dir) + glob1(s2images_pat, tile_out_dir, "filtered")
s2images = glob1(s2images_pat, tile_out_dir, tile_name) + glob1(s2images_pat, tile_out_dir, "filtered", tile_name)
logger.debug(' => S2 products found on %s: %s', tile_name, s2images)
products = [p for p in products
if does_final_product_need_to_be_generated_for(
p, tile_name, polarizations, self.cfg, s2images)
......@@ -710,7 +760,7 @@ class S1FileManager:
lonmin, lonmax, latmin, latmax,
first_date, last_date,
tile_out_dir, tile_name,
orbit_direction, relative_orbit_list, polarization, cover,
platform_list, orbit_direction, relative_orbit_list, polarization, cover,
searched_items_per_page,dryrun, dl_wait, dl_timeout):
"""
Process with the call to eodag search + filter + download.
......@@ -724,7 +774,7 @@ class S1FileManager:
'latmax': latmax
}
products = self._search_products(dag, extent,
first_date, last_date, orbit_direction, relative_orbit_list,
first_date, last_date, platform_list, orbit_direction, relative_orbit_list,
polarization, searched_items_per_page,dryrun)
products = self._filter_products(products, extent, tile_out_dir, tile_name, polarization, cover)
......@@ -781,8 +831,9 @@ class S1FileManager:
downloaded_products += self._download(self._dag,
lonmin, lonmax, latmin, latmax,
self.first_date, self.last_date,
os.path.join(self.cfg.output_preprocess, tiles_list),
tile_name,
tile_out_dir=self.cfg.output_preprocess,
tile_name=tile_name,
platform_list=self.cfg.platform_list,
orbit_direction=self.cfg.orbit_direction,
relative_orbit_list=self.cfg.relative_orbit_list,
polarization=self.cfg.polarisation,
......@@ -818,6 +869,7 @@ class S1FileManager:
def _refresh_s1_product_list(self, new_products=None):
"""
Scan all the available products and filter them according to:
- platform requirements
- orbit requirements
- date requirements
......@@ -830,7 +882,7 @@ class S1FileManager:
if new_products:
logger.debug('new products:')
for np in new_products:
logger.debug('%s -> %s', np.__class__.__name__, np)
logger.debug('-> %s', np)
# content is DirEntry
# NEW is str!! Always
# logger.debug('content[0]: %s -> %s', type(content[0]), content[0])
......@@ -844,9 +896,10 @@ class S1FileManager:
parent_dirs = [os.path.dirname(p) for p in new_products]
content += list(filter(lambda d: d.path in parent_dirs, content0))
logger.debug('dirs found & filtered: %s', content)
logger.debug("products DL'ed: %s", new_products)
assert len(content) == len(new_products), f'Not all new products found in {self.cfg.raw_directory}: {new_products}'
logger.debug('dirs found & filtered: %s', content) # List(DirEntry)
logger.debug("products DL'ed: %s", new_products) # List(str)
if len(content) != len(new_products):
logger.warning(f'Not all new products are found in {self.cfg.raw_directory}: {new_products}. Some products downloaded may be corrupted.')
else:
self._product_list = {}
self._products_info = []
......@@ -862,7 +915,7 @@ class S1FileManager:
# orbit_direction, relative_orbit}
products_info = [ {
'product': p,
# EODAG save SAFEs into {rawdir}/{prod}/{prod}.SAFE
# EODAG saves SAFEs into {rawdir}/{prod}/{prod}.SAFE
'safe_dir': os.path.join(p.path, p.name + '.SAFE'),
} for p in content]
products_info = list(filter(lambda ci: os.path.isdir(ci['safe_dir']), products_info))
......@@ -872,6 +925,7 @@ class S1FileManager:
ci['manifest'] = manifest
ci['orbit_direction'] = get_orbit_direction(manifest)
ci['relative_orbit'] = get_relative_orbit(manifest)
ci['platform'] = ci['product'].name[:3]
# Filter by orbit specification
if self.cfg.orbit_direction or self.cfg.relative_orbit_list:
......@@ -879,9 +933,15 @@ class S1FileManager:
self.cfg.orbit_direction, self.cfg.relative_orbit_list)
logger.debug('%s local products remaining after filtering requested orbits', len(products_info))
# Filter by platform specification
if self.cfg.platform_list:
products_info = _keep_requested_platforms(products_info, self.cfg.platform_list)
logger.debug('%s local products remaining after filtering requested platforms (%s)',
len(products_info), ", ".join(self.cfg.platform_list))
# Final log + extend "global" products_info with newly analysed ones
if products_info:
logger.debug('Time and orbit compatible products found on disk:')
logger.debug('%s time, platform and orbit compatible products found on disk:', len(products_info))
for ci in products_info:
current_content = ci['product']
logger.debug('* %s', current_content.name)
......@@ -1005,9 +1065,7 @@ class S1FileManager:
if l_vv + l_vh + l_hv + l_hh == 0:
# There is not a single file that would have been compatible
# with what is expected
logger.critical("Problem with %s", manifest)
logger.critical("Please remove the raw data for %s SAFE file", manifest)
sys.exit(exits.CORRUPTED_DATA_SAFE)
raise exceptions.CorruptedDataSAFEError(manifest)
self.raw_raster_list.append(acquisition)
......
s1tiling/resources/generate-gpkg/generate-DEM-gpkg.py 0 → 100644
View file @ 67b768f2
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# =========================================================================
# Program: generate-DEM-gpkg.py
#
# All rights reserved.
# Copyright 2017-2023 (c) CNES.
# Copyright 2022-2023 (c) CS GROUP France.
#
# This file is part of S1Tiling project
# https://gitlab.orfeo-toolbox.org/s1-tiling/s1tiling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# =========================================================================
#
# Authors: Thierry KOLECK (CNES)
# Luc HERMITTE (CS Group)
#
# =========================================================================
#
# This code is used to generate a GEOPACKAGE (gpkg) that contains the
# footprint and filename of Copernicus Digital Elevation Model tiles.
# It should be adapted to fit with the organisation of the tiles in the user context
#
# Starting from the SHAPEFILE provide by Copernicus, we need first to convert it to a gpkg file
# Ex: ogr2ogr GEO1988-CopernicusDEM-RP-002_GridFile_I4.0_ESA.gpkg GEO1988-CopernicusDEM-RP-002_GridFile_I4.0_ESA.shp
#
# The SHAPEFILE is availble here:
# https://spacedata.copernicus.eu/documents/20123/122407/GEO1988-CopernicusDEM-RP-002_GridFile_I4.0_ESA.zip/590bb3da-1123-042b-d4d3-021549aabb17?t=1674484982606
#
from osgeo import ogr
def select_columns(input_path, output_path, columns_to_keep):
# Ouvrir le fichier GPKG en lecture seule
input_ds = ogr.Open(input_path, 0)
if input_ds is None:
print("Impossible d'ouvrir le fichier GPKG en lecture seule.")
return
# Créer un nouveau fichier GPKG en sortie
driver = ogr.GetDriverByName("GPKG")
output_ds = driver.CreateDataSource(output_path)
if output_ds is None:
print("Impossible de créer le fichier de sortie GPKG.")
return
# Parcourir chaque couche du fichier d'entrée
print("Layers:",input_ds.GetLayerCount())
for layer_index in range(input_ds.GetLayerCount()):
input_layer = input_ds.GetLayerByIndex(layer_index)
# Créer une couche de sortie avec les colonnes sélectionnées
output_layer = output_ds.CreateLayer(
input_layer.GetName(),
geom_type=input_layer.GetGeomType(),
options=["SPATIAL_INDEX=YES"]
)
# Ajouter les champs sélectionnés à la couche de sortie
print("Fields:",input_layer.GetLayerDefn().GetFieldCount())
for field_index in range(input_layer.GetLayerDefn().GetFieldCount()):
field_defn = input_layer.GetLayerDefn().GetFieldDefn(field_index)
field_name = field_defn.GetName()
if field_name in columns_to_keep:
output_layer.CreateField(field_defn)
file_id_field = ogr.FieldDefn("FileID", ogr.OFTString)
file_id_field.SetWidth(150) # Ajustez la largeur en fonction de vos besoins
output_layer.CreateField(file_id_field)
# Copier les entités avec les champs sélectionnés vers la couche de sortie
for feature in input_layer:
output_feature = ogr.Feature(output_layer.GetLayerDefn())
for field_name in columns_to_keep:
output_feature.SetField(field_name, feature.GetField(field_name))
CellID=feature.GetField("GeoCellID")
print(CellID)
latitudeID=CellID[0:3]
longitudeID=CellID[3:]
output_feature.SetField("FileID", "{}/Copernicus_DSM_10_{}_00_{}_00/DEM/Copernicus_DSM_10_{}_00_{}_00_DEM.tif".format(longitudeID,latitudeID,longitudeID,latitudeID,longitudeID))
output_feature.SetGeometry(feature.GetGeometryRef())
output_layer.CreateFeature(output_feature)
output_feature = None
# Fermer les jeux de données
input_ds = None
output_ds = None
# Chemin vers le fichier GPKG d'entrée
input_file = "/work/scratch/data/koleckt/s1tiling-dev/s1tiling/s1tiling/resources/generate-gpkg/GEO1988-CopernicusDEM-RP-002_GridFile_I4.0_ESA.gpkg"
# Chemin vers le fichier GPKG de sortie
output_file = "/work/scratch/data/koleckt/s1tiling-dev/s1tiling/s1tiling/resources/generate-gpkg/CopernicusDEM-CNES.gpkg"
# Liste des noms de colonnes à conserver
columns_to_keep = [ "GeoCellID"]
# Appeler la fonction pour conserver les colonnes sélectionnées
select_columns(input_file, output_file, columns_to_keep)
print("Colonnes sélectionnées et enregistrées avec succès.")
s1tiling/resources/shapefile/CopernicusDEM-CNES.gpkg 0 → 100644
View file @ 67b768f2
File added