Commit 23d3e11d authored by jbrossar's avatar jbrossar
Browse files

[ENH] Updated reader and writer files to be compliant with the format ARD

Fixed WASP-6
parent 953e74cf
Pipeline #10387 passed with stages
in 26 minutes and 30 seconds
......@@ -37,14 +37,16 @@ struct MuscateHeaderMetadata
std::string SchemaLocation = "";
};
struct MuscateMetadataIdentification{
struct MuscateMetadataIdentification
{
std::string MetadataFormat;
std::string version;
std::string MetadataProfile;
std::string MetadataInformation;
};
struct MuscateDatasetIdentification{
struct MuscateDatasetIdentification
{
std::string Identifier;
std::string Authority;
std::string Producer;
......@@ -53,19 +55,23 @@ struct MuscateDatasetIdentification{
std::string GeographicalZoneType;
std::string Title;
std::string Description;
std::string DataAccess;
};
struct BandList{
struct BandList
{
size_t Count;
std::vector<std::string> bandIDs;
};
struct BandGroup{
struct BandGroup
{
std::string GroupID;
BandList GroupList;
};
struct MuscateProductCharacteristics{
struct MuscateProductCharacteristics
{
std::string ProductID;
std::string AcquisitionDate;
std::string ProductionDate;
......@@ -84,9 +90,11 @@ struct MuscateProductCharacteristics{
std::string DatePrecisionUnit;
BandList BandGlobalList;
std::vector<BandGroup> BandGroupList;
std::string Instrument;
};
struct ImageInformation{
struct ImageInformation
{
std::string path;
bool isMask;
std::string bandIDs;
......@@ -99,7 +107,8 @@ struct ImageInformation{
bool hasDetectorID;
};
struct ImageProperty{
struct ImageProperty
{
std::string Nature;
std::string Format;
std::string Encoding;
......@@ -110,19 +119,22 @@ struct ImageProperty{
std::vector<ImageInformation> ImageFiles;
};
struct MuscateProductOrganisation{
struct MuscateProductOrganisation
{
ImageInformation Quicklook;
std::vector<ImageProperty> ImageProperties;
std::vector<ImageProperty> MaskProperties;
std::vector<ImageProperty> DataProperties;
};
struct CSInfo{
struct CSInfo
{
std::string CSType;
std::string PixelOrigin;
};
struct GeoPoint{
struct GeoPoint
{
std::string name;
std::string lat;
std::string lon;
......@@ -130,7 +142,8 @@ struct GeoPoint{
std::string Ypos;
};
struct GroupPositioning{
struct GroupPositioning
{
std::string groupID;
std::string ULX;
std::string ULY;
......@@ -140,7 +153,8 @@ struct GroupPositioning{
std::string ncols;
};
struct MuscateGeopositionInformations{
struct MuscateGeopositionInformations
{
std::string GeoTables;
std::string HorizontalCSType;
std::string HorizontalCSName;
......@@ -151,14 +165,13 @@ struct MuscateGeopositionInformations{
std::vector<GroupPositioning> GroupPosition;
};
struct MuscateAngleList
{
std::string ColumnUnit;
std::string ColumnStep;
std::string RowUnit;
std::string RowStep;
std::vector<std::vector<double> > Values;
std::vector<std::vector<double>> Values;
};
struct MuscateAngles
......@@ -167,13 +180,13 @@ struct MuscateAngles
MuscateAngleList Azimuth;
};
struct MuscateViewingAnglesGrid{
struct MuscateViewingAnglesGrid
{
std::string bandID;
std::string detectorID;
MuscateAngles ViewIncidenceAnglesGrid;
};
struct MuscateAnglePair
{
std::string ZenithUnit;
......@@ -186,25 +199,28 @@ struct MuscateAnglePair
struct MuscateBandViewingAnglesGrid
{
std::string BandId;
MuscateAngles Angles;
std::string BandId;
MuscateAngles Angles;
};
struct MuscateGeometricInformations{
struct MuscateGeometricInformations
{
MuscateAnglePair MeanSunAngle;
MuscateAngles SolarAngles;
std::vector<MuscateAnglePair> MeanViewingIncidenceAngles;
std::vector<MuscateViewingAnglesGrid> ViewingAngles;
};
struct SpecialValue{
struct SpecialValue
{
std::string name;
std::string value;
SpecialValue(): name(""), value("1") {};
SpecialValue(std::string _name, std::string _value):name(_name), value(_value) {};
SpecialValue() : name(""), value("1"){};
SpecialValue(std::string _name, std::string _value) : name(_name), value(_value){};
};
struct BandInformation{
struct BandInformation
{
std::string bandID;
std::string PhysicalGain;
std::string LuminanceMax;
......@@ -224,30 +240,32 @@ struct BandInformation{
std::string SpectralResponseStep;
std::string SpectralResponseValues;
BandInformation(std::string _ID, std::string _Res, std::string _ResUnit):
bandID(_ID),
SpatialResolution(_Res),
SpatialResolutionUnit(_ResUnit)
{}
BandInformation(std::string _ID, std::string _Res, std::string _ResUnit) : bandID(_ID),
SpatialResolution(_Res),
SpatialResolutionUnit(_ResUnit)
{
}
BandInformation(): bandID(""), SpatialResolution(""), SpatialResolutionUnit("") {}
BandInformation() : bandID(""), SpatialResolution(""), SpatialResolutionUnit("") {}
};
struct MuscateRadiometricInformations{
struct MuscateRadiometricInformations
{
std::vector<SpecialValue> quantificationValues;
std::vector<SpecialValue> specialValues;
std::vector<BandInformation> Bands;
};
struct MuscateQualityIndex{
struct MuscateQualityIndex
{
std::string name;
std::string value;
MuscateQualityIndex(std::string _name, std::string _value): name(_name), value(_value) {}
MuscateQualityIndex(std::string _name, std::string _value) : name(_name), value(_value) {}
};
struct MuscateQualityProduct{
struct MuscateQualityProduct
{
std::string Level;
std::string ProductID;
std::string AcquisitionDate;
......@@ -255,15 +273,20 @@ struct MuscateQualityProduct{
std::vector<MuscateQualityIndex> GlobalIndexList;
};
struct MuscateQualityInformations{
struct MuscateQualityInformations
{
MuscateQualityProduct CurrentProduct;
std::vector<MuscateQualityProduct> ContributingProducts;
};
struct MuscateProductionInformations{
//intentionally empty;
struct MuscateProductionInformations
{
// intentionally empty;
};
struct MuscateAlgorithmInformations
{
// intentionally empty;
};
struct MuscateFileMetadata
{
MuscateHeaderMetadata Header;
......@@ -273,8 +296,9 @@ struct MuscateFileMetadata
MuscateProductOrganisation ProductOrganisation;
MuscateGeopositionInformations GeopositionInformations;
MuscateGeometricInformations GeometricInformations;
MuscateAlgorithmInformations AlgorithmInformations;
MuscateRadiometricInformations RadiometricInformations;
MuscateQualityInformations QualityInformations;
MuscateProductionInformations ProductionInformations;
MuscateProductionInformations ProductionInformations;
std::string ProductPath;
};
......@@ -111,6 +111,11 @@ private:
* @brief Format functions to create an XDoc from the MuscateProductionInformations struct
*/
XElement Format(const MuscateProductionInformations &meta);
/**
* @brief Format functions to create an XDoc from the MuscateAlgorithmInformations struct
*/
XElement Format(const MuscateAlgorithmInformations &meta);
//Specifies the Type of organizational product to be written
typedef enum{
......
......@@ -31,29 +31,33 @@
using namespace ts;
const char * ProductCreatorVenusMuscate::GetNameOfClass() const
const char *ProductCreatorVenusMuscate::GetNameOfClass() const
{
return "ProductCreatorVenusMuscate";
return "ProductCreatorVenusMuscate";
}
bool ProductCreatorVenusMuscate::createProduct(const std::string &destination){
bool ProductCreatorVenusMuscate::createProduct(const std::string &destination)
{
m_strDestRoot = destination;
std::cout << "Venus ProductFormatter: " << destination << std::endl;
LoadAllDescriptors(m_xml);
m_currentDateTime = currentDateTime(); //Production Date
m_currentDateTime = currentDateTime(); // Production Date
std::string productVersion = m_version.toString();
m_strProductDirectoryName = BuildProductDirectoryName(std::string(PRODUCT_DISTRIBUTION_FLAG), productVersion);
m_strProductFullPathOut = m_strDestRoot + "/" + m_strProductDirectoryName;
//create folder hierarchy
// create folder hierarchy
bool bDirStructBuiltOk = createAllFolders(m_strProductFullPathOut);
//Get All composite files: Refls, Weights, Flags, Dates and RGB
// Get All composite files: Refls, Weights, Flags, Dates and RGB
std::string productxs = m_products[0];
if(productxs == NONE_STRING){
if (productxs == NONE_STRING)
{
itkExceptionMacro("Cannot find filename for Venus XS resolution: " << productxs);
}else{
}
else
{
std::cout << "Found products XS: " << productxs << std::endl;
}
......@@ -87,17 +91,16 @@ bool ProductCreatorVenusMuscate::createProduct(const std::string &destination){
bDirStructBuiltOk = copyRaster<ExtractRasterType>(splitterReflsXS, 13, COMPOSITE_REFLECTANCE_RASTER, VNS_B11, false);
bDirStructBuiltOk = copyRaster<ExtractRasterType>(splitterReflsXS, 14, COMPOSITE_REFLECTANCE_RASTER, VNS_B12, false);
//////////////////////////////////
/// Count Cloudy/Snowy Pixels ///
////////////////////////////////
uint64_t cloudCounter = 0, snowCounter = 0, fullCounter = 0;
size_t maskIndex = 2; //FLG is the third image of the UpdateSynthesis raster
size_t maskIndex = 2; // FLG is the third image of the UpdateSynthesis raster
bDirStructBuiltOk = countSpecialPixels(splitterMasksXS->GetOutput()->GetNthElement(maskIndex).GetPointer(), cloudCounter, snowCounter, fullCounter);
double valCloudPercent = 100.0 * cloudCounter/fullCounter;
double valSnowPercent = 100.0 * snowCounter/fullCounter;
std::cout.setf(std::ios::fixed,std::ios::floatfield);
double valCloudPercent = 100.0 * cloudCounter / fullCounter;
double valSnowPercent = 100.0 * snowCounter / fullCounter;
std::cout.setf(std::ios::fixed, std::ios::floatfield);
std::cout.precision(3);
std::cout << "Amount of snow in the final image: " << valSnowPercent << "%" << std::endl;
std::cout << "Amount of clouds in the final image: " << valCloudPercent << "%" << std::endl;
......@@ -106,17 +109,18 @@ bool ProductCreatorVenusMuscate::createProduct(const std::string &destination){
/// Generate Quicklook ///
/////////////////////////
//Quicklook - Has to be an rgb image cause OTBQuicklook can not take multiple separate images as input
// Quicklook - Has to be an rgb image cause OTBQuicklook can not take multiple separate images as input
std::string quicklookPath = m_strProductDirectoryName + "_" + QUICKLOOK_SUFFIX + JPEG_EXTENSION;
std::vector<int> vnsRGBBands = {10,7,6};
std::vector<int> vnsRGBBands = {10, 7, 6};
int offset = 3;
bDirStructBuiltOk = bDirStructBuiltOk? generateQuicklook(productxs, quicklookPath, vnsRGBBands, m_qklMinVal, m_qklMaxVal, offset) : false;
bDirStructBuiltOk = bDirStructBuiltOk ? generateQuicklook(productxs, quicklookPath, vnsRGBBands, m_qklMinVal, m_qklMaxVal, offset) : false;
if(m_gipp.empty() == false){
bDirStructBuiltOk = bDirStructBuiltOk? copyGIPPFile(m_gipp) : false;
if (m_gipp.empty() == false)
{
bDirStructBuiltOk = bDirStructBuiltOk ? copyGIPPFile(m_gipp) : false;
}
if(bDirStructBuiltOk)
if (bDirStructBuiltOk)
{
const std::string strMetadataFileName = m_strProductFullPathOut + "/" + m_strProductDirectoryName + "_" + METADATA_CATEG + XML_EXTENSION;
std::string cloudPercent = std::to_string(int(std::round(valCloudPercent)));
......@@ -125,40 +129,48 @@ bool ProductCreatorVenusMuscate::createProduct(const std::string &destination){
std::cout << "Successfully created the L3A product" << std::endl;
return true;
}
return false;}
return false;
}
void ProductCreatorVenusMuscate::LoadAllDescriptors(const std::vector<std::string> &descriptors){
void ProductCreatorVenusMuscate::LoadAllDescriptors(const std::vector<std::string> &descriptors)
{
// load all descriptors
MuscateMetadataReaderType::Pointer MuscateMetadataReader = MuscateMetadataReaderType::New();
std::string geoInfoXML = NONE_STRING;
for (const std::string& desc : descriptors) {
for (const std::string &desc : descriptors)
{
std::unique_ptr<MuscateMetadataHelper> muscate(new MuscateMetadataHelper);
if (muscate->LoadMetadataFile(desc)) {
if (muscate->LoadMetadataFile(desc))
{
auto meta = muscate->getFullMetadata();
geoInfoXML = desc;
FillMetadataInfo(meta);
} else {
}
else
{
itkExceptionMacro("Unable to read metadata from " << desc);
}
}
//All files have been read
//First, check if all Products operate on the same tile (if not quit):
// All files have been read
// First, check if all Products operate on the same tile (if not quit):
m_productMetadata.DatasetIdentification.GeographicalZone = m_tileID;
//Then determine the platform name
// Then determine the platform name
m_mission = Venus;
//Set first Metadata Infos, cause they're used before the whole xml is created:
m_productMetadata.ProductCharacteristics.Platform = getPlatform(m_mission); //set main platform name
m_productMetadata.ProductCharacteristics.ProductLevel = "L3A"; //set output level
m_productMetadata.RadiometricInformations.Bands = getMissionBands(m_resolutions); //set the bands available for this mission
// Set first Metadata Infos, cause they're used before the whole xml is created:
m_productMetadata.ProductCharacteristics.Platform = getPlatform(m_mission); // set main platform name
m_productMetadata.ProductCharacteristics.ProductLevel = "L3A"; // set output level
m_productMetadata.RadiometricInformations.Bands = getMissionBands(m_resolutions); // set the bands available for this mission
m_productMetadata.GeopositionInformations = GetGeopositionInformations(geoInfoXML);
}
MuscateGeopositionInformations ProductCreatorVenusMuscate::GetGeopositionInformations(const std::string &desc){
MuscateGeopositionInformations ProductCreatorVenusMuscate::GetGeopositionInformations(const std::string &desc)
{
std::unique_ptr<MuscateMetadataHelper> muscate(new MuscateMetadataHelper);
if (muscate->LoadMetadataFile(desc)) {
if (muscate->LoadMetadataFile(desc))
{
auto meta = muscate->getFullMetadata();
if(meta->GeopositionInformations.GroupPosition.size() > 1){
if (meta->GeopositionInformations.GroupPosition.size() > 1)
{
itkExceptionMacro("More than one Group_Geopositioning found for Venus in " << desc);
}
return meta->GeopositionInformations;
......@@ -167,28 +179,30 @@ MuscateGeopositionInformations ProductCreatorVenusMuscate::GetGeopositionInforma
itkExceptionMacro("Error Reading GeopositionInformations from " << desc);
}
std::vector<BandInformation> ProductCreatorVenusMuscate::getMissionBands(const std::vector<std::string>& resolutions){
std::vector<BandInformation> ProductCreatorVenusMuscate::getMissionBands(const std::vector<std::string> &resolutions)
{
return {
BandInformation("B1", resolutions[0], "m"),
BandInformation("B2", resolutions[0], "m"),
BandInformation("B3", resolutions[0], "m"),
BandInformation("B4", resolutions[0], "m"),
BandInformation("B5", resolutions[0], "m"),
BandInformation("B6", resolutions[0], "m"),
BandInformation("B7", resolutions[0], "m"),
BandInformation("B8", resolutions[0], "m"),
BandInformation("B9", resolutions[0], "m"),
BandInformation("B10", resolutions[0], "m"),
BandInformation("B11", resolutions[0], "m"),
BandInformation("B12", resolutions[0], "m")
};
BandInformation("B2", resolutions[0], "m"),
BandInformation("B3", resolutions[0], "m"),
BandInformation("B4", resolutions[0], "m"),
BandInformation("B5", resolutions[0], "m"),
BandInformation("B6", resolutions[0], "m"),
BandInformation("B7", resolutions[0], "m"),
BandInformation("B8", resolutions[0], "m"),
BandInformation("B9", resolutions[0], "m"),
BandInformation("B10", resolutions[0], "m"),
BandInformation("B11", resolutions[0], "m"),
BandInformation("B12", resolutions[0], "m")};
}
std::string ProductCreatorVenusMuscate::BuildProductDirectoryName(const std::string &productType, const std::string &productVersion) {
std::string ProductCreatorVenusMuscate::BuildProductDirectoryName(const std::string &productType, const std::string &productVersion)
{
return BuildFileNameVenus(convertDateTime(m_syntDate), productType, productVersion);
}
std::string ProductCreatorVenusMuscate::BuildFileNameVenus(const std::string &datetime, const std::string &productType, const std::string &productVersion) {
std::string ProductCreatorVenusMuscate::BuildFileNameVenus(const std::string &datetime, const std::string &productType, const std::string &productVersion)
{
std::string strFileName = "{platformName}-XS_{acquisitionDate}_{productLevel}_{zone}_{productType}_V{productVersion}";
strFileName = ReplaceString(strFileName, "{platformName}", m_productMetadata.ProductCharacteristics.Platform);
strFileName = ReplaceString(strFileName, "{acquisitionDate}", datetime);
......@@ -200,45 +214,56 @@ std::string ProductCreatorVenusMuscate::BuildFileNameVenus(const std::string &da
return strFileName;
}
template<typename ExtractorType>
template <typename ExtractorType>
bool ProductCreatorVenusMuscate::copyRaster(typename ExtractorType::Pointer splitter, const size_t &index,
const rasterTypes &rasterType, resolutionTypesVenus resolution, const bool &bIsMask){
const rasterTypes &rasterType, resolutionTypesVenus resolution, const bool &bIsMask)
{
std::unordered_map<int, std::string> rasterNameMap = {
{COMPOSITE_REFLECTANCE_RASTER, std::string(COMPOSITE_REFLECTANCE_SUFFIX)},
{COMPOSITE_DATES_MASK, std::string(COMPOSITE_DATES_SUFFIX)},
{COMPOSITE_FLAGS_MASK, std::string(COMPOSITE_FLAGS_SUFFIX)},
{COMPOSITE_WEIGHTS_RASTER, std::string(COMPOSITE_WEIGHTS_SUFFIX)}};
if(m_productMetadata.RadiometricInformations.Bands.empty()){
{COMPOSITE_REFLECTANCE_RASTER, std::string(COMPOSITE_REFLECTANCE_SUFFIX)},
{COMPOSITE_DATES_MASK, std::string(COMPOSITE_DATES_SUFFIX)},
{COMPOSITE_FLAGS_MASK, std::string(COMPOSITE_FLAGS_SUFFIX)},
{COMPOSITE_WEIGHTS_RASTER, std::string(COMPOSITE_WEIGHTS_SUFFIX)}};
if (m_productMetadata.RadiometricInformations.Bands.empty())
{
itkExceptionMacro("Mission bands not set yet!")
}
std::vector<std::string> bandType;
if(bIsMask || resolution < 0){
if (bIsMask || resolution < 0)
{
resolution = resolutionTypesVenus(int(resolution) + 1);
bandType = {"XS"};
}else{
for(auto band : m_productMetadata.RadiometricInformations.Bands){
}
else
{
for (auto band : m_productMetadata.RadiometricInformations.Bands)
{
bandType.emplace_back(band.bandID);
}
}
//Add MASK-folder path if needed
// Add MASK-folder path if needed
std::string filename = (bIsMask ? std::string(MASKS_FOLDER_NAME) + "/" : "") +
m_strProductDirectoryName + "_" +
rasterNameMap[rasterType] + "_" +
bandType[resolution] +
TIF_EXTENSION;
m_strProductDirectoryName + "_" +
rasterNameMap[rasterType] + "_" +
bandType[resolution] +
TIF_EXTENSION;
std::string maskFullPathOut = m_strProductFullPathOut + "/" + filename;
std::cout << "Writing " << maskFullPathOut << std::endl;
bool copied = false;
if(m_cog){
if (m_cog)
{
copied = writeExtractedImageCoGeoTiff<ExtractorType>(splitter, m_cogtemp, maskFullPathOut, index);
}else{
}
else
{
copied = writeExtractedImage<ExtractorType>(splitter,
maskFullPathOut + (bIsMask ? "?&gdal:co:COMPRESS=DEFLATE" : ""),
index);
maskFullPathOut + (bIsMask ? "?&gdal:co:COMPRESS=DEFLATE" : ""),
index);
}
if(copied){
//create Metadatafile
switch(rasterType){
if (copied)
{
// create Metadatafile
switch (rasterType)
{
case COMPOSITE_REFLECTANCE_RASTER:
{
ImageInformation imageFile;
......@@ -300,43 +325,46 @@ bool ProductCreatorVenusMuscate::copyRaster(typename ExtractorType::Pointer spli
return false;
}
std::vector<BandGroup> ProductCreatorVenusMuscate::getBandGroupList(const std::vector<BandInformation> &missionBands){
std::vector<BandGroup> ProductCreatorVenusMuscate::getBandGroupList(const std::vector<BandInformation> &missionBands)
{
std::vector<BandGroup> result;
if(missionBands.empty()){
if (missionBands.empty())
{
itkExceptionMacro("Mission bands not set!");
}
std::vector<size_t> bandsXS = {0,1,2,3,4,5,6,7,8,9,10,11};
std::vector<size_t> bandsXS = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
BandGroup XS;
XS.GroupID = "XS";
for(size_t i : bandsXS){
for (size_t i : bandsXS)
{
XS.GroupList.bandIDs.emplace_back(missionBands[i].bandID);
}
XS.GroupList.Count = XS.GroupList.bandIDs.size();
result.emplace_back(XS);
return result;
}
void ProductCreatorVenusMuscate::generateMetadataFile(const std::string &strProductMetadataFilePath, const std::string &cloudPercent, const std::string &snowPercent){
void ProductCreatorVenusMuscate::generateMetadataFile(const std::string &strProductMetadataFilePath, const std::string &cloudPercent, const std::string &snowPercent)
{
auto writer = ts::muscate::MuscateMetadataWriter::New();
//Header
// Header
m_productMetadata.Header.SchemaLocation = "Muscate_Metadata.xsd";