OTBAlgorithmProvider.py 14.9 KB
Newer Older
volaya's avatar
volaya committed
1
2
3
# -*- coding: utf-8 -*-

"""
Rashad Kanavath's avatar
Rashad Kanavath committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/***************************************************************************
 OTBAlgorithmProvider.py
 -----------------------
        date                 : 2018-01-30
        copyright            : (C) 2018 by CNES
        email                : rkm
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
volaya's avatar
volaya committed
20
21
"""

Rashad Kanavath's avatar
Rashad Kanavath committed
22
23
24
25
__author__ = 'Rashad Kanavath'
__date__ = '2018-01-30'
__copyright__ = '(C) 2018 by CNES'

volaya's avatar
volaya committed
26
# This will get replaced with a git SHA1 when you do a git archive
Rashad Kanavath's avatar
Rashad Kanavath committed
27

volaya's avatar
volaya committed
28
29
30
__revision__ = '$Format:%H$'

import os
31
import re
volaya's avatar
volaya committed
32
from qgis.PyQt.QtGui import QIcon
Rashad Kanavath's avatar
Rashad Kanavath committed
33
from qgis.PyQt.QtCore import QCoreApplication
34
from qgis.core import (Qgis, QgsApplication, QgsProcessingProvider, QgsMessageLog)
Rashad Kanavath's avatar
Rashad Kanavath committed
35
36
from qgis import utils

volaya's avatar
volaya committed
37
from processing.core.ProcessingConfig import ProcessingConfig, Setting
Rashad Kanavath's avatar
Rashad Kanavath committed
38

39
from otb import OTBUtils
40
from otb.OTBSettings import OTBSettings
41
from otb.OTBAlgorithm import OTBAlgorithm
volaya's avatar
volaya committed
42

43
44
45
46
def otb_exe_file(f):
    if os.name == 'nt':
        return f + '.exe'
    else:
Rashad Kanavath's avatar
Rashad Kanavath committed
47
        return f
48

Rashad Kanavath's avatar
Rashad Kanavath committed
49
class OTBAlgorithmProvider(QgsProcessingProvider):
volaya's avatar
volaya committed
50
51
    def __init__(self):
        super().__init__()
Rashad Kanavath's avatar
Rashad Kanavath committed
52
        self.algs = []
53
54
        #!hack for 6.6!#
        self.version = '6.6.0'
Rashad Kanavath's avatar
Rashad Kanavath committed
55
        self.descriptionFile = ''
Rashad Kanavath's avatar
Rashad Kanavath committed
56
57

    def load(self):
58
59
        group = self.name()
        ProcessingConfig.settingIcons[group] = self.icon()
60
61
        ProcessingConfig.addSetting(Setting(group, OTBSettings.ACTIVATE, self.tr('Activate'), True))
        ProcessingConfig.addSetting(Setting(group, OTBSettings.FOLDER,
Rashad Kanavath's avatar
Rashad Kanavath committed
62
63
64
                                            self.tr("OTB folder"),
                                            OTBUtils.otbFolder(),
                                            valuetype=Setting.FOLDER,
65
                                            validator=self.validateOtbFolder
Rashad Kanavath's avatar
Rashad Kanavath committed
66
                                           ))
67
        ProcessingConfig.addSetting(Setting(group, OTBSettings.APP_FOLDER,
Rashad Kanavath's avatar
Rashad Kanavath committed
68
69
70
                                            self.tr("OTB application folder"),
                                            OTBUtils.appFolder(),
                                            valuetype=Setting.MULTIPLE_FOLDERS,
71
                                            validator=self.validateAppFolders
Rashad Kanavath's avatar
Rashad Kanavath committed
72
                                           ))
73
        ProcessingConfig.addSetting(Setting(group, OTBSettings.SRTM_FOLDER,
Rashad Kanavath's avatar
Rashad Kanavath committed
74
75
76
77
                                            self.tr("SRTM tiles folder"),
                                            OTBUtils.srtmFolder(),
                                            valuetype=Setting.FOLDER
                                           ))
78
        ProcessingConfig.addSetting(Setting(group, OTBSettings.GEOID_FILE,
Rashad Kanavath's avatar
Rashad Kanavath committed
79
80
81
82
                                            self.tr("Geoid file"),
                                            OTBUtils.geoidFile(),
                                            valuetype=Setting.FOLDER
                                           ))
83
84
85
86
87
88
89
90
91
92
93
        ProcessingConfig.addSetting(Setting(group, OTBSettings.MAX_RAM_HINT,
                                            self.tr("Maximum RAM to use"),
                                            OTBUtils.maxRAMHint(),
                                            valuetype=Setting.STRING
                                           ))
        ProcessingConfig.addSetting(Setting(group, OTBSettings.LOGGER_LEVEL,
                                            self.tr("Logger level"),
                                            OTBUtils.loggerLevel(),
                                            valuetype=Setting.STRING,
                                            validator=self.validateLoggerLevel
                                           ))
Rashad Kanavath's avatar
Rashad Kanavath committed
94
        ProcessingConfig.readSettings()
Rashad Kanavath's avatar
Rashad Kanavath committed
95
96
97
98
        self.refreshAlgorithms()
        return True

    def unload(self):
99
        for setting in OTBSettings.keys():
Rashad Kanavath's avatar
Rashad Kanavath committed
100
            ProcessingConfig.removeSetting(setting)
Rashad Kanavath's avatar
Rashad Kanavath committed
101
102

    def isActive(self):
103
        return ProcessingConfig.getSetting(OTBSettings.ACTIVATE)
Rashad Kanavath's avatar
Rashad Kanavath committed
104
105

    def setActive(self, active):
106
        ProcessingConfig.setSettingValue(OTBSettings.ACTIVATE, active)
Rashad Kanavath's avatar
Rashad Kanavath committed
107
108
109
110

    def createAlgsList(self):
        algs = []
        try:
111
112
            folder = OTBUtils.otbFolder()
            alg_names = []
113
            algs_txt = self.algsFile(folder)
Rashad Kanavath's avatar
Rashad Kanavath committed
114
115
116
            with open(algs_txt) as lines:
                line = lines.readline().strip('\n').strip()
                if line != '' and line.startswith('#'):
Rashad Kanavath's avatar
Rashad Kanavath committed
117
                    line = lines.readline().strip('\n').strip()
Rashad Kanavath's avatar
Rashad Kanavath committed
118
                while line != '' and not line.startswith('#'):
Rashad Kanavath's avatar
Rashad Kanavath committed
119
                    data = line.split('|')
Rashad Kanavath's avatar
Rashad Kanavath committed
120
                    self.descriptionFile = self.descrFile(folder, str(data[1]) + '.txt')
121
122
                    group, name = str(data[0]), str(data[1])
                    if name not in alg_names:
Rashad Kanavath's avatar
Rashad Kanavath committed
123
                        algs.append(OTBAlgorithm(group, name, self.descriptionFile))
124
125
                        #avoid duplicate algorithms from algs.txt file (possible but rare)
                        alg_names.append(name)
Rashad Kanavath's avatar
Rashad Kanavath committed
126
127
128
                    line = lines.readline().strip('\n').strip()
        except Exception as e:
            import traceback
Rashad Kanavath's avatar
Rashad Kanavath committed
129
            errmsg = "Could not open OTB algorithm from file: \n" + self.descriptionFile + "\nError:\n" +  traceback.format_exc()
130
            QgsMessageLog.logMessage(self.tr(errmsg), self.tr('Processing'), Qgis.Critical)
Rashad Kanavath's avatar
Rashad Kanavath committed
131
132
133
134
135
        return algs

    def loadAlgorithms(self):
        if not self.canBeActivated():
            return
136

137
138
139
140
141
142
143
144
145
146
147
        version_file = os.path.join(OTBUtils.otbFolder(), 'share', 'doc', 'otb', 'VERSION')
        if not os.path.isfile(version_file):
            version_file = os.path.join(OTBUtils.otbFolder(), 'VERSION')

        if os.path.isfile(version_file):
            with open(version_file) as vf:
                vlines = vf.readlines()
                vlines = [l.strip() for l in vlines]
                vline = vlines[0]
                if 'OTB Version:' in vline:
                    self.version = vline.split(':')[1].strip()
148

149
        QgsMessageLog.logMessage(self.tr("Loading OTB '{}'.".format(self.version)), self.tr('Processing'), Qgis.Info)
Rashad Kanavath's avatar
Rashad Kanavath committed
150
151
        self.algs = self.createAlgsList()
        for a in self.algs:
Rashad Kanavath's avatar
Rashad Kanavath committed
152
            self.addAlgorithm(a)
Rashad Kanavath's avatar
Rashad Kanavath committed
153
        self.algs = []
volaya's avatar
volaya committed
154

155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
        self.create_otbcli()

    def create_otbcli(self):
        otb_folder = self.normalize_path(OTBUtils.otbFolder())
        otb_app_path_env = os.pathsep.join(self.appDirs(OTBUtils.appFolder()))
        gdal_data_dir = None
        geotiff_csv_dir = None
        try:
            if os.name == 'nt':
                app_vargs = " %*"
                export_cmd = 'SET '
                first_line = ':: Setup environment for OTB package. Generated by QGIS plugin'
                otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine.exe')
                gdal_data_dir = os.path.join(otb_folder, 'share', 'data')
                geotiff_csv_dir = os.path.join(otb_folder, 'share', 'epsg_csv')
            else:
                app_vargs = " $@"
                export_cmd = 'export '
                first_line = '#!/bin/sh'
                otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine')
                lines = None
                env_profile = os.path.join(otb_folder, 'otbenv.profile')
                with open(env_profile) as f:
                    lines = f.readlines()
                    lines = [x.strip() for x in lines]
                    for line in lines:
                        if not line or line.startswith('#'):
                            continue
                        if 'GDAL_DATA=' in line:
                            gdal_data_dir = line.split("GDAL_DATA=")[1]
                        if 'GEOTIFF_CSV='in line:
                            geotiff_csv_dir = line.split("GEOTIFF_CSV=")[1]

            otbcli_path = OTBUtils.cliPath()
            with open(otbcli_path, 'w') as otb_cli_file:
                otb_cli_file.write(first_line + os.linesep)
                otb_cli_file.write(export_cmd + "LC_NUMERIC=C" + os.linesep)
                otb_cli_file.write(export_cmd + "GDAL_DRIVER_PATH=disable" + os.linesep)
                if gdal_data_dir:
                    otb_cli_file.write(export_cmd + "GDAL_DATA=" + "\"" + gdal_data_dir + "\"" + os.linesep)
                if geotiff_csv_dir:
                    otb_cli_file.write(export_cmd + "GEOTIFF_CSV=" + "\"" + geotiff_csv_dir + "\"" + os.linesep)
                if OTBUtils.loggerLevel():
                    otb_cli_file.write(export_cmd + "OTB_LOGGER_LEVEL=" + "\"" + OTBUtils.loggerLevel() + "\"" + os.linesep)
                max_ram_hint = OTBUtils.maxRAMHint()
                if max_ram_hint and  not int(max_ram_hint) == 128 :
                    otb_cli_file.write(export_cmd + "OTB_MAX_RAM_HINT=" + "\"" + max_ram_hint + "\"" + os.linesep)
                otb_cli_file.write(export_cmd + "OTB_APPLICATION_PATH=" + "\"" + otb_app_path_env + "\"" + os.linesep)
                otb_cli_file.write("\""  + otb_app_launcher + "\"" + app_vargs  + os.linesep)

            if not os.name == 'nt':
                os.chmod(otbcli_path, 0o744)
        except BaseException as e:
            import traceback
            os.remove(otbcli_path)
            errmsg = "Cannot write:" + otbcli_path + "\nError:\n" +  traceback.format_exc()
            QgsMessageLog.logMessage(self.tr(errmsg), self.tr('Processing'), Qgis.Critical)
            raise e
        QgsMessageLog.logMessage(self.tr("Using otbcli: '{}'.".format(otbcli_path)), self.tr('Processing'), Qgis.Info)

    def canBeActivated(self):
        if not self.isActive():
            return False
        folder = OTBUtils.otbFolder()
        if folder and os.path.exists(folder):
            if os.path.isfile(self.algsFile(folder)):
                return True
            utils.iface.messageBar().pushWarning("OTB", "Cannot find '{}'. OTB provider will be disabled".format(self.algsFile(folder)))
        self.setActive(False)
        return False

    def validateLoggerLevel(self, v):
        allowed_values = ['DEBUG', 'INFO', 'WARNING', 'CRITICAL', 'FATAL']
        if v in allowed_values:
            return True
        else:
            raise ValueError(self.tr("'{}' is not valid. Possible values are '{}'".format(v, ', '.join(allowed_values))))

    def validateAppFolders(self, v):
        if not self.isActive():
            return
        if not v:
            self.setActive(False)
            raise ValueError(self.tr('Cannot activate OTB provider'))

240
        folder = OTBUtils.otbFolder()
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
        otb_app_dirs = self.appDirs(v)
        if len(otb_app_dirs) < 1:
            self.setActive(False)
            raise ValueError(self.tr("'{}' does not exist. OTB provider will be disabled".format(v)))

        #isValid is True if there is atleast one valid otb application is given path
        isValid = False
        descr_folder = self.descrFolder(folder)
        for app_dir in otb_app_dirs:
            if not os.path.exists(app_dir):
                continue
            for otb_app in os.listdir(app_dir):
                if not otb_app.startswith('otbapp_') or \
                    'TestApplication' in otb_app or \
                    'ApplicationExample' in otb_app:
                    continue
                app_name = os.path.basename(otb_app).split('.')[0][7:]
                dfile = os.path.join(descr_folder, app_name + '.txt')
                isValid = True
                if not os.path.exists(dfile):
                    cmdlist = [os.path.join(
                        folder, 'bin',
                        otb_exe_file('otbQgisDescriptor')),
                               app_name, app_dir, descr_folder + '/']
                    commands = ' '.join(cmdlist)
                    QgsMessageLog.logMessage(self.tr(commands), self.tr('Processing'), Qgis.Critical)
                    OTBUtils.executeOtb(commands, feedback=None)

        if isValid:
            utils.iface.messageBar().pushInfo("OTB", "OTB provider is activated from '{}'.".format(folder))
271
        else:
272
273
            self.setActive(False)
            raise ValueError(self.tr("No OTB algorithms found in '{}'. OTB will be disabled".format(','.join(otb_app_dirs))))
274

275
276
277
278
279
280
281
282
283
284
285
286
287
288
    def normalize_path(self, p):
        # https://stackoverflow.com/a/20713238/1003090
        return os.path.normpath(os.sep.join(re.split(r'\\|/', p)))

    def validateOtbFolder(self, v):
        if not self.isActive():
            return
        if not v or not os.path.exists(v):
            self.setActive(False)
            raise ValueError(self.tr("'{}' does not exist. OTB provider will be disabled".format(v)))
        path = self.normalize_path(v)
        if not os.path.exists(os.path.join(path, 'bin', otb_exe_file('otbApplicationLauncherCommandLine'))):
            self.setActive(False)
            raise ValueError(self.tr("Cannot find '{}'. OTB will be disabled".format(os.path.join(v, 'bin', otb_exe_file('otbApplicationLauncherCommandLine')))))
289

290
    def algsFile(self, d):
291
        return os.path.join(self.descrFolder(d), 'algs.txt')
292
293

    def descrFolder(self, d):
294
        #!hack for 6.6!#
295
        if os.path.exists(os.path.join(d, 'description')):
296
297
298
            return os.path.join(d, 'description')
        else:
            return os.path.join(d, 'share', 'otb', 'description')
299
300

    def descrFile(self, d, f):
301
        return os.path.join(self.descrFolder(d), f)
302

303
    def appDirs(self, v):
304
        #!hack needed for QGIS < 3.2!#
Rashad Kanavath's avatar
Rashad Kanavath committed
305
        v = v.replace(';', os.pathsep)
306
307
        #!hack needed for QGIS < 3.2!#
        folders = v.split(os.pathsep)
308
309
310
        app_dirs = []
        for f in folders:
            if f is not None and os.path.exists(f):
311
                app_dirs.append(self.normalize_path(f))
312
        return app_dirs
Rashad Kanavath's avatar
Rashad Kanavath committed
313

volaya's avatar
volaya committed
314
    def name(self):
Rashad Kanavath's avatar
Rashad Kanavath committed
315
316
317
318
        return 'OTB'

    def longName(self):
        return 'OTB ({})'.format(self.version) if self.version is not None else 'OTB'
volaya's avatar
volaya committed
319
320

    def id(self):
Rashad Kanavath's avatar
Rashad Kanavath committed
321
322
323
324
325
326
327
        return 'otb'

    def supportsNonFileBasedOutput(self):
        """
        OTB Provider doesn't support non file based outputs
        """
        return False
volaya's avatar
volaya committed
328
329

    def icon(self):
Rashad Kanavath's avatar
Rashad Kanavath committed
330
        pluginPath = os.path.split(os.path.dirname(__file__))[0]
331
        return QIcon(os.path.join(pluginPath, 'otb', 'otb.png'))
volaya's avatar
volaya committed
332

Rashad Kanavath's avatar
Rashad Kanavath committed
333
334
335
336
    def tr(self, string, context=''):
        if context == '':
            context = 'OTBAlgorithmProvider'
        return QCoreApplication.translate(context, string)
337
338
339
340
341
342
343
344
345

    def defaultVectorFileExtension(self, hasGeometry=True):
        return 'shp'

    def defaultRasterFileExtension(self):
        return 'tif'

    def supportedOutputTableExtensions(self):
        return ['dbf']