OtbAlgorithmProvider.py 14.9 KB
Newer Older
volaya's avatar
volaya committed
1
2
3
# -*- coding: utf-8 -*-

"""
Rashad Kanavath's avatar
Rashad Kanavath committed
4
/***************************************************************************
5
6
7
8
9
10
    OtbAlgorithmProvider.py
    -----------------------
    Date                 : 2018-01-30
    Copyright            : (C) 2018 by CNES
    Email                : rashad dot kanavath at c-s fr
****************************************************************************/
Rashad Kanavath's avatar
Rashad Kanavath committed
11
12
13
14
15
16
17
18
19

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
volaya's avatar
volaya committed
20
21
"""

Rashad Kanavath's avatar
Rashad Kanavath committed
22
23
24
25
__author__ = 'Rashad Kanavath'
__date__ = '2018-01-30'
__copyright__ = '(C) 2018 by CNES'

volaya's avatar
volaya committed
26
# This will get replaced with a git SHA1 when you do a git archive
Rashad Kanavath's avatar
Rashad Kanavath committed
27

volaya's avatar
volaya committed
28
29
30
__revision__ = '$Format:%H$'

import os
31
import re
volaya's avatar
volaya committed
32
from qgis.PyQt.QtGui import QIcon
Rashad Kanavath's avatar
Rashad Kanavath committed
33
from qgis.PyQt.QtCore import QCoreApplication
34
from qgis.core import (Qgis, QgsProcessingProvider, QgsMessageLog)
Rashad Kanavath's avatar
Rashad Kanavath committed
35
36
from qgis import utils

volaya's avatar
volaya committed
37
from processing.core.ProcessingConfig import ProcessingConfig, Setting
38
39
40
from otb import OtbUtils
from otb.OtbSettings import OtbSettings
from otb.OtbAlgorithm import OtbAlgorithm
volaya's avatar
volaya committed
41

42
43
pluginPath = os.path.split(os.path.dirname(__file__))[0]

44
45
46
47
def otb_exe_file(f):
    if os.name == 'nt':
        return f + '.exe'
    else:
Rashad Kanavath's avatar
Rashad Kanavath committed
48
        return f
49

50

51
class OtbAlgorithmProvider(QgsProcessingProvider):
volaya's avatar
volaya committed
52
53
    def __init__(self):
        super().__init__()
Rashad Kanavath's avatar
Rashad Kanavath committed
54
        self.algs = []
55
56
        #!hack for 6.6!#
        self.version = '6.6.0'
Rashad Kanavath's avatar
Rashad Kanavath committed
57
58

    def load(self):
59
60
        group = self.name()
        ProcessingConfig.settingIcons[group] = self.icon()
61
62
        ProcessingConfig.addSetting(Setting(group, OtbSettings.ACTIVATE, self.tr('Activate'), True))
        ProcessingConfig.addSetting(Setting(group, OtbSettings.FOLDER,
Rashad Kanavath's avatar
Rashad Kanavath committed
63
                                            self.tr("OTB folder"),
64
                                            OtbUtils.otbFolder(),
Rashad Kanavath's avatar
Rashad Kanavath committed
65
                                            valuetype=Setting.FOLDER,
66
                                            validator=self.validateOtbFolder
67
                                            ))
68
        ProcessingConfig.addSetting(Setting(group, OtbSettings.APP_FOLDER,
Rashad Kanavath's avatar
Rashad Kanavath committed
69
                                            self.tr("OTB application folder"),
70
                                            OtbUtils.appFolder(),
Rashad Kanavath's avatar
Rashad Kanavath committed
71
                                            valuetype=Setting.MULTIPLE_FOLDERS,
72
                                            validator=self.validateAppFolders
73
                                            ))
74
        ProcessingConfig.addSetting(Setting(group, OtbSettings.SRTM_FOLDER,
Rashad Kanavath's avatar
Rashad Kanavath committed
75
                                            self.tr("SRTM tiles folder"),
76
                                            OtbUtils.srtmFolder(),
Rashad Kanavath's avatar
Rashad Kanavath committed
77
                                            valuetype=Setting.FOLDER
78
                                            ))
79
        ProcessingConfig.addSetting(Setting(group, OtbSettings.GEOID_FILE,
Rashad Kanavath's avatar
Rashad Kanavath committed
80
                                            self.tr("Geoid file"),
81
                                            OtbUtils.geoidFile(),
Rashad Kanavath's avatar
Rashad Kanavath committed
82
                                            valuetype=Setting.FOLDER
83
                                            ))
84
        ProcessingConfig.addSetting(Setting(group, OtbSettings.MAX_RAM_HINT,
85
                                            self.tr("Maximum RAM to use"),
86
                                            OtbUtils.maxRAMHint(),
87
                                            valuetype=Setting.STRING
88
                                            ))
89
        ProcessingConfig.addSetting(Setting(group, OtbSettings.LOGGER_LEVEL,
90
                                            self.tr("Logger level"),
91
                                            OtbUtils.loggerLevel(),
92
93
                                            valuetype=Setting.STRING,
                                            validator=self.validateLoggerLevel
94
                                            ))
Rashad Kanavath's avatar
Rashad Kanavath committed
95
        ProcessingConfig.readSettings()
Rashad Kanavath's avatar
Rashad Kanavath committed
96
97
98
99
        self.refreshAlgorithms()
        return True

    def unload(self):
100
        for setting in OtbSettings.keys():
Rashad Kanavath's avatar
Rashad Kanavath committed
101
            ProcessingConfig.removeSetting(setting)
Rashad Kanavath's avatar
Rashad Kanavath committed
102
103

    def isActive(self):
104
        return ProcessingConfig.getSetting(OtbSettings.ACTIVATE)
Rashad Kanavath's avatar
Rashad Kanavath committed
105
106

    def setActive(self, active):
107
        ProcessingConfig.setSettingValue(OtbSettings.ACTIVATE, active)
Rashad Kanavath's avatar
Rashad Kanavath committed
108
109
110
111

    def createAlgsList(self):
        algs = []
        try:
112
            folder = OtbUtils.otbFolder()
113
            alg_names = []
114
            algs_txt = self.algsFile(folder)
Rashad Kanavath's avatar
Rashad Kanavath committed
115
116
117
            with open(algs_txt) as lines:
                line = lines.readline().strip('\n').strip()
                if line != '' and line.startswith('#'):
Rashad Kanavath's avatar
Rashad Kanavath committed
118
                    line = lines.readline().strip('\n').strip()
Rashad Kanavath's avatar
Rashad Kanavath committed
119
                while line != '' and not line.startswith('#'):
Rashad Kanavath's avatar
Rashad Kanavath committed
120
                    data = line.split('|')
121
                    descriptionFile = self.descrFile(folder, str(data[1]) + '.txt')
122
123
                    group, name = str(data[0]), str(data[1])
                    if name not in alg_names:
124
                        algs.append(OtbAlgorithm(group, name, descriptionFile))
125
126
                        #avoid duplicate algorithms from algs.txt file (possible but rare)
                        alg_names.append(name)
Rashad Kanavath's avatar
Rashad Kanavath committed
127
128
129
                    line = lines.readline().strip('\n').strip()
        except Exception as e:
            import traceback
130
            errmsg = "Could not open OTB algorithm from file: \n" + descriptionFile + "\nError:\n" + traceback.format_exc()
131
            QgsMessageLog.logMessage(self.tr(errmsg), self.tr('Processing'), Qgis.Critical)
Rashad Kanavath's avatar
Rashad Kanavath committed
132
133
134
135
136
        return algs

    def loadAlgorithms(self):
        if not self.canBeActivated():
            return
137

138
        version_file = os.path.join(OtbUtils.otbFolder(), 'share', 'doc', 'otb', 'VERSION')
139
        if not os.path.isfile(version_file):
140
            version_file = os.path.join(OtbUtils.otbFolder(), 'VERSION')
141
142
143
144
145
146
147
148

        if os.path.isfile(version_file):
            with open(version_file) as vf:
                vlines = vf.readlines()
                vlines = [l.strip() for l in vlines]
                vline = vlines[0]
                if 'OTB Version:' in vline:
                    self.version = vline.split(':')[1].strip()
149

150
        QgsMessageLog.logMessage(self.tr("Loading OTB '{}'.".format(self.version)), self.tr('Processing'), Qgis.Info)
Rashad Kanavath's avatar
Rashad Kanavath committed
151
152
        self.algs = self.createAlgsList()
        for a in self.algs:
Rashad Kanavath's avatar
Rashad Kanavath committed
153
            self.addAlgorithm(a)
Rashad Kanavath's avatar
Rashad Kanavath committed
154
        self.algs = []
volaya's avatar
volaya committed
155

156
157
        otb_folder = self.normalize_path(OtbUtils.otbFolder())
        otb_app_path_env = os.pathsep.join(self.appDirs(OtbUtils.appFolder()))
158
159
        gdal_data_dir = None
        geotiff_csv_dir = None
160
        otbcli_path = OtbUtils.cliPath()
161
162
163
164
165
166
167
168
169
        try:
            if os.name == 'nt':
                app_vargs = " %*"
                export_cmd = 'SET '
                first_line = ':: Setup environment for OTB package. Generated by QGIS plugin'
                otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine.exe')
                gdal_data_dir = os.path.join(otb_folder, 'share', 'data')
                geotiff_csv_dir = os.path.join(otb_folder, 'share', 'epsg_csv')
            else:
170
                app_vargs = " \"$@\""
171
172
173
174
175
                export_cmd = 'export '
                first_line = '#!/bin/sh'
                otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine')
                lines = None
                env_profile = os.path.join(otb_folder, 'otbenv.profile')
176
177
178
179
180
181
182
183
184
185
186
                if os.path.exists(env_profile):
                    with open(env_profile) as f:
                        lines = f.readlines()
                        lines = [x.strip() for x in lines]
                        for line in lines:
                            if not line or line.startswith('#'):
                                continue
                            if 'GDAL_DATA=' in line:
                                gdal_data_dir = line.split("GDAL_DATA=")[1]
                            if 'GEOTIFF_CSV='in line:
                                geotiff_csv_dir = line.split("GEOTIFF_CSV=")[1]
187
188
189
190
191
192
193
194
            with open(otbcli_path, 'w') as otb_cli_file:
                otb_cli_file.write(first_line + os.linesep)
                otb_cli_file.write(export_cmd + "LC_NUMERIC=C" + os.linesep)
                otb_cli_file.write(export_cmd + "GDAL_DRIVER_PATH=disable" + os.linesep)
                if gdal_data_dir:
                    otb_cli_file.write(export_cmd + "GDAL_DATA=" + "\"" + gdal_data_dir + "\"" + os.linesep)
                if geotiff_csv_dir:
                    otb_cli_file.write(export_cmd + "GEOTIFF_CSV=" + "\"" + geotiff_csv_dir + "\"" + os.linesep)
195
                if OtbUtils.loggerLevel():
196
                    otb_cli_file.write(export_cmd + "OTB_LOGGER_LEVEL=" + OtbUtils.loggerLevel() + os.linesep)
197
                max_ram_hint = OtbUtils.maxRAMHint()
198
199
                if max_ram_hint and not int(max_ram_hint) == 128:
                    otb_cli_file.write(export_cmd + "OTB_MAX_RAM_HINT=" + max_ram_hint + os.linesep)
200
                otb_cli_file.write(export_cmd + "OTB_APPLICATION_PATH=" + "\"" + otb_app_path_env + "\"" + os.linesep)
201
                otb_cli_file.write("\"" + otb_app_launcher + "\"" + app_vargs + os.linesep)
202
203
204
205
206
207

            if not os.name == 'nt':
                os.chmod(otbcli_path, 0o744)
        except BaseException as e:
            import traceback
            os.remove(otbcli_path)
208
            errmsg = "Cannot write:" + otbcli_path + "\nError:\n" + traceback.format_exc()
209
210
211
212
213
214
215
            QgsMessageLog.logMessage(self.tr(errmsg), self.tr('Processing'), Qgis.Critical)
            raise e
        QgsMessageLog.logMessage(self.tr("Using otbcli: '{}'.".format(otbcli_path)), self.tr('Processing'), Qgis.Info)

    def canBeActivated(self):
        if not self.isActive():
            return False
216
        folder = OtbUtils.otbFolder()
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
        if folder and os.path.exists(folder):
            if os.path.isfile(self.algsFile(folder)):
                return True
            utils.iface.messageBar().pushWarning("OTB", "Cannot find '{}'. OTB provider will be disabled".format(self.algsFile(folder)))
        self.setActive(False)
        return False

    def validateLoggerLevel(self, v):
        allowed_values = ['DEBUG', 'INFO', 'WARNING', 'CRITICAL', 'FATAL']
        if v in allowed_values:
            return True
        else:
            raise ValueError(self.tr("'{}' is not valid. Possible values are '{}'".format(v, ', '.join(allowed_values))))

    def validateAppFolders(self, v):
        if not self.isActive():
            return
        if not v:
            self.setActive(False)
            raise ValueError(self.tr('Cannot activate OTB provider'))

238
        folder = OtbUtils.otbFolder()
239
240
241
242
243
244
245
246
247
248
249
250
251
252
        otb_app_dirs = self.appDirs(v)
        if len(otb_app_dirs) < 1:
            self.setActive(False)
            raise ValueError(self.tr("'{}' does not exist. OTB provider will be disabled".format(v)))

        #isValid is True if there is atleast one valid otb application is given path
        isValid = False
        descr_folder = self.descrFolder(folder)
        for app_dir in otb_app_dirs:
            if not os.path.exists(app_dir):
                continue
            for otb_app in os.listdir(app_dir):
                if not otb_app.startswith('otbapp_') or \
                    'TestApplication' in otb_app or \
253
                        'ApplicationExample' in otb_app:
254
255
256
257
258
259
260
261
                    continue
                app_name = os.path.basename(otb_app).split('.')[0][7:]
                dfile = os.path.join(descr_folder, app_name + '.txt')
                isValid = True
                if not os.path.exists(dfile):
                    cmdlist = [os.path.join(
                        folder, 'bin',
                        otb_exe_file('otbQgisDescriptor')),
262
                        app_name, app_dir, descr_folder + '/']
263
264
                    commands = ' '.join(cmdlist)
                    QgsMessageLog.logMessage(self.tr(commands), self.tr('Processing'), Qgis.Critical)
265
                    OtbUtils.executeOtb(commands, feedback=None)
266
267

        if isValid:
268
269
270
            # if check needed for testsing
            if utils.iface is not None:
                utils.iface.messageBar().pushInfo("OTB", "OTB provider is activated from '{}'.".format(folder))
271
        else:
272
273
            self.setActive(False)
            raise ValueError(self.tr("No OTB algorithms found in '{}'. OTB will be disabled".format(','.join(otb_app_dirs))))
274

275
276
277
278
279
280
281
282
283
284
285
286
287
288
    def normalize_path(self, p):
        # https://stackoverflow.com/a/20713238/1003090
        return os.path.normpath(os.sep.join(re.split(r'\\|/', p)))

    def validateOtbFolder(self, v):
        if not self.isActive():
            return
        if not v or not os.path.exists(v):
            self.setActive(False)
            raise ValueError(self.tr("'{}' does not exist. OTB provider will be disabled".format(v)))
        path = self.normalize_path(v)
        if not os.path.exists(os.path.join(path, 'bin', otb_exe_file('otbApplicationLauncherCommandLine'))):
            self.setActive(False)
            raise ValueError(self.tr("Cannot find '{}'. OTB will be disabled".format(os.path.join(v, 'bin', otb_exe_file('otbApplicationLauncherCommandLine')))))
289

290
    def algsFile(self, d):
291
        return os.path.join(self.descrFolder(d), 'algs.txt')
292
293

    def descrFolder(self, d):
294
        #!hack for 6.6!#
295
        if os.path.exists(os.path.join(d, 'description')):
296
297
298
            return os.path.join(d, 'description')
        else:
            return os.path.join(d, 'share', 'otb', 'description')
299
300

    def descrFile(self, d, f):
301
        return os.path.join(self.descrFolder(d), f)
302

303
    def appDirs(self, v):
304
        app_dirs = []
305
        for f in v.split(';'):
306
            if f is not None and os.path.exists(f):
307
                app_dirs.append(self.normalize_path(f))
308
        return app_dirs
Rashad Kanavath's avatar
Rashad Kanavath committed
309

volaya's avatar
volaya committed
310
    def name(self):
Rashad Kanavath's avatar
Rashad Kanavath committed
311
312
313
314
        return 'OTB'

    def longName(self):
        return 'OTB ({})'.format(self.version) if self.version is not None else 'OTB'
volaya's avatar
volaya committed
315
316

    def id(self):
Rashad Kanavath's avatar
Rashad Kanavath committed
317
318
319
320
321
322
323
        return 'otb'

    def supportsNonFileBasedOutput(self):
        """
        OTB Provider doesn't support non file based outputs
        """
        return False
volaya's avatar
volaya committed
324
325

    def icon(self):
Rashad Kanavath's avatar
Rashad Kanavath committed
326
        pluginPath = os.path.split(os.path.dirname(__file__))[0]
327
        return QIcon(os.path.join(pluginPath, 'otb', 'providerOtb.svg'))
volaya's avatar
volaya committed
328

Rashad Kanavath's avatar
Rashad Kanavath committed
329
330
    def tr(self, string, context=''):
        if context == '':
331
            context = 'OtbAlgorithmProvider'
Rashad Kanavath's avatar
Rashad Kanavath committed
332
        return QCoreApplication.translate(context, string)
333
334
335
336
337
338
339
340
341

    def defaultVectorFileExtension(self, hasGeometry=True):
        return 'shp'

    def defaultRasterFileExtension(self):
        return 'tif'

    def supportedOutputTableExtensions(self):
        return ['dbf']