Commit a9de1218 authored by Rashad Kanavath's avatar Rashad Kanavath

Fixes #28 #29 encoding and env variables in subprocess call

This commits fixes encoding issue using getWindowsCodePage
from Grass7Utils.py

All method in OtbUtils are decorated as static and is now in OtbUtils
class.

Instead of writing a cli_file at startup, provider now pass all
required env_variables in subprocess.popen.

Logging of output from otbalgorithm and updating progress bar is
slightly updated to work with all otb versions.
Algoirthm is launched directly using otbApplicationLauncherCommandLine
`encoding` (on windows) and env arguments passed to subprocess is
logged in QgsMessageLog
parent f7fb2b4d
......@@ -54,7 +54,7 @@ from qgis.core import (Qgis,
from processing.core.parameters import getParameterFromString
from otb.OtbChoiceWidget import OtbParameterChoice
from otb import OtbUtils
from otb.OtbUtils import OtbUtils
class OtbAlgorithm(QgsProcessingAlgorithm):
......@@ -200,8 +200,8 @@ class OtbAlgorithm(QgsProcessingAlgorithm):
return valid_params
def processAlgorithm(self, parameters, context, feedback):
otb_cli_file = OtbUtils.cliPath()
command = '"{}" {} {}'.format(otb_cli_file, self.name(), OtbUtils.appFolder())
app_launcher_path = OtbUtils.getExecutableInPath(OtbUtils.otbFolder(), 'otbApplicationLauncherCommandLine')
command = '"{}" {} {}'.format(app_launcher_path, self.name(), OtbUtils.appFolder())
outputPixelType = None
for k, v in parameters.items():
# if value is None for a parameter we don't have any businees with this key
......@@ -264,11 +264,6 @@ class OtbAlgorithm(QgsProcessingAlgorithm):
else:
command += ' -{} "{}"'.format(out.name(), filePath)
QgsMessageLog.logMessage(self.tr('cmd={}'.format(command)), self.tr('Processing'), Qgis.Info)
if not os.path.exists(otb_cli_file) or not os.path.isfile(otb_cli_file):
import errno
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), otb_cli_file)
OtbUtils.executeOtb(command, feedback)
result = {}
......
......@@ -35,18 +35,12 @@ from qgis.core import (Qgis, QgsProcessingProvider, QgsMessageLog)
from qgis import utils
from processing.core.ProcessingConfig import ProcessingConfig, Setting
from otb import OtbUtils
from otb.OtbUtils import OtbUtils
from otb.OtbSettings import OtbSettings
from otb.OtbAlgorithm import OtbAlgorithm
pluginPath = os.path.split(os.path.dirname(__file__))[0]
def otb_exe_file(f):
if os.name == 'nt':
return f + '.exe'
else:
return f
class OtbAlgorithmProvider(QgsProcessingProvider):
def __init__(self):
......@@ -153,63 +147,6 @@ class OtbAlgorithmProvider(QgsProcessingProvider):
self.addAlgorithm(a)
self.algs = []
otb_folder = self.normalize_path(OtbUtils.otbFolder())
otb_app_path_env = os.pathsep.join(self.appDirs(OtbUtils.appFolder()))
gdal_data_dir = None
geotiff_csv_dir = None
otbcli_path = OtbUtils.cliPath()
try:
if os.name == 'nt':
app_vargs = " %*"
export_cmd = 'SET '
first_line = ':: Setup environment for OTB package. Generated by QGIS plugin'
otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine.exe')
gdal_data_dir = os.path.join(otb_folder, 'share', 'data')
geotiff_csv_dir = os.path.join(otb_folder, 'share', 'epsg_csv')
else:
app_vargs = " \"$@\""
export_cmd = 'export '
first_line = '#!/bin/sh'
otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine')
lines = None
env_profile = os.path.join(otb_folder, 'otbenv.profile')
if os.path.exists(env_profile):
with open(env_profile) as f:
lines = f.readlines()
lines = [x.strip() for x in lines]
for line in lines:
if not line or line.startswith('#'):
continue
if 'GDAL_DATA=' in line:
gdal_data_dir = line.split("GDAL_DATA=")[1]
if 'GEOTIFF_CSV='in line:
geotiff_csv_dir = line.split("GEOTIFF_CSV=")[1]
with open(otbcli_path, 'w') as otb_cli_file:
otb_cli_file.write(first_line + os.linesep)
otb_cli_file.write(export_cmd + "LC_NUMERIC=C" + os.linesep)
otb_cli_file.write(export_cmd + "GDAL_DRIVER_PATH=disable" + os.linesep)
if gdal_data_dir:
otb_cli_file.write(export_cmd + "GDAL_DATA=" + "\"" + gdal_data_dir + "\"" + os.linesep)
if geotiff_csv_dir:
otb_cli_file.write(export_cmd + "GEOTIFF_CSV=" + "\"" + geotiff_csv_dir + "\"" + os.linesep)
if OtbUtils.loggerLevel():
otb_cli_file.write(export_cmd + "OTB_LOGGER_LEVEL=" + OtbUtils.loggerLevel() + os.linesep)
max_ram_hint = OtbUtils.maxRAMHint()
if max_ram_hint and not int(max_ram_hint) == 128:
otb_cli_file.write(export_cmd + "OTB_MAX_RAM_HINT=" + max_ram_hint + os.linesep)
otb_cli_file.write(export_cmd + "OTB_APPLICATION_PATH=" + "\"" + otb_app_path_env + "\"" + os.linesep)
otb_cli_file.write("\"" + otb_app_launcher + "\"" + app_vargs + os.linesep)
if not os.name == 'nt':
os.chmod(otbcli_path, 0o744)
except BaseException as e:
import traceback
os.remove(otbcli_path)
errmsg = "Cannot write:" + otbcli_path + "\nError:\n" + traceback.format_exc()
QgsMessageLog.logMessage(self.tr(errmsg), self.tr('Processing'), Qgis.Critical)
raise e
QgsMessageLog.logMessage(self.tr("Using otbcli: '{}'.".format(otbcli_path)), self.tr('Processing'), Qgis.Info)
def canBeActivated(self):
if not self.isActive():
return False
......@@ -256,10 +193,8 @@ class OtbAlgorithmProvider(QgsProcessingProvider):
dfile = os.path.join(descr_folder, app_name + '.txt')
isValid = True
if not os.path.exists(dfile):
cmdlist = [os.path.join(
folder, 'bin',
otb_exe_file('otbQgisDescriptor')),
app_name, app_dir, descr_folder + '/']
cmdlist = [OtbUtils.getExecutableInPath(folder, 'otbQgisDescriptor'),
app_name, app_dir, descr_folder + '/']
commands = ' '.join(cmdlist)
QgsMessageLog.logMessage(self.tr(commands), self.tr('Processing'), Qgis.Critical)
OtbUtils.executeOtb(commands, feedback=None)
......@@ -283,9 +218,10 @@ class OtbAlgorithmProvider(QgsProcessingProvider):
self.setActive(False)
raise ValueError(self.tr("'{}' does not exist. OTB provider will be disabled".format(v)))
path = self.normalize_path(v)
if not os.path.exists(os.path.join(path, 'bin', otb_exe_file('otbApplicationLauncherCommandLine'))):
app_launcher_path = OtbUtils.getExecutableInPath(path, 'otbApplicationLauncherCommandLine')
if not os.path.exists(app_launcher_path):
self.setActive(False)
raise ValueError(self.tr("Cannot find '{}'. OTB will be disabled".format(os.path.join(v, 'bin', otb_exe_file('otbApplicationLauncherCommandLine')))))
raise ValueError(self.tr("Cannot find '{}'. OTB will be disabled".format(app_launcher_path)))
def algsFile(self, d):
return os.path.join(self.descrFolder(d), 'algs.txt')
......
......@@ -40,92 +40,144 @@ from qgis.PyQt.QtCore import QCoreApplication
from otb.OtbSettings import OtbSettings
def cliPath():
cli_ext = '.bat' if os.name == 'nt' else ''
return os.path.normpath(os.path.join(QgsApplication.qgisSettingsDirPath(),
'processing', 'qgis_otb_cli' + cli_ext))
class OtbUtils:
@staticmethod
def version():
return ProcessingConfig.getSetting(OtbSettings.VERSION) or '0.0.0'
@staticmethod
def loggerLevel():
return ProcessingConfig.getSetting(OtbSettings.LOGGER_LEVEL) or 'INFO'
@staticmethod
def maxRAMHint():
return ProcessingConfig.getSetting(OtbSettings.MAX_RAM_HINT) or ''
@staticmethod
def otbFolder():
if ProcessingConfig.getSetting(OtbSettings.FOLDER):
return os.path.normpath(os.sep.join(re.split(r'\\|/', ProcessingConfig.getSetting(OtbSettings.FOLDER))))
else:
return None
@staticmethod
def appFolder():
app_folder = ProcessingConfig.getSetting(OtbSettings.APP_FOLDER)
if app_folder:
return os.pathsep.join(app_folder.split(';'))
else:
return None
@staticmethod
def srtmFolder():
return ProcessingConfig.getSetting(OtbSettings.SRTM_FOLDER) or ''
@staticmethod
def geoidFile():
return ProcessingConfig.getSetting(OtbSettings.GEOID_FILE) or ''
@staticmethod
def getExecutableInPath(path, exe):
ext = '.exe' if os.name == 'nt' else ''
return os.path.join(path, 'bin', exe + ext)
@staticmethod
def getAuxiliaryDataDirectories():
gdal_data_dir = None
gtiff_csv_dir = None
otb_folder = OtbUtils.otbFolder()
if os.name == 'nt':
gdal_data_dir = os.path.join(otb_folder, 'share', 'data')
gtiff_csv_dir = os.path.join(otb_folder, 'share', 'epsg_csv')
else:
env_profile = os.path.join(otb_folder, 'otbenv.profile')
try:
if os.path.exists(env_profile):
with open(env_profile) as f:
lines = f.readlines()
lines = [x.strip() for x in lines]
for line in lines:
if not line or line.startswith('#'):
continue
if 'GDAL_DATA=' in line:
gdal_data_dir = line.split("GDAL_DATA=")[1]
if 'GEOTIFF_CSV='in line:
gtiff_csv_dir = line.split("GEOTIFF_CSV=")[1]
except BaseException as exc:
errmsg = "Cannot find gdal and geotiff data directory." + str(exc)
QgsMessageLog.logMessage(errmsg, OtbUtils.tr('Processing'), Qgis.Info)
pass
return gdal_data_dir, gtiff_csv_dir
@staticmethod
def executeOtb(commands, feedback, addToLog=True):
otb_env = {
'LC_NUMERIC': 'C',
'GDAL_DRIVER_PATH': 'disable'
}
gdal_data_dir, gtiff_csv_dir = OtbUtils.getAuxiliaryDataDirectories()
if gdal_data_dir and os.path.exists(gdal_data_dir):
otb_env['GDAL_DATA'] = gdal_data_dir
if gtiff_csv_dir and os.path.exists(gtiff_csv_dir):
otb_env['GEOTIFF_CSV'] = gtiff_csv_dir
otb_env['OTB_LOGGER_LEVEL'] = OtbUtils.loggerLevel()
max_ram_hint = OtbUtils.maxRAMHint()
if max_ram_hint and int(max_ram_hint) > 256:
otb_env['OTB_MAX_RAM_HINT'] = max_ram_hint
kw = {}
kw['env'] = otb_env
if os.name == 'nt':
kw['encoding'] = "cp{}".format(OtbUtils.getWindowsCodePage())
QgsMessageLog.logMessage("{}".format(kw), OtbUtils.tr('Processing'), Qgis.Info)
QgsMessageLog.logMessage("cmd={}".format(commands), OtbUtils.tr('Processing'), Qgis.Info)
with subprocess.Popen(
commands,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
**kw
) as proc:
def version():
return ProcessingConfig.getSetting(OtbSettings.VERSION) or '0.0.0'
def loggerLevel():
return ProcessingConfig.getSetting(OtbSettings.LOGGER_LEVEL) or 'INFO'
def maxRAMHint():
return ProcessingConfig.getSetting(OtbSettings.MAX_RAM_HINT) or ''
def otbFolder():
if ProcessingConfig.getSetting(OtbSettings.FOLDER):
return os.path.normpath(os.sep.join(re.split(r'\\|/', ProcessingConfig.getSetting(OtbSettings.FOLDER))))
else:
return None
def appFolder():
app_folder = ProcessingConfig.getSetting(OtbSettings.APP_FOLDER)
if app_folder:
return os.pathsep.join(app_folder.split(';'))
else:
return None
def srtmFolder():
return ProcessingConfig.getSetting(OtbSettings.SRTM_FOLDER) or ''
def geoidFile():
return ProcessingConfig.getSetting(OtbSettings.GEOID_FILE) or ''
def executeOtb(command, feedback, addToLog=True):
loglines = []
with subprocess.Popen(
[command],
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True
) as proc:
try:
for line in iter(proc.stdout.readline, ''):
line = line.strip()
#'* ]' and ' ]' says its some progress update
#print('line[-3:]',line[-3:])
if line[-3:] == "* ]" or line[-3:] == " ]":
if '% [' in line:
part = line.split(':')[1]
percent = part.split('%')[0]
try:
if int(percent) >= 100:
loglines.append(line)
feedback.pushConsoleInfo(line)
feedback.setProgress(int(percent))
except:
pass
else:
loglines.append(line)
except BaseException as e:
loglines.append(str(e))
pass
for logline in loglines:
if feedback is None:
QgsMessageLog.logMessage(logline, 'Processing', Qgis.Info)
else:
feedback.pushConsoleInfo(logline)
# for logline in loglines:
# if 'INFO' in logline or 'FATAL' in logline:
# if feedback is None:
# QgsMessageLog.logMessage(logline, 'Processing', Qgis.Info)
# else:
# feedback.pushConsoleInfo(logline)
def tr(string, context=''):
if context == '':
context = 'OtbUtils'
return QCoreApplication.translate(context, string)
if feedback is None:
QgsMessageLog.logMessage(line, OtbUtils.tr('Processing'), Qgis.Info)
else:
if any([l in line for l in ['(WARNING)', '(FATAL)', 'ERROR']]):
feedback.reportError(line)
else:
feedback.pushConsoleInfo(line.strip())
@staticmethod
def getWindowsCodePage():
"""
Determines MS-Windows CMD.exe shell codepage.
Used into GRASS exec script under MS-Windows.
"""
from ctypes import cdll
return str(cdll.kernel32.GetACP())
@staticmethod
def tr(string, context=''):
if context == '':
context = 'OtbUtils'
return QCoreApplication.translate(context, string)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment