Commit a9de1218 authored by Rashad Kanavath's avatar Rashad Kanavath
Browse files

Fixes #28 #29 encoding and env variables in subprocess call 

This commits fixes encoding issue using getWindowsCodePage
from Grass7Utils.py

All method in OtbUtils are decorated as static and is now in OtbUtils
class.

Instead of writing a cli_file at startup, provider now pass all
required env_variables in subprocess.popen.

Logging of output from otbalgorithm and updating progress bar is
slightly updated to work with all otb versions.
Algoirthm is launched directly using otbApplicationLauncherCommandLine
`encoding` (on windows) and env arguments passed to subprocess is
logged in QgsMessageLog
parent f7fb2b4d

otb/OtbAlgorithm.py

+3 −8
Original line number Diff line number Diff line
@@ -54,7 +54,7 @@ from qgis.core import (Qgis, 


from processing.core.parameters import getParameterFromString

from otb.OtbChoiceWidget import OtbParameterChoice

from otb import OtbUtils

from otb.OtbUtils import OtbUtils





class OtbAlgorithm(QgsProcessingAlgorithm):
@@ -200,8 +200,8 @@ class OtbAlgorithm(QgsProcessingAlgorithm): 
        return valid_params



    def processAlgorithm(self, parameters, context, feedback):

        otb_cli_file = OtbUtils.cliPath()

        command = '"{}" {} {}'.format(otb_cli_file, self.name(), OtbUtils.appFolder())

        app_launcher_path = OtbUtils.getExecutableInPath(OtbUtils.otbFolder(), 'otbApplicationLauncherCommandLine')

        command = '"{}" {} {}'.format(app_launcher_path, self.name(), OtbUtils.appFolder())

        outputPixelType = None

        for k, v in parameters.items():

            # if value is None for a parameter we don't have any businees with this key
@@ -264,11 +264,6 @@ class OtbAlgorithm(QgsProcessingAlgorithm): 
            else:

                command += ' -{} "{}"'.format(out.name(), filePath)



        QgsMessageLog.logMessage(self.tr('cmd={}'.format(command)), self.tr('Processing'), Qgis.Info)

        if not os.path.exists(otb_cli_file) or not os.path.isfile(otb_cli_file):

            import errno

            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), otb_cli_file)



        OtbUtils.executeOtb(command, feedback)



        result = {}

otb/OtbAlgorithmProvider.py

+6 −70
Original line number Diff line number Diff line
@@ -35,18 +35,12 @@ from qgis.core import (Qgis, QgsProcessingProvider, QgsMessageLog) 
from qgis import utils



from processing.core.ProcessingConfig import ProcessingConfig, Setting

from otb import OtbUtils

from otb.OtbUtils import OtbUtils

from otb.OtbSettings import OtbSettings

from otb.OtbAlgorithm import OtbAlgorithm



pluginPath = os.path.split(os.path.dirname(__file__))[0]



def otb_exe_file(f):

    if os.name == 'nt':

        return f + '.exe'

    else:

        return f





class OtbAlgorithmProvider(QgsProcessingProvider):

    def __init__(self):
@@ -153,63 +147,6 @@ class OtbAlgorithmProvider(QgsProcessingProvider): 
            self.addAlgorithm(a)

        self.algs = []



        otb_folder = self.normalize_path(OtbUtils.otbFolder())

        otb_app_path_env = os.pathsep.join(self.appDirs(OtbUtils.appFolder()))

        gdal_data_dir = None

        geotiff_csv_dir = None

        otbcli_path = OtbUtils.cliPath()

        try:

            if os.name == 'nt':

                app_vargs = " %*"

                export_cmd = 'SET '

                first_line = ':: Setup environment for OTB package. Generated by QGIS plugin'

                otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine.exe')

                gdal_data_dir = os.path.join(otb_folder, 'share', 'data')

                geotiff_csv_dir = os.path.join(otb_folder, 'share', 'epsg_csv')

            else:

                app_vargs = " \"$@\""

                export_cmd = 'export '

                first_line = '#!/bin/sh'

                otb_app_launcher = os.path.join(otb_folder, 'bin', 'otbApplicationLauncherCommandLine')

                lines = None

                env_profile = os.path.join(otb_folder, 'otbenv.profile')

                if os.path.exists(env_profile):

                    with open(env_profile) as f:

                        lines = f.readlines()

                        lines = [x.strip() for x in lines]

                        for line in lines:

                            if not line or line.startswith('#'):

                                continue

                            if 'GDAL_DATA=' in line:

                                gdal_data_dir = line.split("GDAL_DATA=")[1]

                            if 'GEOTIFF_CSV='in line:

                                geotiff_csv_dir = line.split("GEOTIFF_CSV=")[1]

            with open(otbcli_path, 'w') as otb_cli_file:

                otb_cli_file.write(first_line + os.linesep)

                otb_cli_file.write(export_cmd + "LC_NUMERIC=C" + os.linesep)

                otb_cli_file.write(export_cmd + "GDAL_DRIVER_PATH=disable" + os.linesep)

                if gdal_data_dir:

                    otb_cli_file.write(export_cmd + "GDAL_DATA=" + "\"" + gdal_data_dir + "\"" + os.linesep)

                if geotiff_csv_dir:

                    otb_cli_file.write(export_cmd + "GEOTIFF_CSV=" + "\"" + geotiff_csv_dir + "\"" + os.linesep)

                if OtbUtils.loggerLevel():

                    otb_cli_file.write(export_cmd + "OTB_LOGGER_LEVEL=" + OtbUtils.loggerLevel() + os.linesep)

                max_ram_hint = OtbUtils.maxRAMHint()

                if max_ram_hint and not int(max_ram_hint) == 128:

                    otb_cli_file.write(export_cmd + "OTB_MAX_RAM_HINT=" + max_ram_hint + os.linesep)

                otb_cli_file.write(export_cmd + "OTB_APPLICATION_PATH=" + "\"" + otb_app_path_env + "\"" + os.linesep)

                otb_cli_file.write("\"" + otb_app_launcher + "\"" + app_vargs + os.linesep)



            if not os.name == 'nt':

                os.chmod(otbcli_path, 0o744)

        except BaseException as e:

            import traceback

            os.remove(otbcli_path)

            errmsg = "Cannot write:" + otbcli_path + "\nError:\n" + traceback.format_exc()

            QgsMessageLog.logMessage(self.tr(errmsg), self.tr('Processing'), Qgis.Critical)

            raise e

        QgsMessageLog.logMessage(self.tr("Using otbcli: '{}'.".format(otbcli_path)), self.tr('Processing'), Qgis.Info)



    def canBeActivated(self):

        if not self.isActive():

            return False
@@ -256,9 +193,7 @@ class OtbAlgorithmProvider(QgsProcessingProvider): 
                dfile = os.path.join(descr_folder, app_name + '.txt')

                isValid = True

                if not os.path.exists(dfile):

                    cmdlist = [os.path.join(

                        folder, 'bin',

                        otb_exe_file('otbQgisDescriptor')),

                    cmdlist = [OtbUtils.getExecutableInPath(folder, 'otbQgisDescriptor'),

                               app_name, app_dir, descr_folder + '/']

                    commands = ' '.join(cmdlist)

                    QgsMessageLog.logMessage(self.tr(commands), self.tr('Processing'), Qgis.Critical)
@@ -283,9 +218,10 @@ class OtbAlgorithmProvider(QgsProcessingProvider): 
            self.setActive(False)

            raise ValueError(self.tr("'{}' does not exist. OTB provider will be disabled".format(v)))

        path = self.normalize_path(v)

        if not os.path.exists(os.path.join(path, 'bin', otb_exe_file('otbApplicationLauncherCommandLine'))):

        app_launcher_path = OtbUtils.getExecutableInPath(path, 'otbApplicationLauncherCommandLine')

        if not os.path.exists(app_launcher_path):

            self.setActive(False)

            raise ValueError(self.tr("Cannot find '{}'. OTB will be disabled".format(os.path.join(v, 'bin', otb_exe_file('otbApplicationLauncherCommandLine')))))

            raise ValueError(self.tr("Cannot find '{}'. OTB will be disabled".format(app_launcher_path)))



    def algsFile(self, d):

        return os.path.join(self.descrFolder(d), 'algs.txt')

otb/OtbUtils.py

+129 −77
Original line number Diff line number Diff line
@@ -40,31 +40,28 @@ from qgis.PyQt.QtCore import QCoreApplication 
from otb.OtbSettings import OtbSettings





def cliPath():

    cli_ext = '.bat' if os.name == 'nt' else ''

    return os.path.normpath(os.path.join(QgsApplication.qgisSettingsDirPath(),

                                         'processing', 'qgis_otb_cli' + cli_ext))



class OtbUtils:



    @staticmethod

    def version():

        return ProcessingConfig.getSetting(OtbSettings.VERSION) or '0.0.0'





    @staticmethod

    def loggerLevel():

        return ProcessingConfig.getSetting(OtbSettings.LOGGER_LEVEL) or 'INFO'





    @staticmethod

    def maxRAMHint():

        return ProcessingConfig.getSetting(OtbSettings.MAX_RAM_HINT) or ''





    @staticmethod

    def otbFolder():

        if ProcessingConfig.getSetting(OtbSettings.FOLDER):

            return os.path.normpath(os.sep.join(re.split(r'\\|/', ProcessingConfig.getSetting(OtbSettings.FOLDER))))

        else:

            return None





    @staticmethod

    def appFolder():

        app_folder = ProcessingConfig.getSetting(OtbSettings.APP_FOLDER)

        if app_folder:
@@ -72,59 +69,114 @@ def appFolder(): 
        else:

            return None





    @staticmethod

    def srtmFolder():

        return ProcessingConfig.getSetting(OtbSettings.SRTM_FOLDER) or ''





    @staticmethod

    def geoidFile():

        return ProcessingConfig.getSetting(OtbSettings.GEOID_FILE) or ''



    @staticmethod

    def getExecutableInPath(path, exe):

        ext = '.exe' if os.name == 'nt' else ''

        return os.path.join(path, 'bin', exe + ext)



    @staticmethod

    def getAuxiliaryDataDirectories():

        gdal_data_dir = None

        gtiff_csv_dir = None

        otb_folder = OtbUtils.otbFolder()

        if os.name == 'nt':

            gdal_data_dir = os.path.join(otb_folder, 'share', 'data')

            gtiff_csv_dir = os.path.join(otb_folder, 'share', 'epsg_csv')

        else:

            env_profile = os.path.join(otb_folder, 'otbenv.profile')

            try:

                if os.path.exists(env_profile):

                    with open(env_profile) as f:

                        lines = f.readlines()

                        lines = [x.strip() for x in lines]

                        for line in lines:

                            if not line or line.startswith('#'):

                                continue

                            if 'GDAL_DATA=' in line:

                                gdal_data_dir = line.split("GDAL_DATA=")[1]

                            if 'GEOTIFF_CSV='in line:

                                gtiff_csv_dir = line.split("GEOTIFF_CSV=")[1]

            except BaseException as exc:

                errmsg = "Cannot find gdal and geotiff data directory." + str(exc)

                QgsMessageLog.logMessage(errmsg, OtbUtils.tr('Processing'), Qgis.Info)

                pass



def executeOtb(command, feedback, addToLog=True):

    loglines = []

        return gdal_data_dir, gtiff_csv_dir



    @staticmethod

    def executeOtb(commands, feedback, addToLog=True):

        otb_env = {

            'LC_NUMERIC': 'C',

            'GDAL_DRIVER_PATH': 'disable'

        }

        gdal_data_dir, gtiff_csv_dir = OtbUtils.getAuxiliaryDataDirectories()

        if gdal_data_dir and os.path.exists(gdal_data_dir):

            otb_env['GDAL_DATA'] = gdal_data_dir

        if gtiff_csv_dir and os.path.exists(gtiff_csv_dir):

            otb_env['GEOTIFF_CSV'] = gtiff_csv_dir



        otb_env['OTB_LOGGER_LEVEL'] = OtbUtils.loggerLevel()

        max_ram_hint = OtbUtils.maxRAMHint()

        if max_ram_hint and int(max_ram_hint) > 256:

            otb_env['OTB_MAX_RAM_HINT'] = max_ram_hint



        kw = {}

        kw['env'] = otb_env

        if os.name == 'nt':

            kw['encoding'] = "cp{}".format(OtbUtils.getWindowsCodePage())



        QgsMessageLog.logMessage("{}".format(kw), OtbUtils.tr('Processing'), Qgis.Info)

        QgsMessageLog.logMessage("cmd={}".format(commands), OtbUtils.tr('Processing'), Qgis.Info)

        with subprocess.Popen(

            [command],

                commands,

                shell=True,

                stdout=subprocess.PIPE,

                stdin=subprocess.DEVNULL,

                stderr=subprocess.STDOUT,

            universal_newlines=True

                universal_newlines=True,

                **kw

        ) as proc:

        try:



            for line in iter(proc.stdout.readline, ''):

                line = line.strip()

                #'* ]' and '  ]' says its some progress update

                #print('line[-3:]',line[-3:])

                if line[-3:] == "* ]" or line[-3:] == "  ]":

                if '% [' in line:

                    part = line.split(':')[1]

                    percent = part.split('%')[0]

                    try:

                        if int(percent) >= 100:

                            loglines.append(line)

                            feedback.pushConsoleInfo(line)

                        feedback.setProgress(int(percent))

                    except:

                        pass

                else:

                    loglines.append(line)

        except BaseException as e:

            loglines.append(str(e))

            pass



        for logline in loglines:

                    if feedback is None:

                QgsMessageLog.logMessage(logline, 'Processing', Qgis.Info)

                        QgsMessageLog.logMessage(line, OtbUtils.tr('Processing'), Qgis.Info)

                    else:

                feedback.pushConsoleInfo(logline)



        # for logline in loglines:

        #     if 'INFO' in logline or 'FATAL' in logline:

        #         if feedback is None:

        #             QgsMessageLog.logMessage(logline, 'Processing', Qgis.Info)

        #         else:

        #             feedback.pushConsoleInfo(logline)

                        if any([l in line for l in ['(WARNING)', '(FATAL)', 'ERROR']]):

                            feedback.reportError(line)

                        else:

                            feedback.pushConsoleInfo(line.strip())



    @staticmethod

    def getWindowsCodePage():

        """

        Determines MS-Windows CMD.exe shell codepage.

        Used into GRASS exec script under MS-Windows.

        """

        from ctypes import cdll

        return str(cdll.kernel32.GetACP())



    @staticmethod

    def tr(string, context=''):

        if context == '':

            context = 'OtbUtils'