otbGenerateWrappersRstDoc.py 20.2 KB
Newer Older
1
#!/usr/bin/env python3
2
#
3
# Copyright (C) 2005-2020 Centre National d'Etudes Spatiales (CNES)
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#
# This file is part of Orfeo Toolbox
#
#     https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
21

22 23
import os
import sys
24
import argparse
25
import re
26 27

import otbApplication
28
from otbApplication import ParameterType_Bool, ParameterType_Int, ParameterType_Radius, ParameterType_RAM, ParameterType_Float, ParameterType_String, ParameterType_StringList, ParameterType_InputFilename, ParameterType_OutputFilename, ParameterType_InputImage, ParameterType_OutputImage, ParameterType_InputVectorData, ParameterType_OutputVectorData, ParameterType_Directory, ParameterType_Choice, ParameterType_InputImageList, ParameterType_InputVectorDataList, ParameterType_InputFilenameList, ParameterType_ListView, ParameterType_Band, ParameterType_Field, ParameterType_Group
29

30 31
from otb_warnings import application_documentation_warnings

32 33
from rst_utils import rst_section, RstPageHeading

34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
linesep = os.linesep

def EncloseString(s):
    if not s.startswith("\"") :
        s = "\"" + s
    if not s.endswith("\""):
        s = s + "\""
    return s

def ExpandPath(filename,path,exp):
    if not exp:
        return filename
    else:
        # Avoid chasing our tails
        (head,tail) = os.path.split(filename)
        if len(tail) > 0:
            filename = tail
        for dir,dirs,files in os.walk(path):
            for file in files:
                if file == filename:
                    return os.path.join(dir,file)
        return os.path.join(path,filename)

def GetPixelType(value):
Victor Poughon's avatar
Victor Poughon committed
58
    pixeltypes = {' uchar' : 1, ' int8' : 0, ' uint8' : 1, ' int16' : 2, ' uint16': 3, ' int32' : 4, ' uint32' : 5, ' float' : 6, ' double': 7}
59 60 61
    # look for type
    foundcode = -1
    foundname = ""
62
    for ptypename, ptypecode in pixeltypes.items():
63 64 65 66 67 68
        if value.endswith(ptypename):
            foundcode = ptypecode
            foundname = ptypename
            break
    return foundcode,foundname

69 70
def GetApplicationExamplePythonSnippet(app,idx,expand = False, inputpath="",outputpath=""):
    appname = "app"
71
    output = ""
72

73
    output += ".. code-block:: python\n\n"
Victor Poughon's avatar
Victor Poughon committed
74 75
    # Render example comment
    if len(app.GetExampleComment(idx)) > 0:
76
        output += "\t# {}\n".format(app.GetExampleComment(idx))
Victor Poughon's avatar
Victor Poughon committed
77

78 79
    output += "\timport otbApplication" + linesep + linesep
    output += "\t" + appname + " = otbApplication.Registry.CreateApplication(\"" + app.GetName() + "\")" + linesep + linesep
80 81 82 83 84
    for i in range(0, app.GetExampleNumberOfParameters(idx)):
        param = app.GetExampleParameterKey(idx,i)
        value = app.GetExampleParameterValue(idx,i)
        paramtype = app.GetParameterType(param)
        paramrole = app.GetParameterRole(param)
85
        if paramtype == ParameterType_ListView or paramtype == ParameterType_Band or paramtype == ParameterType_Field:
86 87 88 89
            if app.GetListViewSingleSelectionMode(param):
                output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(value)+")"
            else:
                output += "\t" + appname + ".SetParameterStringList("+EncloseString(param)+", "+EncloseString(value)+")"
90
        if paramtype == ParameterType_Group:
91
            pass
92
        if paramtype ==  ParameterType_Choice:
93
            #app.SetParameterString(param,value)
94 95 96 97 98 99
            output+= "\t" + appname + ".SetParameterString(" + EncloseString(param) + "," + EncloseString(value) + ")"
        if paramtype == ParameterType_Bool:
            output+= "\t" + appname + ".SetParameterString("+EncloseString(param)+","+EncloseString(value)+")"
        if paramtype == ParameterType_Int \
                or paramtype == ParameterType_Radius \
                or paramtype == ParameterType_RAM:
100
            # app.SetParameterString(param,value)
101 102
            output += "\t" + appname + ".SetParameterInt("+EncloseString(param)+", "+value+")"
        if paramtype == ParameterType_Float:
103
            # app.SetParameterString(param,value)
104 105
            output += "\t" + appname + ".SetParameterFloat("+EncloseString(param)+", "+value + ")"
        if paramtype == ParameterType_String:
106
            # app.SetParameterString(param,value)
107 108
            output+= "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(value)+")"
        if paramtype == ParameterType_StringList:
109 110
            values = value.split(" ")
            # app.SetParameterStringList(param,values)
111 112 113 114
            output += "\t" + appname + ".SetParameterStringList("+EncloseString(param)+", "+str(values)+")"
        if paramtype == ParameterType_InputFilename \
            or paramtype == ParameterType_OutputFilename \
            or paramtype == ParameterType_Directory:
115 116
            if paramrole == 0:
                # app.SetParameterString(param,EncloseString(ExpandPath(value,inputpath,expand)))
117
                output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value,inputpath,expand)) + ")"
118 119
            elif paramrole == 1:
                # app.SetParameterString(param,EncloseString(ExpandPath(value,outputpath,expand)))
120 121
                output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value,outputpath,expand))+")"
        if paramtype == ParameterType_InputImage :
122
            # app.SetParameterString(param,EncloseString(ExpandPath(value,inputpath,expand)))
123 124
            output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value,inputpath,expand))+")"
        if paramtype == ParameterType_InputVectorData:
125
            # app.SetParameterString(param,EncloseString(ExpandPath(value,inputpath,expand)))
126 127
            output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value,inputpath,expand))+")"
        if paramtype == ParameterType_OutputImage :
128 129
            foundcode,foundname = GetPixelType(value)
            if foundcode != -1:
130 131 132
                output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value[:-len(foundname)],outputpath,expand))+")"
                output += "\n"
                output += "\t" + appname + ".SetParameterOutputImagePixelType("+EncloseString(param)+", "+str(foundcode)+")"
133
            else:
134 135
                output += "\t" + appname +".SetParameterString("+EncloseString(param)+", "+ EncloseString(ExpandPath(value,outputpath,expand)) + ")"
        if paramtype == ParameterType_OutputVectorData:
136
            # app.SetParameterString(param,EncloseString(ExpandPath(value,outputpath,expand)))
137 138
            output += "\t" + appname +".SetParameterString("+EncloseString(param)+", "+ EncloseString(ExpandPath(value,outputpath,expand)) + ")"
        if paramtype == ParameterType_InputImageList:
139 140 141
            values = value.split(" ")
            values = [ExpandPath(val,inputpath,expand) for val in values]
            # app.SetParameterStringList(param,values)
142 143
            output += "\t" + appname + ".SetParameterStringList("+EncloseString(param) + ", " + str(values) + ")"
        if paramtype == ParameterType_InputVectorDataList:
144 145 146
            values = value.split(" ")
            values = [ExpandPath(val,inputpath,expand) for val in values]
            #app.SetParameterStringList(param,values)
147
            output += "\t" + appname + ".SetParameterStringList("+EncloseString(param)+ ", " + str(values) + ")"
148
        output+=linesep
149
    output += linesep
Victor Poughon's avatar
Victor Poughon committed
150
    output+= "\t" + appname + ".ExecuteAndWriteOutput()" + linesep + linesep
151
    return output
152

Victor Poughon's avatar
Victor Poughon committed
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
def render_choice(app, key):
    "Render a choice parameter to rst"

    # First render all the choice values
    choice_keys = app.GetChoiceKeys(key)
    choice_names = app.GetChoiceNames(key)

    choice_entries = ""
    for (choice_key, choice_name) in zip(choice_keys, choice_names):
        # For the description, replace newlines by |br| because we are in a bullet list item
        choice_description = app.GetParameterDescription(key + "." + choice_key).replace("\n", " |br| ")
        choice_entries += template_parameter_choice_entry.format(
            name=choice_name,
            #key=choice_key, # if we want to show the key in choice parameter values
            description=choice_description
        )

    # Then render the full choice parameter
    return template_parameter_choice.format(
        name=app.GetParameterName(key),
        key=key,
        value="[" + "|".join(choice_keys) + "]",
        flags=rst_parameter_flags(app, key),
        description=app.GetParameterDescription(key),
        choices=choice_entries,
    )

180 181 182 183 184
def rst_parameter_value(app, key):
    "Render a parameter value to rst"

    type = app.GetParameterType(key)

185 186
    # ListView/Band/Field is a special case depending on its mode
    if type == ParameterType_ListView or type == ParameterType_Band or type == ParameterType_Field:
187 188 189 190 191 192
        if app.GetListViewSingleSelectionMode(key):
            return "string"
        else:
            return "string1 string2..."

    # For all other types it's a simple mapping
193 194 195 196 197 198
    values = {}
    values.update({ParameterType_Bool: "bool"})
    values.update(dict.fromkeys([ParameterType_Int, ParameterType_Radius, ParameterType_RAM], "int"))
    values.update({ParameterType_Float: "float"})
    values.update({ParameterType_String: "string"})
    values.update({ParameterType_StringList: "string1 string2..."})
199
    values.update(dict.fromkeys([ParameterType_InputFilename, ParameterType_OutputFilename], "filename [dtype]"))
200 201
    values.update({ParameterType_InputImage: "image"})
    values.update({ParameterType_OutputImage: "image [dtype]"})
202 203 204 205 206 207
    values.update(dict.fromkeys([ParameterType_InputVectorData, ParameterType_OutputVectorData], "vectorfile"))
    values.update({ParameterType_Directory: "directory"})
    values.update({ParameterType_Choice: "choice"})
    values.update({ParameterType_InputImageList: "image1 image2..."})
    values.update({ParameterType_InputVectorDataList: "vectorfile1 vectorfile2..."})
    values.update({ParameterType_InputFilenameList: "filename1 filename2..."})
208 209 210 211 212 213 214

    if type in values:
        return values[type]
    else:
        raise ValueError("Cannot show parameter value for type ", type)

def rst_parameter_flags(app, key):
215 216 217 218 219 220
    """
    Display the mandatory and default value flags of a parameter
    The display logic tries to follow the logic in WrapperCommandLineLauncher::DisplayParameterHelp
    The default value is formatted using GetParameterAsString to use the same formatting as the cli interface
    """

221 222
    if app.IsMandatory(key) and not app.HasValue(key):
        return "*Mandatory* "
223 224
    elif app.HasValue(key) and app.GetParameterType(key) != ParameterType_Group and app.GetParameterAsString(key) != "":
        return "*Default value: {}* ".format(app.GetParameterAsString(key))
225 226 227 228 229 230 231 232
    else:
        return ""

def detect_abuse(app):
    "Detect choice parameter values which are also used as groups"

    fake_groups = {}
    keys = app.GetParametersKeys()
Victor Poughon's avatar
Victor Poughon committed
233
    choice_keys = [k for k in keys if app.GetParameterType(k) == ParameterType_Choice]
234 235

    # For each choice parameter
Victor Poughon's avatar
Victor Poughon committed
236
    for key in choice_keys:
237

Victor Poughon's avatar
Victor Poughon committed
238 239 240
        # Consider all its possible values
        for choice_key in app.GetChoiceKeys(key):
            fullkey = key + "." + choice_key
241

Victor Poughon's avatar
Victor Poughon committed
242 243 244
            # See if that value is also used as a group anywhere in the application
            for k in keys:
                if k.startswith(fullkey) and k != fullkey:
245

Victor Poughon's avatar
Victor Poughon committed
246 247 248
                    # In that case, mark the first element of that group
                    if fullkey not in fake_groups.values():
                        fake_groups[k] = fullkey
249 250 251

    return fake_groups

Victor Poughon's avatar
Victor Poughon committed
252
def render_parameters(app):
253
    "Render application parameters to rst"
254 255

    output = ""
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346

    fake_markers = detect_abuse(app)

    previous_level = 1
    for key in app.GetParametersKeys():
        type = app.GetParameterType(key)

        # If reducing level not on a group parameter, render a horizontal line
        current_level = 1 + key.count(".")
        if current_level < previous_level and type != ParameterType_Group:
            output += "\n\n------------\n\n"
        previous_level = current_level

        # Choice parameter values can act as groups
        # Detect that case to add a section title
        if key in fake_markers:
            output += rst_section(app.GetParameterName(fake_markers[key]) + " options", "^")

        if type == ParameterType_Group:
            output += template_parameter_group.format(
                name=rst_section(app.GetParameterName(key), "^"),
                description=app.GetParameterDescription(key)
            )

        elif type == ParameterType_Choice:
            output += render_choice(app, key)

        else:
            output += template_parameter.format(
                name=app.GetParameterName(key),
                key=key,
                value=rst_parameter_value(app, key),
                description=app.GetParameterDescription(key),
                flags=rst_parameter_flags(app, key),
            )

    return output

def render_example_cli(app, index):
    "Render a command line example to rst (includes indentation)"

    output = ""

    # Render comment
    if len(app.GetExampleComment(index)) > 0:
        output += "    # " + app.GetExampleComment(index) + "\n"

    output += "    otbcli_" + app.GetName()
    for i in range(app.GetExampleNumberOfParameters(index)):
        output += " -" + app.GetExampleParameterKey(index, i) + " " + app.GetExampleParameterValue(index, i)
    output += "\n"
    return output

def render_all_examples_cli(app):
    "Render all command line examples to rst"

    if app.GetNumberOfExamples() == 0:
        return "    # No example found"
    if app.GetNumberOfExamples() == 1:
        return render_example_cli(app, 0)
    else:
        output = ""
        for i in range(app.GetNumberOfExamples()):
            if i > 0:
                output += "\n"
            output += render_example_cli(app, i)
        return output

def render_all_examples_python(app):
    "Render all python examples to rst"
    output = ""
    for i in range(app.GetNumberOfExamples()):
        output += GetApplicationExamplePythonSnippet(app, i)
    return output

def render_limitations(app):
    "Render app DocLimitations to rst"

    limitations = app.GetDocLimitations()
    if limitations is None or len(limitations) == 0 or limitations == "None":
        return ""
    else:
        return rst_section("Limitations", "-") + limitations

def render_see_also(app):
    "Render app See Also to rst"

    see_also = app.GetDocSeeAlso()
    if see_also is None or len(see_also) < 2:
        return ""
    else:
347
        return rst_section("See also", "-") + "| " + see_also.replace("\n", "\n| ") # use line blocks for see also
348

349 350 351 352 353 354 355 356 357
def multireplace(string, replacements):
    "multiple string replace (from https://stackoverflow.com/a/6117124/5815110)"
    substrs = sorted(replacements, key=len, reverse=True)
    regexp = re.compile('|'.join(map(re.escape, substrs)))
    return regexp.sub(lambda match: replacements[match.group(0)], string)

def make_links(text, allapps):
    "Replace name of applications by internal rst links"

358
    rep = {appname: ":ref:`{}`".format(appname) for appname in allapps}
359 360
    return multireplace(text, rep)

361 362
def render_deprecation_string(app):
    if app.IsDeprecated():
Cédric Traizet's avatar
Cédric Traizet committed
363
        return "This application is deprecated and will be removed in a future release."
364
    else:
Cédric Traizet's avatar
Cédric Traizet committed
365
        return ""
366

367
def render_multiwriting_string(app):
368
    if app.IsMultiWritingEnabled():
369 370 371 372 373 374 375 376 377
        return ("This application has several output images and supports \"multi-writing\". Instead of computing and writing each "
          "image independently, the streamed image blocks are written in a synchronous way for each output. The output images will "
          "be computed strip by strip, using the available RAM to compute the strip size, and a user defined streaming mode can be specified "
          "using the streaming extended filenames (type, mode and value). Note that multi-writing can be disabled using the multi-write extended "
          "filename option: &multiwrite=false, in this case the output images will be written one by one. Note that multi-writing is not supported for "
          "MPI writers.")
    else:
        return ""

378
def render_application(appname, allapps):
379 380
    "Render app to rst"

Cédric Traizet's avatar
Cédric Traizet committed
381
    # Create the application without logger to avoid the deprecation warning log
382
    app = otbApplication.Registry.CreateApplicationWithoutLogger(appname)
383

384 385
    # TODO: remove this when bug 440 is fixed
    app.Init()
386

387 388
    application_documentation_warnings(app)

389
    output = template_application.format(
390
        label=appname,
391
        deprecation_string=render_deprecation_string(app),
392
        multiwriting_string=render_multiwriting_string(app),
393 394
        heading=rst_section(app.GetName(), '='),
        description=app.GetDescription(),
395
        longdescription=make_links(app.GetDocLongDescription(), allapps),
Victor Poughon's avatar
Victor Poughon committed
396
        parameters=render_parameters(app),
397 398 399
        examples_cli=render_all_examples_cli(app),
        examples_python=render_all_examples_python(app),
        limitations=render_limitations(app),
400
        see_also=make_links(render_see_also(app), allapps)
401
    )
402 403 404 405

    return output

def GetApplicationTags(appname):
Cédric Traizet's avatar
Cédric Traizet committed
406 407 408
    # Create the application without logger to avoid the deprecation warning log
    app = otbApplication.Registry.CreateApplicationWithoutLogger(appname)
    return app.GetDocTags()
409

410
def GenerateRstForApplications(rst_dir):
411 412
    "Generate .rst files for all applications"

413
    blackList = ["TestApplication", "Example", "ApplicationExample"]
414
    allApps = otbApplication.Registry.GetAvailableApplications()
415 416

    if not allApps:
417
        raise RuntimeError("No OTB applications available. Please check OTB_APPLICATION_PATH env variable.")
418

419
    writtenTags = []
420 421
    appNames = [app for app in allApps if app not in blackList]

422
    appIndexFile = open(rst_dir + '/Applications.rst', 'w')
423
    appIndexFile.write(RstPageHeading("All Applications", "2", ref="apprefdoc"))
424

425
    print("Generating rst for {} applications".format(len(appNames)))
426

427
    for appName in appNames:
428

429
        # Get application first tag
430 431 432
        tags = list(GetApplicationTags(appName))
        if "Deprecated" in tags:
            tags.remove("Deprecated")
433 434
        if not tags or len(tags) == 0:
            raise RuntimeError("No tags for application: " + appName)
435 436

        tag = tags[0]
Victor Poughon's avatar
Victor Poughon committed
437
        tag_ = tag.replace(" ", "_")
438

439 440
        # Add it to the index (i.e. https://www.orfeo-toolbox.org/CookBook/Applications.html)
        if not tag in writtenTags:
Victor Poughon's avatar
Victor Poughon committed
441
            appIndexFile.write('\tApplications/' + tag_ + '.rst\n')
442
            writtenTags.append(tag)
443

444
        # Create or update tag index file (e.g. https://www.orfeo-toolbox.org/CookBook/Applications/Feature_Extraction.html)
Victor Poughon's avatar
Victor Poughon committed
445
        tagFileName = rst_dir + '/Applications/'  + tag_ + '.rst'
446
        if os.path.isfile(tagFileName):
447 448
            with open(tagFileName, 'a') as tagFile:
                tagFile.write("\tapp_" + appName + "\n")
449
        else:
450 451 452 453 454
            with open(tagFileName, 'w') as tagFile:
                tagFile.write( RstPageHeading(tag, "1") )
                tagFile.write("\tapp_" + appName + "\n")

        # Write application rst
455
        with open(rst_dir + '/Applications/app_'  + appName + '.rst', 'w',encoding='utf-8') as appFile:
456
            appFile.write(render_application(appName, appNames))
457

458
if __name__ == "__main__":
459 460 461
    parser = argparse.ArgumentParser(usage="Export application(s) to rst file")
    parser.add_argument("rst_dir", help="Directory where rst files are generated")
    args = parser.parse_args()
462

463 464 465 466 467 468
    # Load rst templates
    template_application = open("templates/application.rst").read()
    template_parameter = open("templates/parameter.rst").read()
    template_parameter_group = open("templates/parameter_group.rst").read()
    template_parameter_choice_entry = open("templates/parameter_choice_entry.rst").read()
    template_parameter_choice = open("templates/parameter_choice.rst").read()
469

470
    GenerateRstForApplications(args.rst_dir)