otbGenerateWrappersRstDoc.py 20.1 KB
Newer Older
1
#!/usr/bin/env python3
2
#
Julien Michel's avatar
Julien Michel committed
3
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#
# This file is part of Orfeo Toolbox
#
#     https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
21

22
23
import os
import sys
24
import argparse
25
import re
26
27

import otbApplication
28
from otbApplication import ParameterType_Bool, ParameterType_Int, ParameterType_Radius, ParameterType_RAM, ParameterType_Float, ParameterType_String, ParameterType_StringList, ParameterType_InputFilename, ParameterType_OutputFilename, ParameterType_InputImage, ParameterType_OutputImage, ParameterType_InputVectorData, ParameterType_OutputVectorData, ParameterType_Directory, ParameterType_Choice, ParameterType_InputImageList, ParameterType_InputVectorDataList, ParameterType_InputFilenameList, ParameterType_ListView, ParameterType_Group
29

30
31
from otb_warnings import application_documentation_warnings

32
33
from rst_utils import rst_section, RstPageHeading

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
linesep = os.linesep

def EncloseString(s):
    if not s.startswith("\"") :
        s = "\"" + s
    if not s.endswith("\""):
        s = s + "\""
    return s

def ExpandPath(filename,path,exp):
    if not exp:
        return filename
    else:
        # Avoid chasing our tails
        (head,tail) = os.path.split(filename)
        if len(tail) > 0:
            filename = tail
        for dir,dirs,files in os.walk(path):
            for file in files:
                if file == filename:
                    return os.path.join(dir,file)
        return os.path.join(path,filename)

def GetPixelType(value):
Victor Poughon's avatar
Victor Poughon committed
58
    pixeltypes = {' uchar' : 1, ' int8' : 0, ' uint8' : 1, ' int16' : 2, ' uint16': 3, ' int32' : 4, ' uint32' : 5, ' float' : 6, ' double': 7}
59
60
61
    # look for type
    foundcode = -1
    foundname = ""
62
    for ptypename, ptypecode in pixeltypes.items():
63
64
65
66
67
68
        if value.endswith(ptypename):
            foundcode = ptypecode
            foundname = ptypename
            break
    return foundcode,foundname

69
70
def GetApplicationExamplePythonSnippet(app,idx,expand = False, inputpath="",outputpath=""):
    appname = "app"
71
    output = ""
72

73
    output += ".. code-block:: python\n\n"
Victor Poughon's avatar
Victor Poughon committed
74
75
    # Render example comment
    if len(app.GetExampleComment(idx)) > 0:
76
        output += "\t# {}\n".format(app.GetExampleComment(idx))
Victor Poughon's avatar
Victor Poughon committed
77

78
79
    output += "\timport otbApplication" + linesep + linesep
    output += "\t" + appname + " = otbApplication.Registry.CreateApplication(\"" + app.GetName() + "\")" + linesep + linesep
80
81
82
83
84
    for i in range(0, app.GetExampleNumberOfParameters(idx)):
        param = app.GetExampleParameterKey(idx,i)
        value = app.GetExampleParameterValue(idx,i)
        paramtype = app.GetParameterType(param)
        paramrole = app.GetParameterRole(param)
85
        if paramtype == ParameterType_ListView:
86
87
88
89
            if app.GetListViewSingleSelectionMode(param):
                output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(value)+")"
            else:
                output += "\t" + appname + ".SetParameterStringList("+EncloseString(param)+", "+EncloseString(value)+")"
90
        if paramtype == ParameterType_Group:
91
            pass
92
        if paramtype ==  ParameterType_Choice:
93
            #app.SetParameterString(param,value)
94
95
96
97
98
99
            output+= "\t" + appname + ".SetParameterString(" + EncloseString(param) + "," + EncloseString(value) + ")"
        if paramtype == ParameterType_Bool:
            output+= "\t" + appname + ".SetParameterString("+EncloseString(param)+","+EncloseString(value)+")"
        if paramtype == ParameterType_Int \
                or paramtype == ParameterType_Radius \
                or paramtype == ParameterType_RAM:
100
            # app.SetParameterString(param,value)
101
102
            output += "\t" + appname + ".SetParameterInt("+EncloseString(param)+", "+value+")"
        if paramtype == ParameterType_Float:
103
            # app.SetParameterString(param,value)
104
105
            output += "\t" + appname + ".SetParameterFloat("+EncloseString(param)+", "+value + ")"
        if paramtype == ParameterType_String:
106
            # app.SetParameterString(param,value)
107
108
            output+= "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(value)+")"
        if paramtype == ParameterType_StringList:
109
110
            values = value.split(" ")
            # app.SetParameterStringList(param,values)
111
112
113
114
            output += "\t" + appname + ".SetParameterStringList("+EncloseString(param)+", "+str(values)+")"
        if paramtype == ParameterType_InputFilename \
            or paramtype == ParameterType_OutputFilename \
            or paramtype == ParameterType_Directory:
115
116
            if paramrole == 0:
                # app.SetParameterString(param,EncloseString(ExpandPath(value,inputpath,expand)))
117
                output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value,inputpath,expand)) + ")"
118
119
            elif paramrole == 1:
                # app.SetParameterString(param,EncloseString(ExpandPath(value,outputpath,expand)))
120
121
                output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value,outputpath,expand))+")"
        if paramtype == ParameterType_InputImage :
122
            # app.SetParameterString(param,EncloseString(ExpandPath(value,inputpath,expand)))
123
124
            output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value,inputpath,expand))+")"
        if paramtype == ParameterType_InputVectorData:
125
            # app.SetParameterString(param,EncloseString(ExpandPath(value,inputpath,expand)))
126
127
            output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value,inputpath,expand))+")"
        if paramtype == ParameterType_OutputImage :
128
129
            foundcode,foundname = GetPixelType(value)
            if foundcode != -1:
130
131
132
                output += "\t" + appname + ".SetParameterString("+EncloseString(param)+", "+EncloseString(ExpandPath(value[:-len(foundname)],outputpath,expand))+")"
                output += "\n"
                output += "\t" + appname + ".SetParameterOutputImagePixelType("+EncloseString(param)+", "+str(foundcode)+")"
133
            else:
134
135
                output += "\t" + appname +".SetParameterString("+EncloseString(param)+", "+ EncloseString(ExpandPath(value,outputpath,expand)) + ")"
        if paramtype == ParameterType_OutputVectorData:
136
            # app.SetParameterString(param,EncloseString(ExpandPath(value,outputpath,expand)))
137
138
            output += "\t" + appname +".SetParameterString("+EncloseString(param)+", "+ EncloseString(ExpandPath(value,outputpath,expand)) + ")"
        if paramtype == ParameterType_InputImageList:
139
140
141
            values = value.split(" ")
            values = [ExpandPath(val,inputpath,expand) for val in values]
            # app.SetParameterStringList(param,values)
142
143
            output += "\t" + appname + ".SetParameterStringList("+EncloseString(param) + ", " + str(values) + ")"
        if paramtype == ParameterType_InputVectorDataList:
144
145
146
            values = value.split(" ")
            values = [ExpandPath(val,inputpath,expand) for val in values]
            #app.SetParameterStringList(param,values)
147
            output += "\t" + appname + ".SetParameterStringList("+EncloseString(param)+ ", " + str(values) + ")"
148
        output+=linesep
149
    output += linesep
Victor Poughon's avatar
Victor Poughon committed
150
    output+= "\t" + appname + ".ExecuteAndWriteOutput()" + linesep + linesep
151
    return output
152

Victor Poughon's avatar
Victor Poughon committed
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
def render_choice(app, key):
    "Render a choice parameter to rst"

    # First render all the choice values
    choice_keys = app.GetChoiceKeys(key)
    choice_names = app.GetChoiceNames(key)

    choice_entries = ""
    for (choice_key, choice_name) in zip(choice_keys, choice_names):
        # For the description, replace newlines by |br| because we are in a bullet list item
        choice_description = app.GetParameterDescription(key + "." + choice_key).replace("\n", " |br| ")
        choice_entries += template_parameter_choice_entry.format(
            name=choice_name,
            #key=choice_key, # if we want to show the key in choice parameter values
            description=choice_description
        )

    # Then render the full choice parameter
    return template_parameter_choice.format(
        name=app.GetParameterName(key),
        key=key,
        value="[" + "|".join(choice_keys) + "]",
        flags=rst_parameter_flags(app, key),
        description=app.GetParameterDescription(key),
        choices=choice_entries,
    )

180
181
182
183
184
185
186
187
188
189
190
191
192
def rst_parameter_value(app, key):
    "Render a parameter value to rst"

    type = app.GetParameterType(key)

    # ListView is a special case depending on its mode
    if type == ParameterType_ListView:
        if app.GetListViewSingleSelectionMode(key):
            return "string"
        else:
            return "string1 string2..."

    # For all other types it's a simple mapping
193
194
195
196
197
198
    values = {}
    values.update({ParameterType_Bool: "bool"})
    values.update(dict.fromkeys([ParameterType_Int, ParameterType_Radius, ParameterType_RAM], "int"))
    values.update({ParameterType_Float: "float"})
    values.update({ParameterType_String: "string"})
    values.update({ParameterType_StringList: "string1 string2..."})
199
    values.update(dict.fromkeys([ParameterType_InputFilename, ParameterType_OutputFilename], "filename [dtype]"))
200
201
    values.update({ParameterType_InputImage: "image"})
    values.update({ParameterType_OutputImage: "image [dtype]"})
202
203
204
205
206
207
    values.update(dict.fromkeys([ParameterType_InputVectorData, ParameterType_OutputVectorData], "vectorfile"))
    values.update({ParameterType_Directory: "directory"})
    values.update({ParameterType_Choice: "choice"})
    values.update({ParameterType_InputImageList: "image1 image2..."})
    values.update({ParameterType_InputVectorDataList: "vectorfile1 vectorfile2..."})
    values.update({ParameterType_InputFilenameList: "filename1 filename2..."})
208
209
210
211
212
213
214

    if type in values:
        return values[type]
    else:
        raise ValueError("Cannot show parameter value for type ", type)

def rst_parameter_flags(app, key):
215
216
217
218
219
220
    """
    Display the mandatory and default value flags of a parameter
    The display logic tries to follow the logic in WrapperCommandLineLauncher::DisplayParameterHelp
    The default value is formatted using GetParameterAsString to use the same formatting as the cli interface
    """

221
222
    if app.IsMandatory(key) and not app.HasValue(key):
        return "*Mandatory* "
223
224
    elif app.HasValue(key) and app.GetParameterType(key) != ParameterType_Group and app.GetParameterAsString(key) != "":
        return "*Default value: {}* ".format(app.GetParameterAsString(key))
225
226
227
228
229
230
231
232
    else:
        return ""

def detect_abuse(app):
    "Detect choice parameter values which are also used as groups"

    fake_groups = {}
    keys = app.GetParametersKeys()
Victor Poughon's avatar
Victor Poughon committed
233
    choice_keys = [k for k in keys if app.GetParameterType(k) == ParameterType_Choice]
234
235

    # For each choice parameter
Victor Poughon's avatar
Victor Poughon committed
236
    for key in choice_keys:
237

Victor Poughon's avatar
Victor Poughon committed
238
239
240
        # Consider all its possible values
        for choice_key in app.GetChoiceKeys(key):
            fullkey = key + "." + choice_key
241

Victor Poughon's avatar
Victor Poughon committed
242
243
244
            # See if that value is also used as a group anywhere in the application
            for k in keys:
                if k.startswith(fullkey) and k != fullkey:
245

Victor Poughon's avatar
Victor Poughon committed
246
247
248
                    # In that case, mark the first element of that group
                    if fullkey not in fake_groups.values():
                        fake_groups[k] = fullkey
249
250
251

    return fake_groups

Victor Poughon's avatar
Victor Poughon committed
252
def render_parameters(app):
253
    "Render application parameters to rst"
254
255

    output = ""
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346

    fake_markers = detect_abuse(app)

    previous_level = 1
    for key in app.GetParametersKeys():
        type = app.GetParameterType(key)

        # If reducing level not on a group parameter, render a horizontal line
        current_level = 1 + key.count(".")
        if current_level < previous_level and type != ParameterType_Group:
            output += "\n\n------------\n\n"
        previous_level = current_level

        # Choice parameter values can act as groups
        # Detect that case to add a section title
        if key in fake_markers:
            output += rst_section(app.GetParameterName(fake_markers[key]) + " options", "^")

        if type == ParameterType_Group:
            output += template_parameter_group.format(
                name=rst_section(app.GetParameterName(key), "^"),
                description=app.GetParameterDescription(key)
            )

        elif type == ParameterType_Choice:
            output += render_choice(app, key)

        else:
            output += template_parameter.format(
                name=app.GetParameterName(key),
                key=key,
                value=rst_parameter_value(app, key),
                description=app.GetParameterDescription(key),
                flags=rst_parameter_flags(app, key),
            )

    return output

def render_example_cli(app, index):
    "Render a command line example to rst (includes indentation)"

    output = ""

    # Render comment
    if len(app.GetExampleComment(index)) > 0:
        output += "    # " + app.GetExampleComment(index) + "\n"

    output += "    otbcli_" + app.GetName()
    for i in range(app.GetExampleNumberOfParameters(index)):
        output += " -" + app.GetExampleParameterKey(index, i) + " " + app.GetExampleParameterValue(index, i)
    output += "\n"
    return output

def render_all_examples_cli(app):
    "Render all command line examples to rst"

    if app.GetNumberOfExamples() == 0:
        return "    # No example found"
    if app.GetNumberOfExamples() == 1:
        return render_example_cli(app, 0)
    else:
        output = ""
        for i in range(app.GetNumberOfExamples()):
            if i > 0:
                output += "\n"
            output += render_example_cli(app, i)
        return output

def render_all_examples_python(app):
    "Render all python examples to rst"
    output = ""
    for i in range(app.GetNumberOfExamples()):
        output += GetApplicationExamplePythonSnippet(app, i)
    return output

def render_limitations(app):
    "Render app DocLimitations to rst"

    limitations = app.GetDocLimitations()
    if limitations is None or len(limitations) == 0 or limitations == "None":
        return ""
    else:
        return rst_section("Limitations", "-") + limitations

def render_see_also(app):
    "Render app See Also to rst"

    see_also = app.GetDocSeeAlso()
    if see_also is None or len(see_also) < 2:
        return ""
    else:
347
        return rst_section("See also", "-") + "| " + see_also.replace("\n", "\n| ") # use line blocks for see also
348

349
350
351
352
353
354
355
356
357
def multireplace(string, replacements):
    "multiple string replace (from https://stackoverflow.com/a/6117124/5815110)"
    substrs = sorted(replacements, key=len, reverse=True)
    regexp = re.compile('|'.join(map(re.escape, substrs)))
    return regexp.sub(lambda match: replacements[match.group(0)], string)

def make_links(text, allapps):
    "Replace name of applications by internal rst links"

358
    rep = {appname: ":ref:`{}`".format(appname) for appname in allapps}
359
360
    return multireplace(text, rep)

361
362
def render_deprecation_string(app):
    if app.IsDeprecated():
Cédric Traizet's avatar
Cédric Traizet committed
363
        return "This application is deprecated and will be removed in a future release."
364
    else:
Cédric Traizet's avatar
Cédric Traizet committed
365
        return ""
366

367
def render_multiwriting_string(app):
368
    if app.IsMultiWritingEnabled():
369
370
371
372
373
374
375
376
377
        return ("This application has several output images and supports \"multi-writing\". Instead of computing and writing each "
          "image independently, the streamed image blocks are written in a synchronous way for each output. The output images will "
          "be computed strip by strip, using the available RAM to compute the strip size, and a user defined streaming mode can be specified "
          "using the streaming extended filenames (type, mode and value). Note that multi-writing can be disabled using the multi-write extended "
          "filename option: &multiwrite=false, in this case the output images will be written one by one. Note that multi-writing is not supported for "
          "MPI writers.")
    else:
        return ""

378
def render_application(appname, allapps):
379
380
    "Render app to rst"

Cédric Traizet's avatar
Cédric Traizet committed
381
    # Create the application without logger to avoid the deprecation warning log
382
    app = otbApplication.Registry.CreateApplicationWithoutLogger(appname)
383

384
385
    # TODO: remove this when bug 440 is fixed
    app.Init()
386

387
388
    application_documentation_warnings(app)

389
    output = template_application.format(
390
        label=appname,
391
        deprecation_string=render_deprecation_string(app),
392
        multiwriting_string=render_multiwriting_string(app),
393
394
        heading=rst_section(app.GetName(), '='),
        description=app.GetDescription(),
395
        longdescription=make_links(app.GetDocLongDescription(), allapps),
Victor Poughon's avatar
Victor Poughon committed
396
        parameters=render_parameters(app),
397
398
399
        examples_cli=render_all_examples_cli(app),
        examples_python=render_all_examples_python(app),
        limitations=render_limitations(app),
400
        see_also=make_links(render_see_also(app), allapps)
401
    )
402
403
404
405

    return output

def GetApplicationTags(appname):
Cédric Traizet's avatar
Cédric Traizet committed
406
407
408
    # Create the application without logger to avoid the deprecation warning log
    app = otbApplication.Registry.CreateApplicationWithoutLogger(appname)
    return app.GetDocTags()
409

410
def GenerateRstForApplications(rst_dir):
411
412
    "Generate .rst files for all applications"

413
    blackList = ["TestApplication", "Example", "ApplicationExample"]
414
    allApps = otbApplication.Registry.GetAvailableApplications()
415
416

    if not allApps:
417
        raise RuntimeError("No OTB applications available. Please check OTB_APPLICATION_PATH env variable.")
418

419
    writtenTags = []
420
421
    appNames = [app for app in allApps if app not in blackList]

422
    appIndexFile = open(rst_dir + '/Applications.rst', 'w')
423
    appIndexFile.write(RstPageHeading("All Applications", "2", ref="apprefdoc"))
424

425
    print("Generating rst for {} applications".format(len(appNames)))
426

427
    for appName in appNames:
428

429
        # Get application first tag
430
431
432
        tags = list(GetApplicationTags(appName))
        if "Deprecated" in tags:
            tags.remove("Deprecated")
433
434
        if not tags or len(tags) == 0:
            raise RuntimeError("No tags for application: " + appName)
435
436

        tag = tags[0]
Victor Poughon's avatar
Victor Poughon committed
437
        tag_ = tag.replace(" ", "_")
438

439
440
        # Add it to the index (i.e. https://www.orfeo-toolbox.org/CookBook/Applications.html)
        if not tag in writtenTags:
Victor Poughon's avatar
Victor Poughon committed
441
            appIndexFile.write('\tApplications/' + tag_ + '.rst\n')
442
            writtenTags.append(tag)
443

444
        # Create or update tag index file (e.g. https://www.orfeo-toolbox.org/CookBook/Applications/Feature_Extraction.html)
Victor Poughon's avatar
Victor Poughon committed
445
        tagFileName = rst_dir + '/Applications/'  + tag_ + '.rst'
446
        if os.path.isfile(tagFileName):
447
448
            with open(tagFileName, 'a') as tagFile:
                tagFile.write("\tapp_" + appName + "\n")
449
        else:
450
451
452
453
454
            with open(tagFileName, 'w') as tagFile:
                tagFile.write( RstPageHeading(tag, "1") )
                tagFile.write("\tapp_" + appName + "\n")

        # Write application rst
455
        with open(rst_dir + '/Applications/app_'  + appName + '.rst', 'w',encoding='utf-8') as appFile:
456
            appFile.write(render_application(appName, appNames))
457

458
if __name__ == "__main__":
459
460
461
    parser = argparse.ArgumentParser(usage="Export application(s) to rst file")
    parser.add_argument("rst_dir", help="Directory where rst files are generated")
    args = parser.parse_args()
462

463
464
465
466
467
468
    # Load rst templates
    template_application = open("templates/application.rst").read()
    template_parameter = open("templates/parameter.rst").read()
    template_parameter_group = open("templates/parameter_group.rst").read()
    template_parameter_choice_entry = open("templates/parameter_choice_entry.rst").read()
    template_parameter_choice = open("templates/parameter_choice.rst").read()
469

470
    GenerateRstForApplications(args.rst_dir)