arvados.commands.arvcli

  1# Copyright (C) The Arvados Authors. All rights reserved.
  2#
  3# SPDX-License-Identifier: Apache-2.0
  4
  5"""Main executable for Arvados CLI SDK, the `arv` command.
  6
  7This script implements the `arv` command's argument parser. The `arv` command
  8is meant to be invoked in the following manner:
  9
 10$ arv [--flags] subcommand|resource [...options]
 11
 12where `--flags` are common CLI options for the various subcommands.
 13
 14The `ArvCLIArgumentParser` class, specializing the standard Python
 15`argparse.ArgumentParser`, provides the support for this CLI usage.
 16"""
 17
 18
 19import sys
 20import argparse
 21import functools
 22import json
 23import arvados
 24import arvados.commands._util as cmd_util
 25
 26
 27class _ArgTypes:
 28    """Private namespace class for JSON-related CLI argument types."""
 29    @staticmethod
 30    def _validate_type(obj_type, obj):
 31        if isinstance(obj, obj_type):
 32            return obj
 33        raise ValueError(f"{obj!r} is not of type {obj_type!s}.")
 34
 35    json_array = cmd_util.JSONStringArgument(
 36        validator=functools.partial(_validate_type, list),
 37        pretty_name="JSON array"
 38    )
 39
 40    json_object = cmd_util.JSONStringArgument(
 41        validator=functools.partial(_validate_type, dict),
 42        pretty_name="JSON object"
 43    )
 44
 45    json_body = cmd_util.JSONArgument(
 46        validator=json_object.post_validator,
 47        pretty_name="JSON request body object"
 48    )
 49
 50
 51class _ArgUtil:
 52    """Private namespace class for helpful functions (static methods) that
 53    processes the discovery document for the purpose of CLI parser generation.
 54    """
 55    @staticmethod
 56    def singularize_resource(plural: str) -> str:
 57        """Returns the singular form of a resource term in the original
 58        plural.
 59        """
 60        match plural:
 61            case "vocabularies":
 62                return "vocabulary"
 63            case "sys":
 64                return "sys"
 65            case _:
 66                return plural.removesuffix("s")
 67
 68    @staticmethod
 69    def parameter_key_to_argument_name(parameter_key: str) -> str:
 70        """Convert a parameter key in the discovery document to CLI parameter
 71        form, for example, `--foo-bar`.
 72
 73        Arguments:
 74
 75        * parameter_key: str -- Parameter key in the form as they appear in the
 76          discovery document, typically like `foo_bar`.
 77        """
 78        return "--" + parameter_key.replace("_", "-")
 79
 80    @staticmethod
 81    def get_method_options(method_schema):
 82        """Generate command-line options, in the form of "-f/--foo", from the
 83        parameters as defined by the API method schema in the discovery
 84        document.
 85
 86        For each key "foo_bar" in the "parameters" field of the method schema,
 87        command-line options are created according to its definition as
 88        follows.
 89
 90        If the parameter type is "boolean", a pair of options "--no-foo-bar"
 91        and "--foo-bar" are created, with opposite meaning.
 92
 93        If the parameter type is "integer", the CLI input will be interpreted
 94        as a Python int.
 95
 96        All other parameter types are parsed as Python str.
 97
 98        The short form of each option will also be created, by taking the first
 99        letter of the long form, except when that letter is already used, in
100        which case the second letter will be used, and so on. For example,
101        "--foo-bar" will have short form "-f", unless "-f" is already used for
102        another option, in which case "-o" will be used, etc.
103
104        The "negative" form of boolean options ("--no-foo-bar") will not have
105        separate short forms of their own.
106
107        This  generator yields tuples in the form of `(names, kwargs)`, where
108        `names` is a one- or two-element tuple and `kwargs` is a dict, suitable
109        to be passed as
110        `argparse.ArgumentParser.add_argument(*names, **kwargs)`.
111
112        Arguments:
113
114        * method_schema: dict --- Dict object from the parsed discover document
115          that defines a method.
116        """
117        parameters_schema = method_schema.get("parameters", {}).copy()
118        # If the method comes with the "request" field, add another parameter
119        # based on the sole key in the "properties" dict of that field
120        request_schema = method_schema.get("request")
121        if request_schema is not None and request_schema.get("properties"):
122            for parameter_key in request_schema["properties"].keys():
123                parameters_schema[parameter_key] = {
124                    "type": "request",  # special value for request parameter
125                    "required": request_schema.get("required"),
126                    "description": (
127                        f"Either a string representing {parameter_key} as JSON"
128                        f" or a filename from which to read {parameter_key}"
129                        " JSON (use '-' to read from stdin)."
130                    )
131                }
132        argument_key_abbrevs = set("h")  # prevent conflict with "help"
133        for parameter_key, parameter_dict in parameters_schema.items():
134            parameter_kwargs = {
135                "required": parameter_dict.get("required", False)
136            }
137            parameter_kwargs["help"] = parameter_dict.get("description", "")
138            if parameter_kwargs["required"]:
139                parameter_kwargs["help"] += " This option must be specified."
140            # The "type" member refers to one of the JSON values types, out of
141            # string/integer/array/object/boolean.
142            # NOTE: Currently, enum-like value choices are not implemented, as
143            # the enum values cannot be directly inferred from the discover
144            # doc.
145            argument_key = _ArgUtil.parameter_key_to_argument_name(
146                parameter_key
147            )
148            for argument_short_key in argument_key:
149                if (
150                    argument_short_key.isalpha()
151                    and argument_short_key not in argument_key_abbrevs
152                ):
153                    argument_key_abbrevs.add(argument_short_key)
154                    break
155            else:
156                # If the letters of the full argument name are exhausted, fall
157                # back to not using a short argument, indicated by the special
158                # value None:
159                argument_short_key = None
160            default = parameter_dict.get("default")
161            if default is not None:
162                parameter_kwargs["default"] = default
163                if parameter_dict.get("type") != "boolean":
164                    parameter_kwargs["help"] += f" Default: {default!s}."
165            match parameter_dict.get("type"):
166                case "boolean":
167                    # Using the 'action="store_true" (or "store_false")'
168                    # mechanism results in flag-like action rather than an
169                    # option that takes a true or false value. For each bool
170                    # flag "--foo", also generate an additional "negative"
171                    # version "--no-foo".
172                    neg_argument_key = _ArgUtil.parameter_key_to_argument_name(
173                        f"no_{parameter_key}"
174                    )
175                    neg_parameter_kwargs = {}
176                    neg_parameter_kwargs["action"] = "store_false"
177                    neg_parameter_kwargs["required"] = False
178                    neg_parameter_kwargs["dest"] = parameter_key
179                    neg_parameter_kwargs["default"] = json.loads(
180                        parameter_dict.get("default", "null")
181                    )
182                    yield (neg_argument_key,), neg_parameter_kwargs
183
184                    parameter_kwargs["action"] = "store_true"
185                    parameter_kwargs["dest"] = parameter_key
186                    parameter_kwargs["default"] = (
187                        neg_parameter_kwargs["default"]
188                    )
189                case "integer":
190                    parameter_kwargs["type"] = int
191                    parameter_kwargs["metavar"] = "N"
192                case "array":
193                    parameter_kwargs["type"] = _ArgTypes.json_array
194                    parameter_kwargs["metavar"] = "JSON_ARRAY"
195                case "object":
196                    parameter_kwargs["type"] = _ArgTypes.json_object
197                    parameter_kwargs["metavar"] = "JSON_OBJECT"
198                case "request":
199                    parameter_kwargs["type"] = _ArgTypes.json_body
200                    parameter_kwargs["metavar"] = "{JSON,FILE,-}"
201                case _:
202                    parameter_kwargs["type"] = str
203                    parameter_kwargs["metavar"] = "STR"
204            if argument_short_key is None:
205                yield (argument_key,), parameter_kwargs
206            else:
207                yield (
208                    (f"-{argument_short_key}", argument_key), parameter_kwargs
209                )
210
211
212class ArvCLIArgumentParser(argparse.ArgumentParser):
213    """Argument parser for `arv` commands.
214    """
215    def __init__(self, resource_dictionary, **kwargs):
216        """Arguments:
217
218        * resource dictionary: dict --- Dict containing the resources defined
219          in the discovery document; can be obtained as the
220          `_resourceDesc["resources"]` attribute of an Arvados API client
221          object.
222        """
223        super().__init__(description="Arvados command line client", **kwargs)
224        # Common flags to the main command.
225        self.add_argument("-n", "--dry-run", action="store_true",
226                          help="Don't actually do anything")
227        self.add_argument("-v", "--verbose", action="store_true",
228                          help="Print some things on stderr")
229        # Default output format is JSON, while "-s" or "--short" can be
230        # used as a shorthand for "--format=uuid". If both are specified, the
231        # last one takes effect.
232        self.add_argument(
233            "-f", "--format",
234            choices=["json", "yaml", "uuid"],
235            default="json",
236            help="Set output format"
237        )
238        self.add_argument(
239            "-s", "--short",
240            dest="format",
241            action="store_const", const="uuid",
242            help="Return only UUIDs (equivalent to --format=uuid)"
243        )
244
245        subparsers = self.add_subparsers(
246            dest="subcommand",
247            help="Subcommands",
248            required=True,
249            parser_class=functools.partial(
250                argparse.ArgumentParser,
251                add_help=False
252            )
253        )
254
255        keep_parser = subparsers.add_parser("keep")
256        keep_parser.add_argument(
257            "method",
258            choices=["ls", "get", "put", "docker"]
259        )
260
261        ws_parser = subparsers.add_parser("ws")
262        copy_parser = subparsers.add_parser("copy")
263
264        self.subparsers = subparsers
265        self.resource_dictionary = resource_dictionary
266        self._subparser_index = {}
267
268        self.add_resource_subcommands()
269
270    def add_resource_subcommands(self):
271        """Add resources as subcommands, their associated methods as
272        sub-subcommands, and the parameters associated with each method.
273        """
274        for resource, resource_schema in self.resource_dictionary.items():
275            subcommand = _ArgUtil.singularize_resource(resource)
276            resource_subparser = self.subparsers.add_parser(
277                subcommand,
278                # For backward compatibility with legacy Ruby CLI client.
279                aliases=["sy"] if subcommand == "sys" else []
280            )
281            self._subparser_index[subcommand] = resource_subparser
282            if subcommand == "sys":
283                self._subparser_index["sy"] = resource_subparser
284            methods_dict = resource_schema.get("methods")
285            if methods_dict:
286                # Create a collection of "sub-subparsers" under the resource
287                # subparser for the methods.
288                method_subparsers = resource_subparser.add_subparsers(
289                    title="Methods",
290                    dest="method",
291                    parser_class=argparse.ArgumentParser,
292                    help="Methods for subcommand {}".format(subcommand)
293                )
294                for method, method_schema in methods_dict.items():
295                    # Add each specific method as a (sub-)subparser with its
296                    # associated parameters.
297                    method_parser = method_subparsers.add_parser(
298                        method,
299                        help=method_schema.get("description")
300                    )
301                    for parameter_names, kwargs in _ArgUtil.get_method_options(
302                            method_schema
303                    ):
304                        method_parser.add_argument(*parameter_names, **kwargs)
305
306
307def dispatch(arguments=None):
308    api_client = arvados.api("v1")
309    cmd_parser = ArvCLIArgumentParser(api_client._resourceDesc["resources"])
310    args, remaining_args = cmd_parser.parse_known_args(arguments)
311
312    match args.subcommand:
313        case "keep":
314            match args.method:
315                case "ls":
316                    from arvados.commands.ls import main
317                case "get":
318                    from arvados.commands.get import main
319                case "put":
320                    from arvados.commands.put import main
321                case "docker":
322                    from arvados.commands.keepdocker import main
323        case "ws":
324            from arvados.commands.ws import main
325        case "copy":
326            from arvados.commands.arv_copy import main
327        case _:
328            # FIXME
329            print("Called API resource {!r}, method {!r}".format(
330                args.subcommand, args.method
331            ))
332            for k, v in vars(args).items():
333                print("{!r}={!r}".format(k, v))
334            help_wanted = "-h" in remaining_args or "--help" in remaining_args
335            if args.method is None or help_wanted:
336                subparser = cmd_parser._subparser_index.get(args.subcommand)
337                if subparser:
338                    subparser.print_help()
339                sys.exit(0 if help_wanted else 2)
340            sys.exit(0)
341    status = main(remaining_args)
342    sys.exit(status)
343
344
345if __name__ == "__main__":
346    dispatch()