arvados.commands.arvcli

   1# Copyright (C) The Arvados Authors. All rights reserved.
   2#
   3# SPDX-License-Identifier: Apache-2.0
   4
   5"""Main executable for Arvados CLI SDK, the `arv` command.
   6
   7This script implements the `arv` command's argument parser. The `arv` command
   8is meant to be invoked in the following manner:
   9
  10$ arv [--flags] subcommand|resource [...options]
  11
  12where `--flags` are common CLI options for the various subcommands.
  13
  14The `ArvCLIArgumentParser` class, specializing the standard Python
  15`argparse.ArgumentParser`, provides the support for this CLI usage.
  16"""
  17
  18
  19import abc
  20import argparse
  21from collections.abc import Container, Mapping
  22from contextlib import AbstractContextManager
  23from dataclasses import dataclass
  24import functools
  25import importlib
  26import json
  27import os
  28import re
  29import shlex
  30import shutil
  31import subprocess
  32import sys
  33from tempfile import NamedTemporaryFile
  34from typing import Any, NoReturn, TextIO
  35
  36from googleapiclient import discovery
  37from ruamel.yaml import YAML, YAMLError
  38
  39import arvados
  40from arvados._version import __version__
  41import arvados.commands._util as cmd_util
  42
  43
  44yaml = YAML(typ="safe", pure=True)
  45yaml.default_flow_style = False
  46
  47
  48class _ArgTypes:
  49    """Private namespace class for JSON-related CLI argument types."""
  50
  51    @staticmethod
  52    def group_uuid(text: str) -> str:
  53        """Validate an Arvados group UUID as the value of a CLI argument (an
  54        Arvados project being a type of group).
  55        """
  56        # In theory this is a special case of "UUIDInfo" but we mostly need it
  57        # for the nicer error message.
  58        if arvados.util.group_uuid_pattern.fullmatch(text):
  59            return text
  60        raise argparse.ArgumentTypeError(
  61            f"Invalid UUID for Arvados project or group: {text}"
  62        )
  63
  64    @dataclass(frozen=True)
  65    class UUIDInfo:
  66        """'Interpreted' Arvados UUID object with resource type info."""
  67        uuid: str
  68        resource_type: str  # value in CamelCase
  69        rtype_lower: str  # value in snake_case
  70
  71        @classmethod
  72        def parse(
  73            cls, type_map: Mapping[str, str], text: str
  74        ) -> "UUIDInfo":  # self-typing support comes in Python 3.11.
  75            """Parse the UUID argument `text`. If accepted, returns an
  76            `UUIDInfo` instance whose `uuid` attribute is the input UUID
  77            unchanged and the `resource_type` attribute is the type of Arvados
  78            object (in CamelCase), as determined by the input parameter
  79            `type_map`, and `rtype_lower` is the alternative form of
  80            resource type in snake_case.
  81            """
  82            if not arvados.util.uuid_pattern.fullmatch(text):
  83                raise argparse.ArgumentTypeError(
  84                    f"Invalid Arvados object UUID: {text}"
  85                )
  86            type_code = text.split("-")[1]
  87            if type_code not in type_map:
  88                available_types = ", ".join(sorted(
  89                    f"{k} ({v})" for k, v in type_map.items()
  90                ))
  91                raise argparse.ArgumentTypeError(
  92                    f"Invalid object type code {type_code!r} in Arvados"
  93                    f" object UUID {text}: valid type codes are"
  94                    f" {available_types}"
  95                )
  96            type_key = type_map[type_code]
  97            return cls(text, type_key, _ArgUtil.camel_case_to_snake(type_key))
  98
  99    @staticmethod
 100    def _validate_type(obj_type, obj):
 101        if isinstance(obj, obj_type):
 102            return obj
 103        # No details to raise; caller handles error messaging with pretty_name.
 104        raise ValueError()
 105
 106    json_array = cmd_util.JSONStringArgument(
 107        validator=functools.partial(_validate_type, list),
 108        pretty_name="JSON array"
 109    )
 110
 111    json_object = cmd_util.JSONStringArgument(
 112        validator=functools.partial(_validate_type, dict),
 113        pretty_name="JSON object"
 114    )
 115
 116    json_filter = cmd_util.JSONArgument(
 117        validator=cmd_util.validate_filters,
 118        pretty_name="Arvados API filter"
 119    )
 120
 121    json_body = cmd_util.JSONArgument(
 122        validator=json_object.post_validator,
 123        pretty_name="JSON request body object"
 124    )
 125
 126
 127class _ArgUtil:
 128    """Private namespace class for helpful functions (static methods) that
 129    processes the discovery document for the purpose of CLI parser generation.
 130    """
 131    @staticmethod
 132    def singularize_resource(plural: str) -> str:
 133        """Returns the singular form of a resource term in the original
 134        plural.
 135        """
 136        match plural:
 137            case "vocabularies":
 138                return "vocabulary"
 139            case "sys":
 140                return "sys"
 141            case _:
 142                return plural.removesuffix("s")
 143
 144    @staticmethod
 145    def parameter_key_to_argument_name(parameter_key: str) -> str:
 146        """Convert a parameter key in the discovery document to CLI parameter
 147        form, for example, `--foo-bar`.
 148
 149        Arguments:
 150
 151        * parameter_key: str --- Parameter key in the form as they appear in
 152          the discovery document, typically like `foo_bar`.
 153        """
 154        return "--" + parameter_key.replace("_", "-")
 155
 156    @staticmethod
 157    def camel_case_to_snake(text: str) -> str:
 158        """Simple converter of CamelCase text to so-called 'snake_case' (lower
 159        case with underscore). Works if there's no consecutive upper-case
 160        letters such as "API".
 161        """
 162        return text[:1].lower() + "".join(
 163            f"_{c.lower()}" if c.isupper() else c for c in text[1:]
 164        )
 165
 166    @staticmethod
 167    def get_method_options(
 168        method_schema: Mapping[str, Any],
 169        ignored_parameters: Container[str] = ()
 170    ):
 171        """Generate command-line options, in the form of "-f/--foo", from the
 172        parameters as defined by the API method schema in the discovery
 173        document.
 174
 175        For each key "foo_bar" in the "parameters" field of the method schema,
 176        command-line options are created according to its definition as
 177        follows.
 178
 179        If the parameter type is "boolean", a pair of options "--no-foo-bar"
 180        and "--foo-bar" are created, with opposite meaning.
 181
 182        If the parameter type is "integer", the CLI input will be interpreted
 183        as a Python int.
 184
 185        All other parameter types are parsed as Python str.
 186
 187        The short form of each option will also be created, by taking the first
 188        letter of the long form, except when that letter is already used, in
 189        which case the second letter will be used, and so on. For example,
 190        "--foo-bar" will have short form "-f", unless "-f" is already used for
 191        another option, in which case "-o" will be used, etc.
 192
 193        The "negative" form of boolean options ("--no-foo-bar") will not have
 194        separate short forms of their own.
 195
 196        This  generator yields tuples in the form of `(names, kwargs)`, where
 197        `names` is a one- or two-element tuple and `kwargs` is a dict, suitable
 198        to be passed as
 199        `argparse.ArgumentParser.add_argument(*names, **kwargs)`.
 200
 201        Arguments:
 202
 203        * method_schema: Mapping[str, Any] --- Dict object from the parsed
 204          discover document that defines a method.
 205        * ignored_parameters: Container[str] --- If provided, the parameters
 206          that are in `ignored_parameters` will not be processed.
 207        """
 208        parameters_schema = method_schema.get("parameters", {}).copy()
 209        # If the method comes with the "request" field, add another parameter
 210        # based on the sole key in the "properties" dict of that field
 211        request_schema = method_schema.get("request")
 212        if request_schema is not None and request_schema.get("properties"):
 213            for parameter_key in request_schema["properties"].keys():
 214                parameters_schema[parameter_key] = {
 215                    "type": "request",  # special value for request parameter
 216                    "required": request_schema.get("required"),
 217                    "description": (
 218                        f"Either a string representing {parameter_key} as JSON"
 219                        f" or a filename from which to read {parameter_key}"
 220                        " JSON (use '-' to read from stdin)."
 221                    )
 222                }
 223        argument_key_abbrevs = set("h")  # prevent conflict with "help"
 224        for parameter_key, parameter_dict in parameters_schema.items():
 225            if parameter_key in ignored_parameters:
 226                continue
 227            parameter_kwargs = {
 228                "required": parameter_dict.get("required", False)
 229            }
 230            parameter_kwargs["help"] = parameter_dict.get("description", "")
 231            if parameter_kwargs["required"]:
 232                parameter_kwargs["help"] += " This option must be specified."
 233            # The "type" member refers to one of the JSON values types, out of
 234            # string/integer/array/object/boolean.
 235            # NOTE: Currently, enum-like value choices are not implemented, as
 236            # the enum values cannot be directly inferred from the discover
 237            # doc.
 238            argument_key = _ArgUtil.parameter_key_to_argument_name(
 239                parameter_key
 240            )
 241            for argument_short_key in argument_key:
 242                if (
 243                    argument_short_key.isalpha()
 244                    and argument_short_key not in argument_key_abbrevs
 245                ):
 246                    argument_key_abbrevs.add(argument_short_key)
 247                    break
 248            else:
 249                # If the letters of the full argument name are exhausted, fall
 250                # back to not using a short argument, indicated by the special
 251                # value None:
 252                argument_short_key = None
 253            default = parameter_dict.get("default")
 254            if default is not None and parameter_dict.get("type") != "boolean":
 255                parameter_kwargs["help"] += f" Default: {default}."
 256            match parameter_dict.get("type"):
 257                case "boolean":
 258                    # Using the 'action="store_true" (or "store_false")'
 259                    # mechanism results in flag-like action rather than an
 260                    # option that takes a true or false value. For each bool
 261                    # flag "--foo", also generate an additional "negative"
 262                    # version "--no-foo".
 263                    neg_argument_key = _ArgUtil.parameter_key_to_argument_name(
 264                        f"no_{parameter_key}"
 265                    )
 266                    neg_parameter_kwargs = {}
 267                    neg_parameter_kwargs["action"] = "store_false"
 268                    neg_parameter_kwargs["required"] = False
 269                    neg_parameter_kwargs["dest"] = parameter_key
 270                    neg_parameter_kwargs["default"] = json.loads(
 271                        default if default is not None else "null"
 272                    )
 273                    yield (neg_argument_key,), neg_parameter_kwargs
 274
 275                    parameter_kwargs["action"] = "store_true"
 276                    parameter_kwargs["dest"] = parameter_key
 277                    parameter_kwargs["default"] = (
 278                        neg_parameter_kwargs["default"]
 279                    )
 280                case "integer":
 281                    parameter_kwargs["type"] = int
 282                    parameter_kwargs["metavar"] = "N"
 283                case "array":
 284                    # The filters parameter is only used with "getter" methods
 285                    # that doesn't send a request body (which is exclusive to
 286                    # "creator"/"updater" methods). This means it's generally
 287                    # safe to use the "json_filter" type converter which can
 288                    # read from the stdin; it wouldn't conflict with the
 289                    # request body parameter which can also read the stdin.
 290                    if parameter_key == "filters":
 291                        parameter_kwargs["type"] = _ArgTypes.json_filter
 292                        parameter_kwargs["metavar"] = "{JSON,FILE,-}"
 293                        parameter_kwargs["help"] += (
 294                            " This can be a filename from which to read"
 295                            " JSON (use '-' to read from stdin)."
 296                        )
 297                    else:
 298                        parameter_kwargs["type"] = _ArgTypes.json_array
 299                        parameter_kwargs["metavar"] = "JSON_ARRAY"
 300                case "object":
 301                    parameter_kwargs["type"] = _ArgTypes.json_object
 302                    parameter_kwargs["metavar"] = "JSON_OBJECT"
 303                case "request":
 304                    parameter_kwargs["dest"] = "body"
 305                    parameter_kwargs["type"] = _ArgTypes.json_body
 306                    parameter_kwargs["metavar"] = "{JSON,FILE,-}"
 307                case _:
 308                    parameter_kwargs["type"] = str
 309                    parameter_kwargs["metavar"] = "STR"
 310            if argument_short_key is None:
 311                yield (argument_key,), parameter_kwargs
 312            else:
 313                yield (
 314                    (f"-{argument_short_key}", argument_key), parameter_kwargs
 315                )
 316
 317    @staticmethod
 318    def make_uuid_to_resource_map(schemas: dict[str, dict]) -> dict[str, str]:
 319        """Returns a mapping of Arvados object UUID prefixes to resource names
 320        (in the schema-key, CamelCase form, e.g. "ContainerRequest") based on
 321        the input "schemas" portion of the discovery document.
 322        """
 323        result = {}
 324        for schema in schemas.values():
 325            if (
 326                    (prefix := schema.get("uuidPrefix"))
 327                    and (key := schema.get("id"))
 328            ):
 329                result[prefix] = key
 330        return result
 331
 332
 333class ObjectEditingProcessBase(AbstractContextManager, abc.ABC):
 334    """Base class represending a process (in the generic sense, rather than
 335    "a Unix/Linux process") of editing an Arvados object with an external
 336    editor on a temporary file.
 337
 338    The methods `serialize(self, obj, file)` and `deserialize(self, file)` are
 339    abstract methods meant to be overridden. `serialize()` should write to the
 340    open `file` (any file-like object), and `deserialize()` should return an
 341    object loaded from the file.
 342
 343    When initialized, no external file has been created. To do so, enter it as
 344    a context manager.
 345
 346    Upon entering the context, the temporary file will be opened and written to
 347    with proper initial content if necessary. Upon leaving, the temporary file
 348    will be closed and cleaned-up (this normally means the file will be gone
 349    permanently).
 350
 351    Attributes:
 352
 353    * tmp_file: Optional[tempfile.NamedTemporaryFile] --- Temporary file to be
 354      edited.
 355    * prefix: Optional[str] --- Prefix of temporary filename if provided.
 356    * suffix: Optional[str] --- Suffix of temporary filename if provided. This
 357      can be a filename extension with the leading dot/period character `.`,
 358      useful for hinting the external editor with syntax highlighting.
 359    * base_command: list[str] --- Command-line argument list for invoking the
 360      external editor program. See `get_editor_cmdline()` for more.
 361    """
 362    _file_type = None
 363
 364    def __init__(self, initial_object=None, prefix=None, file_extension=None):
 365        """Arguments:
 366
 367        * initial_object: Optional[Any] --- Initial object to be serialized and
 368          written to the temporary file before the editor process is run. If
 369          not provided, the file will be opened empty in the editor.
 370        * prefix: Optional[str] --- String to be used as the prefix
 371          of the temporary file's basename, followed by a hyphen (`-`)
 372          character that will be added automatically. If not provided, the
 373          initial object's `uuid` field will be used if available; otherwise, a
 374          platform-dependent prefix will be chosen automatically. A UUID as
 375          part of the filename is for information only, and it may be displayed
 376          in the editor's UI.
 377        * file_extension: Optional[str] --- Filename extension (without leading
 378          dot) of the temporary file, e.g. "json" or "yml". This information
 379          may be used by the editor to provide syntax highlighting, automatic
 380          indentation, completion, etc.
 381        """
 382        self.initial_object = initial_object
 383
 384        if prefix:
 385            self.prefix = f"{prefix}-"
 386        elif (
 387            isinstance(initial_object, Mapping)
 388            and (obj_uuid := initial_object.get("uuid"))
 389        ):
 390            self.prefix = f"{obj_uuid}-"
 391        else:
 392            self.prefix = None
 393
 394        ext = self._file_type or file_extension
 395        self.suffix = f".{ext.lower()}" if ext else None
 396
 397        self.tmp_file = None
 398        self.base_command = self.get_editor_cmdline()
 399
 400    @staticmethod
 401    def get_editor_cmdline() -> list[str]:
 402        """Returns a partial command-line argument list that begins with the
 403        external editor program. The precedence is the $VISUAL environment
 404        variable, followed by $EDITOR; and if both are missing, then `nano` if
 405        it exists in the $PATH; and finally the hard-coded value `vi` no matter
 406        the command exists or not.
 407        """
 408        if cmd_str := (os.environ.get("VISUAL") or os.environ.get("EDITOR")):
 409            cmd = shlex.split(cmd_str)
 410        elif cmd_str := shutil.which("nano"):
 411            cmd = [cmd_str]
 412        else:
 413            cmd = ["vi"]
 414        return cmd
 415
 416    @abc.abstractmethod
 417    def serialize(self, obj: Any, file: TextIO) -> None:
 418        """Abstract method for serializing any object `obj` to the file-like
 419        object `file` as text.
 420        """
 421
 422    @abc.abstractmethod
 423    def deserialize(self, file: TextIO) -> Any:
 424        """Abstract method for loading from the file-like object `file` as
 425        text. Returns the object deserialized from the text content.
 426        """
 427
 428    def check_tmp_file(self):
 429        """Perform a basic sanity check for the temp file being usable."""
 430        if self.tmp_file is None or self.tmp_file.closed:
 431            raise RuntimeError("Temporary file is not available")
 432
 433    def dump(self, obj: Any) -> None:
 434        """Overwrite the temporary file with the serialized object `obj`."""
 435        self.check_tmp_file()
 436        # The following should not be done while the child process is pending.
 437        self.tmp_file.truncate(0)
 438        self.serialize(obj, self.tmp_file)
 439        self.tmp_file.flush()
 440
 441    def load(self) -> Any:
 442        """Read the temporary file from the beginning. Returns the deserialized
 443        object, or None if the file is empty or only whitespace.
 444        """
 445        self.check_tmp_file()
 446        # Snoop the file to see if it consists of only whitespace characters
 447        # (including empty lines); if so, return the special value None.
 448        with open(self.tmp_file.name, "r") as fdup:
 449            if not fdup.read().strip():
 450                return None
 451
 452        self.tmp_file.seek(0)
 453        return self.deserialize(self.tmp_file)
 454
 455    def edit(self) -> subprocess.CompletedProcess:
 456        """Run external editor and wait for it to finish."""
 457        self.check_tmp_file()
 458        return subprocess.run(
 459            self.base_command + [self.tmp_file.name],
 460            check=False
 461        )  # Wait for child.
 462
 463    def __enter__(self):
 464        self.tmp_file = NamedTemporaryFile(
 465            mode="w+", prefix=self.prefix, suffix=self.suffix
 466        )
 467        if self.initial_object is not None:
 468            self.dump(self.initial_object)
 469        return self
 470
 471    def __exit__(self, exc_type, exc_value, traceback):
 472        self.tmp_file.close()
 473
 474
 475class EditingContentError(ValueError):
 476    """Exception that indicates the content provided by the user via the editor
 477    is invalid for the specific format.
 478    """
 479    def __init__(
 480        self,
 481        path=None, line=0, column=0,
 482        file_type=None,
 483        original_exception=None,
 484    ):
 485        self.path = path
 486        self.line = line
 487        self.column = column
 488        self.file_type = file_type
 489        self.original_exception = original_exception
 490
 491    def __str__(self):
 492        msg = (
 493            f"Error: invalid input file [type {self.file_type or 'unknown'}]:"
 494            f" {self.path}:{self.line}:{self.column}"
 495        )
 496        if (
 497            self.original_exception
 498            and (orig_msg := str(self.original_exception))
 499        ):
 500            msg += f":\n{orig_msg}"
 501        return msg
 502
 503
 504class LoadingHelper:
 505    """Helper class to provide shared error handling and validation traits for
 506    the temp file being edited (whose content shall be an Arvados object).
 507    """
 508    def raise_bad_format(
 509        self, file: TextIO, err: Exception, line: int = 0, column: int = 0
 510    ) -> NoReturn:
 511        path = getattr(file, "name", "<unknown path>")
 512        raise EditingContentError(
 513            path=path, line=line, column=column, file_type=self._file_type,
 514            original_exception=err
 515        )
 516
 517    def validate_mapping(self, obj: Any, file: TextIO) -> Mapping[str, Any]:
 518        if not isinstance(obj, Mapping):
 519            path = getattr(file, "name", "<unknown path>")
 520            format_name = self._file_type or "<unknown format>"
 521            raise EditingContentError(
 522                path=path,
 523                original_exception=ValueError(
 524                    f"{format_name} input has type '{type(obj).__name__}',"
 525                    " not a valid Arvados object"
 526                )
 527            )
 528        return obj
 529
 530
 531class JSONEditingProcess(LoadingHelper, ObjectEditingProcessBase):
 532    """Subclass of editing process tuned for JSON files."""
 533    _file_type = "JSON"
 534
 535    def __init__(self, *args, indent: int = 1, **kwargs):
 536        """Arguments:
 537
 538        * indent: int --- Number of spaces for each indentation level in the
 539          JSON file. Default: 1.
 540        """
 541        super().__init__(*args, **kwargs)
 542        self.indent = indent
 543
 544    def serialize(self, obj: Mapping[str, Any], file: TextIO) -> None:
 545        return json.dump(obj, file, indent=self.indent)
 546
 547    def deserialize(self, file: TextIO) -> Mapping[str, Any]:
 548        try:
 549            obj = json.load(file)
 550        except json.JSONDecodeError as err:
 551            self.raise_bad_format(file, err, err.lineno, err.colno)
 552        return self.validate_mapping(obj, file)
 553
 554
 555class YAMLEditingProcess(LoadingHelper, ObjectEditingProcessBase):
 556    """Subclass of editing process tuned for YAML files."""
 557    _file_type = "YAML"
 558
 559    def serialize(self, obj: Mapping[str, Any], file: TextIO) -> None:
 560        return yaml.dump(obj, file)
 561
 562    def deserialize(self, file: TextIO) -> Mapping[str, Any]:
 563        try:
 564            obj = yaml.load(file)
 565        except YAMLError as err:
 566            # We only do "getattr" because YAMLError is sparsely documented.
 567            problem_mark = getattr(err, "problem_mark", None)
 568            line = getattr(problem_mark, "line", 0)
 569            column = getattr(problem_mark, "column", 0)
 570            self.raise_bad_format(file, err, line, column)
 571        return self.validate_mapping(obj, file)
 572
 573
 574class FullHelpOnErrorArgumentParser(argparse.ArgumentParser):
 575    """Argument parser subclass that customizes the `error()` method.
 576
 577    Intended to be used as a base to a parser with complex subparsers, to print
 578    more-useful information when a required subcommand is missing.
 579    """
 580    def error(self, message, with_help=True):
 581        if with_help:
 582            self.print_help(sys.stderr)
 583            print(file=sys.stderr)
 584        # NOTE: self.prog is to be overridden by child class
 585        print(f"{self.prog}: error: {message}", file=sys.stderr)
 586        sys.exit(2)
 587
 588
 589class ArvCLIArgumentParser(FullHelpOnErrorArgumentParser):
 590    """Argument parser for `arv` commands.
 591    """
 592    prog = "arv"
 593    global_args = frozenset((
 594        "dry_run",
 595        "version",
 596        "verbose",
 597        "format",
 598        "subcommand",
 599        "method"
 600    ))
 601    external_command_modules = {
 602        "keep ls": "arvados.commands.ls",
 603        "keep get": "arvados.commands.get",
 604        "keep put": "arvados.commands.put",
 605        "keep docker": "arvados.commands.keepdocker",
 606        "ws": "arvados.commands.ws",
 607        "copy": "arvados.commands.arv_copy"
 608    }
 609
 610    def __init__(self, discovery_document: dict[str, str | dict], **kwargs):
 611        """Arguments:
 612
 613        * discovery_document: dict --- Dict containing the parsed API discovery
 614          document; can be obtained as the `_rootDesc` attribute of an
 615          Arvados API client object.
 616        """
 617        super().__init__(
 618            description="Arvados command line client",
 619            prog=self.prog,
 620            **kwargs
 621        )
 622        # Common flags to the main command.
 623        self.add_argument("-e", "--version", action="version",
 624                          version=f"%(prog)s {__version__}")
 625        self.add_argument("-n", "--dry-run", action="store_true",
 626                          help="Don't actually do anything")
 627        self.add_argument("-v", "--verbose", action="store_true",
 628                          help="Print some things on stderr")
 629        # Default output format is JSON, while "-s" or "--short" can be
 630        # used as a shorthand for "--format=uuid". If both are specified, the
 631        # last one takes effect.
 632        self.add_argument(
 633            "-f", "--format",
 634            choices=["json", "yaml", "uuid"],
 635            type=str.lower,
 636            default="json",
 637            help="Set output format"
 638        )
 639        self.add_argument(
 640            "-s", "--short",
 641            dest="format",
 642            action="store_const", const="uuid",
 643            help="Return only UUIDs (equivalent to --format=uuid)"
 644        )
 645
 646        subparsers = self.add_subparsers(
 647            dest="subcommand",
 648            description="Available subcommands and resources",
 649            required=True,
 650            metavar="subcommand",  # Suppress huge list in help message.
 651            parser_class=FullHelpOnErrorArgumentParser
 652        )
 653
 654        keep_methods = ["ls", "get", "put", "docker"]
 655        keep_parser = subparsers.add_parser(
 656            "keep", help="Arvados Keep client", add_help=False,
 657            epilog=f"available methods: {', '.join(keep_methods)}"
 658        )
 659        keep_parser.add_argument(
 660            "method",
 661            metavar="METHOD",
 662            choices=keep_methods
 663        )
 664
 665        subparsers.add_parser(
 666            "ws", help="Arvados WebSocket client", add_help=False
 667        )
 668        subparsers.add_parser(
 669            "copy",
 670            help=(
 671                "Copy collection, workflow, or project between Arvados"
 672                " instances"
 673            ),
 674            add_help=False
 675        )
 676
 677        self.subparsers = subparsers
 678        self.discovery_document = discovery_document
 679        # Work around googleapiclient's mutation of _rootDesc/_resourceDesc
 680        # dicts when a resource is created. For instance, currently (as of
 681        # 2026-06-03) "configs.get" resource-method's parameters get mutated at
 682        # init time of the API client object (as a side-effect of getting the
 683        # default storage classes for its KeepClient object).
 684        self._ignored_parameters = frozenset(
 685            discovery_document.get("parameters", {}).keys()
 686            | discovery.STACK_QUERY_PARAMETERS
 687        )
 688        self.resource_schemas = discovery_document.get("resources", {})
 689        self._subparser_index = {}
 690        self._subcommand_to_resource = {}
 691
 692        self.add_resource_subcommands()
 693
 694        if "sys" in self._subcommand_to_resource:
 695            self._subcommand_to_resource["sy"] = (
 696                self._subcommand_to_resource["sys"]
 697            )
 698
 699        self.uuid_parser = functools.partial(
 700            _ArgTypes.UUIDInfo.parse,
 701            _ArgUtil.make_uuid_to_resource_map(
 702                self.discovery_document.get("schemas", {})
 703            )
 704        )
 705
 706        self.add_editor_subcommands()
 707        self.add_get_subcommand()
 708
 709    def add_resource_subcommands(self):
 710        """Add resources as subcommands, their associated methods as
 711        sub-subcommands, and the parameters associated with each method.
 712        """
 713        for resource, resource_schema in self.resource_schemas.items():
 714            subcommand = _ArgUtil.singularize_resource(resource)
 715            self._subcommand_to_resource[subcommand] = resource
 716            # XXX: Below, "{resource}" can be a "word" like
 717            # "api_client_authorizations" that doesn't read well; consider
 718            # retrieving more natural-language-flavored description from the
 719            # "schema" portion of the discovery doc?
 720            subcommand_summary = f"Resource subcommand for {resource}"
 721            resource_subparser = self.subparsers.add_parser(
 722                subcommand,
 723                help=subcommand_summary,
 724                description=subcommand_summary,
 725                # For backward compatibility with legacy Ruby CLI client.
 726                aliases=["sy"] if subcommand == "sys" else []
 727            )
 728            methods_dict = resource_schema.get("methods")
 729            if methods_dict:
 730                # Create a collection of "sub-subparsers" under the resource
 731                # subparser for the methods.
 732                method_subparsers = resource_subparser.add_subparsers(
 733                    title="methods",
 734                    dest="method",
 735                    parser_class=FullHelpOnErrorArgumentParser,
 736                    required=True,
 737                    help=f"Methods for subcommand '{subcommand}'"
 738                )
 739                for method, method_schema in methods_dict.items():
 740                    # Add each specific method as a (sub-)subparser with its
 741                    # associated parameters.
 742                    method_summary = method_schema.get("description")
 743                    method_parser = method_subparsers.add_parser(
 744                        method,
 745                        description=method_summary,
 746                        help=method_summary
 747                    )
 748                    for parameter_names, kwargs in _ArgUtil.get_method_options(
 749                        method_schema,
 750                        ignored_parameters=self._ignored_parameters
 751                    ):
 752                        method_parser.add_argument(*parameter_names, **kwargs)
 753
 754    def add_editor_subcommands(self):
 755        """Add the "create" and "edit" subcommands."""
 756        # Only those resources that support a "create" method can be valid
 757        # for the "create" subcommand.
 758        creatable_targets = set()
 759        for cli_name, resource in self._subcommand_to_resource.items():
 760            if "create" in self.resource_schemas[resource].get("methods", {}):
 761                creatable_targets.add(cli_name)
 762        create_parser = self.subparsers.add_parser(
 763            "create", help="Create Arvados object using external editor"
 764        )
 765        create_parser.add_argument(
 766            "target_resource",
 767            choices=sorted(creatable_targets),
 768            metavar="RESOURCE",
 769            help="Type of the resource to be created"
 770        )
 771        create_parser.add_argument(
 772            "--project-uuid", "-p",
 773            type=_ArgTypes.group_uuid,
 774            metavar="UUID",
 775            help="UUID of the project in which to create the resource"
 776        )
 777
 778        edit_parser = self.subparsers.add_parser(
 779            "edit", help="Edit Arvados object using external editor"
 780        )
 781        edit_parser.add_argument(
 782            "uuid_info",
 783            help="UUID of the object to be edited", metavar="UUID",
 784            type=self.uuid_parser
 785        )
 786        edit_parser.add_argument(
 787            "fields", nargs="*",
 788            type=str.lower,  # "type" applies to individual items.
 789            help="Fields to be edited (case-insensitive)"
 790        )
 791
 792    def add_get_subcommand(self):
 793        get_parser = self.subparsers.add_parser(
 794            "get", help=(
 795                "Fetch the specified Arvados object, select the specified"
 796                " fields, and print a text representation"
 797            )
 798        )
 799        get_parser.add_argument(
 800            "uuid_info",
 801            help="UUID of the object to be fetched", metavar="UUID",
 802            type=self.uuid_parser
 803        )
 804        get_parser.add_argument(
 805            "fields", nargs="*",
 806            type=str.lower,
 807            help="Fields to be fetched (case-insensitive)"
 808        )
 809
 810
 811def _handle_external_command(module_name: str, args: list[str]) -> NoReturn:
 812    """Import the external module for the subcommand, call the module's
 813    `main()` function with given arguments, and exit with the main function's
 814    return value as the exit status code.
 815    """
 816    external_mod = importlib.import_module(module_name)
 817    external_mod.main(args)  # Exits.
 818    raise RuntimeError(
 819        f"Error: {module_name}.main() did not exit when called with {args!r}"
 820    )
 821
 822
 823def _format_api_error_msg(err: arvados.errors.ApiError, method_call) -> str:
 824    """Format API error, with the request-id from the HttpRequest object
 825    `method_call` if 1) it is available and 2) the original message itself
 826    doesn't already contain the request-id.
 827    """
 828    # NOTE: This is not exactly the same output as that generated by the Ruby
 829    # 'arv' command upon error.
 830    msg = str(err)
 831    request_id = method_call.headers.get("X-Request-Id")
 832    if request_id and not re.search(rf"\b{re.escape(request_id)}\b", msg):
 833        msg += f" ({request_id})"
 834    return msg
 835
 836
 837def _call_resource_method(method_obj, method_args: Mapping, fmt: str) -> int:
 838    """Given the API resource method object and parameters, create an API
 839    request, execute it (do the call), and print the response of the API server
 840    in the given format.
 841
 842    Returns 0 if successful, or 1 if any errors are encountered.
 843    """
 844    method_call = method_obj(**method_args)
 845    try:
 846        result = method_call.execute()
 847    except arvados.errors.ApiError as err:
 848        msg = _format_api_error_msg(err, method_call)
 849        print(f"Error: {msg}", file=sys.stderr)
 850        return 1
 851
 852    match fmt:
 853        case "json":
 854            json.dump(result, sys.stdout, indent=1)
 855            print()
 856        case "yaml":
 857            yaml.dump(result, sys.stdout)
 858        case "uuid":
 859            if (
 860                    result.get("kind", "").endswith("List")
 861                    and result.get("items")
 862            ):
 863                for item in result["items"]:
 864                    # The received items may have the "uuid" field filtered out
 865                    # by the "--select" parameter. The ruby "arv" command
 866                    # simply outputs blank lines, which is not desirable.
 867                    obj_uuid = item.get("uuid")
 868                    if obj_uuid is None:
 869                        print(
 870                            (
 871                                "Error: at least one item in response did not"
 872                                " include a uuid. The full response was:"
 873                            ),
 874                            json.dumps(result, indent=1),
 875                            sep="\n",
 876                            file=sys.stderr
 877                        )
 878                        return 1
 879                    print(item["uuid"])
 880            else:
 881                obj_uuid = result.get("uuid")
 882                if obj_uuid is None:
 883                    print(
 884                        "Error: response did not include a uuid:",
 885                        json.dumps(result, indent=1),
 886                        sep="\n",
 887                        file=sys.stderr
 888                    )
 889                    return 1
 890                print(obj_uuid)
 891    return 0
 892
 893
 894def _handle_resource_method(api_client, resource, args) -> NoReturn:
 895    """Prepare API request by resource name and the already-parsed arguments,
 896    send the request, and analyze & print out the result.
 897    """
 898    arv_resource = getattr(api_client, resource)()
 899    arv_method = getattr(arv_resource, args.method)
 900    method_args = {
 901        k: v
 902        for k, v in vars(args).items()
 903        if k not in ArvCLIArgumentParser.global_args
 904    }
 905
 906    sys.exit(_call_resource_method(arv_method, method_args, args.format))
 907
 908
 909def _select_fields(
 910    src: Mapping[str, Any], fields: Container[str]
 911) -> Mapping[str, Any]:
 912    """Select the items of input dict `src` whose keys are in `fields` (a
 913    subset of the keys of `src`, or, if `fields` is empty, return the input
 914    `src` unmodified.
 915    """
 916    return {k: src[k] for k in fields} or src
 917
 918
 919def _get_obj_by_uuid_info(api_client, parser, args) -> tuple[int, dict | str]:
 920    """Obtain the object for the "get" subcommand and the initial object for
 921    the "arv edit" subcommand, based on the commandline args and the current
 922    API client & CLI parser instances. If the "fields" argument (`args.fields`)
 923    is provided, only those fields that are specified are returned.
 924
 925    Return value:
 926
 927    * status: int --- Status code indicating the result of operation:
 928      * 0: Success; the second return value is the initial object as a dict.
 929      * 1: API error; the second return value is the error message.
 930      * 2: Invalid input; the second return value is the error message.
 931    * value: dict | str --- Returned, filtered initial object `dict` in case of
 932      success, or an error-message string in case of failure.
 933    """
 934    resource_name = args.uuid_info.rtype_lower
 935
 936    # Filter the fields for any invalid keys of the particular resource.
 937    valid_fields = parser.discovery_document.get("schemas", {})[
 938        args.uuid_info.resource_type
 939    ]["properties"]
 940    # Sets doesn't remember insertion order, but we want to put invalid keys in
 941    # the order given by the user for consistency, so we do a dedup with dict.
 942    invalid_fields = {f: None for f in args.fields if f not in valid_fields}
 943    if invalid_fields:
 944        return 2, (
 945            f"invalid fields for resource {resource_name!r}:"
 946            f" {', '.join(map(repr, invalid_fields))}"
 947        )
 948
 949    method_call = getattr(
 950        api_client, parser._subcommand_to_resource[resource_name]
 951    )().get(uuid=args.uuid_info.uuid)
 952    try:
 953        arv_obj = method_call.execute()
 954    except arvados.errors.ApiError as err:
 955        return 1, _format_api_error_msg(err, method_call)
 956
 957    return 0, _select_fields(arv_obj, args.fields)
 958
 959
 960def _handle_external_editor_command(api_client, parser, args) -> NoReturn:
 961    """Handle the subcommands "create" or "edit"."""
 962    if args.subcommand == "create":
 963        init_obj = {
 964            "owner_uuid": args.project_uuid
 965        } if args.project_uuid else {}
 966        # Tempfile name resembling "new-collection-{random}.{json|yml}".
 967        prefix = f"new-{args.target_resource}"
 968    else:
 969        status, obj_or_msg = _get_obj_by_uuid_info(
 970            api_client, parser, args
 971        )
 972        if status != 0:
 973            print(f"Error: {obj_or_msg}", file=sys.stderr)
 974            sys.exit(status)
 975        # Tempfile name resembling
 976        # "collection-clstr-4zz18-{15chars}-{random}.{json|yml}".
 977        init_obj = obj_or_msg
 978        prefix = f"{args.uuid_info.rtype_lower}-{args.uuid_info.uuid}"
 979
 980    match args.format:
 981        case "json":
 982            editing_class = JSONEditingProcess
 983        case "yaml":
 984            editing_class = YAMLEditingProcess
 985        case _:
 986            raise RuntimeError(
 987                f"Error: unexpected value for format option: {args.format}"
 988            )
 989
 990    with editing_class(initial_object=init_obj, prefix=prefix) as editing:
 991        api_call_status = None
 992        while api_call_status is None:
 993            try:
 994                editing.edit()
 995            except OSError as err:
 996                cmd_str = shlex.join(
 997                    editing.base_command + [editing.tmp_file.name]
 998                )
 999                print(
1000                    f"Error: failed to execute editor `{cmd_str}`: {err}",
1001                    file=sys.stderr
1002                )
1003                sys.exit(1)
1004
1005            try:
1006                edited_obj = editing.load()
1007            except EditingContentError as err:
1008                # Invalid input from editor; emit error message and let the
1009                # user try again.
1010                print(str(err), file=sys.stderr)
1011                while (wants_retry := _ask_reedit()) is None:
1012                    pass
1013                if wants_retry:
1014                    # NOTE: Back to the start of the editing loop!
1015                    continue
1016                sys.exit(1)  # User won't retry; exit with failure.
1017            if not edited_obj:
1018                print(
1019                    "notice: input is empty; exiting without changes",
1020                    file=sys.stderr
1021                )
1022                sys.exit(0)
1023
1024            if args.subcommand == "create":
1025                resource = parser._subcommand_to_resource[args.target_resource]
1026            else:
1027                resource = parser._subcommand_to_resource[
1028                    args.uuid_info.rtype_lower
1029                ]
1030
1031            arv_resource = getattr(api_client, resource)()
1032
1033            if args.subcommand == "create":
1034                api_call_status = _call_resource_method(
1035                    arv_resource.create, {"body": edited_obj}, args.format
1036                )
1037            else:
1038                obj_delta = {
1039                    k: v
1040                    for k, v in edited_obj.items()
1041                    if k not in init_obj or v != init_obj[k]
1042                }
1043                if not obj_delta:
1044                    print(
1045                        "notice: object is unchanged; did not update",
1046                        file=sys.stderr
1047                    )
1048                    sys.exit(0)
1049                api_call_status = _call_resource_method(
1050                    arv_resource.update,
1051                    {"uuid": args.uuid_info.uuid, "body": obj_delta},
1052                    args.format
1053                )
1054
1055            if api_call_status != 0:
1056                # If the API request failed, try editing again if the user so
1057                # desires.
1058                wants_retry = None
1059                while wants_retry is None:
1060                    wants_retry = _ask_reedit()
1061                    if wants_retry:
1062                        # Editing loop to be restarted; clear last API call
1063                        # status.
1064                        api_call_status = None
1065                        continue
1066            # End of the editing loop.
1067        sys.exit(api_call_status)
1068    # End of the NoReturn function.
1069
1070
1071def _ask_reedit() -> bool | None:
1072    """Ask the user if they'd like to continue editing. Returns True for "yes"
1073    (default, applies also when the user types in a blank newline), False for
1074    "no", and None for any other answer.
1075    """
1076    # Put the prompt to the stderr rather than the stdout because we would like
1077    # to keep the stdout clean for API server output, which makes testing
1078    # simpler, too. Note that if we are ever to import `readline`, which we're
1079    # not doing now, this customized prompting behavior might break cursor
1080    # positioning and would have to be revisited.
1081    print(
1082        "Edit and try again? ([Y]es/no) ", end="", file=sys.stderr, flush=True
1083    )
1084    match input().strip().lower():
1085        case "" | "y" | "ye" | "yes":
1086            return True
1087        case "n" | "no":
1088            return False
1089        case _:
1090            return None
1091
1092
1093def _handle_get_subcommand(api_client, parser, args) -> NoReturn:
1094    status, obj_or_msg = _get_obj_by_uuid_info(api_client, parser, args)
1095    if status != 0:
1096        # "obj_or_msg" is a message.
1097        print(f"Error: {obj_or_msg}", file=sys.stderr)
1098    else:
1099        # "obj_or_msg" is a real Arvados object.
1100        match args.format:
1101            case "json":
1102                json.dump(obj_or_msg, sys.stdout, indent=1)
1103                print()
1104            case "yaml":
1105                yaml.dump(obj_or_msg, sys.stdout)
1106            case _:
1107                # This must not happen, as "--format=uuid" is an invalid global
1108                # option value for "get" subcommand.
1109                raise RuntimeError(
1110                    f"Error: unexpected value for format option: {args.format}"
1111                )
1112    sys.exit(status)
1113
1114
1115def dispatch(arguments=None):
1116    api_client = arvados.api("v1")
1117    cmd_parser = ArvCLIArgumentParser(api_client._rootDesc)
1118    args, remaining_args = cmd_parser.parse_known_args(arguments)
1119
1120    # There's always args.subcommand if we reach here, because "subcommand" is
1121    # required by the parser. But "method" may be absent, as is in the case of
1122    # external commands like "ws" or "copy".
1123    method = getattr(args, "method", "")
1124    command_key = f"{args.subcommand} {method}" if method else args.subcommand
1125
1126    # Are we calling an external command?
1127    ext_module = cmd_parser.external_command_modules.get(command_key)
1128    if ext_module is not None:
1129        sys.argv[0] = f"arv {command_key}"
1130        _handle_external_command(ext_module, remaining_args)  # Exits.
1131
1132    # Are we doing an API resource call?
1133    resource = cmd_parser._subcommand_to_resource.get(args.subcommand)
1134    if resource is not None:
1135        # Any further remaining args indicate either malformed or unrecognized
1136        # global args (e.g. "arvcli.py --bad-arg resource method") or undefined
1137        # parameters to a valid resouce-method combination.
1138        if remaining_args:
1139            cmd_parser.error(
1140                f"unrecognized arguments: {', '.join(remaining_args)}\n"
1141                f"Try: {cmd_parser.prog} --help\n"
1142                f"     {cmd_parser.prog} {command_key} --help",
1143                with_help=False
1144            )  # Exits with status 2.
1145        _handle_resource_method(api_client, resource, args)  # Exits.
1146
1147    # Are we starting an external editor program?
1148    if args.subcommand in ("create", "edit"):
1149        if args.format == "uuid":
1150            cmd_parser.error(
1151                "--format=uuid or -s option is not supported when creating or"
1152                " editing Arvados objects with external editor. Please"
1153                " choose --format=json (default) or --format=yaml.",
1154                with_help=False
1155            )  # Exits with status 2.
1156        _handle_external_editor_command(api_client, cmd_parser, args)  # Exits.
1157
1158    # Are we running "arv get"?
1159    if args.subcommand == "get":
1160        if args.format == "uuid":
1161            cmd_parser.error(
1162                "--format=uuid or -s option is not supported for the 'arv get'"
1163                " command. Please choose --format=json (default) or"
1164                " --format=yaml.",
1165                with_help=False
1166            )  # Exits with status 2.
1167        _handle_get_subcommand(api_client, cmd_parser, args)  # Exits.
1168
1169    # NOTE: The code immediately below is not reachable.
1170    raise RuntimeError(f"Unexpected arguments: {arguments!r}")
1171
1172
1173if __name__ == "__main__":
1174    dispatch()