arvados.commands.arvcli

   1# Copyright (C) The Arvados Authors. All rights reserved.
   2#
   3# SPDX-License-Identifier: Apache-2.0
   4
   5"""Main executable for Arvados CLI SDK, the `arv` command.
   6
   7This script implements the `arv` command's argument parser. The `arv` command
   8is meant to be invoked in the following manner:
   9
  10$ arv [--flags] subcommand|resource [...options]
  11
  12where `--flags` are common CLI options for the various subcommands.
  13
  14The `ArvCLIArgumentParser` class, specializing the standard Python
  15`argparse.ArgumentParser`, provides the support for this CLI usage.
  16"""
  17
  18
  19import abc
  20import argparse
  21from collections.abc import Container, Mapping
  22from contextlib import AbstractContextManager
  23from dataclasses import dataclass
  24import functools
  25import importlib
  26import json
  27import os
  28import re
  29import shlex
  30import shutil
  31import subprocess
  32import sys
  33from tempfile import NamedTemporaryFile
  34from typing import Any, NoReturn, TextIO
  35
  36from googleapiclient import discovery
  37from ruamel.yaml import YAML, YAMLError
  38
  39import arvados
  40import arvados.commands._util as cmd_util
  41
  42
  43yaml = YAML(typ="safe", pure=True)
  44yaml.default_flow_style = False
  45
  46
  47class _ArgTypes:
  48    """Private namespace class for JSON-related CLI argument types."""
  49
  50    @staticmethod
  51    def group_uuid(text: str) -> str:
  52        """Validate an Arvados group UUID as the value of a CLI argument (an
  53        Arvados project being a type of group).
  54        """
  55        # In theory this is a special case of "UUIDInfo" but we mostly need it
  56        # for the nicer error message.
  57        if arvados.util.group_uuid_pattern.fullmatch(text):
  58            return text
  59        raise argparse.ArgumentTypeError(
  60            f"Invalid UUID for Arvados project or group: {text}"
  61        )
  62
  63    @dataclass(frozen=True)
  64    class UUIDInfo:
  65        """'Interpreted' Arvados UUID object with resource type info."""
  66        uuid: str
  67        resource_type: str  # value in CamelCase
  68        rtype_lower: str  # value in snake_case
  69
  70        @classmethod
  71        def parse(
  72            cls, type_map: Mapping[str, str], text: str
  73        ) -> "UUIDInfo":  # self-typing support comes in Python 3.11.
  74            """Parse the UUID argument `text`. If accepted, returns an
  75            `UUIDInfo` instance whose `uuid` attribute is the input UUID
  76            unchanged and the `resource_type` attribute is the type of Arvados
  77            object (in CamelCase), as determined by the input parameter
  78            `type_map`, and `rtype_lower` is the alternative form of
  79            resource type in snake_case.
  80            """
  81            if not arvados.util.uuid_pattern.fullmatch(text):
  82                raise argparse.ArgumentTypeError(
  83                    f"Invalid Arvados object UUID: {text}"
  84                )
  85            type_code = text.split("-")[1]
  86            if type_code not in type_map:
  87                available_types = ", ".join(sorted(
  88                    f"{k} ({v})" for k, v in type_map.items()
  89                ))
  90                raise argparse.ArgumentTypeError(
  91                    f"Invalid object type code {type_code!r} in Arvados"
  92                    f" object UUID {text}: valid type codes are"
  93                    f" {available_types}"
  94                )
  95            type_key = type_map[type_code]
  96            return cls(text, type_key, _ArgUtil.camel_case_to_snake(type_key))
  97
  98    @staticmethod
  99    def _validate_type(obj_type, obj):
 100        if isinstance(obj, obj_type):
 101            return obj
 102        # No details to raise; caller handles error messaging with pretty_name.
 103        raise ValueError()
 104
 105    json_array = cmd_util.JSONStringArgument(
 106        validator=functools.partial(_validate_type, list),
 107        pretty_name="JSON array"
 108    )
 109
 110    json_object = cmd_util.JSONStringArgument(
 111        validator=functools.partial(_validate_type, dict),
 112        pretty_name="JSON object"
 113    )
 114
 115    json_filter = cmd_util.JSONArgument(
 116        validator=cmd_util.validate_filters,
 117        pretty_name="Arvados API filter"
 118    )
 119
 120    json_body = cmd_util.JSONArgument(
 121        validator=json_object.post_validator,
 122        pretty_name="JSON request body object"
 123    )
 124
 125
 126class _ArgUtil:
 127    """Private namespace class for helpful functions (static methods) that
 128    processes the discovery document for the purpose of CLI parser generation.
 129    """
 130    @staticmethod
 131    def singularize_resource(plural: str) -> str:
 132        """Returns the singular form of a resource term in the original
 133        plural.
 134        """
 135        match plural:
 136            case "vocabularies":
 137                return "vocabulary"
 138            case "sys":
 139                return "sys"
 140            case _:
 141                return plural.removesuffix("s")
 142
 143    @staticmethod
 144    def parameter_key_to_argument_name(parameter_key: str) -> str:
 145        """Convert a parameter key in the discovery document to CLI parameter
 146        form, for example, `--foo-bar`.
 147
 148        Arguments:
 149
 150        * parameter_key: str --- Parameter key in the form as they appear in
 151          the discovery document, typically like `foo_bar`.
 152        """
 153        return "--" + parameter_key.replace("_", "-")
 154
 155    @staticmethod
 156    def camel_case_to_snake(text: str) -> str:
 157        """Simple converter of CamelCase text to so-called 'snake_case' (lower
 158        case with underscore). Works if there's no consecutive upper-case
 159        letters such as "API".
 160        """
 161        return text[:1].lower() + "".join(
 162            f"_{c.lower()}" if c.isupper() else c for c in text[1:]
 163        )
 164
 165    @staticmethod
 166    def get_method_options(
 167        method_schema: Mapping[str, Any],
 168        ignored_parameters: Container[str] = ()
 169    ):
 170        """Generate command-line options, in the form of "-f/--foo", from the
 171        parameters as defined by the API method schema in the discovery
 172        document.
 173
 174        For each key "foo_bar" in the "parameters" field of the method schema,
 175        command-line options are created according to its definition as
 176        follows.
 177
 178        If the parameter type is "boolean", a pair of options "--no-foo-bar"
 179        and "--foo-bar" are created, with opposite meaning.
 180
 181        If the parameter type is "integer", the CLI input will be interpreted
 182        as a Python int.
 183
 184        All other parameter types are parsed as Python str.
 185
 186        The short form of each option will also be created, by taking the first
 187        letter of the long form, except when that letter is already used, in
 188        which case the second letter will be used, and so on. For example,
 189        "--foo-bar" will have short form "-f", unless "-f" is already used for
 190        another option, in which case "-o" will be used, etc.
 191
 192        The "negative" form of boolean options ("--no-foo-bar") will not have
 193        separate short forms of their own.
 194
 195        This  generator yields tuples in the form of `(names, kwargs)`, where
 196        `names` is a one- or two-element tuple and `kwargs` is a dict, suitable
 197        to be passed as
 198        `argparse.ArgumentParser.add_argument(*names, **kwargs)`.
 199
 200        Arguments:
 201
 202        * method_schema: Mapping[str, Any] --- Dict object from the parsed
 203          discover document that defines a method.
 204        * ignored_parameters: Container[str] --- If provided, the parameters
 205          that are in `ignored_parameters` will not be processed.
 206        """
 207        parameters_schema = method_schema.get("parameters", {}).copy()
 208        # If the method comes with the "request" field, add another parameter
 209        # based on the sole key in the "properties" dict of that field
 210        request_schema = method_schema.get("request")
 211        if request_schema is not None and request_schema.get("properties"):
 212            for parameter_key in request_schema["properties"].keys():
 213                parameters_schema[parameter_key] = {
 214                    "type": "request",  # special value for request parameter
 215                    "required": request_schema.get("required"),
 216                    "description": (
 217                        f"Either a string representing {parameter_key} as JSON"
 218                        f" or a filename from which to read {parameter_key}"
 219                        " JSON (use '-' to read from stdin)."
 220                    )
 221                }
 222        argument_key_abbrevs = set("h")  # prevent conflict with "help"
 223        for parameter_key, parameter_dict in parameters_schema.items():
 224            if parameter_key in ignored_parameters:
 225                continue
 226            parameter_kwargs = {
 227                "required": parameter_dict.get("required", False)
 228            }
 229            parameter_kwargs["help"] = parameter_dict.get("description", "")
 230            if parameter_kwargs["required"]:
 231                parameter_kwargs["help"] += " This option must be specified."
 232            # The "type" member refers to one of the JSON values types, out of
 233            # string/integer/array/object/boolean.
 234            # NOTE: Currently, enum-like value choices are not implemented, as
 235            # the enum values cannot be directly inferred from the discover
 236            # doc.
 237            argument_key = _ArgUtil.parameter_key_to_argument_name(
 238                parameter_key
 239            )
 240            for argument_short_key in argument_key:
 241                if (
 242                    argument_short_key.isalpha()
 243                    and argument_short_key not in argument_key_abbrevs
 244                ):
 245                    argument_key_abbrevs.add(argument_short_key)
 246                    break
 247            else:
 248                # If the letters of the full argument name are exhausted, fall
 249                # back to not using a short argument, indicated by the special
 250                # value None:
 251                argument_short_key = None
 252            default = parameter_dict.get("default")
 253            if default is not None and parameter_dict.get("type") != "boolean":
 254                parameter_kwargs["help"] += f" Default: {default}."
 255            match parameter_dict.get("type"):
 256                case "boolean":
 257                    # Using the 'action="store_true" (or "store_false")'
 258                    # mechanism results in flag-like action rather than an
 259                    # option that takes a true or false value. For each bool
 260                    # flag "--foo", also generate an additional "negative"
 261                    # version "--no-foo".
 262                    neg_argument_key = _ArgUtil.parameter_key_to_argument_name(
 263                        f"no_{parameter_key}"
 264                    )
 265                    neg_parameter_kwargs = {}
 266                    neg_parameter_kwargs["action"] = "store_false"
 267                    neg_parameter_kwargs["required"] = False
 268                    neg_parameter_kwargs["dest"] = parameter_key
 269                    neg_parameter_kwargs["default"] = json.loads(
 270                        default if default is not None else "null"
 271                    )
 272                    yield (neg_argument_key,), neg_parameter_kwargs
 273
 274                    parameter_kwargs["action"] = "store_true"
 275                    parameter_kwargs["dest"] = parameter_key
 276                    parameter_kwargs["default"] = (
 277                        neg_parameter_kwargs["default"]
 278                    )
 279                case "integer":
 280                    parameter_kwargs["type"] = int
 281                    parameter_kwargs["metavar"] = "N"
 282                case "array":
 283                    # The filters parameter is only used with "getter" methods
 284                    # that doesn't send a request body (which is exclusive to
 285                    # "creator"/"updater" methods). This means it's generally
 286                    # safe to use the "json_filter" type converter which can
 287                    # read from the stdin; it wouldn't conflict with the
 288                    # request body parameter which can also read the stdin.
 289                    if parameter_key == "filters":
 290                        parameter_kwargs["type"] = _ArgTypes.json_filter
 291                        parameter_kwargs["metavar"] = "{JSON,FILE,-}"
 292                        parameter_kwargs["help"] += (
 293                            " This can be a filename from which to read"
 294                            " JSON (use '-' to read from stdin)."
 295                        )
 296                    else:
 297                        parameter_kwargs["type"] = _ArgTypes.json_array
 298                        parameter_kwargs["metavar"] = "JSON_ARRAY"
 299                case "object":
 300                    parameter_kwargs["type"] = _ArgTypes.json_object
 301                    parameter_kwargs["metavar"] = "JSON_OBJECT"
 302                case "request":
 303                    parameter_kwargs["dest"] = "body"
 304                    parameter_kwargs["type"] = _ArgTypes.json_body
 305                    parameter_kwargs["metavar"] = "{JSON,FILE,-}"
 306                case _:
 307                    parameter_kwargs["type"] = str
 308                    parameter_kwargs["metavar"] = "STR"
 309            if argument_short_key is None:
 310                yield (argument_key,), parameter_kwargs
 311            else:
 312                yield (
 313                    (f"-{argument_short_key}", argument_key), parameter_kwargs
 314                )
 315
 316    @staticmethod
 317    def make_uuid_to_resource_map(schemas: dict[str, dict]) -> dict[str, str]:
 318        """Returns a mapping of Arvados object UUID prefixes to resource names
 319        (in the schema-key, CamelCase form, e.g. "ContainerRequest") based on
 320        the input "schemas" portion of the discovery document.
 321        """
 322        result = {}
 323        for schema in schemas.values():
 324            if (
 325                    (prefix := schema.get("uuidPrefix"))
 326                    and (key := schema.get("id"))
 327            ):
 328                result[prefix] = key
 329        return result
 330
 331
 332class ObjectEditingProcessBase(AbstractContextManager, abc.ABC):
 333    """Base class represending a process (in the generic sense, rather than
 334    "a Unix/Linux process") of editing an Arvados object with an external
 335    editor on a temporary file.
 336
 337    The methods `serialize(self, obj, file)` and `deserialize(self, file)` are
 338    abstract methods meant to be overridden. `serialize()` should write to the
 339    open `file` (any file-like object), and `deserialize()` should return an
 340    object loaded from the file.
 341
 342    When initialized, no external file has been created. To do so, enter it as
 343    a context manager.
 344
 345    Upon entering the context, the temporary file will be opened and written to
 346    with proper initial content if necessary. Upon leaving, the temporary file
 347    will be closed and cleaned-up (this normally means the file will be gone
 348    permanently).
 349
 350    Attributes:
 351
 352    * tmp_file: Optional[tempfile.NamedTemporaryFile] --- Temporary file to be
 353      edited.
 354    * prefix: Optional[str] --- Prefix of temporary filename if provided.
 355    * suffix: Optional[str] --- Suffix of temporary filename if provided. This
 356      can be a filename extension with the leading dot/period character `.`,
 357      useful for hinting the external editor with syntax highlighting.
 358    * base_command: list[str] --- Command-line argument list for invoking the
 359      external editor program. See `get_editor_cmdline()` for more.
 360    """
 361    _tmpfile_extension = None
 362
 363    def __init__(self, initial_object=None, prefix=None, file_extension=None):
 364        """Arguments:
 365
 366        * initial_object: Optional[Any] --- Initial object to be serialized and
 367          written to the temporary file before the editor process is run. If
 368          not provided, the file will be opened empty in the editor.
 369        * prefix: Optional[str] --- String to be used as the prefix
 370          of the temporary file's basename, followed by a hyphen (`-`)
 371          character that will be added automatically. If not provided, the
 372          initial object's `uuid` field will be used if available; otherwise, a
 373          platform-dependent prefix will be chosen automatically. A UUID as
 374          part of the filename is for information only, and it may be displayed
 375          in the editor's UI.
 376        * file_extension: Optional[str] --- Filename extension (without leading
 377          dot) of the temporary file, e.g. "json" or "yml". This information
 378          may be used by the editor to provide syntax highlighting, automatic
 379          indentation, completion, etc.
 380        """
 381        self.initial_object = initial_object
 382
 383        if prefix:
 384            self.prefix = f"{prefix}-"
 385        elif (
 386            isinstance(initial_object, Mapping)
 387            and (obj_uuid := initial_object.get("uuid"))
 388        ):
 389            self.prefix = f"{obj_uuid}-"
 390        else:
 391            self.prefix = None
 392
 393        ext = self._tmpfile_extension or file_extension
 394        self.suffix = f".{ext}" if ext else None
 395
 396        self.tmp_file = None
 397        self.base_command = self.get_editor_cmdline()
 398
 399    @staticmethod
 400    def get_editor_cmdline() -> list[str]:
 401        """Returns a partial command-line argument list that begins with the
 402        external editor program. The precedence is the $VISUAL environment
 403        variable, followed by $EDITOR; and if both are missing, then `nano` if
 404        it exists in the $PATH; and finally the hard-coded value `vi` no matter
 405        the command exists or not.
 406        """
 407        if cmd_str := (os.environ.get("VISUAL") or os.environ.get("EDITOR")):
 408            cmd = shlex.split(cmd_str)
 409        elif cmd_str := shutil.which("nano"):
 410            cmd = [cmd_str]
 411        else:
 412            cmd = ["vi"]
 413        return cmd
 414
 415    @abc.abstractmethod
 416    def serialize(self, obj: Any, file: TextIO) -> None:
 417        """Abstract method for serializing any object `obj` to the file-like
 418        object `file` as text.
 419        """
 420
 421    @abc.abstractmethod
 422    def deserialize(self, file: TextIO) -> Any:
 423        """Abstract method for loading from the file-like object `file` as
 424        text. Returns the object deserialized from the text content.
 425        """
 426
 427    def check_tmp_file(self):
 428        """Perform a basic sanity check for the temp file being usable."""
 429        if self.tmp_file is None or self.tmp_file.closed:
 430            raise RuntimeError("Temporary file is not available")
 431
 432    def dump(self, obj: Any) -> None:
 433        """Overwrite the temporary file with the serialized object `obj`."""
 434        self.check_tmp_file()
 435        # The following should not be done while the child process is pending.
 436        self.tmp_file.truncate(0)
 437        self.serialize(obj, self.tmp_file)
 438        self.tmp_file.flush()
 439
 440    def load(self) -> Any:
 441        """Read the temporary file from the beginning. Returns the deserialized
 442        object, or None if the file is empty or only whitespace.
 443        """
 444        self.check_tmp_file()
 445        # Snoop the file to see if it consists of only whitespace characters
 446        # (including empty lines); if so, return the special value None.
 447        with open(self.tmp_file.name, "r") as fdup:
 448            if not fdup.read().strip():
 449                return None
 450
 451        self.tmp_file.seek(0)
 452        return self.deserialize(self.tmp_file)
 453
 454    def edit(self) -> subprocess.CompletedProcess:
 455        """Run external editor and wait for it to finish."""
 456        self.check_tmp_file()
 457        return subprocess.run(
 458            self.base_command + [self.tmp_file.name],
 459            check=False
 460        )  # Wait for child.
 461
 462    def __enter__(self):
 463        self.tmp_file = NamedTemporaryFile(
 464            mode="w+", prefix=self.prefix, suffix=self.suffix
 465        )
 466        if self.initial_object is not None:
 467            self.dump(self.initial_object)
 468        return self
 469
 470    def __exit__(self, exc_type, exc_value, traceback):
 471        self.tmp_file.close()
 472
 473
 474class EditingContentError(ValueError):
 475    """Exception that indicates the content provided by the user via the editor
 476    is invalid for the specific format.
 477    """
 478    def __init__(
 479        self,
 480        path=None, line=0, column=0,
 481        file_type=None,
 482        original_exception=None,
 483    ):
 484        self.path = path
 485        self.line = line
 486        self.column = column
 487        self.file_type = file_type
 488        self.original_exception = original_exception
 489
 490    def __str__(self):
 491        msg = (
 492            f"Error: invalid input file [type {self.file_type or 'unknown'}]:"
 493            f" {self.path}:{self.line}:{self.column}"
 494        )
 495        if (
 496            self.original_exception
 497            and (orig_msg := str(self.original_exception))
 498        ):
 499            msg += f":\n{orig_msg}"
 500        return msg
 501
 502
 503class JSONEditingProcess(ObjectEditingProcessBase):
 504    """Subclass of editing process tuned for JSON files."""
 505    _tmpfile_extension = "json"
 506    input_error_type = functools.partial(
 507        EditingContentError, file_type="JSON"
 508    )
 509
 510    def __init__(self, *args, indent: int = 1, **kwargs):
 511        """Arguments:
 512
 513        * indent: int --- Number of spaces for each indentation level in the
 514          JSON file. Default: 1.
 515        """
 516        super().__init__(*args, **kwargs)
 517        self.indent = indent
 518
 519    def serialize(self, obj: Mapping[str, Any], file: TextIO) -> None:
 520        return json.dump(obj, file, indent=self.indent)
 521
 522    def deserialize(self, file: TextIO) -> Mapping[str, Any]:
 523        path = getattr(file, "name", "<unknown path>")
 524        try:
 525            obj = json.load(file)
 526        except json.JSONDecodeError as err:
 527            line = getattr(err, "lineno", 0)
 528            column = getattr(err, "colno", 0)
 529            raise self.input_error_type(
 530                path=path, line=line, column=column,
 531                original_exception=err
 532            )
 533        if not isinstance(obj, Mapping):
 534            raise self.input_error_type(
 535                path=path,
 536                original_exception=ValueError(
 537                    f"JSON input has type '{type(obj).__name__}',"
 538                    " not a valid Arvados object"
 539                )
 540            )
 541        return obj
 542
 543
 544class YAMLEditingProcess(ObjectEditingProcessBase):
 545    """Subclass of editing process tuned for YAML files."""
 546    _tmpfile_extension = "yml"
 547    input_error_type = functools.partial(
 548        EditingContentError, file_type="YAML"
 549    )
 550
 551    def serialize(self, obj: Mapping[str, Any], file: TextIO) -> None:
 552        return yaml.dump(obj, file)
 553
 554    def deserialize(self, file: TextIO) -> Mapping[str, Any]:
 555        path = getattr(file, "name", "<unknown path>")
 556        try:
 557            obj = yaml.load(file)
 558        except YAMLError as err:
 559            if problem_mark := getattr(err, "problem_mark", None):
 560                line = getattr(problem_mark, "line", 0)
 561                column = getattr(problem_mark, "column", 0)
 562            else:
 563                line = 0
 564                column = 0
 565            raise self.input_error_type(
 566                path=path, line=line, column=column,
 567                original_exception=err
 568            )
 569        if not isinstance(obj, Mapping):
 570            raise self.input_error_type(
 571                path=path,
 572                original_exception=ValueError(
 573                    f"YAML input has type '{type(obj).__name__}',"
 574                    " not a valid Arvados object"
 575                )
 576            )
 577        return obj
 578
 579
 580class FullHelpOnErrorArgumentParser(argparse.ArgumentParser):
 581    """Argument parser subclass that customizes the `error()` method.
 582
 583    Intended to be used as a base to a parser with complex subparsers, to print
 584    more-useful information when a required subcommand is missing.
 585    """
 586    def error(self, message, with_help=True):
 587        if with_help:
 588            self.print_help(sys.stderr)
 589            print(file=sys.stderr)
 590        # NOTE: self.prog is to be overridden by child class
 591        print(f"{self.prog}: error: {message}", file=sys.stderr)
 592        sys.exit(2)
 593
 594
 595class ArvCLIArgumentParser(FullHelpOnErrorArgumentParser):
 596    """Argument parser for `arv` commands.
 597    """
 598    prog = "arv"
 599    global_args = frozenset((
 600        "dry_run",
 601        "verbose",
 602        "format",
 603        "subcommand",
 604        "method"
 605    ))
 606    external_command_modules = {
 607        "keep ls": "arvados.commands.ls",
 608        "keep get": "arvados.commands.get",
 609        "keep put": "arvados.commands.put",
 610        "keep docker": "arvados.commands.keepdocker",
 611        "ws": "arvados.commands.ws",
 612        "copy": "arvados.commands.arv_copy"
 613    }
 614
 615    def __init__(self, discovery_document: dict[str, str | dict], **kwargs):
 616        """Arguments:
 617
 618        * discovery_document: dict --- Dict containing the parsed API discovery
 619          document; can be obtained as the `_rootDesc` attribute of an
 620          Arvados API client object.
 621        """
 622        super().__init__(
 623            description="Arvados command line client",
 624            prog=self.prog,
 625            **kwargs
 626        )
 627        # Common flags to the main command.
 628        self.add_argument("-n", "--dry-run", action="store_true",
 629                          help="Don't actually do anything")
 630        self.add_argument("-v", "--verbose", action="store_true",
 631                          help="Print some things on stderr")
 632        # Default output format is JSON, while "-s" or "--short" can be
 633        # used as a shorthand for "--format=uuid". If both are specified, the
 634        # last one takes effect.
 635        self.add_argument(
 636            "-f", "--format",
 637            choices=["json", "yaml", "uuid"],
 638            type=str.lower,
 639            default="json",
 640            help="Set output format"
 641        )
 642        self.add_argument(
 643            "-s", "--short",
 644            dest="format",
 645            action="store_const", const="uuid",
 646            help="Return only UUIDs (equivalent to --format=uuid)"
 647        )
 648
 649        subparsers = self.add_subparsers(
 650            dest="subcommand",
 651            description="Available subcommands and resources",
 652            required=True,
 653            metavar="subcommand",  # Suppress huge list in help message.
 654            parser_class=FullHelpOnErrorArgumentParser
 655        )
 656
 657        keep_methods = ["ls", "get", "put", "docker"]
 658        keep_parser = subparsers.add_parser(
 659            "keep", help="Arvados Keep client", add_help=False,
 660            epilog=f"available methods: {', '.join(keep_methods)}"
 661        )
 662        keep_parser.add_argument(
 663            "method",
 664            metavar="METHOD",
 665            choices=keep_methods
 666        )
 667
 668        subparsers.add_parser(
 669            "ws", help="Arvados WebSocket client", add_help=False
 670        )
 671        subparsers.add_parser(
 672            "copy",
 673            help=(
 674                "Copy collection, workflow, or project between Arvados"
 675                " instances"
 676            ),
 677            add_help=False
 678        )
 679
 680        self.subparsers = subparsers
 681        self.discovery_document = discovery_document
 682        # Work around googleapiclient's mutation of _rootDesc/_resourceDesc
 683        # dicts when a resource is created. For instance, currently (as of
 684        # 2026-06-03) "configs.get" resource-method's parameters get mutated at
 685        # init time of the API client object (as a side-effect of getting the
 686        # default storage classes for its KeepClient object).
 687        self._ignored_parameters = frozenset(
 688            discovery_document.get("parameters", {}).keys()
 689            | discovery.STACK_QUERY_PARAMETERS
 690        )
 691        self.resource_schemas = discovery_document.get("resources", {})
 692        self._subparser_index = {}
 693        self._subcommand_to_resource = {}
 694
 695        self.add_resource_subcommands()
 696
 697        if "sys" in self._subcommand_to_resource:
 698            self._subcommand_to_resource["sy"] = (
 699                self._subcommand_to_resource["sys"]
 700            )
 701
 702        self.uuid_parser = functools.partial(
 703            _ArgTypes.UUIDInfo.parse,
 704            _ArgUtil.make_uuid_to_resource_map(
 705                self.discovery_document.get("schemas", {})
 706            )
 707        )
 708
 709        self.add_editor_subcommands()
 710        self.add_get_subcommand()
 711
 712    def add_resource_subcommands(self):
 713        """Add resources as subcommands, their associated methods as
 714        sub-subcommands, and the parameters associated with each method.
 715        """
 716        for resource, resource_schema in self.resource_schemas.items():
 717            subcommand = _ArgUtil.singularize_resource(resource)
 718            self._subcommand_to_resource[subcommand] = resource
 719            # XXX: Below, "{resource}" can be a "word" like
 720            # "api_client_authorizations" that doesn't read well; consider
 721            # retrieving more natural-language-flavored description from the
 722            # "schema" portion of the discovery doc?
 723            subcommand_summary = f"Resource subcommand for {resource}"
 724            resource_subparser = self.subparsers.add_parser(
 725                subcommand,
 726                help=subcommand_summary,
 727                description=subcommand_summary,
 728                # For backward compatibility with legacy Ruby CLI client.
 729                aliases=["sy"] if subcommand == "sys" else []
 730            )
 731            methods_dict = resource_schema.get("methods")
 732            if methods_dict:
 733                # Create a collection of "sub-subparsers" under the resource
 734                # subparser for the methods.
 735                method_subparsers = resource_subparser.add_subparsers(
 736                    title="methods",
 737                    dest="method",
 738                    parser_class=FullHelpOnErrorArgumentParser,
 739                    required=True,
 740                    help=f"Methods for subcommand '{subcommand}'"
 741                )
 742                for method, method_schema in methods_dict.items():
 743                    # Add each specific method as a (sub-)subparser with its
 744                    # associated parameters.
 745                    method_summary = method_schema.get("description")
 746                    method_parser = method_subparsers.add_parser(
 747                        method,
 748                        description=method_summary,
 749                        help=method_summary
 750                    )
 751                    for parameter_names, kwargs in _ArgUtil.get_method_options(
 752                        method_schema,
 753                        ignored_parameters=self._ignored_parameters
 754                    ):
 755                        method_parser.add_argument(*parameter_names, **kwargs)
 756
 757    def add_editor_subcommands(self):
 758        """Add the "create" and "edit" subcommands."""
 759        # Only those resources that support a "create" method can be valid
 760        # for the "create" subcommand.
 761        creatable_targets = set()
 762        for cli_name, resource in self._subcommand_to_resource.items():
 763            if "create" in self.resource_schemas[resource].get("methods", {}):
 764                creatable_targets.add(cli_name)
 765        create_parser = self.subparsers.add_parser(
 766            "create", help="Create Arvados object using external editor"
 767        )
 768        create_parser.add_argument(
 769            "target_resource",
 770            choices=sorted(creatable_targets),
 771            metavar="RESOURCE",
 772            help="Type of the resource to be created"
 773        )
 774        create_parser.add_argument(
 775            "--project-uuid", "-p",
 776            type=_ArgTypes.group_uuid,
 777            metavar="UUID",
 778            help="UUID of the project in which to create the resource"
 779        )
 780
 781        edit_parser = self.subparsers.add_parser(
 782            "edit", help="Edit Arvados object using external editor"
 783        )
 784        edit_parser.add_argument(
 785            "uuid_info",
 786            help="UUID of the object to be edited", metavar="UUID",
 787            type=self.uuid_parser
 788        )
 789        edit_parser.add_argument(
 790            "fields", nargs="*",
 791            type=str.lower,  # "type" applies to individual items.
 792            help="Fields to be edited (case-insensitive)"
 793        )
 794
 795    def add_get_subcommand(self):
 796        get_parser = self.subparsers.add_parser(
 797            "get", help=(
 798                "Fetch the specified Arvados object, select the specified"
 799                " fields, and print a text representation"
 800            )
 801        )
 802        get_parser.add_argument(
 803            "uuid_info",
 804            help="UUID of the object to be fetched", metavar="UUID",
 805            type=self.uuid_parser
 806        )
 807        get_parser.add_argument(
 808            "fields", nargs="*",
 809            type=str.lower,
 810            help="Fields to be fetched (case-insensitive)"
 811        )
 812
 813
 814def _handle_external_command(module_name: str, args: list[str]) -> NoReturn:
 815    """Import the external module for the subcommand, call the module's
 816    `main()` function with given arguments, and exit with the main function's
 817    return value as the exit status code.
 818    """
 819    external_mod = importlib.import_module(module_name)
 820    sys.exit(external_mod.main(args))
 821
 822
 823def _format_api_error_msg(err: arvados.errors.ApiError, method_call) -> str:
 824    """Format API error, with the request-id from the HttpRequest object
 825    `method_call` if 1) it is available and 2) the original message itself
 826    doesn't already contain the request-id.
 827    """
 828    # NOTE: This is not exactly the same output as that generated by the Ruby
 829    # 'arv' command upon error.
 830    msg = str(err)
 831    request_id = method_call.headers.get("X-Request-Id")
 832    if request_id and not re.search(rf"\b{re.escape(request_id)}\b", msg):
 833        msg += f" ({request_id})"
 834    return msg
 835
 836
 837def _call_resource_method(method_obj, method_args: Mapping, fmt: str) -> int:
 838    """Given the API resource method object and parameters, create an API
 839    request, execute it (do the call), and print the response of the API server
 840    in the given format.
 841
 842    Returns 0 if successful, or 1 if any errors are encountered.
 843    """
 844    method_call = method_obj(**method_args)
 845    try:
 846        result = method_call.execute()
 847    except arvados.errors.ApiError as err:
 848        msg = _format_api_error_msg(err, method_call)
 849        print(f"Error: {msg}", file=sys.stderr)
 850        return 1
 851
 852    match fmt:
 853        case "json":
 854            json.dump(result, sys.stdout, indent=1)
 855            print()
 856        case "yaml":
 857            yaml.dump(result, sys.stdout)
 858        case "uuid":
 859            if (
 860                    result.get("kind", "").endswith("List")
 861                    and result.get("items")
 862            ):
 863                for item in result["items"]:
 864                    # The received items may have the "uuid" field filtered out
 865                    # by the "--select" parameter. The ruby "arv" command
 866                    # simply outputs blank lines, which is not desirable.
 867                    obj_uuid = item.get("uuid")
 868                    if obj_uuid is None:
 869                        print(
 870                            (
 871                                "Error: at least one item in response did not"
 872                                " include a uuid. The full response was:"
 873                            ),
 874                            json.dumps(result, indent=1),
 875                            sep="\n",
 876                            file=sys.stderr
 877                        )
 878                        return 1
 879                    print(item["uuid"])
 880            else:
 881                obj_uuid = result.get("uuid")
 882                if obj_uuid is None:
 883                    print(
 884                        "Error: response did not include a uuid:",
 885                        json.dumps(result, indent=1),
 886                        sep="\n",
 887                        file=sys.stderr
 888                    )
 889                    return 1
 890                print(obj_uuid)
 891    return 0
 892
 893
 894def _handle_resource_method(api_client, resource, args) -> NoReturn:
 895    """Prepare API request by resource name and the already-parsed arguments,
 896    send the request, and analyze & print out the result.
 897    """
 898    arv_resource = getattr(api_client, resource)()
 899    arv_method = getattr(arv_resource, args.method)
 900    method_args = {
 901        k: v
 902        for k, v in vars(args).items()
 903        if k not in ArvCLIArgumentParser.global_args
 904    }
 905
 906    sys.exit(_call_resource_method(arv_method, method_args, args.format))
 907
 908
 909def _select_fields(
 910    src: Mapping[str, Any], fields: Container[str]
 911) -> Mapping[str, Any]:
 912    """Select the items of input dict `src` whose keys are in `fields` (a
 913    subset of the keys of `src`, or, if `fields` is empty, return the input
 914    `src` unmodified.
 915    """
 916    return {k: src[k] for k in fields} or src
 917
 918
 919def _get_obj_by_uuid_info(api_client, parser, args) -> tuple[int, dict | str]:
 920    """Obtain the object for the "get" subcommand and the initial object for
 921    the "arv edit" subcommand, based on the commandline args and the current
 922    API client & CLI parser instances. If the "fields" argument (`args.fields`)
 923    is provided, only those fields that are specified are returned.
 924
 925    Return value:
 926
 927    * status: int --- Status code indicating the result of operation:
 928      * 0: Success; the second return value is the initial object as a dict.
 929      * 1: API error; the second return value is the error message.
 930      * 2: Invalid input; the second return value is the error message.
 931    * value: dict | str --- Returned, filtered initial object `dict` in case of
 932      success, or an error-message string in case of failure.
 933    """
 934    resource_name = args.uuid_info.rtype_lower
 935
 936    # Filter the fields for any invalid keys of the particular resource.
 937    valid_fields = parser.discovery_document.get("schemas", {})[
 938        args.uuid_info.resource_type
 939    ]["properties"]
 940    # Sets doesn't remember insertion order, but we want to put invalid keys in
 941    # the order given by the user for consistency, so we do a dedup with dict.
 942    invalid_fields = {f: None for f in args.fields if f not in valid_fields}
 943    if invalid_fields:
 944        return 2, (
 945            f"invalid fields for resource {resource_name!r}:"
 946            f" {', '.join(map(repr, invalid_fields))}"
 947        )
 948
 949    method_call = getattr(
 950        api_client, parser._subcommand_to_resource[resource_name]
 951    )().get(uuid=args.uuid_info.uuid)
 952    try:
 953        arv_obj = method_call.execute()
 954    except arvados.errors.ApiError as err:
 955        return 1, _format_api_error_msg(err, method_call)
 956
 957    return 0, _select_fields(arv_obj, args.fields)
 958
 959
 960def _handle_external_editor_command(api_client, parser, args) -> NoReturn:
 961    """Handle the subcommands "create" or "edit"."""
 962    if args.subcommand == "create":
 963        init_obj = {
 964            "owner_uuid": args.project_uuid
 965        } if args.project_uuid else {}
 966        # Tempfile name resembling "new-collection-{random}.{json|yml}".
 967        prefix = f"new-{args.target_resource}"
 968    else:
 969        status, obj_or_msg = _get_obj_by_uuid_info(
 970            api_client, parser, args
 971        )
 972        if status != 0:
 973            print(f"Error: {obj_or_msg}", file=sys.stderr)
 974            sys.exit(status)
 975        # Tempfile name resembling
 976        # "collection-clstr-4zz18-{15chars}-{random}.{json|yml}".
 977        init_obj = obj_or_msg
 978        prefix = f"{args.uuid_info.rtype_lower}-{args.uuid_info.uuid}"
 979
 980    match args.format:
 981        case "json":
 982            editing_class = JSONEditingProcess
 983        case "yaml":
 984            editing_class = YAMLEditingProcess
 985        case _:
 986            raise RuntimeError(
 987                f"Error: unexpected value for format option: {args.format}"
 988            )
 989
 990    with editing_class(initial_object=init_obj, prefix=prefix) as editing:
 991        api_call_status = None
 992        while api_call_status is None:
 993            try:
 994                editing.edit()
 995            except OSError as err:
 996                cmd_str = shlex.join(
 997                    editing.base_command + [editing.tmp_file.name]
 998                )
 999                print(
1000                    f"Error: failed to execute editor `{cmd_str}`: {err}",
1001                    file=sys.stderr
1002                )
1003                sys.exit(1)
1004
1005            try:
1006                edited_obj = editing.load()
1007            except EditingContentError as err:
1008                # Invalid input from editor; emit error message and let the
1009                # user try again.
1010                print(str(err), file=sys.stderr)
1011                while (wants_retry := _ask_reedit()) is None:
1012                    pass
1013                if wants_retry:
1014                    # NOTE: Back to the start of the editing loop!
1015                    continue
1016                sys.exit(1)  # User won't retry; exit with failure.
1017            if not edited_obj:
1018                print(
1019                    "notice: input is empty; exiting without changes",
1020                    file=sys.stderr
1021                )
1022                sys.exit(0)
1023
1024            if args.subcommand == "create":
1025                resource = parser._subcommand_to_resource[args.target_resource]
1026            else:
1027                resource = parser._subcommand_to_resource[
1028                    args.uuid_info.rtype_lower
1029                ]
1030
1031            arv_resource = getattr(api_client, resource)()
1032
1033            if args.subcommand == "create":
1034                api_call_status = _call_resource_method(
1035                    arv_resource.create, {"body": edited_obj}, args.format
1036                )
1037            else:
1038                obj_delta = {
1039                    k: v
1040                    for k, v in edited_obj.items()
1041                    if k not in init_obj or v != init_obj[k]
1042                }
1043                if not obj_delta:
1044                    print(
1045                        "notice: object is unchanged; did not update",
1046                        file=sys.stderr
1047                    )
1048                    sys.exit(0)
1049                api_call_status = _call_resource_method(
1050                    arv_resource.update,
1051                    {"uuid": args.uuid_info.uuid, "body": obj_delta},
1052                    args.format
1053                )
1054
1055            if api_call_status != 0:
1056                # If the API request failed, try editing again if the user so
1057                # desires.
1058                wants_retry = None
1059                while wants_retry is None:
1060                    wants_retry = _ask_reedit()
1061                    if wants_retry:
1062                        # Editing loop to be restarted; clear last API call
1063                        # status.
1064                        api_call_status = None
1065                        continue
1066            # End of the editing loop.
1067        sys.exit(api_call_status)
1068    # End of the NoReturn function.
1069
1070
1071def _ask_reedit() -> bool | None:
1072    """Ask the user if they'd like to continue editing. Returns True for "yes"
1073    (default, applies also when the user types in a blank newline), False for
1074    "no", and None for any other answer.
1075    """
1076    # Put the prompt to the stderr rather than the stdout because we would like
1077    # to keep the stdout clean for API server output, which makes testing
1078    # simpler, too. Note that if we are ever to import `readline`, which we're
1079    # not doing now, this customized prompting behavior might break cursor
1080    # positioning and would have to be revisited.
1081    print(
1082        "Edit and try again? ([Y]es/no) ", end="", file=sys.stderr, flush=True
1083    )
1084    match input().strip().lower():
1085        case "" | "y" | "ye" | "yes":
1086            return True
1087        case "n" | "no":
1088            return False
1089        case _:
1090            return None
1091
1092
1093def _handle_get_subcommand(api_client, parser, args) -> NoReturn:
1094    status, obj_or_msg = _get_obj_by_uuid_info(api_client, parser, args)
1095    if status != 0:
1096        # "obj_or_msg" is a message.
1097        print(f"Error: {obj_or_msg}", file=sys.stderr)
1098    else:
1099        # "obj_or_msg" is a real Arvados object.
1100        match args.format:
1101            case "json":
1102                json.dump(obj_or_msg, sys.stdout, indent=1)
1103                print()
1104            case "yaml":
1105                yaml.dump(obj_or_msg, sys.stdout)
1106            case _:
1107                # This must not happen, as "--format=uuid" is an invalid global
1108                # option value for "get" subcommand.
1109                raise RuntimeError(
1110                    f"Error: unexpected value for format option: {args.format}"
1111                )
1112    sys.exit(status)
1113
1114
1115def dispatch(arguments=None):
1116    api_client = arvados.api("v1")
1117    cmd_parser = ArvCLIArgumentParser(api_client._rootDesc)
1118    args, remaining_args = cmd_parser.parse_known_args(arguments)
1119
1120    # There's always args.subcommand if we reach here, because "subcommand" is
1121    # required by the parser. But "method" may be absent, as is in the case of
1122    # external commands like "ws" or "copy".
1123    method = getattr(args, "method", "")
1124    command_key = f"{args.subcommand} {method}" if method else args.subcommand
1125
1126    # Are we calling an external command?
1127    ext_module = cmd_parser.external_command_modules.get(command_key)
1128    if ext_module is not None:
1129        sys.argv[0] = f"arv {command_key}"
1130        _handle_external_command(ext_module, remaining_args)  # Exits.
1131
1132    # Are we doing an API resource call?
1133    resource = cmd_parser._subcommand_to_resource.get(args.subcommand)
1134    if resource is not None:
1135        # Any further remaining args indicate either malformed or unrecognized
1136        # global args (e.g. "arvcli.py --bad-arg resource method") or undefined
1137        # parameters to a valid resouce-method combination.
1138        if remaining_args:
1139            cmd_parser.error(
1140                f"unrecognized arguments: {', '.join(remaining_args)}\n"
1141                f"Try: {cmd_parser.prog} --help\n"
1142                f"     {cmd_parser.prog} {command_key} --help",
1143                with_help=False
1144            )  # Exits with status 2.
1145        _handle_resource_method(api_client, resource, args)  # Exits.
1146
1147    # Are we starting an external editor program?
1148    if args.subcommand in ("create", "edit"):
1149        if args.format == "uuid":
1150            cmd_parser.error(
1151                "--format=uuid or -s option is not supported when creating or"
1152                " editing Arvados objects with external editor. Please"
1153                " choose --format=json (default) or --format=yaml.",
1154                with_help=False
1155            )  # Exits with status 2.
1156        _handle_external_editor_command(api_client, cmd_parser, args)  # Exits.
1157
1158    # Are we running "arv get"?
1159    if args.subcommand == "get":
1160        if args.format == "uuid":
1161            cmd_parser.error(
1162                "--format=uuid or -s option is not supported for the 'arv get'"
1163                " command. Please choose --format=json (default) or"
1164                " --format=yaml.",
1165                with_help=False
1166            )  # Exits with status 2.
1167        _handle_get_subcommand(api_client, cmd_parser, args)  # Exits.
1168
1169    # NOTE: The code immediately below is not reachable.
1170    raise RuntimeError("Unexpected arguments: {arguments!r}")
1171
1172
1173if __name__ == "__main__":
1174    dispatch()