arvados.api

  1# Copyright (C) The Arvados Authors. All rights reserved.
  2#
  3# SPDX-License-Identifier: Apache-2.0
  4"""Arvados REST API client
  5
  6This module provides classes and functions to construct an Arvados REST API
  7client. Most users will want to use one of these constructor functions, in
  8order of preference:
  9
 10* `arvados.api.api` provides a high-level interface to construct a client from
 11  either arguments or user configuration. You can call this module just like
 12  a function as a shortcut for calling `arvados.api.api`.
 13
 14* `arvados.api.api_from_config` constructs a client from user configuration in
 15  a dictionary.
 16
 17* `arvados.api.api_client` provides a lower-level interface to construct a
 18  simpler client object that is not threadsafe.
 19
 20Other classes and functions is this module support creating and customizing
 21the client for specialized use-cases.
 22
 23The methods on an Arvados REST API client are generated dynamically at
 24runtime. The `arvados.api_resources` module documents those methods and
 25return values for the current version of Arvados. It does not
 26implement anything so you don't need to import it, but it's a helpful
 27reference to understand how to use the Arvados REST API client.
 28"""
 29
 30import collections
 31import errno
 32import hashlib
 33import httplib2
 34import json
 35import logging
 36import os
 37import pathlib
 38import re
 39import socket
 40import ssl
 41import sys
 42import tempfile
 43import threading
 44import time
 45import types
 46
 47from typing import (
 48    Any,
 49    Dict,
 50    List,
 51    Mapping,
 52    Optional,
 53)
 54
 55import apiclient
 56import apiclient.http
 57from apiclient import discovery as apiclient_discovery
 58from apiclient import errors as apiclient_errors
 59from . import config
 60from . import errors
 61from . import keep
 62from . import retry
 63from . import util
 64from ._internal import basedirs
 65from .logging import GoogleHTTPClientFilter, log_handler
 66
 67_logger = logging.getLogger('arvados.api')
 68_googleapiclient_log_lock = threading.Lock()
 69
 70MAX_IDLE_CONNECTION_DURATION = 30
 71"""
 72Number of seconds that API client HTTP connections should be allowed to idle
 73in keepalive state before they are forced closed. Client code can adjust this
 74constant, and it will be used for all Arvados API clients constructed after
 75that point.
 76"""
 77
 78# An unused HTTP 5xx status code to request a retry internally.
 79# See _intercept_http_request. This should not be user-visible.
 80_RETRY_4XX_STATUS = 545
 81
 82if sys.version_info >= (3,):
 83    httplib2.SSLHandshakeError = None
 84
 85def _reset_googleapiclient_logging() -> None:
 86    """Make the next API client constructor log googleapiclient retries
 87
 88    This function is for main-level CLI tools like arv-copy to ensure retries
 89    are logged when constructing different API clients.
 90    """
 91    try:
 92        _googleapiclient_log_lock.release()
 93    except RuntimeError:
 94        pass
 95
 96_orig_retry_request = apiclient.http._retry_request
 97def _retry_request(http, num_retries, *args, **kwargs):
 98    try:
 99        num_retries = max(num_retries, http.num_retries)
100    except AttributeError:
101        # `http` client object does not have a `num_retries` attribute.
102        # It apparently hasn't gone through _patch_http_request, possibly
103        # because this isn't an Arvados API client. Pass through to
104        # avoid interfering with other Google API clients.
105        return _orig_retry_request(http, num_retries, *args, **kwargs)
106    response, body = _orig_retry_request(http, num_retries, *args, **kwargs)
107    # If _intercept_http_request ran out of retries for a 4xx response,
108    # restore the original status code.
109    if response.status == _RETRY_4XX_STATUS:
110        response.status = int(response['status'])
111    return (response, body)
112apiclient.http._retry_request = _retry_request
113
114def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
115    if not headers.get('X-Request-Id'):
116        headers['X-Request-Id'] = self._request_id()
117    try:
118        if (self.max_request_size and
119            kwargs.get('body') and
120            self.max_request_size < len(kwargs['body'])):
121            raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
122
123        headers['Authorization'] = 'Bearer %s' % self.arvados_api_token
124
125        if (time.time() - self._last_request_time) > self._max_keepalive_idle:
126            # High probability of failure due to connection atrophy. Make
127            # sure this request [re]opens a new connection by closing and
128            # forgetting all cached connections first.
129            for conn in self.connections.values():
130                conn.close()
131            self.connections.clear()
132
133        self._last_request_time = time.time()
134        try:
135            response, body = self.orig_http_request(uri, method, headers=headers, **kwargs)
136        except ssl.CertificateError as e:
137            raise ssl.CertificateError(e.args[0], "Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) from None
138        # googleapiclient only retries 403, 429, and 5xx status codes.
139        # If we got another 4xx status that we want to retry, convert it into
140        # 5xx so googleapiclient handles it the way we want.
141        if response.status in retry._HTTP_CAN_RETRY and response.status < 500:
142            response.status = _RETRY_4XX_STATUS
143        return (response, body)
144    except Exception as e:
145        # Prepend "[request_id] " to the error message, which we
146        # assume is the first string argument passed to the exception
147        # constructor.
148        for i in range(len(e.args or ())):
149            if type(e.args[i]) == type(""):
150                e.args = e.args[:i] + ("[{}] {}".format(headers['X-Request-Id'], e.args[i]),) + e.args[i+1:]
151                raise type(e)(*e.args)
152        raise
153
154def _patch_http_request(http, api_token, num_retries):
155    http.arvados_api_token = api_token
156    http.max_request_size = 0
157    http.num_retries = num_retries
158    http.orig_http_request = http.request
159    http.request = types.MethodType(_intercept_http_request, http)
160    http._last_request_time = 0
161    http._max_keepalive_idle = MAX_IDLE_CONNECTION_DURATION
162    http._request_id = util.new_request_id
163    return http
164
165def _close_connections(self):
166    for conn in self._http.connections.values():
167        conn.close()
168
169# Monkey patch discovery._cast() so objects and arrays get serialized
170# with json.dumps() instead of str().
171_cast_orig = apiclient_discovery._cast
172def _cast_objects_too(value, schema_type):
173    global _cast_orig
174    if (type(value) != type('') and
175        type(value) != type(b'') and
176        (schema_type == 'object' or schema_type == 'array')):
177        return json.dumps(value)
178    else:
179        return _cast_orig(value, schema_type)
180apiclient_discovery._cast = _cast_objects_too
181
182# Convert apiclient's HttpErrors into our own API error subclass for better
183# error reporting.
184# Reassigning apiclient_errors.HttpError is not sufficient because most of the
185# apiclient submodules import the class into their own namespace.
186def _new_http_error(cls, *args, **kwargs):
187    return super(apiclient_errors.HttpError, cls).__new__(
188        errors.ApiError, *args, **kwargs)
189apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
190
191class ThreadSafeHTTPCache:
192    """Thread-safe replacement for `httplib2.FileCache`
193
194    `arvados.api.http_cache` is the preferred way to construct this object.
195    Refer to that function's docstring for details.
196    """
197
198    def __init__(self, path=None, max_age=None):
199        self._dir = path
200        if max_age is not None:
201            try:
202                self._clean(threshold=time.time() - max_age)
203            except:
204                pass
205
206    def _clean(self, threshold=0):
207        for ent in os.listdir(self._dir):
208            fnm = os.path.join(self._dir, ent)
209            if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
210                continue
211            stat = os.lstat(fnm)
212            if stat.st_mtime < threshold:
213                try:
214                    os.unlink(fnm)
215                except OSError as err:
216                    if err.errno != errno.ENOENT:
217                        raise
218
219    def __str__(self):
220        return self._dir
221
222    def _filename(self, url):
223        return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
224
225    def get(self, url):
226        filename = self._filename(url)
227        try:
228            with open(filename, 'rb') as f:
229                return f.read()
230        except (IOError, OSError):
231            return None
232
233    def set(self, url, content):
234        try:
235            fd, tempname = tempfile.mkstemp(dir=self._dir)
236        except:
237            return None
238        try:
239            try:
240                f = os.fdopen(fd, 'wb')
241            except:
242                os.close(fd)
243                raise
244            try:
245                f.write(content)
246            finally:
247                f.close()
248            os.rename(tempname, self._filename(url))
249            tempname = None
250        finally:
251            if tempname:
252                os.unlink(tempname)
253
254    def delete(self, url):
255        try:
256            os.unlink(self._filename(url))
257        except OSError as err:
258            if err.errno != errno.ENOENT:
259                raise
260
261
262class ThreadSafeAPIClient(object):
263    """Thread-safe wrapper for an Arvados API client
264
265    This class takes all the arguments necessary to build a lower-level
266    Arvados API client `googleapiclient.discovery.Resource`, then
267    transparently builds and wraps a unique object per thread. This works
268    around the fact that the client's underlying HTTP client object is not
269    thread-safe.
270
271    Arguments:
272
273    * apiconfig: Mapping[str, str] | None --- A mapping with entries for
274      `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
275      `ARVADOS_API_HOST_INSECURE`. If not provided, uses
276      `arvados.config.settings` to get these parameters from user
277      configuration.  You can pass an empty mapping to build the client
278      solely from `api_params`.
279
280    * keep_params: Mapping[str, Any] --- Keyword arguments used to construct
281      an associated `arvados.keep.KeepClient`.
282
283    * api_params: Mapping[str, Any] --- Keyword arguments used to construct
284      each thread's API client. These have the same meaning as in the
285      `arvados.api.api` function.
286
287    * version: str | None --- A string naming the version of the Arvados API
288      to use. If not specified, the code will log a warning and fall back to
289      `'v1'`.
290    """
291    def __init__(
292            self,
293            apiconfig: Optional[Mapping[str, str]]=None,
294            keep_params: Optional[Mapping[str, Any]]={},
295            api_params: Optional[Mapping[str, Any]]={},
296            version: Optional[str]=None,
297    ) -> None:
298        if apiconfig or apiconfig is None:
299            self._api_kwargs = api_kwargs_from_config(version, apiconfig, **api_params)
300        else:
301            self._api_kwargs = normalize_api_kwargs(version, **api_params)
302        self.api_token = self._api_kwargs['token']
303        self.request_id = self._api_kwargs.get('request_id')
304        self.local = threading.local()
305        self.keep = keep.KeepClient(api_client=self, **keep_params)
306
307    def localapi(self) -> 'googleapiclient.discovery.Resource':
308        try:
309            client = self.local.api
310        except AttributeError:
311            client = api_client(**self._api_kwargs)
312            client._http._request_id = lambda: self.request_id or util.new_request_id()
313            self.local.api = client
314        return client
315
316    def __getattr__(self, name: str) -> Any:
317        # Proxy nonexistent attributes to the thread-local API client.
318        return getattr(self.localapi(), name)
319
320
321def http_cache(data_type: str) -> Optional[ThreadSafeHTTPCache]:
322    """Set up an HTTP file cache
323
324    This function constructs and returns an `arvados.api.ThreadSafeHTTPCache`
325    backed by the filesystem under a cache directory from the environment, or
326    `None` if the directory cannot be set up. The return value can be passed to
327    `httplib2.Http` as the `cache` argument.
328
329    Arguments:
330
331    * data_type: str --- The name of the subdirectory
332      where data is cached.
333    """
334    try:
335        path = basedirs.BaseDirectories('CACHE').storage_path(data_type)
336    except (OSError, RuntimeError):
337        return None
338    else:
339        return ThreadSafeHTTPCache(str(path), max_age=60*60*24*2)
340
341def api_client(
342        version: str,
343        discoveryServiceUrl: str,
344        token: str,
345        *,
346        cache: bool=True,
347        http: Optional[httplib2.Http]=None,
348        insecure: bool=False,
349        num_retries: int=10,
350        request_id: Optional[str]=None,
351        timeout: int=5*60,
352        **kwargs: Any,
353) -> apiclient_discovery.Resource:
354    """Build an Arvados API client
355
356    This function returns a `googleapiclient.discovery.Resource` object
357    constructed from the given arguments. This is a relatively low-level
358    interface that requires all the necessary inputs as arguments. Most
359    users will prefer to use `api` which can accept more flexible inputs.
360
361    Arguments:
362
363    * version: str --- A string naming the version of the Arvados API to use.
364
365    * discoveryServiceUrl: str --- The URL used to discover APIs passed
366      directly to `googleapiclient.discovery.build`.
367
368    * token: str --- The authentication token to send with each API call.
369
370    Keyword-only arguments:
371
372    * cache: bool --- If true, loads the API discovery document from, or
373      saves it to, a cache on disk.
374
375    * http: httplib2.Http | None --- The HTTP client object the API client
376      object will use to make requests.  If not provided, this function will
377      build its own to use. Either way, the object will be patched as part
378      of the build process.
379
380    * insecure: bool --- If true, ignore SSL certificate validation
381      errors. Default `False`.
382
383    * num_retries: int --- The number of times to retry each API request if
384      it encounters a temporary failure. Default 10.
385
386    * request_id: str | None --- Default `X-Request-Id` header value for
387      outgoing requests that don't already provide one. If `None` or
388      omitted, generate a random ID. When retrying failed requests, the same
389      ID is used on all attempts.
390
391    * timeout: int --- A timeout value for HTTP requests in seconds. Default
392      300 (5 minutes).
393
394    Additional keyword arguments will be passed directly to
395    `googleapiclient.discovery.build`.
396    """
397    if http is None:
398        http = httplib2.Http(
399            ca_certs=util.ca_certs_path(),
400            cache=http_cache('discovery') if cache else None,
401            disable_ssl_certificate_validation=bool(insecure),
402        )
403    if http.timeout is None:
404        http.timeout = timeout
405    http = _patch_http_request(http, token, num_retries)
406
407    # The first time a client is instantiated, temporarily route
408    # googleapiclient.http retry logs if they're not already. These are
409    # important because temporary problems fetching the discovery document
410    # can cause clients to appear to hang early. This can be removed after
411    # we have a more general story for handling googleapiclient logs (#20521).
412    client_logger = logging.getLogger('googleapiclient.http')
413    # "first time a client is instantiated" = thread that acquires this lock
414    # It is only released explicitly by calling _reset_googleapiclient_logging.
415    # googleapiclient sets up its own NullHandler so we detect if logging is
416    # configured by looking for a real handler anywhere in the hierarchy.
417    client_logger_unconfigured = _googleapiclient_log_lock.acquire(blocking=False) and all(
418        isinstance(handler, logging.NullHandler)
419        for logger_name in ['', 'googleapiclient', 'googleapiclient.http']
420        for handler in logging.getLogger(logger_name).handlers
421    )
422    if client_logger_unconfigured:
423        client_level = client_logger.level
424        client_filter = GoogleHTTPClientFilter()
425        client_logger.addFilter(client_filter)
426        client_logger.addHandler(log_handler)
427        if logging.NOTSET < client_level < client_filter.retry_levelno:
428            client_logger.setLevel(client_level)
429        else:
430            client_logger.setLevel(client_filter.retry_levelno)
431    try:
432        svc = apiclient_discovery.build(
433            'arvados', version,
434            cache_discovery=False,
435            discoveryServiceUrl=discoveryServiceUrl,
436            http=http,
437            num_retries=num_retries,
438            **kwargs,
439        )
440    finally:
441        if client_logger_unconfigured:
442            client_logger.removeHandler(log_handler)
443            client_logger.removeFilter(client_filter)
444            client_logger.setLevel(client_level)
445    svc.api_token = token
446    svc.insecure = insecure
447    svc.request_id = request_id
448    svc.config = lambda: util.get_config_once(svc)
449    svc.vocabulary = lambda: util.get_vocabulary_once(svc)
450    svc.close_connections = types.MethodType(_close_connections, svc)
451    http.max_request_size = svc._rootDesc.get('maxRequestSize', 0)
452    http.cache = None
453    http._request_id = lambda: svc.request_id or util.new_request_id()
454    return svc
455
456def normalize_api_kwargs(
457        version: Optional[str]=None,
458        discoveryServiceUrl: Optional[str]=None,
459        host: Optional[str]=None,
460        token: Optional[str]=None,
461        **kwargs: Any,
462) -> Dict[str, Any]:
463    """Validate kwargs from `api` and build kwargs for `api_client`
464
465    This method takes high-level keyword arguments passed to the `api`
466    constructor and normalizes them into a new dictionary that can be passed
467    as keyword arguments to `api_client`. It raises `ValueError` if required
468    arguments are missing or conflict.
469
470    Arguments:
471
472    * version: str | None --- A string naming the version of the Arvados API
473      to use. If not specified, the code will log a warning and fall back to
474      'v1'.
475
476    * discoveryServiceUrl: str | None --- The URL used to discover APIs
477      passed directly to `googleapiclient.discovery.build`. It is an error
478      to pass both `discoveryServiceUrl` and `host`.
479
480    * host: str | None --- The hostname and optional port number of the
481      Arvados API server. Used to build `discoveryServiceUrl`. It is an
482      error to pass both `discoveryServiceUrl` and `host`.
483
484    * token: str --- The authentication token to send with each API call.
485
486    Additional keyword arguments will be included in the return value.
487    """
488    if discoveryServiceUrl and host:
489        raise ValueError("both discoveryServiceUrl and host provided")
490    elif discoveryServiceUrl:
491        url_src = "discoveryServiceUrl"
492    elif host:
493        url_src = "host argument"
494        discoveryServiceUrl = 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,)
495    elif token:
496        # This specific error message gets priority for backwards compatibility.
497        raise ValueError("token argument provided, but host missing.")
498    else:
499        raise ValueError("neither discoveryServiceUrl nor host provided")
500    if not token:
501        raise ValueError("%s provided, but token missing" % (url_src,))
502    if not version:
503        version = 'v1'
504        _logger.info(
505            "Using default API version. Call arvados.api(%r) instead.",
506            version,
507        )
508    return {
509        'discoveryServiceUrl': discoveryServiceUrl,
510        'token': token,
511        'version': version,
512        **kwargs,
513    }
514
515def api_kwargs_from_config(
516        version: Optional[str]=None,
517        apiconfig: Optional[Mapping[str, str]]=None,
518        **kwargs: Any
519) -> Dict[str, Any]:
520    """Build `api_client` keyword arguments from configuration
521
522    This function accepts a mapping with Arvados configuration settings like
523    `ARVADOS_API_HOST` and converts them into a mapping of keyword arguments
524    that can be passed to `api_client`. If `ARVADOS_API_HOST` or
525    `ARVADOS_API_TOKEN` are not configured, it raises `ValueError`.
526
527    Arguments:
528
529    * version: str | None --- A string naming the version of the Arvados API
530      to use. If not specified, the code will log a warning and fall back to
531      'v1'.
532
533    * apiconfig: Mapping[str, str] | None --- A mapping with entries for
534      `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
535      `ARVADOS_API_HOST_INSECURE`. If not provided, calls
536      `arvados.config.settings` to get these parameters from user
537      configuration.
538
539    Additional keyword arguments will be included in the return value.
540    """
541    if apiconfig is None:
542        apiconfig = config.settings()
543    missing = " and ".join(
544        key
545        for key in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']
546        if key not in apiconfig
547    )
548    if missing:
549        raise ValueError(
550            "%s not set.\nPlease set in %s or export environment variable." %
551            (missing, config.default_config_file),
552        )
553    return normalize_api_kwargs(
554        version,
555        None,
556        apiconfig['ARVADOS_API_HOST'],
557        apiconfig['ARVADOS_API_TOKEN'],
558        insecure=config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig),
559        **kwargs,
560    )
561
562def api(
563        version: Optional[str]=None,
564        cache: bool=True,
565        host: Optional[str]=None,
566        token: Optional[str]=None,
567        insecure: bool=False,
568        request_id: Optional[str]=None,
569        timeout: int=5*60,
570        *,
571        discoveryServiceUrl: Optional[str]=None,
572        **kwargs: Any,
573) -> ThreadSafeAPIClient:
574    """Dynamically build an Arvados API client
575
576    This function provides a high-level "do what I mean" interface to build an
577    Arvados API client object. You can call it with no arguments to build a
578    client from user configuration; pass `host` and `token` arguments just
579    like you would write in user configuration; or pass additional arguments
580    for lower-level control over the client.
581
582    This function returns a `arvados.api.ThreadSafeAPIClient`, an
583    API-compatible wrapper around `googleapiclient.discovery.Resource`. If
584    you're handling concurrency yourself and/or your application is very
585    performance-sensitive, consider calling `api_client` directly.
586
587    Arguments:
588
589    * version: str | None --- A string naming the version of the Arvados API
590      to use. If not specified, the code will log a warning and fall back to
591      'v1'.
592
593    * host: str | None --- The hostname and optional port number of the
594      Arvados API server.
595
596    * token: str | None --- The authentication token to send with each API
597      call.
598
599    * discoveryServiceUrl: str | None --- The URL used to discover APIs
600      passed directly to `googleapiclient.discovery.build`.
601
602    If `host`, `token`, and `discoveryServiceUrl` are all omitted, `host` and
603    `token` will be loaded from the user's configuration. Otherwise, you must
604    pass `token` and one of `host` or `discoveryServiceUrl`. It is an error to
605    pass both `host` and `discoveryServiceUrl`.
606
607    Other arguments are passed directly to `api_client`. See that function's
608    docstring for more information about their meaning.
609    """
610    kwargs.update(
611        cache=cache,
612        insecure=insecure,
613        request_id=request_id,
614        timeout=timeout,
615    )
616    if discoveryServiceUrl or host or token:
617        kwargs.update(normalize_api_kwargs(version, discoveryServiceUrl, host, token))
618    else:
619        kwargs.update(api_kwargs_from_config(version))
620    version = kwargs.pop('version')
621    return ThreadSafeAPIClient({}, {}, kwargs, version)
622
623def api_from_config(
624        version: Optional[str]=None,
625        apiconfig: Optional[Mapping[str, str]]=None,
626        **kwargs: Any
627) -> ThreadSafeAPIClient:
628    """Build an Arvados API client from a configuration mapping
629
630    This function builds an Arvados API client from a mapping with user
631    configuration. It accepts that mapping as an argument, so you can use a
632    configuration that's different from what the user has set up.
633
634    This function returns a `arvados.api.ThreadSafeAPIClient`, an
635    API-compatible wrapper around `googleapiclient.discovery.Resource`. If
636    you're handling concurrency yourself and/or your application is very
637    performance-sensitive, consider calling `api_client` directly.
638
639    Arguments:
640
641    * version: str | None --- A string naming the version of the Arvados API
642      to use. If not specified, the code will log a warning and fall back to
643      'v1'.
644
645    * apiconfig: Mapping[str, str] | None --- A mapping with entries for
646      `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
647      `ARVADOS_API_HOST_INSECURE`. If not provided, calls
648      `arvados.config.settings` to get these parameters from user
649      configuration.
650
651    Other arguments are passed directly to `api_client`. See that function's
652    docstring for more information about their meaning.
653    """
654    return api(**api_kwargs_from_config(version, apiconfig, **kwargs))