Package arvados :: Module api
[hide private]
[frames] | no frames]

Source Code for Module arvados.api

  1  # Copyright (C) The Arvados Authors. All rights reserved. 
  2  # 
  3  # SPDX-License-Identifier: Apache-2.0 
  4   
  5  from __future__ import absolute_import 
  6  from future import standard_library 
  7  standard_library.install_aliases() 
  8  from builtins import range 
  9  import collections 
 10  import http.client 
 11  import httplib2 
 12  import json 
 13  import logging 
 14  import os 
 15  import re 
 16  import socket 
 17  import time 
 18  import types 
 19   
 20  import apiclient 
 21  from apiclient import discovery as apiclient_discovery 
 22  from apiclient import errors as apiclient_errors 
 23  from . import config 
 24  from . import errors 
 25  from . import util 
 26  from . import cache 
 27   
 28  _logger = logging.getLogger('arvados.api') 
 29   
 30  MAX_IDLE_CONNECTION_DURATION = 30 
 31  RETRY_DELAY_INITIAL = 2 
 32  RETRY_DELAY_BACKOFF = 2 
 33  RETRY_COUNT = 2 
 34   
35 -class OrderedJsonModel(apiclient.model.JsonModel):
36 """Model class for JSON that preserves the contents' order. 37 38 API clients that care about preserving the order of fields in API 39 server responses can use this model to do so, like this:: 40 41 from arvados.api import OrderedJsonModel 42 client = arvados.api('v1', ..., model=OrderedJsonModel()) 43 """ 44
45 - def deserialize(self, content):
46 # This is a very slightly modified version of the parent class' 47 # implementation. Copyright (c) 2010 Google. 48 content = content.decode('utf-8') 49 body = json.loads(content, object_pairs_hook=collections.OrderedDict) 50 if self._data_wrapper and isinstance(body, dict) and 'data' in body: 51 body = body['data'] 52 return body
53 54
55 -def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
56 if (self.max_request_size and 57 kwargs.get('body') and 58 self.max_request_size < len(kwargs['body'])): 59 raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size)) 60 61 if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true": 62 headers['X-External-Client'] = '1' 63 64 headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token 65 if not headers.get('X-Request-Id'): 66 headers['X-Request-Id'] = self._request_id() 67 68 retryable = method in [ 69 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT'] 70 retry_count = self._retry_count if retryable else 0 71 72 if (not retryable and 73 time.time() - self._last_request_time > self._max_keepalive_idle): 74 # High probability of failure due to connection atrophy. Make 75 # sure this request [re]opens a new connection by closing and 76 # forgetting all cached connections first. 77 for conn in self.connections.values(): 78 conn.close() 79 self.connections.clear() 80 81 delay = self._retry_delay_initial 82 for _ in range(retry_count): 83 self._last_request_time = time.time() 84 try: 85 return self.orig_http_request(uri, method, headers=headers, **kwargs) 86 except http.client.HTTPException: 87 _logger.debug("Retrying API request in %d s after HTTP error", 88 delay, exc_info=True) 89 except socket.error: 90 # This is the one case where httplib2 doesn't close the 91 # underlying connection first. Close all open 92 # connections, expecting this object only has the one 93 # connection to the API server. This is safe because 94 # httplib2 reopens connections when needed. 95 _logger.debug("Retrying API request in %d s after socket error", 96 delay, exc_info=True) 97 for conn in self.connections.values(): 98 conn.close() 99 except httplib2.SSLHandshakeError as e: 100 # Intercept and re-raise with a better error message. 101 raise httplib2.SSLHandshakeError("Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) 102 103 time.sleep(delay) 104 delay = delay * self._retry_delay_backoff 105 106 self._last_request_time = time.time() 107 return self.orig_http_request(uri, method, headers=headers, **kwargs)
108
109 -def _patch_http_request(http, api_token):
110 http.arvados_api_token = api_token 111 http.max_request_size = 0 112 http.orig_http_request = http.request 113 http.request = types.MethodType(_intercept_http_request, http) 114 http._last_request_time = 0 115 http._max_keepalive_idle = MAX_IDLE_CONNECTION_DURATION 116 http._retry_delay_initial = RETRY_DELAY_INITIAL 117 http._retry_delay_backoff = RETRY_DELAY_BACKOFF 118 http._retry_count = RETRY_COUNT 119 http._request_id = util.new_request_id 120 return http
121 122 # Monkey patch discovery._cast() so objects and arrays get serialized 123 # with json.dumps() instead of str(). 124 _cast_orig = apiclient_discovery._cast
125 -def _cast_objects_too(value, schema_type):
126 global _cast_orig 127 if (type(value) != type('') and 128 type(value) != type(b'') and 129 (schema_type == 'object' or schema_type == 'array')): 130 return json.dumps(value) 131 else: 132 return _cast_orig(value, schema_type)
133 apiclient_discovery._cast = _cast_objects_too 134 135 # Convert apiclient's HttpErrors into our own API error subclass for better 136 # error reporting. 137 # Reassigning apiclient_errors.HttpError is not sufficient because most of the 138 # apiclient submodules import the class into their own namespace.
139 -def _new_http_error(cls, *args, **kwargs):
140 return super(apiclient_errors.HttpError, cls).__new__( 141 errors.ApiError, *args, **kwargs)
142 apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error) 143
144 -def http_cache(data_type):
145 homedir = os.environ.get('HOME') 146 if not homedir or len(homedir) == 0: 147 return None 148 path = homedir + '/.cache/arvados/' + data_type 149 try: 150 util.mkdir_dash_p(path) 151 except OSError: 152 return None 153 return cache.SafeHTTPCache(path, max_age=60*60*24*2)
154
155 -def api(version=None, cache=True, host=None, token=None, insecure=False, 156 request_id=None, **kwargs):
157 """Return an apiclient Resources object for an Arvados instance. 158 159 :version: 160 A string naming the version of the Arvados API to use (for 161 example, 'v1'). 162 163 :cache: 164 Use a cache (~/.cache/arvados/discovery) for the discovery 165 document. 166 167 :host: 168 The Arvados API server host (and optional :port) to connect to. 169 170 :token: 171 The authentication token to send with each API call. 172 173 :insecure: 174 If True, ignore SSL certificate validation errors. 175 176 :request_id: 177 Default X-Request-Id header value for outgoing requests that 178 don't already provide one. If None or omitted, generate a random 179 ID. When retrying failed requests, the same ID is used on all 180 attempts. 181 182 Additional keyword arguments will be passed directly to 183 `apiclient_discovery.build` if a new Resource object is created. 184 If the `discoveryServiceUrl` or `http` keyword arguments are 185 missing, this function will set default values for them, based on 186 the current Arvados configuration settings. 187 188 """ 189 190 if not version: 191 version = 'v1' 192 _logger.info("Using default API version. " + 193 "Call arvados.api('%s') instead." % 194 version) 195 if 'discoveryServiceUrl' in kwargs: 196 if host: 197 raise ValueError("both discoveryServiceUrl and host provided") 198 # Here we can't use a token from environment, config file, 199 # etc. Those probably have nothing to do with the host 200 # provided by the caller. 201 if not token: 202 raise ValueError("discoveryServiceUrl provided, but token missing") 203 elif host and token: 204 pass 205 elif not host and not token: 206 return api_from_config( 207 version=version, cache=cache, request_id=request_id, **kwargs) 208 else: 209 # Caller provided one but not the other 210 if not host: 211 raise ValueError("token argument provided, but host missing.") 212 else: 213 raise ValueError("host argument provided, but token missing.") 214 215 if host: 216 # Caller wants us to build the discoveryServiceUrl 217 kwargs['discoveryServiceUrl'] = ( 218 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,)) 219 220 if 'http' not in kwargs: 221 http_kwargs = {'ca_certs': util.ca_certs_path()} 222 if cache: 223 http_kwargs['cache'] = http_cache('discovery') 224 if insecure: 225 http_kwargs['disable_ssl_certificate_validation'] = True 226 kwargs['http'] = httplib2.Http(**http_kwargs) 227 228 kwargs['http'] = _patch_http_request(kwargs['http'], token) 229 230 svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs) 231 svc.api_token = token 232 svc.insecure = insecure 233 svc.request_id = request_id 234 kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0) 235 kwargs['http'].cache = None 236 kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id() 237 return svc
238
239 -def api_from_config(version=None, apiconfig=None, **kwargs):
240 """Return an apiclient Resources object enabling access to an Arvados server 241 instance. 242 243 :version: 244 A string naming the version of the Arvados REST API to use (for 245 example, 'v1'). 246 247 :apiconfig: 248 If provided, this should be a dict-like object (must support the get() 249 method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and 250 optionally ARVADOS_API_HOST_INSECURE. If not provided, use 251 arvados.config (which gets these parameters from the environment by 252 default.) 253 254 Other keyword arguments such as `cache` will be passed along `api()` 255 256 """ 257 # Load from user configuration or environment 258 if apiconfig is None: 259 apiconfig = config.settings() 260 261 errors = [] 262 for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']: 263 if x not in apiconfig: 264 errors.append(x) 265 if errors: 266 raise ValueError(" and ".join(errors)+" not set.\nPlease set in %s or export environment variable." % config.default_config_file) 267 host = apiconfig.get('ARVADOS_API_HOST') 268 token = apiconfig.get('ARVADOS_API_TOKEN') 269 insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig) 270 271 return api(version=version, host=host, token=token, insecure=insecure, **kwargs)
272