1
2
3
4
5 from __future__ import absolute_import
6 from future import standard_library
7 standard_library.install_aliases()
8 from builtins import range
9 import collections
10 import http.client
11 import httplib2
12 import json
13 import logging
14 import os
15 import re
16 import socket
17 import time
18 import types
19
20 import apiclient
21 from apiclient import discovery as apiclient_discovery
22 from apiclient import errors as apiclient_errors
23 from . import config
24 from . import errors
25 from . import util
26 from . import cache
27
28 _logger = logging.getLogger('arvados.api')
29
30 MAX_IDLE_CONNECTION_DURATION = 30
31 RETRY_DELAY_INITIAL = 2
32 RETRY_DELAY_BACKOFF = 2
33 RETRY_COUNT = 2
34
36 """Model class for JSON that preserves the contents' order.
37
38 API clients that care about preserving the order of fields in API
39 server responses can use this model to do so, like this::
40
41 from arvados.api import OrderedJsonModel
42 client = arvados.api('v1', ..., model=OrderedJsonModel())
43 """
44
46
47
48 content = content.decode('utf-8')
49 body = json.loads(content, object_pairs_hook=collections.OrderedDict)
50 if self._data_wrapper and isinstance(body, dict) and 'data' in body:
51 body = body['data']
52 return body
53
54
56 if (self.max_request_size and
57 kwargs.get('body') and
58 self.max_request_size < len(kwargs['body'])):
59 raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
60
61 if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
62 headers['X-External-Client'] = '1'
63
64 headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
65 if not headers.get('X-Request-Id'):
66 headers['X-Request-Id'] = self._request_id()
67
68 retryable = method in [
69 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
70 retry_count = self._retry_count if retryable else 0
71
72 if (not retryable and
73 time.time() - self._last_request_time > self._max_keepalive_idle):
74
75
76
77 for conn in self.connections.values():
78 conn.close()
79 self.connections.clear()
80
81 delay = self._retry_delay_initial
82 for _ in range(retry_count):
83 self._last_request_time = time.time()
84 try:
85 return self.orig_http_request(uri, method, headers=headers, **kwargs)
86 except http.client.HTTPException:
87 _logger.debug("Retrying API request in %d s after HTTP error",
88 delay, exc_info=True)
89 except socket.error:
90
91
92
93
94
95 _logger.debug("Retrying API request in %d s after socket error",
96 delay, exc_info=True)
97 for conn in self.connections.values():
98 conn.close()
99 except httplib2.SSLHandshakeError as e:
100
101 raise httplib2.SSLHandshakeError("Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e))
102
103 time.sleep(delay)
104 delay = delay * self._retry_delay_backoff
105
106 self._last_request_time = time.time()
107 return self.orig_http_request(uri, method, headers=headers, **kwargs)
108
110 http.arvados_api_token = api_token
111 http.max_request_size = 0
112 http.orig_http_request = http.request
113 http.request = types.MethodType(_intercept_http_request, http)
114 http._last_request_time = 0
115 http._max_keepalive_idle = MAX_IDLE_CONNECTION_DURATION
116 http._retry_delay_initial = RETRY_DELAY_INITIAL
117 http._retry_delay_backoff = RETRY_DELAY_BACKOFF
118 http._retry_count = RETRY_COUNT
119 http._request_id = util.new_request_id
120 return http
121
122
123
124 _cast_orig = apiclient_discovery._cast
126 global _cast_orig
127 if (type(value) != type('') and
128 type(value) != type(b'') and
129 (schema_type == 'object' or schema_type == 'array')):
130 return json.dumps(value)
131 else:
132 return _cast_orig(value, schema_type)
133 apiclient_discovery._cast = _cast_objects_too
134
135
136
137
138
142 apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
143
145 homedir = os.environ.get('HOME')
146 if not homedir or len(homedir) == 0:
147 return None
148 path = homedir + '/.cache/arvados/' + data_type
149 try:
150 util.mkdir_dash_p(path)
151 except OSError:
152 return None
153 return cache.SafeHTTPCache(path, max_age=60*60*24*2)
154
155 -def api(version=None, cache=True, host=None, token=None, insecure=False,
156 request_id=None, **kwargs):
157 """Return an apiclient Resources object for an Arvados instance.
158
159 :version:
160 A string naming the version of the Arvados API to use (for
161 example, 'v1').
162
163 :cache:
164 Use a cache (~/.cache/arvados/discovery) for the discovery
165 document.
166
167 :host:
168 The Arvados API server host (and optional :port) to connect to.
169
170 :token:
171 The authentication token to send with each API call.
172
173 :insecure:
174 If True, ignore SSL certificate validation errors.
175
176 :request_id:
177 Default X-Request-Id header value for outgoing requests that
178 don't already provide one. If None or omitted, generate a random
179 ID. When retrying failed requests, the same ID is used on all
180 attempts.
181
182 Additional keyword arguments will be passed directly to
183 `apiclient_discovery.build` if a new Resource object is created.
184 If the `discoveryServiceUrl` or `http` keyword arguments are
185 missing, this function will set default values for them, based on
186 the current Arvados configuration settings.
187
188 """
189
190 if not version:
191 version = 'v1'
192 _logger.info("Using default API version. " +
193 "Call arvados.api('%s') instead." %
194 version)
195 if 'discoveryServiceUrl' in kwargs:
196 if host:
197 raise ValueError("both discoveryServiceUrl and host provided")
198
199
200
201 if not token:
202 raise ValueError("discoveryServiceUrl provided, but token missing")
203 elif host and token:
204 pass
205 elif not host and not token:
206 return api_from_config(
207 version=version, cache=cache, request_id=request_id, **kwargs)
208 else:
209
210 if not host:
211 raise ValueError("token argument provided, but host missing.")
212 else:
213 raise ValueError("host argument provided, but token missing.")
214
215 if host:
216
217 kwargs['discoveryServiceUrl'] = (
218 'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,))
219
220 if 'http' not in kwargs:
221 http_kwargs = {'ca_certs': util.ca_certs_path()}
222 if cache:
223 http_kwargs['cache'] = http_cache('discovery')
224 if insecure:
225 http_kwargs['disable_ssl_certificate_validation'] = True
226 kwargs['http'] = httplib2.Http(**http_kwargs)
227
228 kwargs['http'] = _patch_http_request(kwargs['http'], token)
229
230 svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs)
231 svc.api_token = token
232 svc.insecure = insecure
233 svc.request_id = request_id
234 kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
235 kwargs['http'].cache = None
236 kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
237 return svc
238
240 """Return an apiclient Resources object enabling access to an Arvados server
241 instance.
242
243 :version:
244 A string naming the version of the Arvados REST API to use (for
245 example, 'v1').
246
247 :apiconfig:
248 If provided, this should be a dict-like object (must support the get()
249 method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and
250 optionally ARVADOS_API_HOST_INSECURE. If not provided, use
251 arvados.config (which gets these parameters from the environment by
252 default.)
253
254 Other keyword arguments such as `cache` will be passed along `api()`
255
256 """
257
258 if apiconfig is None:
259 apiconfig = config.settings()
260
261 errors = []
262 for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
263 if x not in apiconfig:
264 errors.append(x)
265 if errors:
266 raise ValueError(" and ".join(errors)+" not set.\nPlease set in %s or export environment variable." % config.default_config_file)
267 host = apiconfig.get('ARVADOS_API_HOST')
268 token = apiconfig.get('ARVADOS_API_TOKEN')
269 insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
270
271 return api(version=version, host=host, token=token, insecure=insecure, **kwargs)
272