Module arvados.cache
Expand source code
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
from builtins import object
import errno
import hashlib
import os
import tempfile
import time
class SafeHTTPCache(object):
"""Thread-safe replacement for httplib2.FileCache"""
def __init__(self, path=None, max_age=None):
self._dir = path
if max_age is not None:
try:
self._clean(threshold=time.time() - max_age)
except:
pass
def _clean(self, threshold=0):
for ent in os.listdir(self._dir):
fnm = os.path.join(self._dir, ent)
if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
continue
stat = os.lstat(fnm)
if stat.st_mtime < threshold:
try:
os.unlink(fnm)
except OSError as err:
if err.errno != errno.ENOENT:
raise
def __str__(self):
return self._dir
def _filename(self, url):
return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
def get(self, url):
filename = self._filename(url)
try:
with open(filename, 'rb') as f:
return f.read()
except (IOError, OSError):
return None
def set(self, url, content):
try:
fd, tempname = tempfile.mkstemp(dir=self._dir)
except:
return None
try:
try:
f = os.fdopen(fd, 'wb')
except:
os.close(fd)
raise
try:
f.write(content)
finally:
f.close()
os.rename(tempname, self._filename(url))
tempname = None
finally:
if tempname:
os.unlink(tempname)
def delete(self, url):
try:
os.unlink(self._filename(url))
except OSError as err:
if err.errno != errno.ENOENT:
raise
Classes
class SafeHTTPCache (path=None, max_age=None)
-
Thread-safe replacement for httplib2.FileCache
Expand source code
class SafeHTTPCache(object): """Thread-safe replacement for httplib2.FileCache""" def __init__(self, path=None, max_age=None): self._dir = path if max_age is not None: try: self._clean(threshold=time.time() - max_age) except: pass def _clean(self, threshold=0): for ent in os.listdir(self._dir): fnm = os.path.join(self._dir, ent) if os.path.isdir(fnm) or not fnm.endswith('.tmp'): continue stat = os.lstat(fnm) if stat.st_mtime < threshold: try: os.unlink(fnm) except OSError as err: if err.errno != errno.ENOENT: raise def __str__(self): return self._dir def _filename(self, url): return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp') def get(self, url): filename = self._filename(url) try: with open(filename, 'rb') as f: return f.read() except (IOError, OSError): return None def set(self, url, content): try: fd, tempname = tempfile.mkstemp(dir=self._dir) except: return None try: try: f = os.fdopen(fd, 'wb') except: os.close(fd) raise try: f.write(content) finally: f.close() os.rename(tempname, self._filename(url)) tempname = None finally: if tempname: os.unlink(tempname) def delete(self, url): try: os.unlink(self._filename(url)) except OSError as err: if err.errno != errno.ENOENT: raise
Methods
def delete(self, url)
-
Expand source code
def delete(self, url): try: os.unlink(self._filename(url)) except OSError as err: if err.errno != errno.ENOENT: raise
def get(self, url)
-
Expand source code
def get(self, url): filename = self._filename(url) try: with open(filename, 'rb') as f: return f.read() except (IOError, OSError): return None
def set(self, url, content)
-
Expand source code
def set(self, url, content): try: fd, tempname = tempfile.mkstemp(dir=self._dir) except: return None try: try: f = os.fdopen(fd, 'wb') except: os.close(fd) raise try: f.write(content) finally: f.close() os.rename(tempname, self._filename(url)) tempname = None finally: if tempname: os.unlink(tempname)