arvados.cache 

 1# Copyright (C) The Arvados Authors. All rights reserved.
 2#
 3# SPDX-License-Identifier: Apache-2.0
 4
 5from builtins import object
 6import errno
 7import hashlib
 8import os
 9import tempfile
10import time
11
12class SafeHTTPCache(object):
13    """Thread-safe replacement for httplib2.FileCache"""
14
15    def __init__(self, path=None, max_age=None):
16        self._dir = path
17        if max_age is not None:
18            try:
19                self._clean(threshold=time.time() - max_age)
20            except:
21                pass
22
23    def _clean(self, threshold=0):
24        for ent in os.listdir(self._dir):
25            fnm = os.path.join(self._dir, ent)
26            if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
27                continue
28            stat = os.lstat(fnm)
29            if stat.st_mtime < threshold:
30                try:
31                    os.unlink(fnm)
32                except OSError as err:
33                    if err.errno != errno.ENOENT:
34                        raise
35
36    def __str__(self):
37        return self._dir
38
39    def _filename(self, url):
40        return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
41
42    def get(self, url):
43        filename = self._filename(url)
44        try:
45            with open(filename, 'rb') as f:
46                return f.read()
47        except (IOError, OSError):
48            return None
49
50    def set(self, url, content):
51        try:
52            fd, tempname = tempfile.mkstemp(dir=self._dir)
53        except:
54            return None
55        try:
56            try:
57                f = os.fdopen(fd, 'wb')
58            except:
59                os.close(fd)
60                raise
61            try:
62                f.write(content)
63            finally:
64                f.close()
65            os.rename(tempname, self._filename(url))
66            tempname = None
67        finally:
68            if tempname:
69                os.unlink(tempname)
70
71    def delete(self, url):
72        try:
73            os.unlink(self._filename(url))
74        except OSError as err:
75            if err.errno != errno.ENOENT:
76                raise
class SafeHTTPCache: 
13class SafeHTTPCache(object):
14    """Thread-safe replacement for httplib2.FileCache"""
15
16    def __init__(self, path=None, max_age=None):
17        self._dir = path
18        if max_age is not None:
19            try:
20                self._clean(threshold=time.time() - max_age)
21            except:
22                pass
23
24    def _clean(self, threshold=0):
25        for ent in os.listdir(self._dir):
26            fnm = os.path.join(self._dir, ent)
27            if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
28                continue
29            stat = os.lstat(fnm)
30            if stat.st_mtime < threshold:
31                try:
32                    os.unlink(fnm)
33                except OSError as err:
34                    if err.errno != errno.ENOENT:
35                        raise
36
37    def __str__(self):
38        return self._dir
39
40    def _filename(self, url):
41        return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
42
43    def get(self, url):
44        filename = self._filename(url)
45        try:
46            with open(filename, 'rb') as f:
47                return f.read()
48        except (IOError, OSError):
49            return None
50
51    def set(self, url, content):
52        try:
53            fd, tempname = tempfile.mkstemp(dir=self._dir)
54        except:
55            return None
56        try:
57            try:
58                f = os.fdopen(fd, 'wb')
59            except:
60                os.close(fd)
61                raise
62            try:
63                f.write(content)
64            finally:
65                f.close()
66            os.rename(tempname, self._filename(url))
67            tempname = None
68        finally:
69            if tempname:
70                os.unlink(tempname)
71
72    def delete(self, url):
73        try:
74            os.unlink(self._filename(url))
75        except OSError as err:
76            if err.errno != errno.ENOENT:
77                raise

Thread-safe replacement for httplib2.FileCache

SafeHTTPCache(path=None, max_age=None) 
16    def __init__(self, path=None, max_age=None):
17        self._dir = path
18        if max_age is not None:
19            try:
20                self._clean(threshold=time.time() - max_age)
21            except:
22                pass
def get(self, url): 
43    def get(self, url):
44        filename = self._filename(url)
45        try:
46            with open(filename, 'rb') as f:
47                return f.read()
48        except (IOError, OSError):
49            return None
def set(self, url, content): 
51    def set(self, url, content):
52        try:
53            fd, tempname = tempfile.mkstemp(dir=self._dir)
54        except:
55            return None
56        try:
57            try:
58                f = os.fdopen(fd, 'wb')
59            except:
60                os.close(fd)
61                raise
62            try:
63                f.write(content)
64            finally:
65                f.close()
66            os.rename(tempname, self._filename(url))
67            tempname = None
68        finally:
69            if tempname:
70                os.unlink(tempname)
def delete(self, url): 
72    def delete(self, url):
73        try:
74            os.unlink(self._filename(url))
75        except OSError as err:
76            if err.errno != errno.ENOENT:
77                raise