arvados.cache 

 1# Copyright (C) The Arvados Authors. All rights reserved.
 2#
 3# SPDX-License-Identifier: Apache-2.0
 4
 5import errno
 6import hashlib
 7import os
 8import tempfile
 9import time
10
11class SafeHTTPCache(object):
12    """Thread-safe replacement for httplib2.FileCache"""
13
14    def __init__(self, path=None, max_age=None):
15        self._dir = path
16        if max_age is not None:
17            try:
18                self._clean(threshold=time.time() - max_age)
19            except:
20                pass
21
22    def _clean(self, threshold=0):
23        for ent in os.listdir(self._dir):
24            fnm = os.path.join(self._dir, ent)
25            if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
26                continue
27            stat = os.lstat(fnm)
28            if stat.st_mtime < threshold:
29                try:
30                    os.unlink(fnm)
31                except OSError as err:
32                    if err.errno != errno.ENOENT:
33                        raise
34
35    def __str__(self):
36        return self._dir
37
38    def _filename(self, url):
39        return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
40
41    def get(self, url):
42        filename = self._filename(url)
43        try:
44            with open(filename, 'rb') as f:
45                return f.read()
46        except (IOError, OSError):
47            return None
48
49    def set(self, url, content):
50        try:
51            fd, tempname = tempfile.mkstemp(dir=self._dir)
52        except:
53            return None
54        try:
55            try:
56                f = os.fdopen(fd, 'wb')
57            except:
58                os.close(fd)
59                raise
60            try:
61                f.write(content)
62            finally:
63                f.close()
64            os.rename(tempname, self._filename(url))
65            tempname = None
66        finally:
67            if tempname:
68                os.unlink(tempname)
69
70    def delete(self, url):
71        try:
72            os.unlink(self._filename(url))
73        except OSError as err:
74            if err.errno != errno.ENOENT:
75                raise
class SafeHTTPCache: 
12class SafeHTTPCache(object):
13    """Thread-safe replacement for httplib2.FileCache"""
14
15    def __init__(self, path=None, max_age=None):
16        self._dir = path
17        if max_age is not None:
18            try:
19                self._clean(threshold=time.time() - max_age)
20            except:
21                pass
22
23    def _clean(self, threshold=0):
24        for ent in os.listdir(self._dir):
25            fnm = os.path.join(self._dir, ent)
26            if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
27                continue
28            stat = os.lstat(fnm)
29            if stat.st_mtime < threshold:
30                try:
31                    os.unlink(fnm)
32                except OSError as err:
33                    if err.errno != errno.ENOENT:
34                        raise
35
36    def __str__(self):
37        return self._dir
38
39    def _filename(self, url):
40        return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
41
42    def get(self, url):
43        filename = self._filename(url)
44        try:
45            with open(filename, 'rb') as f:
46                return f.read()
47        except (IOError, OSError):
48            return None
49
50    def set(self, url, content):
51        try:
52            fd, tempname = tempfile.mkstemp(dir=self._dir)
53        except:
54            return None
55        try:
56            try:
57                f = os.fdopen(fd, 'wb')
58            except:
59                os.close(fd)
60                raise
61            try:
62                f.write(content)
63            finally:
64                f.close()
65            os.rename(tempname, self._filename(url))
66            tempname = None
67        finally:
68            if tempname:
69                os.unlink(tempname)
70
71    def delete(self, url):
72        try:
73            os.unlink(self._filename(url))
74        except OSError as err:
75            if err.errno != errno.ENOENT:
76                raise

Thread-safe replacement for httplib2.FileCache

SafeHTTPCache(path=None, max_age=None) 
15    def __init__(self, path=None, max_age=None):
16        self._dir = path
17        if max_age is not None:
18            try:
19                self._clean(threshold=time.time() - max_age)
20            except:
21                pass
def get(self, url): 
42    def get(self, url):
43        filename = self._filename(url)
44        try:
45            with open(filename, 'rb') as f:
46                return f.read()
47        except (IOError, OSError):
48            return None
def set(self, url, content): 
50    def set(self, url, content):
51        try:
52            fd, tempname = tempfile.mkstemp(dir=self._dir)
53        except:
54            return None
55        try:
56            try:
57                f = os.fdopen(fd, 'wb')
58            except:
59                os.close(fd)
60                raise
61            try:
62                f.write(content)
63            finally:
64                f.close()
65            os.rename(tempname, self._filename(url))
66            tempname = None
67        finally:
68            if tempname:
69                os.unlink(tempname)
def delete(self, url): 
71    def delete(self, url):
72        try:
73            os.unlink(self._filename(url))
74        except OSError as err:
75            if err.errno != errno.ENOENT:
76                raise