Coverage for girder/utility/filesystem_assetstore_adapter: 91%

# Coverage for girder/utility/filesystem_assetstore_adapter : 91%

## 94 statements 86 run 8 missing 0 excluded

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 | #!/usr/bin/env python # -*- coding: utf-8 -*- ############################################################################### # Copyright 2013 Kitware Inc. # # Licensed under the Apache License, Version 2.0 ( the "License" ); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### import cherrypy import os import stat import tempfile from hashlib import sha512 from . import sha512_state from .abstract_assetstore_adapter import AbstractAssetstoreAdapter from .model_importer import ModelImporter BUF_SIZE = 65536 class FilesystemAssetstoreAdapter(AbstractAssetstoreAdapter): """ This assetstore type stores files on the filesystem underneath a root directory. Files are named by their SHA-512 hash, which avoids duplication of file content. """ @staticmethod def fileIndexFields(): """ File documents should have an index on their sha512 field. """ return ['sha512'] def __init__(self, assetstore): """ :param assetstore: The assetstore to act on. """ self.assetstore = assetstore self.tempDir = os.path.join(assetstore['root'], 'temp') if not os.path.exists(self.tempDir): os.makedirs(self.tempDir) def capacityInfo(self): """ For filesystem assetstores, we just need to report the free and total space on the filesystem where the assetstore lives. """ stat = os.statvfs(self.assetstore['root']) return { 'free': stat.f_bavail * stat.f_frsize, 'total': stat.f_blocks * stat.f_frsize } def initUpload(self, upload): """ Generates a temporary file and sets its location in the upload document as tempFile. This is the file that the chunks will be appended to. """ fd, path = tempfile.mkstemp(dir=self.tempDir) os.close(fd) # Must close this file descriptor or it will leak upload['tempFile'] = path upload['sha512state'] = sha512_state.serializeHex(sha512()) return upload def uploadChunk(self, upload, chunk): """ Appends the chunk into the temporary file. """ # Restore the internal state of the streaming SHA-512 checksum checksum = sha512_state.restoreHex(upload['sha512state']) if self.requestOffset(upload) > upload['received']: # This probably means the server died midway through writing last # chunk to disk, and the database record was not updated. This means # we need to update the sha512 state with the difference. with open(upload['tempFile'], 'rb') as tempFile: tempFile.seek(upload['received']) while True: data = tempFile.read(BUF_SIZE) if not data: break checksum.update(data) with open(upload['tempFile'], 'a+b') as tempFile: size = 0 while True: data = chunk.read(BUF_SIZE) if not data: break size += len(data) tempFile.write(data) checksum.update(data) chunk.close() # Persist the internal state of the checksum upload['sha512state'] = sha512_state.serializeHex(checksum) upload['received'] += size return upload def requestOffset(self, upload): """ Returns the size of the temp file. """ return os.stat(upload['tempFile']).st_size def finalizeUpload(self, upload, file): """ Moves the file into its permanent content-addressed location within the assetstore. Directory hierarchy yields 256^2 buckets. """ hash = sha512_state.restoreHex(upload['sha512state']).hexdigest() dir = os.path.join(hash[0:2], hash[2:4]) absdir = os.path.join(self.assetstore['root'], dir) path = os.path.join(dir, hash) abspath = os.path.join(self.assetstore['root'], path) if not os.path.exists(absdir): os.makedirs(absdir) if os.path.exists(abspath): # Already have this file stored, just delete temp file. os.remove(upload['tempFile']) else: # Move the temp file to permanent location in the assetstore. os.rename(upload['tempFile'], abspath) os.chmod(abspath, stat.S_IRUSR | stat.S_IWUSR) file['sha512'] = hash file['path'] = path return file def downloadFile(self, file, offset=0, headers=True): """ Returns a generator function that will be used to stream the file from disk to the response. """ path = os.path.join(self.assetstore['root'], file['path']) if not os.path.isfile(path): raise Exception('File %s does not exist.' % path) if headers: mimeType = file.get('mimeType', 'application/octet-stream') if not mimeType: mimeType = 'application/octet-stream' cherrypy.response.headers['Content-Type'] = mimeType cherrypy.response.headers['Content-Length'] = file['size'] - offset cherrypy.response.headers['Content-Disposition'] = \ 'attachment; filename="%s"' % file['name'] def stream(): with open(path, 'rb') as f: if offset > 0: f.seek(offset) while True: data = f.read(BUF_SIZE) if not data: break yield data return stream def deleteFile(self, file): """ Deletes the file from disk if it is the only File in this assetstore with the given sha512. """ q = { 'sha512': file['sha512'], 'assetstoreId': self.assetstore['_id'] } matching = ModelImporter().model('file').find(q, limit=2, fields=[]) if matching.count(True) == 1: path = os.path.join(self.assetstore['root'], file['path']) if os.path.isfile(path): os.remove(path)