Coverage for soxspipe/commonutils/set_of_files.py: 13%

# Coverage for soxspipe/commonutils/set_of_files.py : 13%

## 117 statements 15 run 102 missing 0 excluded

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/bin/env python

2# encoding: utf-8

3"""

4*Tools for working with 'set-of-files' (sof) files*

5

6:Author:

7 David Young & Marco Landoni

8

9:Date Created:

10 January 22, 2020

11"""

12################# GLOBAL IMPORTS ####################

13 from builtins import object

14 import sys

15 import os

16 os.environ['TERM'] = 'vt100'

17 from fundamentals import tools

18 from astropy.io import fits

19 from ccdproc import ImageFileCollection

20 import codecs

21 from soxspipe.commonutils.keyword_lookup import keyword_lookup

22

23

24 class set_of_files(object):

25 """

26 *The worker class for the sof module used to homogenise various frame input formats (sof file, directory of fits fits, list of fits file paths) into a CCDProc ImageFileCollection*

27

28 **Key Arguments:**

29 - ``log`` -- logger

30 - ``settings`` -- the settings dictionary

31 - ``inputFrames`` -- can be a directory, a set-of-files (SOF) file or a list of fits frame paths. Default []

32 - ``keys`` -- key aliases to report in the ImageFileCollection. Default ['MJDOBS', 'CDELT1', 'CDELT2', 'PSZX', 'DPR_TYPE', 'SEQ_ARM', 'EXPTIME', 'NAXIS1', 'NAXIS2', 'DET_READ_SPEED']

33

34 **Usage**

35

36 To initiate a sof object, use the following:

37

38 ```python

39 # inputFrames = "/path/to/a/directory"

40 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits']

41 inputFrames = '/path/to/myfiles.sof'

42 from soxspipe.commonutils import set_of_files

43 sof = set_of_files(

44 log=log,

45 settings=settings,

46 inputFrames=inputFrames

47 )

48 ```

49

50 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file

51 """

52 # Initialisation

53

54 def __init__(

55 self,

56 log,

57 settings=False,

58 inputFrames=[],

59 keys=['MJDOBS', 'CDELT1', 'CDELT2', 'PSZX',

60 'DPR_TYPE', 'DPR_CATG', 'DPR_TECH', 'SEQ_ARM', 'EXPTIME', 'NAXIS1', 'NAXIS2', 'DET_READ_SPEED', 'CONAD', 'DET_GAIN', 'RON', 'CHIP_RON', 'BUNIT']

61 ):

62 self.log = log

63 log.debug("instansiating a new 'sof' object")

64 self.settings = settings

65 self.inputFrames = inputFrames

66

67 # KEYWORD LOOKUP OBJECT - LOOKUP KEYWORD FROM DICTIONARY IN RESOURCES

68 # FOLDER

69 kw = keyword_lookup(

70 log=self.log,

71 settings=self.settings

72 ).get

73 keys = kw(keys)

74 self.keys = []

75 self.keys[:] = [k.lower() for k in keys]

76

77 # Initial Actions

78 # FIX RELATIVE HOME PATHS

79 from os.path import expanduser

80 home = expanduser("~")

81 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~":

82 self.inputFrames = home + "/" + self.inputFrames[1:]

83

84 return None

85

86 def _generate_sof_file_from_directory(

87 self,

88 directory,

89 sofPath):

90 """*generate an sof file from a directory of FITS frames*

91

92 **Key Arguments:**

93 - ``directory`` -- the path to the directory to containing the FITS files.

94 - ``sofPath`` -- the path to generate the sof file to

95

96 **Return:**

97 - ``sofPath`` -- the path to the sof file

98

99 **Usage**

100

101 ```python

102 from soxspipe.commonutils import set_of_files

103 sof = set_of_files(

104 log=log,

105 settings=settings

106 )

107 sofFile = sof._generate_sof_file_from_directory(

108 directory="path/to/directory", sofPath="/path/to/myFile.sof")

109 ```

110 """

111 self.log.debug(

112 'starting the ``_generate_sof_file_from_directory`` method')

113

114 from soxspipe.commonutils import keyword_lookup

115 kw = keyword_lookup(

116 log=self.log,

117 settings=self.settings

118 ).get

119

120 # MAKE RELATIVE HOME PATH ABSOLUTE

121 from os.path import expanduser

122 home = expanduser("~")

123 if directory[0] == "~":

124 directory = directory.replace("~", home)

125 if sofPath[0] == "~":

126 sofPath = sofPath.replace("~", home)

127

128 content = ""

129 for d in sorted(os.listdir(directory)):

130 if os.path.isfile(os.path.join(directory, d)) and (os.path.splitext(d)[-1].lower() == ".fits"):

131 fitsPath = os.path.abspath(os.path.join(directory, d))

132 # OPEN FITS FILE AT HDULIST - HDU (HEADER DATA UNIT) CONTAINS A HEADER AND A DATA ARRAY (IMAGE) OR

133 # TABLE.

134 with fits.open(fitsPath) as hdul:

135 # READ HEADER INTO MEMORY

136 hdr = hdul[0].header

137 # PRINT FULL FITS HEADER TO STDOUT

138 # print(repr(hdr).strip())

139 dpr_type = hdr[kw("DPR_TYPE")].strip()

140 # CHECK ARM

141 arm = hdr[kw("SEQ_ARM")]

142 # CHECK BINNING

143 if kw('CDELT1') in hdr:

144 xbin = str(int(hdr[kw('CDELT1')]))

145 ybin = str(int(hdr[kw('CDELT2')]))

146 catagory = dpr_type + "_" + arm.strip()

147 if kw('CDELT1') in hdr:

148 catagory += "_" + \

149 xbin.strip() + "x" + ybin.strip()

150

151 content += "%(fitsPath)s %(catagory)s\n" % locals()

152

153 # Recursively create missing directories

154 moduleDirectory = os.path.dirname(sofPath)

155 if not os.path.exists(moduleDirectory):

156 os.makedirs(moduleDirectory)

157

158 # WRITE TO FILE

159 with open(sofPath, 'w') as myFile:

160 myFile.write(content)

161

162 self.log.debug(

163 'completed the ``_generate_sof_file_from_directory`` method')

164 return sofPath

165

166 def get(

167 self):

168 """*return the set-of-files as a CCDProc ImageFileCollection*

169

170 **Return:**

171 - ``sof`` -- a ccdproc ImageFileCollection of the frames

172

173 **Usage**

174

175 To generate a ImageFileCollection from a directory, a list of fits filepaths or a set-of-files (SOF) file try the following:

176

177 ```python

178 # inputFrames = "/path/to/a/directory"

179 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits']

180 inputFrames = '/path/to/myfiles.sof'

181 from soxspipe.commonutils import set_of_files

182 sof = set_of_files(

183 log=log,

184 settings=settings,

185 inputFrames=inputFrames

186 )

187 sofFile, supplementarySof = sof.get()

188 print(sofFile.summary)

189 ```

190

191 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file.

192 """

193 self.log.debug('starting the ``get`` method')

194

195 from os.path import expanduser

196 home = expanduser("~")

197 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~":

198 self.inputFrames = home + "/" + self.inputFrames[1:]

199

200 # DIRECTORY OF FRAMES

201 if isinstance(self.inputFrames, str) and os.path.isdir(self.inputFrames):

202 sof = ImageFileCollection(self.inputFrames, keywords=self.keys)

203 supplementaryFilepaths = []

204 for d in os.listdir(self.inputFrames):

205 filepath = os.path.join(self.inputFrames, d)

206 if os.path.isfile(filepath) and ".fits" not in d.lower() and d[0] != ".":

207 supplementaryFilepaths.append(filepath)

208

209 elif isinstance(self.inputFrames, str) and os.path.isfile(self.inputFrames) and '.sof' in self.inputFrames:

210 readFile = codecs.open(

211 self.inputFrames, encoding='utf-8', mode='r')

212 thisData = readFile.read()

213 readFile.close()

214 lines = thisData.split("\n")

215

216 # REMOVE COMMENTED LINES

217 lines = [l for l in lines if len(l) and l[0] != "#"]

218

219 fitsFiles = []

220 fitsFiles[:] = [l.split(".fits")[0].replace("~/", home + "/") +

221 ".fits" for l in lines if ".fits" in l]

222 supplementaryFilepaths = [

223 l.replace("~/", home + "/") for l in lines if ".fits" not in l.lower() and len(l) > 3]

224 # MAKE SURE FILES EXIST

225 allFiles = fitsFiles.extend(supplementaryFilepaths)

226 for f in fitsFiles + supplementaryFilepaths:

227 exists = os.path.exists(f)

228 if not exists:

229 raise FileNotFoundError(f"the input file `{f}` does not appear to exist")

230

231 locations = [os.path.dirname(f) for f in fitsFiles]

232 if len(set(locations)) == 1:

233 location = locations[0]

234 fitsFiles = [os.path.basename(

235 f) for f in fitsFiles]

236 else:

237 location = None

238 sof = ImageFileCollection(

239 filenames=fitsFiles, location=location, keywords=self.keys)

240 elif isinstance(self.inputFrames, list):

241 fitsFiles = [f for f in self.inputFrames if ".fits" in f.lower()]

242 # FIND UNIQUE FILE LOCATIONS

243 locations = [os.path.dirname(f) for f in fitsFiles]

244 if len(set(locations)) == 1:

245 location = locations[0]

246 fitsFiles = [os.path.basename(

247 f) for f in fitsFiles]

248 else:

249 location = None

250 sof = ImageFileCollection(

251 filenames=fitsFiles, location=location, keywords=self.keys)

252 supplementaryFilepaths = [

253 f for f in self.inputFrames if ".fits" not in f.lower() and f[0] != "."]

254 else:

255 raise TypeError(

256 "'inputFrames' should be the path to a directory of files, an SOF file or a list of FITS frame paths")

257

258 supplementary_sof = self.create_supplimentary_file_dictionary(

259 supplementaryFilepaths)

260

261 self.log.debug('completed the ``get`` method')

262 return sof, supplementary_sof

263

264 def create_supplimentary_file_dictionary(

265 self,

266 supplementaryFilepaths):

267 """*create supplimentary file dictionary*

268

269 **Key Arguments:**

270 - ``supplementaryFilepaths`` -- the list of filepaths to genereate the dictionary for

271

272 **Return:**

273 - ``supplementary_sof`` -- a dictionary of non-fits files needed for recipe

274 """

275 self.log.debug(

276 'starting the ``create_supplimentary_file_dictionary`` method')

277

278 supplementary_sof = {}

279 for f in supplementaryFilepaths:

280 for a in ["NIR", "UVB", "VIS"]:

281 if a.lower() in f.lower() and a not in supplementary_sof.keys():

282 supplementary_sof[a] = {}

283

284 for f in supplementaryFilepaths:

285 if "disp_map" in f.lower():

286 for a in ["NIR", "UVB", "VIS"]:

287 if a.lower() in f.lower():

288 supplementary_sof[a]["DISP_MAP"] = f

289

290 self.log.debug(

291 'completed the ``create_supplimentary_file_dictionary`` method')

292 return supplementary_sof

293

294 # use the tab-trigger below for new method

295 # xt-class-method