Coverage for soxspipe/commonutils/set_of_files.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2# encoding: utf-8
3"""
4*Tools for working with 'set-of-files' (sof) files*
6:Author:
7 David Young & Marco Landoni
9:Date Created:
10 January 22, 2020
11"""
12################# GLOBAL IMPORTS ####################
13from builtins import object
14import sys
15import os
16os.environ['TERM'] = 'vt100'
17from fundamentals import tools
18from astropy.io import fits
19from ccdproc import ImageFileCollection
20import codecs
21from soxspipe.commonutils.keyword_lookup import keyword_lookup
24class set_of_files(object):
25 """
26 *The worker class for the sof module used to homogenise various frame input formats (sof file, directory of fits fits, list of fits file paths) into a CCDProc ImageFileCollection*
28 **Key Arguments:**
29 - ``log`` -- logger
30 - ``settings`` -- the settings dictionary
31 - ``inputFrames`` -- can be a directory, a set-of-files (SOF) file or a list of fits frame paths. Default []
32 - ``keys`` -- key aliases to report in the ImageFileCollection. Default ['MJDOBS', 'CDELT1', 'CDELT2', 'PSZX', 'DPR_TYPE', 'SEQ_ARM', 'EXPTIME', 'NAXIS1', 'NAXIS2', 'DET_READ_SPEED']
34 **Usage**
36 To initiate a sof object, use the following:
38 ```python
39 # inputFrames = "/path/to/a/directory"
40 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits']
41 inputFrames = '/path/to/myfiles.sof'
42 from soxspipe.commonutils import set_of_files
43 sof = set_of_files(
44 log=log,
45 settings=settings,
46 inputFrames=inputFrames
47 )
48 ```
50 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file
51 """
52 # Initialisation
54 def __init__(
55 self,
56 log,
57 settings=False,
58 inputFrames=[],
59 keys=['MJDOBS', 'CDELT1', 'CDELT2', 'PSZX',
60 'DPR_TYPE', 'DPR_CATG', 'DPR_TECH', 'SEQ_ARM', 'EXPTIME', 'NAXIS1', 'NAXIS2', 'DET_READ_SPEED', 'CONAD', 'DET_GAIN', 'RON', 'CHIP_RON', 'BUNIT']
61 ):
62 self.log = log
63 log.debug("instansiating a new 'sof' object")
64 self.settings = settings
65 self.inputFrames = inputFrames
67 # KEYWORD LOOKUP OBJECT - LOOKUP KEYWORD FROM DICTIONARY IN RESOURCES
68 # FOLDER
69 kw = keyword_lookup(
70 log=self.log,
71 settings=self.settings
72 ).get
73 keys = kw(keys)
74 self.keys = []
75 self.keys[:] = [k.lower() for k in keys]
77 # Initial Actions
78 # FIX RELATIVE HOME PATHS
79 from os.path import expanduser
80 home = expanduser("~")
81 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~":
82 self.inputFrames = home + "/" + self.inputFrames[1:]
84 return None
86 def _generate_sof_file_from_directory(
87 self,
88 directory,
89 sofPath):
90 """*generate an sof file from a directory of FITS frames*
92 **Key Arguments:**
93 - ``directory`` -- the path to the directory to containing the FITS files.
94 - ``sofPath`` -- the path to generate the sof file to
96 **Return:**
97 - ``sofPath`` -- the path to the sof file
99 **Usage**
101 ```python
102 from soxspipe.commonutils import set_of_files
103 sof = set_of_files(
104 log=log,
105 settings=settings
106 )
107 sofFile = sof._generate_sof_file_from_directory(
108 directory="path/to/directory", sofPath="/path/to/myFile.sof")
109 ```
110 """
111 self.log.debug(
112 'starting the ``_generate_sof_file_from_directory`` method')
114 from soxspipe.commonutils import keyword_lookup
115 kw = keyword_lookup(
116 log=self.log,
117 settings=self.settings
118 ).get
120 # MAKE RELATIVE HOME PATH ABSOLUTE
121 from os.path import expanduser
122 home = expanduser("~")
123 if directory[0] == "~":
124 directory = directory.replace("~", home)
125 if sofPath[0] == "~":
126 sofPath = sofPath.replace("~", home)
128 content = ""
129 for d in sorted(os.listdir(directory)):
130 if os.path.isfile(os.path.join(directory, d)) and (os.path.splitext(d)[-1].lower() == ".fits"):
131 fitsPath = os.path.abspath(os.path.join(directory, d))
132 # OPEN FITS FILE AT HDULIST - HDU (HEADER DATA UNIT) CONTAINS A HEADER AND A DATA ARRAY (IMAGE) OR
133 # TABLE.
134 with fits.open(fitsPath) as hdul:
135 # READ HEADER INTO MEMORY
136 hdr = hdul[0].header
137 # PRINT FULL FITS HEADER TO STDOUT
138 # print(repr(hdr).strip())
139 dpr_type = hdr[kw("DPR_TYPE")].strip()
140 # CHECK ARM
141 arm = hdr[kw("SEQ_ARM")]
142 # CHECK BINNING
143 if kw('CDELT1') in hdr:
144 xbin = str(int(hdr[kw('CDELT1')]))
145 ybin = str(int(hdr[kw('CDELT2')]))
146 catagory = dpr_type + "_" + arm.strip()
147 if kw('CDELT1') in hdr:
148 catagory += "_" + \
149 xbin.strip() + "x" + ybin.strip()
151 content += "%(fitsPath)s %(catagory)s\n" % locals()
153 # Recursively create missing directories
154 moduleDirectory = os.path.dirname(sofPath)
155 if not os.path.exists(moduleDirectory):
156 os.makedirs(moduleDirectory)
158 # WRITE TO FILE
159 with open(sofPath, 'w') as myFile:
160 myFile.write(content)
162 self.log.debug(
163 'completed the ``_generate_sof_file_from_directory`` method')
164 return sofPath
166 def get(
167 self):
168 """*return the set-of-files as a CCDProc ImageFileCollection*
170 **Return:**
171 - ``sof`` -- a ccdproc ImageFileCollection of the frames
173 **Usage**
175 To generate a ImageFileCollection from a directory, a list of fits filepaths or a set-of-files (SOF) file try the following:
177 ```python
178 # inputFrames = "/path/to/a/directory"
179 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits']
180 inputFrames = '/path/to/myfiles.sof'
181 from soxspipe.commonutils import set_of_files
182 sof = set_of_files(
183 log=log,
184 settings=settings,
185 inputFrames=inputFrames
186 )
187 sofFile, supplementarySof = sof.get()
188 print(sofFile.summary)
189 ```
191 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file.
192 """
193 self.log.debug('starting the ``get`` method')
195 from os.path import expanduser
196 home = expanduser("~")
197 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~":
198 self.inputFrames = home + "/" + self.inputFrames[1:]
200 # DIRECTORY OF FRAMES
201 if isinstance(self.inputFrames, str) and os.path.isdir(self.inputFrames):
202 sof = ImageFileCollection(self.inputFrames, keywords=self.keys)
203 supplementaryFilepaths = []
204 for d in os.listdir(self.inputFrames):
205 filepath = os.path.join(self.inputFrames, d)
206 if os.path.isfile(filepath) and ".fits" not in d.lower() and d[0] != ".":
207 supplementaryFilepaths.append(filepath)
209 elif isinstance(self.inputFrames, str) and os.path.isfile(self.inputFrames) and '.sof' in self.inputFrames:
210 readFile = codecs.open(
211 self.inputFrames, encoding='utf-8', mode='r')
212 thisData = readFile.read()
213 readFile.close()
214 lines = thisData.split("\n")
216 # REMOVE COMMENTED LINES
217 lines = [l for l in lines if len(l) and l[0] != "#"]
219 fitsFiles = []
220 fitsFiles[:] = [l.split(".fits")[0].replace("~/", home + "/") +
221 ".fits" for l in lines if ".fits" in l]
222 supplementaryFilepaths = [
223 l.replace("~/", home + "/") for l in lines if ".fits" not in l.lower() and len(l) > 3]
224 # MAKE SURE FILES EXIST
225 allFiles = fitsFiles.extend(supplementaryFilepaths)
226 for f in fitsFiles + supplementaryFilepaths:
227 exists = os.path.exists(f)
228 if not exists:
229 raise FileNotFoundError(f"the input file `{f}` does not appear to exist")
231 locations = [os.path.dirname(f) for f in fitsFiles]
232 if len(set(locations)) == 1:
233 location = locations[0]
234 fitsFiles = [os.path.basename(
235 f) for f in fitsFiles]
236 else:
237 location = None
238 sof = ImageFileCollection(
239 filenames=fitsFiles, location=location, keywords=self.keys)
240 elif isinstance(self.inputFrames, list):
241 fitsFiles = [f for f in self.inputFrames if ".fits" in f.lower()]
242 # FIND UNIQUE FILE LOCATIONS
243 locations = [os.path.dirname(f) for f in fitsFiles]
244 if len(set(locations)) == 1:
245 location = locations[0]
246 fitsFiles = [os.path.basename(
247 f) for f in fitsFiles]
248 else:
249 location = None
250 sof = ImageFileCollection(
251 filenames=fitsFiles, location=location, keywords=self.keys)
252 supplementaryFilepaths = [
253 f for f in self.inputFrames if ".fits" not in f.lower() and f[0] != "."]
254 else:
255 raise TypeError(
256 "'inputFrames' should be the path to a directory of files, an SOF file or a list of FITS frame paths")
258 supplementary_sof = self.create_supplimentary_file_dictionary(
259 supplementaryFilepaths)
261 self.log.debug('completed the ``get`` method')
262 return sof, supplementary_sof
264 def create_supplimentary_file_dictionary(
265 self,
266 supplementaryFilepaths):
267 """*create supplimentary file dictionary*
269 **Key Arguments:**
270 - ``supplementaryFilepaths`` -- the list of filepaths to genereate the dictionary for
272 **Return:**
273 - ``supplementary_sof`` -- a dictionary of non-fits files needed for recipe
274 """
275 self.log.debug(
276 'starting the ``create_supplimentary_file_dictionary`` method')
278 supplementary_sof = {}
279 for f in supplementaryFilepaths:
280 for a in ["NIR", "UVB", "VIS"]:
281 if a.lower() in f.lower() and a not in supplementary_sof.keys():
282 supplementary_sof[a] = {}
284 for f in supplementaryFilepaths:
285 if "disp_map" in f.lower():
286 for a in ["NIR", "UVB", "VIS"]:
287 if a.lower() in f.lower():
288 supplementary_sof[a]["DISP_MAP"] = f
290 self.log.debug(
291 'completed the ``create_supplimentary_file_dictionary`` method')
292 return supplementary_sof
294 # use the tab-trigger below for new method
295 # xt-class-method