Coverage for sherlock/cl_utils.py: 32%
159 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
1#!/usr/bin/env python
2# encoding: utf-8
3"""
4Documentation for sherlock can be found here: http://sherlock.readthedocs.org
6.. todo ::
8 - docuument cl_utils module
9 - tidy usage text
11Usage:
12 sherlock init
13 sherlock info [-s <pathToSettingsFile>]
14 sherlock [-NA] dbmatch [--update] [-s <pathToSettingsFile>]
15 sherlock [-bN] match -- <ra> <dec> [<pathToSettingsFile>]
16 sherlock clean [-s <pathToSettingsFile>]
17 sherlock wiki [-s <pathToSettingsFile>]
18 sherlock import ned <ra> <dec> <radiusArcsec> [-s <pathToSettingsFile>]
19 sherlock import cat <cat_name> <pathToDataFile> <cat_version> [-s <pathToSettingsFile>]
20 sherlock import stream <stream_name> [-s <pathToSettingsFile>]
22Options:
23 init setup the sherlock settings file for the first time
24 match XXXX
25 dbmatch database match
26 clean XXXX
27 wiki XXXX
28 import XXXX
29 ned use the online NED database as the source catalogue
30 cat import a static catalogue into the sherlock-catalogues database
31 stream download/stream new data from a give source catalogue into the sherlock sherlock-catalogues database
32 info print an overview of the current catalogues, views and streams in the sherlock database ready for crossmatching
34 ra the right-ascension coordinate with which to perform a conesearch (sexegesimal or decimal degrees)
35 dec the declination coordinate with which to perform a conesearch (sexegesimal or decimal degrees)
36 radiusArcsec radius in arcsec of the footprint to download from the online NED database
37 cat_name name of the catalogue being imported (veron|ned_d)
38 stream_name name of the stream to import into the sherlock-catalogues database (ifs)
40 -N, --skipNedUpdate do not update the NED database before classification
41 -A, --skipMagUpdate do not update the peak magnitudes and human readable text annotations of objects (can eat up some time)
42 -h, --help show this help message
43 -s, --settings the settings file
44 -b, --verbose print more details to stdout
45 -u, --update update the transient database with new classifications and crossmatches
46 -v, --version print the version of sherlock
47"""
48from __future__ import print_function
49from __future__ import absolute_import
50import readline
51from sherlock import transient_classifier
52from sherlock.commonutils import update_wiki_pages
53from sherlock.imports import ned as nedStreamImporter
54from sherlock.imports import ned_d as nedImporter
55from sherlock.imports import ifs as ifsImporter
56from sherlock.imports import veron as veronImporter
57from .commonutils import update_wiki_pages
58from .database_cleaner import database_cleaner
59from fundamentals.renderer import list_of_dictionaries
60from subprocess import Popen, PIPE, STDOUT
61from fundamentals import tools, times
62from docopt import docopt
63import pickle
64import glob
66import sys
67import os
68os.environ['TERM'] = 'vt100'
71def tab_complete(text, state):
72 return (glob.glob(text + '*') + [None])[state]
75def main(arguments=None):
76 """
77 *The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
78 """
79 # setup the command-line util settings
80 su = tools(
81 arguments=arguments,
82 docString=__doc__,
83 logLevel="WARNING",
84 options_first=False,
85 distributionName="qub-sherlock",
86 projectName="sherlock",
87 defaultSettingsFile=True
88 )
89 arguments, settings, log, dbConn = su.setup()
91 # tab completion for raw_input
92 readline.set_completer_delims(' \t\n;')
93 readline.parse_and_bind("tab: complete")
94 readline.set_completer(tab_complete)
96 # UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
97 # AUTOMATICALLY
98 a = {}
99 for arg, val in list(arguments.items()):
100 if arg[0] == "-":
101 varname = arg.replace("-", "") + "Flag"
102 else:
103 varname = arg.replace("<", "").replace(">", "")
104 a[varname] = val
105 if arg == "--dbConn":
106 dbConn = val
107 a["dbConn"] = val
108 log.debug('%s = %s' % (varname, val,))
110 ## START LOGGING ##
111 startTime = times.get_now_sql_datetime()
112 log.info(
113 '--- STARTING TO RUN THE cl_utils.py AT %s' %
114 (startTime,))
116 # set options interactively if user requests
117 if "interactiveFlag" in a and a["interactiveFlag"]:
119 # load previous settings
120 moduleDirectory = os.path.dirname(__file__) + "/resources"
121 pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
122 try:
123 with open(pathToPickleFile):
124 pass
125 previousSettingsExist = True
126 except:
127 previousSettingsExist = False
128 previousSettings = {}
129 if previousSettingsExist:
130 previousSettings = pickle.load(open(pathToPickleFile, "rb"))
132 # x-raw-input
133 # x-boolean-raw-input
134 # x-raw-input-with-default-value-from-previous-settings
136 # save the most recently used requests
137 pickleMeObjects = []
138 pickleMe = {}
139 theseLocals = locals()
140 for k in pickleMeObjects:
141 pickleMe[k] = theseLocals[k]
142 pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
144 if a["init"]:
145 from os.path import expanduser
146 home = expanduser("~")
147 filepath = home + "/.config/sherlock/sherlock.yaml"
148 try:
149 cmd = """open %(filepath)s""" % locals()
150 p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
151 except:
152 pass
153 try:
154 cmd = """start %(filepath)s""" % locals()
155 p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
156 except:
157 pass
158 return
160 init = a["init"]
161 match = a["match"]
162 dbmatch = a["dbmatch"]
163 clean = a["clean"]
164 wiki = a["wiki"]
165 iimport = a["import"]
166 ned = a["ned"]
167 cat = a["cat"]
168 stream = a["stream"]
169 info = a["info"]
170 ra = a["ra"]
171 dec = a["dec"]
172 radiusArcsec = a["radiusArcsec"]
173 cat_name = a["cat_name"]
174 cat_version = a["cat_version"]
175 stream_name = a["stream_name"]
176 pathToDataFile = a["pathToDataFile"]
177 skipNedUpdateFlag = a["skipNedUpdateFlag"]
178 skipMagUpdateFlag = a["skipMagUpdateFlag"]
179 settingsFlag = a["settingsFlag"]
180 verboseFlag = a["verboseFlag"]
181 updateFlag = a["updateFlag"]
183 # CALL FUNCTIONS/OBJECTS
184 if match or dbmatch:
185 if verboseFlag:
186 verbose = 2
187 else:
188 verbose = 1
190 if skipNedUpdateFlag:
191 updateNed = False
192 else:
193 updateNed = True
195 if skipMagUpdateFlag:
196 updatePeakMags = False
197 else:
198 updatePeakMags = True
200 classifier = transient_classifier.transient_classifier(
201 log=log,
202 settings=settings,
203 ra=ra,
204 dec=dec,
205 name=False,
206 verbose=verbose,
207 update=updateFlag,
208 updateNed=updateNed,
209 updatePeakMags=updatePeakMags
210 )
212 classifier.classify()
214 if clean:
215 cleaner = database_cleaner(
216 log=log,
217 settings=settings
218 )
219 cleaner.clean()
220 if wiki:
221 updateWiki = update_wiki_pages(
222 log=log,
223 settings=settings
224 )
225 updateWiki.update()
227 if iimport and ned:
228 ned = nedStreamImporter(
229 log=log,
230 settings=settings,
231 coordinateList=["%(ra)s %(dec)s" % locals()],
232 radiusArcsec=radiusArcsec
233 )
234 ned.ingest()
236 if iimport and cat:
238 if cat_name == "veron":
239 catalogue = veronImporter(
240 log=log,
241 settings=settings,
242 pathToDataFile=pathToDataFile,
243 version=cat_version,
244 catalogueName=cat_name
245 )
246 catalogue.ingest()
248 if "ned_d" in cat_name:
249 catalogue = nedImporter(
250 log=log,
251 settings=settings,
252 pathToDataFile=pathToDataFile,
253 version=cat_version,
254 catalogueName=cat_name
255 )
256 catalogue.ingest()
257 if iimport and stream:
258 if "ifs" in stream_name:
259 stream = ifsImporter(
260 log=log,
261 settings=settings
262 )
263 stream.ingest()
264 if not init and not match and not clean and not wiki and not iimport and ra:
266 classifier = transient_classifier.transient_classifier(
267 log=log,
268 settings=settings,
269 ra=ra,
270 dec=dec,
271 name=False,
272 verbose=verboseFlag
273 )
274 classifier.classify()
276 if info:
277 print("sherlock-catalogues")
278 wiki = update_wiki_pages(
279 log=log,
280 settings=settings
281 )
282 table = list(wiki._get_table_infos(trimmed=True))
284 dataSet = list_of_dictionaries(
285 log=log,
286 listOfDictionaries=table
287 )
288 tableData = dataSet.reST(filepath=None)
289 print(tableData)
290 print()
292 print("Crossmatch Streams")
293 table = list(wiki._get_stream_view_infos(trimmed=True))
294 dataSet = list_of_dictionaries(
295 log=log,
296 listOfDictionaries=table
297 )
298 tableData = dataSet.reST(filepath=None)
299 print(tableData)
300 print()
302 print("Views on Catalogues and Streams")
304 table = list(wiki._get_view_infos(trimmed=True))
305 dataSet = list_of_dictionaries(
306 log=log,
307 listOfDictionaries=table
308 )
309 tableData = dataSet.reST(filepath=None)
310 print(tableData)
312 if "dbConn" in locals() and dbConn:
313 dbConn.commit()
314 dbConn.close()
315 ## FINISH LOGGING ##
316 endTime = times.get_now_sql_datetime()
317 runningTime = times.calculate_time_difference(startTime, endTime)
318 log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
319 (endTime, runningTime, ))
321 return
324if __name__ == '__main__':
325 main()