Coverage for sherlock/transient

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4*classify a set of transients defined by a database query in the sherlock settings file*

6:Author:

7 David Young

8"""

9from __future__ import print_function

10from __future__ import division

11from astrocalc.coords import unit_conversion

12import copy

13from fundamentals.mysql import insert_list_of_dictionaries_into_database_tables

14from fundamentals import fmultiprocess

15import psutil

16from sherlock.commonutils import get_crossmatch_catalogues_column_map

17from sherlock.imports import ned

18from HMpTy.htm import sets

19from HMpTy.mysql import conesearch

20from fundamentals.renderer import list_of_dictionaries

21from fundamentals.mysql import readquery, directory_script_runner, writequery

22from fundamentals import tools

23import numpy as np

24from operator import itemgetter

25from datetime import datetime, date, time, timedelta

27from builtins import zip

28from builtins import str

29from builtins import range

30from builtins import object

31from past.utils import old_div

32import sys

33import os

34import collections

35import codecs

36import re

37import math

38import time

39import inspect

40import yaml

41from random import randint

42os.environ['TERM'] = 'vt100'

44theseBatches = []

45crossmatchArray = []

48class transient_classifier(object):

49 """

50 *The Sherlock Transient Classifier*

52 **Key Arguments**

54 - ``log`` -- logger

55 - ``settings`` -- the settings dictionary

56 - ``update`` -- update the transient database with crossmatch results (boolean)

57 - ``ra`` -- right ascension of a single transient source. Default *False*

58 - ``dec`` -- declination of a single transient source. Default *False*

59 - ``name`` -- the ID of a single transient source. Default *False*

60 - ``verbose`` -- amount of details to print about crossmatches to stdout. 0|1|2 Default *0*

61 - ``updateNed`` -- update the local NED database before running the classifier. Classification will not be as accuracte the NED database is not up-to-date. Default *True*.

62 - ``daemonMode`` -- run sherlock in daemon mode. In daemon mode sherlock remains live and classifies sources as they come into the database. Default *True*

63 - ``updatePeakMags`` -- update peak magnitudes in human-readable annotation of objects (can take some time - best to run occationally)

64 - ``lite`` -- return only a lite version of the results with the topped ranked matches only. Default *False*

65 - ``oneRun`` -- only process one batch of transients, usful for unit testing. Default *False*

67 **Usage**

69 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).

71 To initiate a transient_classifier object, use the following:

73 .. todo::

75 - update the package tutorial if needed

77 The sherlock classifier can be run in one of two ways. The first is to pass into the coordinates of an object you wish to classify:

79 ```python

80 from sherlock import transient_classifier

81 classifier = transient_classifier(

82 log=log,

83 settings=settings,

84 ra="08:57:57.19",

85 dec="+43:25:44.1",

86 name="PS17gx",

87 verbose=0

88 )

89 classifications, crossmatches = classifier.classify()

90 ```

92 The crossmatches returned are a list of dictionaries giving details of the crossmatched sources. The classifications returned are a list of classifications resulting from these crossmatches. The lists are ordered from most to least likely classification and the indicies for the crossmatch and the classification lists are synced.

94 The second way to run the classifier is to not pass in a coordinate set and therefore cause sherlock to run the classifier on the transient database referenced in the sherlock settings file:

96 ```python

97 from sherlock import transient_classifier

98 classifier = transient_classifier(

99 log=log,

100 settings=settings,

101 update=True

102 )

103 classifier.classify()

104 ```

105

106 Here the transient list is selected out of the database using the ``transient query`` value in the settings file:

107

108 ```yaml

109 database settings:

110 transients:

111 user: myusername

112 password: mypassword

113 db: nice_transients

114 host: 127.0.0.1

115 transient table: transientBucket

116 transient query: "select primaryKeyId as 'id', transientBucketId as 'alt_id', raDeg 'ra', decDeg 'dec', name 'name', sherlockClassification as 'object_classification'

117 from transientBucket where object_classification is null

118 transient primary id column: primaryKeyId

119 transient classification column: sherlockClassification

120 tunnel: False

121 ```

122

123 By setting ``update=True`` the classifier will update the ``sherlockClassification`` column of the ``transient table`` with new classification and populate the ``sherlock_crossmatches`` table with key details of the crossmatched sources from the catalogues database. By setting ``update=False`` results are printed to stdout but the database is not updated (useful for dry runs and testing new algorithms),

124

125

126 .. todo ::

127

128 - update key arguments values and definitions with defaults

129 - update return values and definitions

130 - update usage examples and text

131 - update docstring text

132 - check sublime snippet exists

133 - clip any useful text to docs mindmap

134 - regenerate the docs and check redendering of this docstring

135 """

136 # INITIALISATION

137

138 def __init__(

139 self,

140 log,

141 settings=False,

142 update=False,

143 ra=False,

144 dec=False,

145 name=False,

146 verbose=0,

147 updateNed=True,

148 daemonMode=False,

149 updatePeakMags=True,

150 oneRun=False,

151 lite=False

152 ):

153 self.log = log

154 log.debug("instansiating a new 'classifier' object")

155 self.settings = settings

156 self.update = update

157 self.ra = ra

158 self.dec = dec

159 self.name = name

160 self.cl = False

161 self.verbose = verbose

162 self.updateNed = updateNed

163 self.daemonMode = daemonMode

164 self.updatePeakMags = updatePeakMags

165 self.oneRun = oneRun

166 self.lite = lite

167 self.filterPreference = [

168 "R", "_r", "G", "V", "_g", "B", "I", "_i", "_z", "J", "H", "K", "U", "_u", "_y", "W1", "unkMag"

169 ]

170

171 # COLLECT ADVANCED SETTINGS IF AVAILABLE

172 parentDirectory = os.path.dirname(__file__)

173 advs = parentDirectory + "/advanced_settings.yaml"

174 level = 0

175 exists = False

176 count = 1

177 while not exists and len(advs) and count < 10:

178 count += 1

179 level -= 1

180 exists = os.path.exists(advs)

181 if not exists:

182 advs = "/".join(parentDirectory.split("/")

183 [:level]) + "/advanced_settings.yaml"

184 print(advs)

185 if not exists:

186 advs = {}

187 else:

188 with open(advs, 'r') as stream:

189 advs = yaml.safe_load(stream)

190 # MERGE ADVANCED SETTINGS AND USER SETTINGS (USER SETTINGS OVERRIDE)

191 self.settings = {**advs, **self.settings}

192

193 # INITIAL ACTIONS

194 # SETUP DATABASE CONNECTIONS

195 # SETUP ALL DATABASE CONNECTIONS

196 from sherlock import database

197 db = database(

198 log=self.log,

199 settings=self.settings

200 )

201 dbConns, dbVersions = db.connect()

202 self.dbVersions = dbVersions

203 self.transientsDbConn = dbConns["transients"]

204 self.cataloguesDbConn = dbConns["catalogues"]

205

206 # SIZE OF BATCHES TO SPLIT TRANSIENT INTO BEFORE CLASSIFYING

207 self.largeBatchSize = self.settings["database-batch-size"]

208 self.miniBatchSize = 1000

209

210 # LITE VERSION CANNOT BE RUN ON A DATABASE QUERY AS YET

211 if self.ra == False:

212 self.lite = False

213

214 # IS SHERLOCK CLASSIFIER BEING QUERIED FROM THE COMMAND-LINE?

215 if self.ra and self.dec:

216 self.cl = True

217 if not self.name:

218 self.name = "Transient"

219

220 # ASTROCALC UNIT CONVERTER OBJECT

221 self.converter = unit_conversion(

222 log=self.log

223 )

224

225 if self.ra and not isinstance(self.ra, float) and ":" in self.ra:

226 # ASTROCALC UNIT CONVERTER OBJECT

227 self.ra = self.converter.ra_sexegesimal_to_decimal(

228 ra=self.ra

229 )

230 self.dec = self.converter.dec_sexegesimal_to_decimal(

231 dec=self.dec

232 )

233

234 # DATETIME REGEX - EXPENSIVE OPERATION, LET"S JUST DO IT ONCE

235 self.reDatetime = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T')

236

237 return None

238

239 def classify(self):

240 """

241 *classify the transients selected from the transient selection query in the settings file or passed in via the CL or other code*

242

243 **Return**

244

245 - ``crossmatches`` -- list of dictionaries of crossmatched associated sources

246 - ``classifications`` -- the classifications assigned to the transients post-crossmatches (dictionary of rank ordered list of classifications)

247

248

249 See class docstring for usage.

250

251 .. todo ::

252

253 - update key arguments values and definitions with defaults

254 - update return values and definitions

255 - update usage examples and text

256 - update docstring text

257 - check sublime snippet exists

258 - clip any useful text to docs mindmap

259 - regenerate the docs and check redendering of this docstring

260 """

261

262 global theseBatches

263 global crossmatchArray

264

265 self.log.debug('starting the ``classify`` method')

266

267 remaining = 1

268

269 # THE COLUMN MAPS - WHICH COLUMNS IN THE CATALOGUE TABLES = RA, DEC,

270 # REDSHIFT, MAG ETC

271 colMaps = get_crossmatch_catalogues_column_map(

272 log=self.log,

273 dbConn=self.cataloguesDbConn

274 )

275

276 if self.transientsDbConn and self.update:

277 self._create_tables_if_not_exist()

278

279 import time

280 start_time = time.time()

281

282 # COUNT SEARCHES

283 sa = self.settings["search algorithm"]

284 searchCount = 0

285 brightnessFilters = ["bright", "faint", "general"]

286 for search_name, searchPara in list(sa.items()):

287 for bf in brightnessFilters:

288 if bf in searchPara:

289 searchCount += 1

290

291 cpuCount = psutil.cpu_count()

292 if searchCount > cpuCount:

293 searchCount = cpuCount

294

295 miniBatchSize = self.miniBatchSize

296

297 while remaining:

298

299 # IF A TRANSIENT HAS NOT BEEN PASSED IN VIA THE COMMAND-LINE, THEN

300 # QUERY THE TRANSIENT DATABASE

301 if not self.ra and not self.dec:

302

303 # COUNT REMAINING TRANSIENTS

304 from fundamentals.mysql import readquery

305 sqlQuery = self.settings["database settings"][

306 "transients"]["transient count"]

307 thisInt = randint(0, 100)

308 if "where" in sqlQuery:

309 sqlQuery = sqlQuery.replace(

310 "where", "where %(thisInt)s=%(thisInt)s and " % locals())

311

312 if remaining == 1 or remaining < self.largeBatchSize:

313 rows = readquery(

314 log=self.log,

315 sqlQuery=sqlQuery,

316 dbConn=self.transientsDbConn,

317 )

318 remaining = rows[0]["count(*)"]

319 else:

320 remaining = remaining - self.largeBatchSize

321

322 print(

323 "%(remaining)s transient sources requiring a classification remain" % locals())

324

325 # START THE TIME TO TRACK CLASSIFICATION SPPED

326 start_time = time.time()

327

328 # A LIST OF DICTIONARIES OF TRANSIENT METADATA

329 transientsMetadataList = self._get_transient_metadata_from_database_list()

330

331 count = len(transientsMetadataList)

332 print(

333 " now classifying the next %(count)s transient sources" % locals())

334

335 # EXAMPLE OF TRANSIENT METADATA

336 # { 'name': 'PS17gx',

337 # 'alt_id': 'PS17gx',

338 # 'object_classification': 'SN',

339 # 'dec': '+43:25:44.1',

340 # 'id': 1,

341 # 'ra': '08:57:57.19'}

342 # TRANSIENT PASSED VIA COMMAND-LINE

343 else:

344 # CONVERT SINGLE TRANSIENTS TO LIST

345 if not isinstance(self.ra, list):

346 self.ra = [self.ra]

347 self.dec = [self.dec]

348 self.name = [self.name]

349

350 # GIVEN TRANSIENTS UNIQUE NAMES IF NOT PROVIDED

351 if not self.name[0]:

352 self.name = []

353 for i, v in enumerate(self.ra):

354 self.name.append("transient_%(i)05d" % locals())

355

356 transientsMetadataList = []

357 for r, d, n in zip(self.ra, self.dec, self.name):

358 transient = {

359 'name': n,

360 'object_classification': None,

361 'dec': d,

362 'id': n,

363 'ra': r

364 }

365 transientsMetadataList.append(transient)

366 remaining = 0

367

368 if self.oneRun:

369 remaining = 0

370

371 if len(transientsMetadataList) == 0:

372 if self.daemonMode == False:

373 remaining = 0

374 print("No transients need classified")

375 return None, None

376 else:

377 print(

378 "No remaining transients need classified, will try again in 5 mins")

379 time.sleep("10")

380

381 # FROM THE LOCATIONS OF THE TRANSIENTS, CHECK IF OUR LOCAL NED DATABASE

382 # NEEDS UPDATED

383 if self.updateNed:

384

385 self._update_ned_stream(

386 transientsMetadataList=transientsMetadataList

387 )

388

389 # SOME TESTING SHOWED THAT 25 IS GOOD

390 total = len(transientsMetadataList)

391 batches = int((old_div(float(total), float(miniBatchSize))) + 1.)

392

393 if batches == 0:

394 batches = 1

395

396 start = 0

397 end = 0

398 theseBatches = []

399 for i in range(batches):

400 end = end + miniBatchSize

401 start = i * miniBatchSize

402 thisBatch = transientsMetadataList[start:end]

403 theseBatches.append(thisBatch)

404

405 if self.verbose > 1:

406 print("BATCH SIZE = %(total)s" % locals())

407 print("MINI BATCH SIZE = %(batches)s x %(miniBatchSize)s" % locals())

408

409 poolSize = self.settings["cpu-pool-size"]

410 if poolSize and batches < poolSize:

411 poolSize = batches

412

413 start_time2 = time.time()

414

415 if self.verbose > 1:

416 print("START CROSSMATCH")

417

418 crossmatchArray = fmultiprocess(log=self.log, function=_crossmatch_transients_against_catalogues,

419 inputArray=list(range(len(theseBatches))), poolSize=poolSize, settings=self.settings, colMaps=colMaps)

420

421 if self.verbose > 1:

422 print("FINISH CROSSMATCH/START RANKING: %d" %

423 (time.time() - start_time2,))

424 start_time2 = time.time()

425

426 classifications = {}

427 crossmatches = []

428

429 for sublist in crossmatchArray:

430 sublist = sorted(

431 sublist, key=itemgetter('transient_object_id'))

432

433 # REORGANISE INTO INDIVIDUAL TRANSIENTS FOR RANKING AND

434 # TOP-LEVEL CLASSIFICATION EXTRACTION

435

436 batch = []

437 if len(sublist) != 0:

438 transientId = sublist[0]['transient_object_id']

439 for s in sublist:

440 if s['transient_object_id'] != transientId:

441 # RANK TRANSIENT CROSSMATCH BATCH

442 cl, cr = self._rank_classifications(

443 batch, colMaps)

444 crossmatches.extend(cr)

445 classifications = dict(

446 list(classifications.items()) + list(cl.items()))

447

448 transientId = s['transient_object_id']

449 batch = [s]

450 else:

451 batch.append(s)

452

453 # RANK FINAL BATCH

454 cl, cr = self._rank_classifications(

455 batch, colMaps)

456 classifications = dict(

457 list(classifications.items()) + list(cl.items()))

458 crossmatches.extend(cr)

459

460 for t in transientsMetadataList:

461 if t["id"] not in classifications:

462 classifications[t["id"]] = ["ORPHAN"]

463

464 # UPDATE THE TRANSIENT DATABASE IF UPDATE REQUESTED (ADD DATA TO

465 # tcs_crossmatch_table AND A CLASSIFICATION TO THE ORIGINAL TRANSIENT

466 # TABLE)

467 if self.verbose > 1:

468 print("FINISH RANKING/START UPDATING TRANSIENT DB: %d" %

469 (time.time() - start_time2,))

470 start_time2 = time.time()

471 if self.update and not self.ra:

472 self._update_transient_database(

473 crossmatches=crossmatches,

474 classifications=classifications,

475 transientsMetadataList=transientsMetadataList,

476 colMaps=colMaps

477 )

478

479 if self.verbose > 1:

480 print("FINISH UPDATING TRANSIENT DB/START ANNOTATING TRANSIENT DB: %d" %

481 (time.time() - start_time2,))

482 start_time2 = time.time()

483

484 # COMMAND-LINE SINGLE CLASSIFICATION

485 if self.ra:

486 classifications = self.update_classification_annotations_and_summaries(

487 False, True, crossmatches, classifications)

488 for k, v in classifications.items():

489 if len(v) == 1 and v[0] == "ORPHAN":

490 v.append(

491 "No contexual information is available for this transient")

492

493 if self.lite != False:

494 crossmatches = self._lighten_return(crossmatches)

495

496 if self.cl:

497 self._print_results_to_stdout(

498 classifications=classifications,

499 crossmatches=crossmatches

500 )

501

502 return classifications, crossmatches

503

504 if self.updatePeakMags and self.settings["database settings"]["transients"]["transient peak magnitude query"]:

505 self.update_peak_magnitudes()

506

507 # BULK RUN -- NOT A COMMAND-LINE SINGLE CLASSIFICATION

508 self.update_classification_annotations_and_summaries(

509 self.updatePeakMags)

510

511 print("FINISH ANNOTATING TRANSIENT DB: %d" %

512 (time.time() - start_time2,))

513 start_time2 = time.time()

514

515 classificationRate = old_div(count, (time.time() - start_time))

516 print(

517 "Sherlock is classify at a rate of %(classificationRate)2.1f transients/sec" % locals())

518

519 self.log.debug('completed the ``classify`` method')

520 return None, None

521

522 def _get_transient_metadata_from_database_list(

523 self):

524 """use the transient query in the settings file to generate a list of transients to corssmatch and classify

525

526 **Return**

527

528

529 - ``transientsMetadataList``

530

531 .. todo ::

532

533 - update key arguments values and definitions with defaults

534 - update return values and definitions

535 - update usage examples and text

536 - update docstring text

537 - check sublime snippet exists

538 - clip any useful text to docs mindmap

539 - regenerate the docs and check redendering of this docstring

540 """

541 self.log.debug(

542 'starting the ``_get_transient_metadata_from_database_list`` method')

543

544 sqlQuery = self.settings["database settings"][

545 "transients"]["transient query"] + " limit " + str(self.largeBatchSize)

546

547 thisInt = randint(0, 100)

548 if "where" in sqlQuery:

549 sqlQuery = sqlQuery.replace(

550 "where", "where %(thisInt)s=%(thisInt)s and " % locals())

551

552 transientsMetadataList = readquery(

553 log=self.log,

554 sqlQuery=sqlQuery,

555 dbConn=self.transientsDbConn,

556 quiet=False

557 )

558

559 self.log.debug(

560 'completed the ``_get_transient_metadata_from_database_list`` method')

561 return transientsMetadataList

562

563 def _update_ned_stream(

564 self,

565 transientsMetadataList

566 ):

567 """ update the NED stream within the catalogues database at the locations of the transients

568

569 **Key Arguments**

570

571 - ``transientsMetadataList`` -- the list of transient metadata lifted from the database.

572

573

574 .. todo ::

575

576 - update key arguments values and definitions with defaults

577 - update return values and definitions

578 - update usage examples and text

579 - update docstring text

580 - check sublime snippet exists

581 - clip any useful text to docs mindmap

582 - regenerate the docs and check redendering of this docstring

583 """

584 self.log.debug('starting the ``_update_ned_stream`` method')

585

586 coordinateList = []

587 for i in transientsMetadataList:

588 # thisList = str(i["ra"]) + " " + str(i["dec"])

589 thisList = (i["ra"], i["dec"])

590 coordinateList.append(thisList)

591

592 coordinateList = self._remove_previous_ned_queries(

593 coordinateList=coordinateList

594 )

595

596 # MINIMISE COORDINATES IN LIST TO REDUCE NUMBER OF REQUIRE NED QUERIES

597 coordinateList = self._consolidate_coordinateList(

598 coordinateList=coordinateList

599 )

600

601 stream = ned(

602 log=self.log,

603 settings=self.settings,

604 coordinateList=coordinateList,

605 radiusArcsec=self.settings["ned stream search radius arcec"]

606 )

607 stream.ingest()

608

609 sqlQuery = """SET session sql_mode = "";""" % locals(

610 )

611 writequery(

612 log=self.log,

613 sqlQuery=sqlQuery,

614 dbConn=self.cataloguesDbConn

615 )

616 sqlQuery = """update tcs_cat_ned_stream set magnitude = CAST(`magnitude_filter` AS DECIMAL(5,2)) where magnitude is null and magnitude_filter is not null;""" % locals(

617 )

618 writequery(

619 log=self.log,

620 sqlQuery=sqlQuery,

621 dbConn=self.cataloguesDbConn

622 )

623

624 self.log.debug('completed the ``_update_ned_stream`` method')

625 return None

626

627 def _remove_previous_ned_queries(

628 self,

629 coordinateList):

630 """iterate through the transient locations to see if we have recent local NED coverage of that area already

631

632 **Key Arguments**

633

634 - ``coordinateList`` -- set of coordinate to check for previous queries

635

636

637 **Return**

638

639 - ``updatedCoordinateList`` -- coordinate list with previous queries removed

640

641

642 .. todo ::

643

644 - update key arguments values and definitions with defaults

645 - update return values and definitions

646 - update usage examples and text

647 - update docstring text

648 - check sublime snippet exists

649 - clip any useful text to docs mindmap

650 - regenerate the docs and check redendering of this docstring

651 """

652 self.log.debug('starting the ``_remove_previous_ned_queries`` method')

653

654 # 1 DEGREE QUERY RADIUS

655 radius = 60. * 60.

656 updatedCoordinateList = []

657 keepers = []

658

659 # CALCULATE THE OLDEST RESULTS LIMIT

660 now = datetime.now()

661 td = timedelta(

662 days=self.settings["ned stream refresh rate in days"])

663 refreshLimit = now - td

664 refreshLimit = refreshLimit.strftime("%Y-%m-%d %H:%M:%S")

665

666 raList = []

667 raList[:] = [c[0] for c in coordinateList]

668 decList = []

669 decList[:] = [c[1] for c in coordinateList]

670

671 # MATCH COORDINATES AGAINST PREVIOUS NED SEARCHES

672 cs = conesearch(

673 log=self.log,

674 dbConn=self.cataloguesDbConn,

675 tableName="tcs_helper_ned_query_history",

676 columns="*",

677 ra=raList,

678 dec=decList,

679 radiusArcsec=radius,

680 separations=True,

681 distinct=True,

682 sqlWhere="dateQueried > '%(refreshLimit)s'" % locals(),

683 closest=False

684 )

685 matchIndies, matches = cs.search()

686

687 # DETERMINE WHICH COORDINATES REQUIRE A NED QUERY

688 curatedMatchIndices = []

689 curatedMatches = []

690 for i, m in zip(matchIndies, matches.list):

691 match = False

692 row = m

693 row["separationArcsec"] = row["cmSepArcsec"]

694 raStream = row["raDeg"]

695 decStream = row["decDeg"]

696 radiusStream = row["arcsecRadius"]

697 dateStream = row["dateQueried"]

698 angularSeparation = row["separationArcsec"]

699

700 if angularSeparation + self.settings["first pass ned search radius arcec"] < radiusStream:

701 curatedMatchIndices.append(i)

702 curatedMatches.append(m)

703

704 # NON MATCHES

705 for i, v in enumerate(coordinateList):

706 if i not in curatedMatchIndices:

707 updatedCoordinateList.append(v)

708

709 self.log.debug('completed the ``_remove_previous_ned_queries`` method')

710 return updatedCoordinateList

711

712 def _update_transient_database(

713 self,

714 crossmatches,

715 classifications,

716 transientsMetadataList,

717 colMaps):

718 """ update transient database with classifications and crossmatch results

719

720 **Key Arguments**

721

722 - ``crossmatches`` -- the crossmatches and associations resulting from the catlaogue crossmatches

723 - ``classifications`` -- the classifications assigned to the transients post-crossmatches (dictionary of rank ordered list of classifications)

724 - ``transientsMetadataList`` -- the list of transient metadata lifted from the database.

725 - ``colMaps`` -- maps of the important column names for each table/view in the crossmatch-catalogues database

726

727

728 .. todo ::

729

730 - update key arguments values and definitions with defaults

731 - update return values and definitions

732 - update usage examples and text

733 - update docstring text

734 - check sublime snippet exists

735 - clip any useful text to docs mindmap

736 - regenerate the docs and check redendering of this docstring

737 """

738

739 self.log.debug('starting the ``_update_transient_database`` method')

740

741 import time

742 start_time = time.time()

743 print("UPDATING TRANSIENTS DATABASE WITH RESULTS")

744 print("DELETING OLD RESULTS")

745

746 now = datetime.now()

747 now = now.strftime("%Y-%m-%d_%H-%M-%S-%f")

748

749 transientTable = self.settings["database settings"][

750 "transients"]["transient table"]

751 transientTableClassCol = self.settings["database settings"][

752 "transients"]["transient classification column"]

753 transientTableIdCol = self.settings["database settings"][

754 "transients"]["transient primary id column"]

755

756 # COMBINE ALL CROSSMATCHES INTO A LIST OF DICTIONARIES TO DUMP INTO

757 # DATABASE TABLE

758 transientIDs = [str(c)

759 for c in list(classifications.keys())]

760 transientIDs = ",".join(transientIDs)

761

762 # REMOVE PREVIOUS MATCHES

763 sqlQuery = """delete from sherlock_crossmatches where transient_object_id in (%(transientIDs)s);""" % locals(

764 )

765 writequery(

766 log=self.log,

767 sqlQuery=sqlQuery,

768 dbConn=self.transientsDbConn,

769 )

770 sqlQuery = """delete from sherlock_classifications where transient_object_id in (%(transientIDs)s);""" % locals(

771 )

772 writequery(

773 log=self.log,

774 sqlQuery=sqlQuery,

775 dbConn=self.transientsDbConn,

776 )

777

778 print("FINISHED DELETING OLD RESULTS/ADDING TO CROSSMATCHES: %d" %

779 (time.time() - start_time,))

780 start_time = time.time()

781

782 if len(crossmatches):

783 insert_list_of_dictionaries_into_database_tables(

784 dbConn=self.transientsDbConn,

785 log=self.log,

786 dictList=crossmatches,

787 dbTableName="sherlock_crossmatches",

788 dateModified=True,

789 batchSize=10000,

790 replace=True,

791 dbSettings=self.settings["database settings"][

792 "transients"]

793 )

794

795 print("FINISHED ADDING TO CROSSMATCHES/UPDATING CLASSIFICATIONS IN TRANSIENT TABLE: %d" %

796 (time.time() - start_time,))

797 start_time = time.time()

798

799 sqlQuery = ""

800 inserts = []

801 for k, v in list(classifications.items()):

802 thisInsert = {

803 "transient_object_id": k,

804 "classification": v[0]

805 }

806 inserts.append(thisInsert)

807

808 print("FINISHED UPDATING CLASSIFICATIONS IN TRANSIENT TABLE/UPDATING sherlock_classifications TABLE: %d" %

809 (time.time() - start_time,))

810 start_time = time.time()

811

812 insert_list_of_dictionaries_into_database_tables(

813 dbConn=self.transientsDbConn,

814 log=self.log,

815 dictList=inserts,

816 dbTableName="sherlock_classifications",

817 dateModified=True,

818 batchSize=10000,

819 replace=True,

820 dbSettings=self.settings["database settings"][

821 "transients"]

822 )

823

824 print("FINISHED UPDATING sherlock_classifications TABLE: %d" %

825 (time.time() - start_time,))

826 start_time = time.time()

827

828 self.log.debug('completed the ``_update_transient_database`` method')

829 return None

830

831 def _rank_classifications(

832 self,

833 crossmatchArray,

834 colMaps):

835 """*rank the classifications returned from the catalogue crossmatcher, annotate the results with a classification rank-number (most likely = 1) and a rank-score (weight of classification)*

836

837 **Key Arguments**

838

839 - ``crossmatchArrayIndex`` -- the index of list of unranked crossmatch classifications

840 - ``colMaps`` -- dictionary of dictionaries with the name of the database-view (e.g. `tcs_view_agn_milliquas_v4_5`) as the key and the column-name dictary map as value (`{view_name: {columnMap}}`).

841

842

843 **Return**

844

845 - ``classifications`` -- the classifications assigned to the transients post-crossmatches

846 - ``crossmatches`` -- the crossmatches annotated with rankings and rank-scores

847

848

849 .. todo ::

850

851 - update key arguments values and definitions with defaults

852 - update return values and definitions

853 - update usage examples and text

854 - update docstring text

855 - check sublime snippet exists

856 - clip any useful text to docs mindmap

857 - regenerate the docs and check redendering of this docstring

858 """

859 self.log.debug('starting the ``_rank_classifications`` method')

860

861 crossmatches = crossmatchArray

862

863 # GROUP CROSSMATCHES INTO DISTINCT SOURCES (DUPLICATE ENTRIES OF THE

864 # SAME ASTROPHYSICAL SOURCE ACROSS MULTIPLE CATALOGUES)

865 ra, dec = list(zip(*[(r["raDeg"], r["decDeg"]) for r in crossmatches]))

866

867 from HMpTy.htm import sets

868 xmatcher = sets(

869 log=self.log,

870 ra=ra,

871 dec=dec,

872 radius=1. / (60. * 60.), # in degrees

873 sourceList=crossmatches

874 )

875 groupedMatches = xmatcher.match

876

877 associatationTypeOrder = ["AGN", "CV", "NT", "SN", "VS", "BS"]

878

879 # ADD DISTINCT-SOURCE KEY

880 dupKey = 0

881 distinctMatches = []

882 for x in groupedMatches:

883 dupKey += 1

884 mergedMatch = copy.deepcopy(x[0])

885 mergedMatch["merged_rank"] = int(dupKey)

886

887 # ADD OTHER ESSENTIAL KEYS

888 for e in ['z', 'photoZ', 'photoZErr']:

889 if e not in mergedMatch:

890 mergedMatch[e] = None

891 bestQualityCatalogue = colMaps[mergedMatch[

892 "catalogue_view_name"]]["object_type_accuracy"]

893 bestDirectDistance = {

894 "direct_distance": mergedMatch["direct_distance"],

895 "direct_distance_modulus": mergedMatch["direct_distance_modulus"],

896 "direct_distance_scale": mergedMatch["direct_distance_scale"],

897 "qual": colMaps[mergedMatch["catalogue_view_name"]]["object_type_accuracy"]

898 }

899 if not mergedMatch["direct_distance"]:

900 bestDirectDistance["qual"] = 0

901

902 bestSpecz = {

903 "z": mergedMatch["z"],

904 "distance": mergedMatch["distance"],

905 "distance_modulus": mergedMatch["distance_modulus"],

906 "scale": mergedMatch["scale"],

907 "qual": colMaps[mergedMatch["catalogue_view_name"]]["object_type_accuracy"]

908 }

909 if not mergedMatch["distance"]:

910 bestSpecz["qual"] = 0

911

912 bestPhotoz = {

913 "photoZ": mergedMatch["photoZ"],

914 "photoZErr": mergedMatch["photoZErr"],

915 "qual": colMaps[mergedMatch["catalogue_view_name"]]["object_type_accuracy"]

916 }

917 if not mergedMatch["photoZ"]:

918 bestPhotoz["qual"] = 0

919

920 # ORDER THESE FIRST IN NAME LISTING

921 mergedMatch["search_name"] = None

922 mergedMatch["catalogue_object_id"] = None

923 primeCats = ["NED", "SDSS", "MILLIQUAS"]

924 for cat in primeCats:

925 for i, m in enumerate(x):

926 # MERGE SEARCH NAMES

927 snippet = m["search_name"].split(" ")[0].upper()

928 if cat.upper() in snippet:

929 if not mergedMatch["search_name"]:

930 mergedMatch["search_name"] = m["search_name"].split(" ")[

931 0].upper()

932 elif "/" not in mergedMatch["search_name"] and snippet not in mergedMatch["search_name"].upper():

933 mergedMatch["search_name"] = mergedMatch["search_name"].split(

934 " ")[0].upper() + "/" + m["search_name"].split(" ")[0].upper()

935 elif snippet not in mergedMatch["search_name"].upper():

936 mergedMatch[

937 "search_name"] += "/" + m["search_name"].split(" ")[0].upper()

938 elif "/" not in mergedMatch["search_name"]:

939 mergedMatch["search_name"] = mergedMatch["search_name"].split(

940 " ")[0].upper()

941 mergedMatch["catalogue_table_name"] = mergedMatch[

942 "search_name"]

943

944 # MERGE CATALOGUE SOURCE NAMES

945 if not mergedMatch["catalogue_object_id"]:

946 mergedMatch["catalogue_object_id"] = str(

947 m["catalogue_object_id"])

948

949 # NOW ADD THE REST

950 for i, m in enumerate(x):

951 # MERGE SEARCH NAMES

952 snippet = m["search_name"].split(" ")[0].upper()

953 if snippet not in primeCats:

954 if not mergedMatch["search_name"]:

955 mergedMatch["search_name"] = m["search_name"].split(" ")[

956 0].upper()

957 elif "/" not in mergedMatch["search_name"] and snippet not in mergedMatch["search_name"].upper():

958 mergedMatch["search_name"] = mergedMatch["search_name"].split(

959 " ")[0].upper() + "/" + m["search_name"].split(" ")[0].upper()

960 elif snippet not in mergedMatch["search_name"].upper():

961 mergedMatch[

962 "search_name"] += "/" + m["search_name"].split(" ")[0].upper()

963 elif "/" not in mergedMatch["search_name"]:

964 mergedMatch["search_name"] = mergedMatch["search_name"].split(

965 " ")[0].upper()

966 mergedMatch["catalogue_table_name"] = mergedMatch[

967 "search_name"]

968

969 # MERGE CATALOGUE SOURCE NAMES

970 if not mergedMatch["catalogue_object_id"]:

971 mergedMatch["catalogue_object_id"] = str(

972 m["catalogue_object_id"])

973 # else:

974 # mergedMatch["catalogue_object_id"] = str(

975 # mergedMatch["catalogue_object_id"])

976 # m["catalogue_object_id"] = str(

977 # m["catalogue_object_id"])

978 # if m["catalogue_object_id"].replace(" ", "").lower() not in mergedMatch["catalogue_object_id"].replace(" ", "").lower():

979 # mergedMatch["catalogue_object_id"] += "/" + \

980 # m["catalogue_object_id"]

981

982 for i, m in enumerate(x):

983 m["merged_rank"] = int(dupKey)

984 if i > 0:

985 # MERGE ALL BEST MAGNITUDE MEASUREMENTS

986 for f in self.filterPreference:

987

988 if f in m and m[f] and (f not in mergedMatch or (f + "Err" in mergedMatch and f + "Err" in m and (mergedMatch[f + "Err"] == None or (m[f + "Err"] and mergedMatch[f + "Err"] > m[f + "Err"])))):

989 mergedMatch[f] = m[f]

990 try:

991 mergedMatch[f + "Err"] = m[f + "Err"]

992 except:

993 pass

994 mergedMatch["original_search_radius_arcsec"] = "multiple"

995 mergedMatch["catalogue_object_subtype"] = "multiple"

996 mergedMatch["catalogue_view_name"] = "multiple"

997

998 # DETERMINE BEST CLASSIFICATION

999 if mergedMatch["classificationReliability"] == 3 and m["classificationReliability"] < 3:

1000 mergedMatch["association_type"] = m["association_type"]

1001 mergedMatch["catalogue_object_type"] = m[

1002 "catalogue_object_type"]

1003 mergedMatch["classificationReliability"] = m[

1004 "classificationReliability"]

1005

1006 if m["classificationReliability"] != 3 and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] > bestQualityCatalogue:

1007 bestQualityCatalogue = colMaps[

1008 m["catalogue_view_name"]]["object_type_accuracy"]

1009 mergedMatch["association_type"] = m["association_type"]

1010 mergedMatch["catalogue_object_type"] = m[

1011 "catalogue_object_type"]

1012 mergedMatch["classificationReliability"] = m[

1013 "classificationReliability"]

1014

1015 if m["classificationReliability"] != 3 and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] == bestQualityCatalogue and m["association_type"] in associatationTypeOrder and (mergedMatch["association_type"] not in associatationTypeOrder or associatationTypeOrder.index(m["association_type"]) < associatationTypeOrder.index(mergedMatch["association_type"])):

1016 mergedMatch["association_type"] = m["association_type"]

1017 mergedMatch["catalogue_object_type"] = m[

1018 "catalogue_object_type"]

1019 mergedMatch["classificationReliability"] = m[

1020 "classificationReliability"]

1021

1022 # FIND BEST DISTANCES

1023 if "direct_distance" in m and m["direct_distance"] and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] > bestDirectDistance["qual"]:

1024 bestDirectDistance = {

1025 "direct_distance": m["direct_distance"],

1026 "direct_distance_modulus": m["direct_distance_modulus"],

1027 "direct_distance_scale": m["direct_distance_scale"],

1028 "catalogue_object_type": m["catalogue_object_type"],

1029 "qual": colMaps[m["catalogue_view_name"]]["object_type_accuracy"]

1030 }

1031 # FIND BEST SPEC-Z

1032 if "z" in m and m["z"] and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] > bestSpecz["qual"]:

1033 bestSpecz = {

1034 "z": m["z"],

1035 "distance": m["distance"],

1036 "distance_modulus": m["distance_modulus"],

1037 "scale": m["scale"],

1038 "catalogue_object_type": m["catalogue_object_type"],

1039 "qual": colMaps[m["catalogue_view_name"]]["object_type_accuracy"]

1040 }

1041 # FIND BEST PHOT-Z

1042 if "photoZ" in m and m["photoZ"] and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] > bestPhotoz["qual"]:

1043 bestPhotoz = {

1044 "photoZ": m["photoZ"],

1045 "photoZErr": m["photoZErr"],

1046 "distance": m["distance"],

1047 "distance_modulus": m["distance_modulus"],

1048 "scale": m["scale"],

1049 "catalogue_object_type": m["catalogue_object_type"],

1050 "qual": colMaps[m["catalogue_view_name"]]["object_type_accuracy"]

1051 }

1052 # CLOSEST ANGULAR SEP & COORDINATES

1053 if m["separationArcsec"] < mergedMatch["separationArcsec"]:

1054 mergedMatch["separationArcsec"] = m["separationArcsec"]

1055 mergedMatch["raDeg"] = m["raDeg"]

1056 mergedMatch["decDeg"] = m["decDeg"]

1057

1058 # MERGE THE BEST RESULTS

1059 for l in [bestPhotoz, bestSpecz, bestDirectDistance]:

1060 for k, v in list(l.items()):

1061 if k != "qual" and v:

1062 mergedMatch[k] = v

1063

1064 mergedMatch["catalogue_object_id"] = str(mergedMatch[

1065 "catalogue_object_id"]).replace(" ", "")

1066

1067 # RECALULATE PHYSICAL DISTANCE SEPARATION

1068 if mergedMatch["direct_distance_scale"]:

1069 mergedMatch["physical_separation_kpc"] = mergedMatch[

1070 "direct_distance_scale"] * mergedMatch["separationArcsec"]

1071

1072 elif mergedMatch["scale"]:

1073 mergedMatch["physical_separation_kpc"] = mergedMatch[

1074 "scale"] * mergedMatch["separationArcsec"]

1075

1076 if "/" in mergedMatch["search_name"]:

1077 mergedMatch["search_name"] = "multiple"

1078

1079 distinctMatches.append(mergedMatch)

1080

1081 crossmatches = []

1082 for xm, gm in zip(distinctMatches, groupedMatches):

1083 # SPEC-Z GALAXIES

1084 if (xm["physical_separation_kpc"] is not None and xm["physical_separation_kpc"] != "null" and xm["physical_separation_kpc"] < 20. and (("z" in xm and xm["z"] is not None) or "photoZ" not in xm or xm["photoZ"] is None or xm["photoZ"] < 0.)):

1085 rankScore = xm["classificationReliability"] * 1000 + 2. - \

1086 (50 - old_div(xm["physical_separation_kpc"], 20))

1087 # PHOTO-Z GALAXIES

1088 elif (xm["physical_separation_kpc"] is not None and xm["physical_separation_kpc"] != "null" and xm["physical_separation_kpc"] < 20. and xm["association_type"] == "SN"):

1089 rankScore = xm["classificationReliability"] * 1000 + 5 - \

1090 (50 - old_div(xm["physical_separation_kpc"], 20))

1091 # NOT SPEC-Z, NON PHOTO-Z GALAXIES & PHOTO-Z GALAXIES

1092 elif (xm["association_type"] == "SN"):

1093 rankScore = xm["classificationReliability"] * 1000 + 5.

1094 # VS

1095 elif (xm["association_type"] == "VS"):

1096 rankScore = xm["classificationReliability"] * \

1097 1000 + xm["separationArcsec"] + 2.

1098 # BS

1099 elif (xm["association_type"] == "BS"):

1100 rankScore = xm["classificationReliability"] * \

1101 1000 + xm["separationArcsec"]

1102 else:

1103 rankScore = xm["classificationReliability"] * \

1104 1000 + xm["separationArcsec"] + 10.

1105 xm["rankScore"] = rankScore

1106 crossmatches.append(xm)

1107 if len(gm) > 1:

1108 for g in gm:

1109 g["rankScore"] = rankScore

1110

1111 crossmatches = sorted(

1112 crossmatches, key=itemgetter('rankScore'), reverse=False)

1113 crossmatches = sorted(

1114 crossmatches, key=itemgetter('transient_object_id'))

1115

1116 transient_object_id = None

1117 uniqueIndexCheck = []

1118 classifications = {}

1119 crossmatchesKeep = []

1120 rank = 0

1121 transClass = []

1122 for xm in crossmatches:

1123 rank += 1

1124 if rank == 1:

1125 transClass.append(xm["association_type"])

1126 classifications[xm["transient_object_id"]] = transClass

1127 if rank == 1 or self.lite == False:

1128 xm["rank"] = rank

1129 crossmatchesKeep.append(xm)

1130 crossmatches = crossmatchesKeep

1131

1132 crossmatchesKeep = []

1133 if self.lite == False:

1134 for xm in crossmatches:

1135 group = groupedMatches[xm["merged_rank"] - 1]

1136 xm["merged_rank"] = None

1137 crossmatchesKeep.append(xm)

1138

1139 if len(group) > 1:

1140 groupKeep = []

1141 uniqueIndexCheck = []

1142 for g in group:

1143 g["merged_rank"] = xm["rank"]

1144 g["rankScore"] = xm["rankScore"]

1145 index = "%(catalogue_table_name)s%(catalogue_object_id)s" % g

1146 # IF WE HAVE HIT A NEW SOURCE

1147 if index not in uniqueIndexCheck:

1148 uniqueIndexCheck.append(index)

1149 crossmatchesKeep.append(g)

1150

1151 crossmatches = crossmatchesKeep

1152

1153 self.log.debug('completed the ``_rank_classifications`` method')

1154

1155 return classifications, crossmatches

1156

1157 def _print_results_to_stdout(

1158 self,

1159 classifications,

1160 crossmatches):

1161 """*print the classification and crossmatch results for a single transient object to stdout*

1162

1163 **Key Arguments**

1164

1165 - ``crossmatches`` -- the unranked crossmatch classifications

1166 - ``classifications`` -- the classifications assigned to the transients post-crossmatches (dictionary of rank ordered list of classifications)

1167

1168

1169 .. todo ::

1170

1171 - update key arguments values and definitions with defaults

1172 - update return values and definitions

1173 - update usage examples and text

1174 - update docstring text

1175 - check sublime snippet exists

1176 - clip any useful text to docs mindmap

1177 - regenerate the docs and check redendering of this docstring

1178 """

1179 self.log.debug('starting the ``_print_results_to_stdout`` method')

1180

1181 if self.verbose == 0:

1182 return

1183

1184 crossmatchesCopy = copy.deepcopy(crossmatches)

1185

1186 # REPORT ONLY THE MOST PREFERED MAGNITUDE VALUE

1187 basic = ["association_type", "rank", "rankScore", "catalogue_table_name", "catalogue_object_id", "catalogue_object_type", "catalogue_object_subtype",

1188 "raDeg", "decDeg", "separationArcsec", "physical_separation_kpc", "direct_distance", "distance", "z", "photoZ", "photoZErr", "Mag", "MagFilter", "MagErr", "classificationReliability", "merged_rank"]

1189 verbose = ["search_name", "catalogue_view_name", "original_search_radius_arcsec", "direct_distance_modulus", "distance_modulus", "direct_distance_scale", "major_axis_arcsec", "scale", "U", "UErr",

1190 "B", "BErr", "V", "VErr", "R", "RErr", "I", "IErr", "J", "JErr", "H", "HErr", "K", "KErr", "_u", "_uErr", "_g", "_gErr", "_r", "_rErr", "_i", "_iErr", "_z", "_zErr", "_y", "G", "GErr", "_yErr", "unkMag"]

1191 dontFormat = ["decDeg", "raDeg", "rank",

1192 "catalogue_object_id", "catalogue_object_subtype", "merged_rank"]

1193 if self.verbose == 2:

1194 basic = basic + verbose

1195

1196 for n in self.name:

1197

1198 if n in classifications:

1199 headline = "\n" + n + "'s Predicted Classification: " + \

1200 classifications[n][0]

1201 else:

1202 headline = n + "'s Predicted Classification: ORPHAN"

1203 print(headline)

1204 print("Suggested Associations:")

1205

1206 myCrossmatches = []

1207 myCrossmatches[:] = [c for c in crossmatchesCopy if c[

1208 "transient_object_id"] == n]

1209

1210 for c in myCrossmatches:

1211 for f in self.filterPreference:

1212 if f in c and c[f]:

1213 c["Mag"] = c[f]

1214 c["MagFilter"] = f.replace("_", "").replace("Mag", "")

1215 if f + "Err" in c:

1216 c["MagErr"] = c[f + "Err"]

1217 else:

1218 c["MagErr"] = None

1219 break

1220

1221 allKeys = []

1222 for c in myCrossmatches:

1223 for k, v in list(c.items()):

1224 if k not in allKeys:

1225 allKeys.append(k)

1226

1227 for c in myCrossmatches:

1228 for k in allKeys:

1229 if k not in c:

1230 c[k] = None

1231

1232 printCrossmatches = []

1233 for c in myCrossmatches:

1234 ordDict = collections.OrderedDict(sorted({}.items()))

1235 for k in basic:

1236 if k in c:

1237 if k == "catalogue_table_name":

1238 c[k] = c[k].replace(

1239 "tcs_cat_", "").replace("_", " ")

1240 if k == "classificationReliability":

1241 if c[k] == 1:

1242 c["classification reliability"] = "synonym"

1243 elif c[k] == 2:

1244 c["classification reliability"] = "association"

1245 elif c[k] == 3:

1246 c["classification reliability"] = "annotation"

1247 k = "classification reliability"

1248 if k == "catalogue_object_subtype" and "sdss" in c["catalogue_table_name"]:

1249 if c[k] == 6:

1250 c[k] = "galaxy"

1251 elif c[k] == 3:

1252 c[k] = "star"

1253 columnName = k.replace(

1254 "tcs_cat_", "").replace("_", " ")

1255 value = c[k]

1256 if k not in dontFormat:

1257 try:

1258 ordDict[columnName] = "%(value)0.3f" % locals()

1259 except:

1260 ordDict[columnName] = value

1261 else:

1262 ordDict[columnName] = value

1263

1264 printCrossmatches.append(ordDict)

1265

1266 outputFormat = None

1267 # outputFormat = "csv"

1268

1269 from fundamentals.renderer import list_of_dictionaries

1270 dataSet = list_of_dictionaries(

1271 log=self.log,

1272 listOfDictionaries=printCrossmatches

1273 )

1274

1275 if outputFormat == "csv":

1276 tableData = dataSet.csv(filepath=None)

1277 else:

1278 tableData = dataSet.table(filepath=None)

1279

1280 print(tableData)

1281

1282 self.log.debug('completed the ``_print_results_to_stdout`` method')

1283 return None

1284

1285 def _lighten_return(

1286 self,

1287 crossmatches):

1288 """*lighten the classification and crossmatch results for smaller database footprint*

1289

1290 **Key Arguments**

1291

1292 - ``classifications`` -- the classifications assigned to the transients post-crossmatches (dictionary of rank ordered list of classifications)

1293 """

1294 self.log.debug('starting the ``_lighten_return`` method')

1295

1296 # REPORT ONLY THE MOST PREFERED MAGNITUDE VALUE

1297 basic = ["transient_object_id", "association_type", "catalogue_table_name", "catalogue_object_id", "catalogue_object_type",

1298 "raDeg", "decDeg", "separationArcsec", "northSeparationArcsec", "eastSeparationArcsec", "physical_separation_kpc", "direct_distance", "distance", "z", "photoZ", "photoZErr", "Mag", "MagFilter", "MagErr", "classificationReliability", "major_axis_arcsec"]

1299 verbose = ["search_name", "catalogue_view_name", "original_search_radius_arcsec", "direct_distance_modulus", "distance_modulus", "direct_distance_scale", "scale", "U", "UErr",

1300 "B", "BErr", "V", "VErr", "R", "RErr", "I", "IErr", "J", "JErr", "H", "HErr", "K", "KErr", "_u", "_uErr", "_g", "_gErr", "_r", "_rErr", "_i", "_iErr", "_z", "_zErr", "_y", "G", "GErr", "_yErr", "unkMag"]

1301 dontFormat = ["decDeg", "raDeg", "rank",

1302 "catalogue_object_id", "catalogue_object_subtype", "merged_rank", "classificationReliability"]

1303 if self.verbose == 2:

1304 basic = basic + verbose

1305

1306 for c in crossmatches:

1307 for f in self.filterPreference:

1308 if f in c and c[f]:

1309 c["Mag"] = c[f]

1310 c["MagFilter"] = f.replace("_", "").replace("Mag", "")

1311 if f + "Err" in c:

1312 c["MagErr"] = c[f + "Err"]

1313 else:

1314 c["MagErr"] = None

1315 break

1316

1317 allKeys = []

1318 for c in crossmatches:

1319 for k, v in list(c.items()):

1320 if k not in allKeys:

1321 allKeys.append(k)

1322

1323 for c in crossmatches:

1324 for k in allKeys:

1325 if k not in c:

1326 c[k] = None

1327

1328 liteCrossmatches = []

1329 for c in crossmatches:

1330 ordDict = collections.OrderedDict(sorted({}.items()))

1331 for k in basic:

1332 if k in c:

1333 if k == "catalogue_table_name":

1334 c[k] = c[k].replace(

1335 "tcs_cat_", "").replace("_", " ")

1336 if k == "catalogue_object_subtype" and "sdss" in c["catalogue_table_name"]:

1337 if c[k] == 6:

1338 c[k] = "galaxy"

1339 elif c[k] == 3:

1340 c[k] = "star"

1341 columnName = k.replace(

1342 "tcs_cat_", "")

1343 value = c[k]

1344 if k not in dontFormat:

1345 try:

1346 ordDict[columnName] = float(f'{value:0.3f}')

1347 except:

1348 ordDict[columnName] = value

1349 else:

1350 ordDict[columnName] = value

1351

1352 liteCrossmatches.append(ordDict)

1353

1354 self.log.debug('completed the ``_lighten_return`` method')

1355 return liteCrossmatches

1356

1357 def _consolidate_coordinateList(

1358 self,

1359 coordinateList):

1360 """*match the coordinate list against itself with the parameters of the NED search queries to minimise duplicated NED queries*

1361

1362 **Key Arguments**

1363

1364 - ``coordinateList`` -- the original coordinateList.

1365

1366

1367 **Return**

1368

1369 - ``updatedCoordinateList`` -- the coordinate list with duplicated search areas removed

1372 **Usage**

1374 .. todo::

1376 - add usage info

1377 - create a sublime snippet for usage

1378 - update package tutorial if needed

1379

1380 ```python

1381 usage code

1382 ```

1383

1384

1385 .. todo ::

1386

1387 - update key arguments values and definitions with defaults

1388 - update return values and definitions

1389 - update usage examples and text

1390 - update docstring text

1391 - check sublime snippet exists

1392 - clip any useful text to docs mindmap

1393 - regenerate the docs and check redendering of this docstring

1394 """

1395 self.log.debug('starting the ``_consolidate_coordinateList`` method')

1396

1397 raList = []

1398 raList[:] = np.array([c[0] for c in coordinateList])

1399 decList = []

1400 decList[:] = np.array([c[1] for c in coordinateList])

1401

1402 nedStreamRadius = old_div(self.settings[

1403 "ned stream search radius arcec"], (60. * 60.))

1404 firstPassNedSearchRadius = old_div(self.settings[

1405 "first pass ned search radius arcec"], (60. * 60.))

1406 radius = nedStreamRadius - firstPassNedSearchRadius

1407

1408 # LET'S BE CONSERVATIVE

1409 # radius = radius * 0.9

1410

1411 xmatcher = sets(

1412 log=self.log,

1413 ra=raList,

1414 dec=decList,

1415 radius=radius, # in degrees

1416 sourceList=coordinateList,

1417 convertToArray=False

1418 )

1419 allMatches = xmatcher.match

1420

1421 updatedCoordianteList = []

1422 for aSet in allMatches:

1423 updatedCoordianteList.append(aSet[0])

1424

1425 self.log.debug('completed the ``_consolidate_coordinateList`` method')

1426 return updatedCoordianteList

1427

1428 def classification_annotations(

1429 self):

1430 """*add a detialed classification annotation to each classification in the sherlock_classifications table*

1431

1432 **Key Arguments**

1434 # -

1437 **Return**

1439 - None

1442 **Usage**

1444 .. todo::

1446 - add usage info

1447 - create a sublime snippet for usage

1448 - write a command-line tool for this method

1449 - update package tutorial with command-line tool info if needed

1450

1451 ```python

1452 usage code

1453 ```

1454

1455

1456 .. todo ::

1457

1458 - update key arguments values and definitions with defaults

1459 - update return values and definitions

1460 - update usage examples and text

1461 - update docstring text

1462 - check sublime snippet exists

1463 - clip any useful text to docs mindmap

1464 - regenerate the docs and check redendering of this docstring

1465 """

1466 self.log.debug('starting the ``classification_annotations`` method')

1467

1468 from fundamentals.mysql import readquery

1469 sqlQuery = u"""

1470 select * from sherlock_classifications cl, sherlock_crossmatches xm where cl.transient_object_id=xm.transient_object_id and cl.annotation is null

1471 """ % locals()

1472 topXMs = readquery(

1473 log=self.log,

1474 sqlQuery=sqlQuery,

1475 dbConn=self.transientsDbConn

1476 )

1477

1478 for xm in topXMs:

1479 annotation = []

1480 classType = xm["classificationReliability"]

1481 if classType == 1:

1482 annotation.append("is synonymous with")

1483 elif classType in [2, 3]:

1484 annotation.append("is possibly associated with")

1485

1486 self.log.debug('completed the ``classification_annotations`` method')

1487 return None

1488

1489 def update_classification_annotations_and_summaries(

1490 self,

1491 updatePeakMagnitudes=True,

1492 cl=False,

1493 crossmatches=False,

1494 classifications=False):

1495 """*update classification annotations and summaries*

1496

1497 **Key Arguments**

1498

1499 - ``updatePeakMagnitudes`` -- update the peak magnitudes in the annotations to give absolute magnitudes. Default *True*

1500 - ``cl`` -- reporting only to the command-line, do not update database. Default *False*

1501 - ``crossmatches`` -- crossmatches will be passed for the single classifications to report annotations from command-line

1502 - ``classifications`` -- classifications will be passed for the single classifications to have annotation appended to the dictionary for stand-alone non-database scripts

1504 **Return**

1506 - None

1509 **Usage**

1511 .. todo::

1513 - add usage info

1514 - create a sublime snippet for usage

1515 - write a command-line tool for this method

1516 - update package tutorial with command-line tool info if needed

1517

1518 ```python

1519 usage code

1520 ```

1521

1522

1523 .. todo ::

1524

1525 - update key arguments values and definitions with defaults

1526 - update return values and definitions

1527 - update usage examples and text

1528 - update docstring text

1529 - check sublime snippet exists

1530 - clip any useful text to docs mindmap

1531 - regenerate the docs and check redendering of this docstring

1532 """

1533 self.log.debug(

1534 'starting the ``update_classification_annotations_and_summaries`` method')

1535

1536 # import time

1537 # start_time = time.time()

1538 # print "COLLECTING TRANSIENTS WITH NO ANNOTATIONS"

1539

1540 # BULK RUN

1541 if crossmatches == False:

1542 if updatePeakMagnitudes:

1543 sqlQuery = u"""

1544 SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id= cm.transient_object_id and ((cl.classification not in ("AGN","CV","BS","VS") AND cm.dateLastModified > DATE_SUB(NOW(), INTERVAL 1 Day)) or cl.annotation is null)

1545 -- SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id= cm.transient_object_id and (cl.annotation is null or cl.dateLastModified is null or cl.dateLastModified > DATE_SUB(NOW(), INTERVAL 30 DAY)) order by cl.dateLastModified asc limit 100000

1546 """ % locals()

1547 else:

1548 sqlQuery = u"""

1549 SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id=cm.transient_object_id and cl.summary is null

1550 """ % locals()

1551

1552 rows = readquery(

1553 log=self.log,

1554 sqlQuery=sqlQuery,

1555 dbConn=self.transientsDbConn

1556 )

1557 # COMMAND-LINE SINGLE CLASSIFICATION

1558 else:

1559 rows = crossmatches

1560

1561 # print "FINISHED COLLECTING TRANSIENTS WITH NO ANNOTATIONS/GENERATING ANNOTATIONS: %d" % (time.time() - start_time,)

1562 # start_time = time.time()

1563

1564 updates = []

1565

1566 for row in rows:

1567 annotation, summary, sep = self.generate_match_annotation(

1568 match=row, updatePeakMagnitudes=updatePeakMagnitudes)

1569

1570 if cl and "rank" in row and row["rank"] == 1:

1571 if classifications != False:

1572 classifications[

1573 row["transient_object_id"]].append(annotation)

1574 if self.verbose != 0:

1575 print("\n" + annotation)

1576

1577 update = {

1578 "transient_object_id": row["transient_object_id"],

1579 "annotation": annotation,

1580 "summary": summary,

1581 "separationArcsec": sep

1582 }

1583 updates.append(update)

1584

1585 if cl:

1586 return classifications

1587

1588 # print "FINISHED GENERATING ANNOTATIONS/ADDING ANNOTATIONS TO TRANSIENT DATABASE: %d" % (time.time() - start_time,)

1589 # start_time = time.time()

1590

1591 insert_list_of_dictionaries_into_database_tables(

1592 dbConn=self.transientsDbConn,

1593 log=self.log,

1594 dictList=updates,

1595 dbTableName="sherlock_classifications",

1596 dateModified=True,

1597 batchSize=10000,

1598 replace=True,

1599 dbSettings=self.settings["database settings"]["transients"]

1600 )

1601

1602 # print "FINISHED ADDING ANNOTATIONS TO TRANSIENT DATABASE/UPDATING ORPHAN ANNOTATIONS: %d" % (time.time() - start_time,)

1603 # start_time = time.time()

1604

1605 sqlQuery = """update sherlock_classifications set annotation = "The transient location is not matched against any known catalogued source", summary = "No catalogued match" where classification = 'ORPHAN' and summary is null """ % locals()

1606 writequery(

1607 log=self.log,

1608 sqlQuery=sqlQuery,

1609 dbConn=self.transientsDbConn,

1610 )

1611

1612 # print "FINISHED UPDATING ORPHAN ANNOTATIONS: %d" % (time.time() - start_time,)

1613 # start_time = time.time()

1614

1615 self.log.debug(

1616 'completed the ``update_classification_annotations_and_summaries`` method')

1617 return None

1618

1619 # use the tab-trigger below for new method

1620 def update_peak_magnitudes(

1621 self):

1622 """*update peak magnitudes*

1623

1624 **Key Arguments**

1626 # -

1629 **Return**

1631 - None

1634 **Usage**

1636 .. todo::

1638 - add usage info

1639 - create a sublime snippet for usage

1640 - write a command-line tool for this method

1641 - update package tutorial with command-line tool info if needed

1642

1643 ```python

1644 usage code

1645 ```

1646

1647

1648 .. todo ::

1649

1650 - update key arguments values and definitions with defaults

1651 - update return values and definitions

1652 - update usage examples and text

1653 - update docstring text

1654 - check sublime snippet exists

1655 - clip any useful text to docs mindmap

1656 - regenerate the docs and check redendering of this docstring

1657 """

1658 self.log.debug('starting the ``update_peak_magnitudes`` method')

1659

1660 sqlQuery = self.settings["database settings"][

1661 "transients"]["transient peak magnitude query"]

1662

1663 sqlQuery = """UPDATE sherlock_crossmatches s,

1664 (%(sqlQuery)s) t

1665 SET

1666 s.transientAbsMag = ROUND(t.mag - IFNULL(direct_distance_modulus,

1667 distance_modulus),

1668 2)

1669 WHERE

1670 IFNULL(direct_distance_modulus,

1671 distance_modulus) IS NOT NULL

1672 AND (s.association_type not in ("AGN","CV","BS","VS")

1673 or s.transientAbsMag is null)

1674 AND t.id = s.transient_object_id

1675 AND (s.dateLastModified > DATE_SUB(NOW(), INTERVAL 1 DAY));""" % locals()

1676

1677 writequery(

1678 log=self.log,

1679 sqlQuery=sqlQuery,

1680 dbConn=self.transientsDbConn,

1681 )

1682

1683 self.log.debug('completed the ``update_peak_magnitudes`` method')

1684 return None

1685

1686 def _create_tables_if_not_exist(

1687 self):

1688 """*create the sherlock helper tables if they don't yet exist*

1689

1690 **Key Arguments**

1692 # -

1695 **Return**

1697 - None

1700 **Usage**

1702 .. todo::

1704 - add usage info

1705 - create a sublime snippet for usage

1706 - write a command-line tool for this method

1707 - update package tutorial with command-line tool info if needed

1708

1709 ```python

1710 usage code

1711 ```

1712

1713

1714 .. todo ::

1715

1716 - update key arguments values and definitions with defaults

1717 - update return values and definitions

1718 - update usage examples and text

1719 - update docstring text

1720 - check sublime snippet exists

1721 - clip any useful text to docs mindmap

1722 - regenerate the docs and check redendering of this docstring

1723 """

1724 self.log.debug('starting the ``_create_tables_if_not_exist`` method')

1725

1726 transientTable = self.settings["database settings"][

1727 "transients"]["transient table"]

1728 transientTableClassCol = self.settings["database settings"][

1729 "transients"]["transient classification column"]

1730 transientTableIdCol = self.settings["database settings"][

1731 "transients"]["transient primary id column"]

1732

1733 crossmatchTable = "sherlock_crossmatches"

1734 createStatement = """

1735CREATE TABLE IF NOT EXISTS `sherlock_crossmatches` (

1736 `transient_object_id` bigint(20) unsigned DEFAULT NULL,

1737 `catalogue_object_id` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL,

1738 `catalogue_table_id` smallint(5) unsigned DEFAULT NULL,

1739 `separationArcsec` double DEFAULT NULL,

1740 `northSeparationArcsec` double DEFAULT NULL,

1741 `eastSeparationArcsec` double DEFAULT NULL,

1742 `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,

1743 `z` double DEFAULT NULL,

1744 `scale` double DEFAULT NULL,

1745 `distance` double DEFAULT NULL,

1746 `distance_modulus` double DEFAULT NULL,

1747 `photoZ` double DEFAULT NULL,

1748 `photoZErr` double DEFAULT NULL,

1749 `association_type` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,

1750 `dateCreated` datetime DEFAULT NULL,

1751 `physical_separation_kpc` double DEFAULT NULL,

1752 `catalogue_object_type` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,

1753 `catalogue_object_subtype` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,

1754 `association_rank` int(11) DEFAULT NULL,

1755 `catalogue_table_name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,

1756 `catalogue_view_name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,

1757 `rank` int(11) DEFAULT NULL,

1758 `rankScore` double DEFAULT NULL,

1759 `search_name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,

1760 `major_axis_arcsec` double DEFAULT NULL,

1761 `direct_distance` double DEFAULT NULL,

1762 `direct_distance_scale` double DEFAULT NULL,

1763 `direct_distance_modulus` double DEFAULT NULL,

1764 `raDeg` double DEFAULT NULL,

1765 `decDeg` double DEFAULT NULL,

1766 `original_search_radius_arcsec` double DEFAULT NULL,

1767 `catalogue_view_id` int(11) DEFAULT NULL,

1768 `U` double DEFAULT NULL,

1769 `UErr` double DEFAULT NULL,

1770 `B` double DEFAULT NULL,

1771 `BErr` double DEFAULT NULL,

1772 `V` double DEFAULT NULL,

1773 `VErr` double DEFAULT NULL,

1774 `R` double DEFAULT NULL,

1775 `RErr` double DEFAULT NULL,

1776 `I` double DEFAULT NULL,

1777 `IErr` double DEFAULT NULL,

1778 `J` double DEFAULT NULL,

1779 `JErr` double DEFAULT NULL,

1780 `H` double DEFAULT NULL,

1781 `HErr` double DEFAULT NULL,

1782 `K` double DEFAULT NULL,

1783 `KErr` double DEFAULT NULL,

1784 `_u` double DEFAULT NULL,

1785 `_uErr` double DEFAULT NULL,

1786 `_g` double DEFAULT NULL,

1787 `_gErr` double DEFAULT NULL,

1788 `_r` double DEFAULT NULL,

1789 `_rErr` double DEFAULT NULL,

1790 `_i` double DEFAULT NULL,

1791 `_iErr` double DEFAULT NULL,

1792 `_z` double DEFAULT NULL,

1793 `_zErr` double DEFAULT NULL,

1794 `_y` double DEFAULT NULL,

1795 `_yErr` double DEFAULT NULL,

1796 `G` double DEFAULT NULL,

1797 `GErr` double DEFAULT NULL,

1798 `W1` double DEFAULT NULL,

1799 `W1Err` double DEFAULT NULL,

1800 `unkMag` double DEFAULT NULL,

1801 `unkMagErr` double DEFAULT NULL,

1802 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,

1803 `updated` tinyint(4) DEFAULT '0',

1804 `classificationReliability` tinyint(4) DEFAULT NULL,

1805 `transientAbsMag` double DEFAULT NULL,

1806 `merged_rank` tinyint(4) DEFAULT NULL,

1807 PRIMARY KEY (`id`),

1808 KEY `key_transient_object_id` (`transient_object_id`),

1809 KEY `key_catalogue_object_id` (`catalogue_object_id`),

1810 KEY `idx_separationArcsec` (`separationArcsec`),

1811 KEY `idx_rank` (`rank`)

1812) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

1813

1814CREATE TABLE IF NOT EXISTS `sherlock_classifications` (

1815 `transient_object_id` bigint(20) NOT NULL,

1816 `classification` varchar(45) DEFAULT NULL,

1817 `annotation` TEXT COLLATE utf8_unicode_ci DEFAULT NULL,

1818 `summary` VARCHAR(50) COLLATE utf8_unicode_ci DEFAULT NULL,

1819 `separationArcsec` DOUBLE DEFAULT NULL,

1820 `matchVerified` TINYINT NULL DEFAULT NULL,

1821 `developmentComment` VARCHAR(100) NULL,

1822 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,

1823 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,

1824 `updated` varchar(45) DEFAULT '0',

1825 PRIMARY KEY (`transient_object_id`),

1826 KEY `key_transient_object_id` (`transient_object_id`),

1827 KEY `idx_summary` (`summary`),

1828 KEY `idx_classification` (`classification`),

1829 KEY `idx_dateLastModified` (`dateLastModified`)

1830) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

1831

1832""" % locals()

1833

1834 # A FIX FOR MYSQL VERSIONS < 5.6

1835 triggers = []

1836 if float(self.dbVersions["transients"][:3]) < 5.6:

1837 createStatement = createStatement.replace(

1838 "`dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,", "`dateLastModified` datetime DEFAULT NULL,")

1839 createStatement = createStatement.replace(

1840 "`dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,", "`dateCreated` datetime DEFAULT NULL,")

1841

1842 triggers.append("""

1843CREATE TRIGGER dateCreated

1844BEFORE INSERT ON `%(crossmatchTable)s`

1845FOR EACH ROW

1846BEGIN

1847 IF NEW.dateCreated IS NULL THEN

1848 SET NEW.dateCreated = NOW();

1849 SET NEW.dateLastModified = NOW();

1850 END IF;

1851END""" % locals())

1852

1853 try:

1854 writequery(

1855 log=self.log,

1856 sqlQuery=createStatement,

1857 dbConn=self.transientsDbConn,

1858 Force=True

1859 )

1860 except:

1861 self.log.info(

1862 "Could not create table (`%(crossmatchTable)s`). Probably already exist." % locals())

1863

1864 sqlQuery = u"""

1865 SHOW TRIGGERS;

1866 """ % locals()

1867 rows = readquery(

1868 log=self.log,

1869 sqlQuery=sqlQuery,

1870 dbConn=self.transientsDbConn,

1871 )

1872

1873 # DON'T ADD TRIGGERS IF THEY ALREADY EXIST

1874 for r in rows:

1875 if r["Trigger"] in ("sherlock_classifications_BEFORE_INSERT", "sherlock_classifications_AFTER_INSERT"):

1876 return None

1877

1878 triggers.append("""CREATE TRIGGER `sherlock_classifications_BEFORE_INSERT` BEFORE INSERT ON `sherlock_classifications` FOR EACH ROW

1879BEGIN

1880 IF new.classification = "ORPHAN" THEN

1881 SET new.annotation = "The transient location is not matched against any known catalogued source", new.summary = "No catalogued match";

1882 END IF;

1883END""" % locals())

1884

1885 triggers.append("""CREATE TRIGGER `sherlock_classifications_AFTER_INSERT` AFTER INSERT ON `sherlock_classifications` FOR EACH ROW

1886BEGIN

1887 update `%(transientTable)s` set `%(transientTableClassCol)s` = new.classification

1888 where `%(transientTableIdCol)s` = new.transient_object_id;

1889END""" % locals())

1890

1891 for t in triggers:

1892 try:

1893 writequery(

1894 log=self.log,

1895 sqlQuery=t,

1896 dbConn=self.transientsDbConn,

1897 Force=True

1898 )

1899 except:

1900 self.log.info(

1901 "Could not create trigger (`%(crossmatchTable)s`). Probably already exist." % locals())

1902

1903 self.log.debug('completed the ``_create_tables_if_not_exist`` method')

1904 return None

1905

1906 # use the tab-trigger below for new method

1907 def generate_match_annotation(

1908 self,

1909 match,

1910 updatePeakMagnitudes=False):

1911 """*generate a human readale annotation for the transient-catalogue source match*

1912

1913 **Key Arguments**

1914

1915 - ``match`` -- the source crossmatched against the transient

1916 - ``updatePeakMagnitudes`` -- update the peak magnitudes in the annotations to give absolute magnitudes. Default *False*

1919 **Return**

1921 - None

1924 **Usage**

1928 ```python

1929 usage code

1930 ```

1931

1932 ---

1933

1934 ```eval_rst

1935 .. todo::

1936

1937 - add usage info

1938 - create a sublime snippet for usage

1939 - write a command-line tool for this method

1940 - update package tutorial with command-line tool info if needed

1941 ```

1942 """

1943 self.log.debug('starting the ``generate_match_annotation`` method')

1944

1945 if "catalogue_object_subtype" not in match:

1946 match["catalogue_object_subtype"] = None

1947 catalogue = match["catalogue_table_name"]

1948 objectId = match["catalogue_object_id"]

1949 objectType = match["catalogue_object_type"]

1950 objectSubtype = match["catalogue_object_subtype"]

1951 catalogueString = catalogue

1952 if catalogueString is None:

1953 badGuy = match["transient_object_id"]

1954 print(f"Issue with object {badGuy}")

1955 raise TypeError(f"Issue with object {badGuy}")

1956 if "catalogue" not in catalogueString.lower():

1957 catalogueString = catalogue + " catalogue"

1958 if "/" in catalogueString:

1959 catalogueString += "s"

1960

1961 if "ned" in catalogue.lower():

1962 objectId = objectId.replace("+", "%2B")

1963 objectId = '''<a href="https://ned.ipac.caltech.edu/cgi-bin/objsearch?objname=%(objectId)s&extend=no&hconst=73&omegam=0.27&omegav=0.73&corr_z=1&out_csys=Equatorial&out_equinox=J2000.0&obj_sort=RA+or+Longitude&of=pre_text&zv_breaker=30000.0&list_limit=5&img_stamp=YES">%(objectId)s</a>''' % locals()

1964 elif "sdss" in catalogue.lower():

1965 objectId = "http://skyserver.sdss.org/dr12/en/tools/explore/Summary.aspx?id=%(objectId)s" % locals(

1966 )

1967

1968 ra = self.converter.ra_decimal_to_sexegesimal(

1969 ra=match["raDeg"],

1970 delimiter=""

1971 )

1972 dec = self.converter.dec_decimal_to_sexegesimal(

1973 dec=match["decDeg"],

1974 delimiter=""

1975 )

1976 betterName = "SDSS J" + ra[0:9] + dec[0:9]

1977 objectId = '''<a href="%(objectId)s">%(betterName)s</a>''' % locals()

1978 elif "milliquas" in catalogue.lower():

1979 thisName = objectId

1980 objectId = objectId.replace(" ", "+")

1981 objectId = '''<a href="https://heasarc.gsfc.nasa.gov/db-perl/W3Browse/w3table.pl?popupFrom=Query+Results&tablehead=name%%3Dheasarc_milliquas%%26description%%3DMillion+Quasars+Catalog+%%28MILLIQUAS%%29%%2C+Version+4.8+%%2822+June+2016%%29%%26url%%3Dhttp%%3A%%2F%%2Fheasarc.gsfc.nasa.gov%%2FW3Browse%%2Fgalaxy-catalog%%2Fmilliquas.html%%26archive%%3DN%%26radius%%3D1%%26mission%%3DGALAXY+CATALOG%%26priority%%3D5%%26tabletype%%3DObject&dummy=Examples+of+query+constraints%%3A&varon=name&bparam_name=%%3D%%22%(objectId)s%%22&bparam_name%%3A%%3Aunit=+&bparam_name%%3A%%3Aformat=char25&varon=ra&bparam_ra=&bparam_ra%%3A%%3Aunit=degree&bparam_ra%%3A%%3Aformat=float8%%3A.5f&varon=dec&bparam_dec=&bparam_dec%%3A%%3Aunit=degree&bparam_dec%%3A%%3Aformat=float8%%3A.5f&varon=bmag&bparam_bmag=&bparam_bmag%%3A%%3Aunit=mag&bparam_bmag%%3A%%3Aformat=float8%%3A4.1f&varon=rmag&bparam_rmag=&bparam_rmag%%3A%%3Aunit=mag&bparam_rmag%%3A%%3Aformat=float8%%3A4.1f&varon=redshift&bparam_redshift=&bparam_redshift%%3A%%3Aunit=+&bparam_redshift%%3A%%3Aformat=float8%%3A6.3f&varon=radio_name&bparam_radio_name=&bparam_radio_name%%3A%%3Aunit=+&bparam_radio_name%%3A%%3Aformat=char22&varon=xray_name&bparam_xray_name=&bparam_xray_name%%3A%%3Aunit=+&bparam_xray_name%%3A%%3Aformat=char22&bparam_lii=&bparam_lii%%3A%%3Aunit=degree&bparam_lii%%3A%%3Aformat=float8%%3A.5f&bparam_bii=&bparam_bii%%3A%%3Aunit=degree&bparam_bii%%3A%%3Aformat=float8%%3A.5f&bparam_broad_type=&bparam_broad_type%%3A%%3Aunit=+&bparam_broad_type%%3A%%3Aformat=char4&bparam_optical_flag=&bparam_optical_flag%%3A%%3Aunit=+&bparam_optical_flag%%3A%%3Aformat=char3&bparam_red_psf_flag=&bparam_red_psf_flag%%3A%%3Aunit=+&bparam_red_psf_flag%%3A%%3Aformat=char1&bparam_blue_psf_flag=&bparam_blue_psf_flag%%3A%%3Aunit=+&bparam_blue_psf_flag%%3A%%3Aformat=char1&bparam_ref_name=&bparam_ref_name%%3A%%3Aunit=+&bparam_ref_name%%3A%%3Aformat=char6&bparam_ref_redshift=&bparam_ref_redshift%%3A%%3Aunit=+&bparam_ref_redshift%%3A%%3Aformat=char6&bparam_qso_prob=&bparam_qso_prob%%3A%%3Aunit=percent&bparam_qso_prob%%3A%%3Aformat=int2%%3A3d&bparam_alt_name_1=&bparam_alt_name_1%%3A%%3Aunit=+&bparam_alt_name_1%%3A%%3Aformat=char22&bparam_alt_name_2=&bparam_alt_name_2%%3A%%3Aunit=+&bparam_alt_name_2%%3A%%3Aformat=char22&Entry=&Coordinates=J2000&Radius=Default&Radius_unit=arcsec&NR=CheckCaches%%2FGRB%%2FSIMBAD%%2BSesame%%2FNED&Time=&ResultMax=1000&displaymode=Display&Action=Start+Search&table=heasarc_milliquas">%(thisName)s</a>''' % locals()

1982

1983 if objectSubtype and str(objectSubtype).lower() in ["uvs", "radios", "xray", "qso", "irs", 'uves', 'viss', 'hii', 'gclstr', 'ggroup', 'gpair', 'gtrpl']:

1984 objectType = objectSubtype

1985

1986 if objectType == "star":

1987 objectType = "stellar source"

1988 elif objectType == "agn":

1989 objectType = "AGN"

1990 elif objectType == "cb":

1991 objectType = "CV"

1992 elif objectType == "unknown":

1993 objectType = "unclassified source"

1994

1995 sep = match["separationArcsec"]

1996 if match["classificationReliability"] == 1:

1997 classificationReliability = "synonymous"

1998 psep = match["physical_separation_kpc"]

1999 if psep:

2000 location = '%(sep)0.1f" (%(psep)0.1f Kpc) from the %(objectType)s core' % locals(

2001 )

2002 else:

2003 location = '%(sep)0.1f" from the %(objectType)s core' % locals(

2004 )

2005 else:

2006 # elif match["classificationReliability"] in (2, 3):

2007 classificationReliability = "possibly associated"

2008 n = float(match["northSeparationArcsec"])

2009 if n > 0:

2010 nd = "S"

2011 else:

2012 nd = "N"

2013 e = float(match["eastSeparationArcsec"])

2014 if e > 0:

2015 ed = "W"

2016 else:

2017 ed = "E"

2018 n = math.fabs(float(n))

2019 e = math.fabs(float(e))

2020 psep = match["physical_separation_kpc"]

2021 if psep:

2022 location = '%(n)0.2f" %(nd)s, %(e)0.2f" %(ed)s (%(psep)0.1f Kpc) from the %(objectType)s centre' % locals(

2023 )

2024 else:

2025 location = '%(n)0.2f" %(nd)s, %(e)0.2f" %(ed)s from the %(objectType)s centre' % locals(

2026 )

2027 location = location.replace("unclassified", "object's")

2028

2029 best_mag = None

2030 best_mag_error = None

2031 best_mag_filter = None

2032 filters = ["R", "V", "B", "I", "J", "G", "H", "K", "U",

2033 "_r", "_g", "_i", "_g", "_z", "_y", "_u", "W1", "unkMag"]

2034 for f in filters:

2035 if f in match and match[f] and not best_mag:

2036 best_mag = match[f]

2037 try:

2038 best_mag_error = match[f + "Err"]

2039 except:

2040 pass

2041 subfilter = f.replace(

2042 "_", "").replace("Mag", "")

2043 best_mag_filter = f.replace(

2044 "_", "").replace("Mag", "") + "="

2045 if "unk" in best_mag_filter:

2046 best_mag_filter = ""

2047 subfilter = ''

2048

2049 if not best_mag_filter:

2050 if str(best_mag).lower() in ("8", "11", "18"):

2051 best_mag_filter = "an "

2052 else:

2053 best_mag_filter = "a "

2054 else:

2055 if str(best_mag_filter)[0].lower() in ("r", "i", "h"):

2056 best_mag_filter = "an " + best_mag_filter

2057 else:

2058 best_mag_filter = "a " + best_mag_filter

2059 if not best_mag:

2060 best_mag = "an unknown-"

2061 best_mag_filter = ""

2062 else:

2063 best_mag = "%(best_mag)0.2f " % locals()

2064

2065 distance = None

2066 if "direct_distance" in match and match["direct_distance"]:

2067 d = match["direct_distance"]

2068 distance = "distance of %(d)0.1f Mpc" % locals()

2069

2070 if match["z"]:

2071 z = match["z"]

2072 distance += "(z=%(z)0.3f)" % locals()

2073 elif "z" in match and match["z"]:

2074 z = match["z"]

2075 distance = "z=%(z)0.3f" % locals()

2076 elif "photoZ" in match and match["photoZ"]:

2077 z = match["photoZ"]

2078 zErr = match["photoZErr"]

2079 if not zErr:

2080 distance = "photoZ=%(z)0.3f" % locals()

2081 else:

2082 distance = "photoZ=%(z)0.3f (&plusmn%(zErr)0.3f)" % locals()

2083

2084 if distance:

2085 distance = "%(distance)s" % locals()

2086

2087 distance_modulus = None

2088 if match["direct_distance_modulus"]:

2089 distance_modulus = match["direct_distance_modulus"]

2090 elif match["distance_modulus"]:

2091 distance_modulus = match["distance_modulus"]

2092

2093 if updatePeakMagnitudes:

2094 if distance:

2095 absMag = match["transientAbsMag"]

2096 absMag = """ A host %(distance)s implies a transient M = %(absMag)s mag.""" % locals(

2097 )

2098 else:

2099 absMag = ""

2100 else:

2101 if distance and distance_modulus:

2102 absMag = "%(distance_modulus)0.2f" % locals()

2103 absMag = """ A host %(distance)s implies a m - M = %(absMag)s.""" % locals(

2104 )

2105 else:

2106 absMag = ""

2107

2108 annotation = "The transient is %(classificationReliability)s with %(objectId)s; %(best_mag_filter)s%(best_mag)smag %(objectType)s found in the %(catalogueString)s. It's located %(location)s.%(absMag)s" % locals()

2109 try:

2110 summary = '%(sep)0.1f" from %(objectType)s in %(catalogue)s' % locals()

2111 except:

2112 badGuy = match["transient_object_id"]

2113 print(f"Issue with object {badGuy}")

2114 raise TypeError(f"Issue with object {badGuy}")

2115

2116 self.log.debug('completed the ``generate_match_annotation`` method')

2117 return annotation, summary, sep

2118

2119 # use the tab-trigger below for new method

2120 # xt-class-method

2121

2122

2123def _crossmatch_transients_against_catalogues(

2124 transientsMetadataListIndex,

2125 log,

2126 settings,

2127 colMaps):

2128 """run the transients through the crossmatch algorithm in the settings file

2129

2130 **Key Arguments**

2131

2132

2133 - ``transientsMetadataListIndex`` -- the list of transient metadata lifted from the database.

2134 - ``colMaps`` -- dictionary of dictionaries with the name of the database-view (e.g. `tcs_view_agn_milliquas_v4_5`) as the key and the column-name dictary map as value (`{view_name: {columnMap}}`).

2135

2136 **Return**

2137

2138 - ``crossmatches`` -- a list of dictionaries of the associated sources crossmatched from the catalogues database

2139

2140

2141 .. todo ::

2142

2143 - update key arguments values and definitions with defaults

2144 - update return values and definitions

2145 - update usage examples and text

2146 - update docstring text

2147 - check sublime snippet exists

2148 - clip any useful text to docs mindmap

2149 - regenerate the docs and check redendering of this docstring

2150 """

2151

2152 from fundamentals.mysql import database

2153 from sherlock import transient_catalogue_crossmatch

2154

2155 global theseBatches

2156

2157 log.debug(

2158 'starting the ``_crossmatch_transients_against_catalogues`` method')

2159

2160 # SETUP ALL DATABASE CONNECTIONS

2161

2162 transientsMetadataList = theseBatches[transientsMetadataListIndex]

2163

2164 dbConn = database(

2165 log=log,

2166 dbSettings=settings["database settings"]["static catalogues"]

2167 ).connect()

2168

2169 cm = transient_catalogue_crossmatch(

2170 log=log,

2171 dbConn=dbConn,

2172 transients=transientsMetadataList,

2173 settings=settings,

2174 colMaps=colMaps

2175 )

2176 crossmatches = cm.match()

2177

2178 log.debug(

2179 'completed the ``_crossmatch_transients_against_catalogues`` method')

2180

2181 return crossmatches

Coverage for sherlock/transient_classifier.py: 70%

832 statements