Coverage for sherlock/imports/ned

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4*Import ned_d catalogue into sherlock-catalogues database*

6:Author:

7 David Young

8"""

9from __future__ import print_function

10from __future__ import division

11from ._base_importer import _base_importer

12from docopt import docopt

13from neddy import namesearch

14from sloancone import check_coverage

15from astrocalc.coords import unit_conversion

16from fundamentals.mysql import writequery, readquery

17import re

18import string

19import codecs

20import pickle

21import glob

22import time

23import csv

24import readline

26from builtins import zip

27from past.utils import old_div

28import sys

29import os

30os.environ['TERM'] = 'vt100'

33class ned_d(_base_importer):

34 """

35 *Import the * `NED-D <https://ned.ipac.caltech.edu/Library/Distances/>`_ *galaxy catalogue in to the sherlock-catalogues database*

37 **Key Arguments**

39 - ``log`` -- logger

40 - ``settings`` -- the settings dictionary

41 - ``pathToDataFile`` -- path to the ned_d data file

42 - ``version`` -- version of the ned_d catalogue

43 - ``catalogueName`` -- the name of the catalogue

46 **Usage**

48 To import the ned_d catalogue catalogue, run the following:

50 ```python

51 from sherlock.imports import ned_d

52 catalogue = ned_d(

53 log=log,

54 settings=settings,

55 pathToDataFile="/path/to/ned_d.txt",

56 version="1.0",

57 catalogueName="ned_d"

58 )

59 catalogue.ingest()

60 ```

63 .. todo ::

65 - abstract this module out into its own stand alone script

66 """

67 # INITIALIZATION

69 def ingest(self):

70 """Import the ned_d catalogue into the catalogues database

72 The method first generates a list of python dictionaries from the ned_d datafile, imports this list of dictionaries into a database table and then generates the HTMIDs for that table.

74 **Usage**

76 See class docstring for usage

79 .. todo ::

81 - update docstring text

82 - check sublime snippet exists

83 - clip any useful text to docs mindmap

84 - regenerate the docs and check redendering of this docstring

85 """

86 self.log.debug('starting the ``get`` method')

88 dictList = self._create_dictionary_of_ned_d()

89 self.primaryIdColumnName = "primaryId"

90 self.raColName = "raDeg"

91 self.declColName = "decDeg"

93 tableName = self.dbTableName

94 viewName = tableName.replace("tcs_cat_", "tcs_view_galaxy_")

95 createStatement = u"""

96 CREATE TABLE `%(tableName)s` (

97 `primaryId` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'An internal counter',

98 `Method` varchar(150) DEFAULT NULL,

99 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,

100 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,

101 `updated` varchar(45) DEFAULT '0',

102 `dist_derived_from_sn` varchar(150) DEFAULT NULL,

103 `dist_in_ned_flag` varchar(10) DEFAULT NULL,

104 `dist_index_id` mediumint(9) DEFAULT NULL,

105 `dist_mod` double DEFAULT NULL,

106 `dist_mod_err` double DEFAULT NULL,

107 `dist_mpc` double DEFAULT NULL,

108 `galaxy_index_id` mediumint(9) DEFAULT NULL,

109 `hubble_const` double DEFAULT NULL,

110 `lmc_mod` double DEFAULT NULL,

111 `notes` varchar(500) DEFAULT NULL,

112 `primary_ned_id` varchar(150) DEFAULT NULL,

113 `redshift` double DEFAULT NULL,

114 `ref` varchar(150) DEFAULT NULL,

115 `ref_date` int(11) DEFAULT NULL,

116 `master_row` tinyint(4) DEFAULT '0',

117 `major_diameter_arcmin` double DEFAULT NULL,

118 `ned_notes` varchar(700) DEFAULT NULL,

119 `object_type` varchar(100) DEFAULT NULL,

120 `redshift_err` double DEFAULT NULL,

121 `redshift_quality` varchar(100) DEFAULT NULL,

122 `magnitude_filter` varchar(10) DEFAULT NULL,

123 `minor_diameter_arcmin` double DEFAULT NULL,

124 `morphology` varchar(50) DEFAULT NULL,

125 `hierarchy` varchar(50) DEFAULT NULL,

126 `galaxy_morphology` varchar(50) DEFAULT NULL,

127 `radio_morphology` varchar(50) DEFAULT NULL,

128 `activity_type` varchar(50) DEFAULT NULL,

129 `in_ned` tinyint(4) DEFAULT NULL,

130 `raDeg` double DEFAULT NULL,

131 `decDeg` double DEFAULT NULL,

132 `eb_v` double DEFAULT NULL,

133 `sdss_coverage` tinyint(4) DEFAULT NULL,

134 `htm16ID` bigint(20) DEFAULT NULL,

135 `htm13ID` int(11) DEFAULT NULL,

136 `htm10ID` int(11) DEFAULT NULL,

137 `magnitude` double DEFAULT NULL,

138 PRIMARY KEY (`primaryId`),

139 UNIQUE KEY `galaxy_index_id_dist_index_id` (`galaxy_index_id`,`dist_index_id`),

140 KEY `idx_htm10ID` (`htm10ID`),

141 KEY `idx_htm13ID` (`htm13ID`),

142 KEY `idx_htm16ID` (`htm16ID`),

143 KEY `idx_dist` (`dist_mpc`),

144 KEY `idx_redshift` (`redshift`),

145 KEY `idx_major_diameter` (`major_diameter_arcmin`),

146 KEY `idx_master_name` (`master_row`),

147 KEY `idx_magnitude` (`magnitude`),

148 KEY `idx_in_ned` (`in_ned`),

149 KEY `idx_type` (`object_type`),

150 KEY `idx_dist_from_sn` (`dist_derived_from_sn`)

151 ) ENGINE=MyISAM AUTO_INCREMENT=0 DEFAULT CHARSET=latin1;

152 DROP VIEW IF EXISTS `%(viewName)s`;

153 CREATE

154 VIEW `%(viewName)s` AS

155 (SELECT

156 `%(tableName)s`.`primaryId` AS `primaryId`,

157 `%(tableName)s`.`Method` AS `Method`,

158 `%(tableName)s`.`dateCreated` AS `dateCreated`,

159 `%(tableName)s`.`dist_derived_from_sn` AS `dist_derived_from_sn`,

160 `%(tableName)s`.`dist_in_ned_flag` AS `dist_in_ned_flag`,

161 `%(tableName)s`.`dist_index_id` AS `dist_index_id`,

162 `%(tableName)s`.`dist_mod_median` AS `dist_mod`,

163 `%(tableName)s`.`dist_mod_err` AS `dist_mod_err`,

164 `%(tableName)s`.`dist_mpc_median` AS `dist_mpc`,

165 `%(tableName)s`.`galaxy_index_id` AS `galaxy_index_id`,

166 `%(tableName)s`.`hubble_const` AS `hubble_const`,

167 `%(tableName)s`.`lmc_mod` AS `lmc_mod`,

168 `%(tableName)s`.`notes` AS `notes`,

169 `%(tableName)s`.`primary_ned_id` AS `primary_ned_id`,

170 `%(tableName)s`.`redshift` AS `redshift`,

171 `%(tableName)s`.`ref` AS `ref`,

172 `%(tableName)s`.`ref_date` AS `ref_date`,

173 `%(tableName)s`.`master_row` AS `master_row`,

174 `%(tableName)s`.`major_diameter_arcmin` AS `major_diameter_arcmin`,

175 `%(tableName)s`.`ned_notes` AS `ned_notes`,

176 `%(tableName)s`.`object_type` AS `object_type`,

177 `%(tableName)s`.`redshift_err` AS `redshift_err`,

178 `%(tableName)s`.`redshift_quality` AS `redshift_quality`,

179 `%(tableName)s`.`magnitude` AS `magnitude`,

180 `%(tableName)s`.`minor_diameter_arcmin` AS `minor_diameter_arcmin`,

181 `%(tableName)s`.`morphology` AS `morphology`,

182 `%(tableName)s`.`hierarchy` AS `hierarchy`,

183 `%(tableName)s`.`galaxy_morphology` AS `galaxy_morphology`,

184 `%(tableName)s`.`radio_morphology` AS `radio_morphology`,

185 `%(tableName)s`.`activity_type` AS `activity_type`,

186 `%(tableName)s`.`in_ned` AS `in_ned`,

187 `%(tableName)s`.`raDeg` AS `raDeg`,

188 `%(tableName)s`.`decDeg` AS `decDeg`,

189 `%(tableName)s`.`eb_v` AS `eb_v`,

190 `%(tableName)s`.`htm16ID` AS `htm16ID`,

191 `%(tableName)s`.`htm10ID` AS `htm10ID`,

192 `%(tableName)s`.`htm13ID` AS `htm13ID`,

193 `%(tableName)s`.`sdss_coverage` AS `sdss_coverage`

194 FROM

195 `%(tableName)s`

196 WHERE

197 `%(tableName)s`.`master_row` = 1);

198 """ % locals()

199

200 self.add_data_to_database_table(

201 dictList=dictList,

202 createStatement=createStatement

203 )

204

205 self._clean_up_columns()

206 self._get_metadata_for_galaxies()

207 # self._update_sdss_coverage()

208

209 self.log.debug('completed the ``get`` method')

210 return None

211

212 def _create_dictionary_of_ned_d(

213 self):

214 """create a list of dictionaries containing all the rows in the ned_d catalogue

215

216 **Return**

217

218 - ``dictList`` - a list of dictionaries containing all the rows in the ned_d catalogue

219

220

221 .. todo ::

222

223 - update key arguments values and definitions with defaults

224 - update return values and definitions

225 - update usage examples and text

226 - update docstring text

227 - check sublime snippet exists

228 - clip any useful text to docs mindmap

229 - regenerate the docs and check redendering of this docstring

230 """

231 self.log.debug(

232 'starting the ``_create_dictionary_of_ned_d`` method')

233

234 import pandas as pd

235

236 count = 0

237 with open(self.pathToDataFile, 'r') as csvFile:

238 csvReader = csv.reader(

239 csvFile, dialect='excel', delimiter=',', quotechar='"')

240 totalRows = sum(1 for row in csvReader)

241 csvFile.close()

242 totalCount = totalRows

243

244 with open(self.pathToDataFile, 'r') as csvFile:

245 csvReader = csv.reader(

246 csvFile, dialect='excel', delimiter=',', quotechar='"')

247 theseKeys = []

248 dictList = []

249 for row in csvReader:

250 if len(theseKeys) == 0:

251 totalRows -= 1

252 if "Exclusion Code" in row and "Hubble const." in row:

253 for i in row:

254 if i == "redshift (z)":

255 theseKeys.append("redshift")

256 elif i == "Hubble const.":

257 theseKeys.append("hubble_const")

258 elif i == "G":

259 theseKeys.append("galaxy_index_id")

260 elif i == "err":

261 theseKeys.append("dist_mod_err")

262 elif i == "D (Mpc)":

263 theseKeys.append("dist_mpc")

264 elif i == "Date (Yr. - 1980)":

265 theseKeys.append("ref_date")

266 elif i == "REFCODE":

267 theseKeys.append("ref")

268 elif i == "Exclusion Code":

269 theseKeys.append("dist_in_ned_flag")

270 elif i == "Adopted LMC modulus":

271 theseKeys.append("lmc_mod")

272 elif i == "m-M":

273 theseKeys.append("dist_mod")

274 elif i == "Notes":

275 theseKeys.append("notes")

276 elif i == "SN ID":

277 theseKeys.append("dist_derived_from_sn")

278 elif i == "method":

279 theseKeys.append("dist_method")

280 elif i == "Galaxy ID":

281 theseKeys.append("primary_ned_id")

282 elif i == "D":

283 theseKeys.append("dist_index_id")

284 else:

285 theseKeys.append(i)

286 continue

287

288 if len(theseKeys):

289 count += 1

290 if count > 1:

291 # Cursor up one line and clear line

292 sys.stdout.write("\x1b[1A\x1b[2K")

293 if count > totalCount:

294 count = totalCount

295 percent = (old_div(float(count), float(totalCount))) * 100.

296 print(

297 "%(count)s / %(totalCount)s (%(percent)1.1f%%) rows added to memory" % locals())

298 rowDict = {}

299 for t, r in zip(theseKeys, row):

300 rowDict[t] = r

301 if t == "ref_date":

302 try:

303 rowDict[t] = int(r) + 1980

304 except:

305 rowDict[t] = None

306

307 if rowDict["dist_index_id"] != "999999":

308 dictList.append(rowDict)

309

310 csvFile.close()

311

312 df = pd.DataFrame(dictList)

313 df['galaxy_index_id'] = df['galaxy_index_id'].values.astype(int)

314 # GROUP RESULTS

315 dfGroups = df.groupby(['galaxy_index_id'])

316

317 medianDistDf = dfGroups[["dist_mpc", "dist_mod"]].median().reset_index()

318 # RENAME COLUMNS

319 medianDistDf.rename(columns={"dist_mpc": "dist_mpc_median"}, inplace=True)

320 medianDistDf.rename(columns={"dist_mod": "dist_mod_median"}, inplace=True)

321

322 # MERGE DATAFRAMES

323 df = df.merge(medianDistDf, on=['galaxy_index_id'], how='outer')

324

325 # SORT BY COLUMN NAME

326 df.sort_values(['galaxy_index_id'],

327 ascending=[True], inplace=True)

328

329 dictList = df.to_dict('records')

330 self.log.debug(

331 'completed the ``_create_dictionary_of_ned_d`` method')

332 return dictList

333

334 def _clean_up_columns(

335 self):

336 """clean up columns of the NED table

337

338 .. todo ::

339

340 - update key arguments values and definitions with defaults

341 - update return values and definitions

342 - update usage examples and text

343 - update docstring text

344 - check sublime snippet exists

345 - clip any useful text to docs mindmap

346 - regenerate the docs and check redendering of this docstring

347 """

348 self.log.debug('starting the ``_clean_up_columns`` method')

349

350 tableName = self.dbTableName

351

352 print("cleaning up %(tableName)s columns" % locals())

353

354 sqlQuery = u"""

355 set sql_mode="STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION";

356 """ % locals()

357 writequery(

358 log=self.log,

359 sqlQuery=sqlQuery,

360 dbConn=self.cataloguesDbConn,

361 )

362

363 sqlQuery = u"""

364 update %(tableName)s set dist_mod_err = null where dist_mod_err = 0;

365 update %(tableName)s set dist_in_ned_flag = null where dist_in_ned_flag = "";

366 update %(tableName)s set notes = null where notes = "";

367 update %(tableName)s set redshift = null where redshift = 0;

368 update %(tableName)s set dist_derived_from_sn = null where dist_derived_from_sn = "";

369 update %(tableName)s set hubble_const = null where hubble_const = 0;

370 update %(tableName)s set lmc_mod = null where lmc_mod = 0;

371 update %(tableName)s set master_row = 0;

372 update %(tableName)s set master_row = 1 where primaryId in (select * from (select distinct primaryId from %(tableName)s group by galaxy_index_id) as alias);

373 """ % locals()

374 writequery(

375 log=self.log,

376 sqlQuery=sqlQuery,

377 dbConn=self.cataloguesDbConn,

378 )

379

380 self.log.debug('completed the ``_clean_up_columns`` method')

381 return None

382

383 def _get_metadata_for_galaxies(

384 self):

385 """get metadata for galaxies

386

387 .. todo ::

388

389 - update key arguments values and definitions with defaults

390 - update return values and definitions

391 - update usage examples and text

392 - update docstring text

393 - check sublime snippet exists

394 - clip any useful text to docs mindmap

395 - regenerate the docs and check redendering of this docstring

396 """

397 self.log.debug('starting the ``_get_metadata_for_galaxies`` method')

398

399 total, batches = self._count_galaxies_requiring_metadata()

400 print("%(total)s galaxies require metadata. Need to send %(batches)s batch requests to NED." % locals())

401

402 totalBatches = self.batches

403 thisCount = 0

404

405 # FOR EACH BATCH, GET THE GALAXY IDs, QUERY NED AND UPDATE THE DATABASE

406 while self.total:

407 thisCount += 1

408 self._get_3000_galaxies_needing_metadata()

409 dictList = self._query_ned_and_add_results_to_database(thisCount)

410

411 self.add_data_to_database_table(

412 dictList=dictList,

413 createStatement=False

414 )

415

416 self._count_galaxies_requiring_metadata()

417

418 self.log.debug('completed the ``_get_metadata_for_galaxies`` method')

419 return None

420

421 def _count_galaxies_requiring_metadata(

422 self):

423 """ count galaxies requiring metadata

424

425 **Return**

426

427 - ``self.total``, ``self.batches`` -- total number of galaxies needing metadata & the number of batches required to be sent to NED

428

429

430 .. todo ::

431

432 - update key arguments values and definitions with defaults

433 - update return values and definitions

434 - update usage examples and text

435 - update docstring text

436 - check sublime snippet exists

437 - clip any useful text to docs mindmap

438 - regenerate the docs and check redendering of this docstring

439 """

440 self.log.debug(

441 'starting the ``_count_galaxies_requiring_metadata`` method')

442

443 tableName = self.dbTableName

444

445 sqlQuery = u"""

446 select count(*) as count from %(tableName)s where master_row = 1 and in_ned is null

447 """ % locals()

448 rows = readquery(

449 log=self.log,

450 sqlQuery=sqlQuery,

451 dbConn=self.cataloguesDbConn,

452 quiet=False

453 )

454 self.total = rows[0]["count"]

455 self.batches = int(old_div(self.total, 3000.)) + 1

456

457 if self.total == 0:

458 self.batches = 0

459

460 self.log.debug(

461 'completed the ``_count_galaxies_requiring_metadata`` method')

462 return self.total, self.batches

463

464 def _get_3000_galaxies_needing_metadata(

465 self):

466 """ get 3000 galaxies needing metadata

467

468 **Return**

469

470 - ``len(self.theseIds)`` -- the number of NED IDs returned

471

472

473 .. todo ::

474

475 - update key arguments values and definitions with defaults

476 - update return values and definitions

477 - update usage examples and text

478 - update docstring text

479 - check sublime snippet exists

480 - clip any useful text to docs mindmap

481 - regenerate the docs and check redendering of this docstring

482 """

483 self.log.debug(

484 'starting the ``_get_3000_galaxies_needing_metadata`` method')

485

486 tableName = self.dbTableName

487

488 # SELECT THE DATA FROM NED TABLE

489 self.theseIds = {}

490 sqlQuery = u"""

491 select primaryId, primary_ned_id from %(tableName)s where master_row = 1 and in_ned is null limit 3000;

492 """ % locals()

493 rows = readquery(

494 log=self.log,

495 sqlQuery=sqlQuery,

496 dbConn=self.cataloguesDbConn,

497 quiet=False

498 )

499 for row in rows:

500 self.theseIds[row["primary_ned_id"]] = row["primaryId"]

501

502 self.log.debug(

503 'completed the ``_get_3000_galaxies_needing_metadata`` method')

504

505 return len(self.theseIds)

506

507 def _query_ned_and_add_results_to_database(

508 self,

509 batchCount):

510 """ query ned and add results to database

511

512 **Key Arguments**

513

514 - ``batchCount`` - the index number of the batch sent to NED

515

516

517 .. todo ::

518

519 - update key arguments values and definitions with defaults

520 - update return values and definitions

521 - update usage examples and text

522 - update docstring text

523 - check sublime snippet exists

524 - clip any useful text to docs mindmap

525 - regenerate the docs and check redendering of this docstring

526 """

527 self.log.debug(

528 'starting the ``_query_ned_and_add_results_to_database`` method')

529

530 import re

531 regex = re.compile(r'[^\d\.]')

532

533 tableName = self.dbTableName

534 # ASTROCALC UNIT CONVERTER OBJECT

535 converter = unit_conversion(

536 log=self.log

537 )

538

539 # QUERY NED WITH BATCH

540 totalCount = len(self.theseIds)

541 print("requesting metadata from NED for %(totalCount)s galaxies (batch %(batchCount)s)" % locals())

542 search = namesearch(

543 log=self.log,

544 names=list(self.theseIds.keys()),

545 quiet=True

546 )

547 results = search.get()

548 print("results returned from ned -- starting to add to database" % locals())

549

550 # CLEAN THE RETURNED DATA AND UPDATE DATABASE

551 totalCount = len(results)

552 count = 0

553 sqlQuery = ""

554 dictList = []

555

556 colList = ["redshift_quality", "redshift", "hierarchy", "object_type", "major_diameter_arcmin", "morphology", "magnitude_filter",

557 "ned_notes", "eb_v", "raDeg", "radio_morphology", "activity_type", "minor_diameter_arcmin", "decDeg", "redshift_err", "in_ned", "magnitude"]

558

559 if not len(results):

560 for k, v in list(self.theseIds.items()):

561 dictList.append({

562 "in_ned": 0,

563 "primaryID": v

564 })

565 for thisDict in results:

566 thisDict["magnitude"] = thisDict["magnitude_filter"]

567 thisDict["tableName"] = tableName

568 count += 1

569 for k, v in list(thisDict.items()):

570 if not v or len(v) == 0:

571 thisDict[k] = "null"

572 if k in ["major_diameter_arcmin", "minor_diameter_arcmin"] and (":" in v or "?" in v or "<" in v):

573 thisDict[k] = v.replace(":", "").replace(

574 "?", "").replace("<", "")

575 if isinstance(v, ("".__class__, u"".__class__)) and '"' in v:

576 thisDict[k] = v.replace('"', '\\"')

577 if k in ["magnitude"] and thisDict[k] != "null":

578 thisDict[k] = regex.sub("", thisDict[k])

579

580 if "Input name not" not in thisDict["input_note"] and "Same object as" not in thisDict["input_note"]:

581 if thisDict["ra"] != "null" and thisDict["dec"] != "null":

582 thisDict["raDeg"] = converter.ra_sexegesimal_to_decimal(

583 ra=thisDict["ra"]

584 )

585 thisDict["decDeg"] = converter.dec_sexegesimal_to_decimal(

586 dec=thisDict["dec"]

587 )

588 else:

589 thisDict["raDeg"] = None

590 thisDict["decDeg"] = None

591 thisDict["in_ned"] = 1

592 thisDict["eb_v"] = thisDict["eb-v"]

593

594 row = {}

595 row["primary_ned_id"] = thisDict["input_name"]

596

597 try:

598 row["primaryID"] = self.theseIds[thisDict["input_name"]]

599 for c in colList:

600 if thisDict[c] == "null":

601 row[c] = None

602 else:

603 row[c] = thisDict[c]

604 dictList.append(row)

605 except:

606 g = thisDict["input_name"]

607 self.log.error(

608 "Cannot find database table %(tableName)s primaryID for '%(g)s'\n\n" % locals())

609 dictList.append({

610 "in_ned": 0,

611 "primary_ned_id": thisDict["input_name"]

612 })

613

614 else:

615 dictList.append({

616 "primary_ned_id": thisDict["input_name"],

617 "in_ned": 0,

618 "primaryID": self.theseIds[thisDict["input_name"]]

619 })

620

621 self.log.debug(

622 'completed the ``_query_ned_and_add_results_to_database`` method')

623 return dictList

624

625 def _update_sdss_coverage(

626 self):

627 """ update sdss coverage

628

629 .. todo ::

630

631 - update key arguments values and definitions with defaults

632 - update return values and definitions

633 - update usage examples and text

634 - update docstring text

635 - check sublime snippet exists

636 - clip any useful text to docs mindmap

637 - regenerate the docs and check redendering of this docstring

638 """

639 self.log.debug('starting the ``_update_sdss_coverage`` method')

640

641 tableName = self.dbTableName

642

643 # SELECT THE LOCATIONS NEEDING TO BE CHECKED

644 sqlQuery = u"""

645 select primary_ned_id, primaryID, raDeg, decDeg, sdss_coverage from %(tableName)s where sdss_coverage is null and master_row = 1 and in_ned = 1 order by dist_mpc;

646 """ % locals()

647 rows = readquery(

648 log=self.log,

649 sqlQuery=sqlQuery,

650 dbConn=self.cataloguesDbConn,

651 quiet=False

652 )

653

654 totalCount = len(rows)

655 count = 0

656 for row in rows:

657 count += 1

658 if count > 1:

659 # Cursor up three lines and clear

660 sys.stdout.write("\x1b[1A\x1b[2K")

661 sys.stdout.write("\x1b[1A\x1b[2K")

662 sys.stdout.write("\x1b[1A\x1b[2K")

663

664 if count > totalCount:

665 count = totalCount

666 percent = (old_div(float(count), float(totalCount))) * 100.

667

668 primaryID = row["primaryID"]

669 raDeg = float(row["raDeg"])

670 decDeg = float(row["decDeg"])

671 primary_ned_id = row["primary_ned_id"]

672

673 # SDSS CAN ONLY ACCEPT 60 QUERIES/MIN

674 time.sleep(1.1)

675 print("%(count)s / %(totalCount)s (%(percent)1.1f%%) NED galaxies checked for SDSS coverage" % locals())

676 print("NED NAME: ", primary_ned_id)

677

678 # covered = True | False | 999 (i.e. not sure)

679 sdss_coverage = check_coverage(

680 log=self.log,

681 ra=raDeg,

682 dec=decDeg

683 ).get()

684

685 if sdss_coverage == 999:

686 sdss_coverage_flag = "null"

687 elif sdss_coverage == True:

688 sdss_coverage_flag = 1

689 elif sdss_coverage == False:

690 sdss_coverage_flag = 0

691 else:

692 self.log.error('could not get sdss coverage' % locals())

693 sys.exit(0)

694

695 # UPDATE THE DATABASE FLAG

696 sqlQuery = u"""

697 update %(tableName)s set sdss_coverage = %(sdss_coverage_flag)s where primaryID = %(primaryID)s

698 """ % locals()

699 writequery(

700 log=self.log,

701 sqlQuery=sqlQuery,

702 dbConn=self.cataloguesDbConn,

703 )

704

705 self.log.debug('completed the ``_update_sdss_coverage`` method')

706 return None

707

708 # use the tab-trigger below for new method

709 # xt-class-method

Coverage for sherlock/imports/ned_d.py: 12%

254 statements