Source code for lightkurve.search

"""Defines tools to retrieve Kepler data from the archive at MAST."""
from __future__ import division
import os
import glob
import logging
import re
import warnings
from requests import HTTPError

from memoization import cached
import numpy as np
from astropy.table import join, Table, Row
from astropy.coordinates import SkyCoord
from astropy.io import ascii
from astropy import units as u
from astropy.utils import deprecated
from astropy.time import Time

from .targetpixelfile import TargetPixelFile
from .collections import TargetPixelFileCollection, LightCurveCollection
from .utils import LightkurveError, suppress_stdout, LightkurveWarning, LightkurveDeprecationWarning
from .io import read
from . import conf
from . import PACKAGEDIR

log = logging.getLogger(__name__)

__all__ = [
    "search_targetpixelfile",
    "search_lightcurve",
    "search_lightcurvefile",
    "search_tesscut",
    "SearchResult",
]


# Which external links should we display in the SearchResult repr?
AUTHOR_LINKS = {
    "Kepler": "https://archive.stsci.edu/kepler/data_products.html",
    "K2": "https://archive.stsci.edu/k2/data_products.html",
    "SPOC": "https://heasarc.gsfc.nasa.gov/docs/tess/pipeline.html",
    "TESS-SPOC": "https://archive.stsci.edu/hlsp/tess-spoc",
    "QLP": "https://archive.stsci.edu/hlsp/qlp",
    "TASOC": "https://archive.stsci.edu/hlsp/tasoc",
    "PATHOS": "https://archive.stsci.edu/hlsp/pathos",
    "CDIPS": "https://archive.stsci.edu/hlsp/cdips",
    "K2SFF": "https://archive.stsci.edu/hlsp/k2sff",
    "EVEREST": "https://archive.stsci.edu/hlsp/everest",
    "TESScut": "https://mast.stsci.edu/tesscut/",
    "GSFC-ELEANOR-LITE": "https://archive.stsci.edu/hlsp/gsfc-eleanor-lite",
}

REPR_COLUMNS_BASE = [
    "#",
    "mission",
    "year",
    "author",
    "exptime",
    "target_name",
    "distance",
]


class SearchError(Exception):
    pass


[docs]class SearchResult(object): """Container for the results returned by the search functions. The purpose of this class is to provide a convenient way to inspect and download products that have been identified using one of the data search functions. Parameters ---------- table : `~astropy.table.Table` object Astropy table returned by a join of the astroquery `Observations.query_criteria()` and `Observations.get_product_list()` methods. """ table = None """`~astropy.table.Table` containing the full search results returned by the MAST API.""" display_extra_columns = [] """A list of extra columns to be included in the default display of the search result. It can be configured in a few different ways. For example, to include ``proposal_id`` in the default display, users can set it: 1. in the user's ``lightkurve.cfg`` file:: [search] # The extra comma at the end is needed for a single extra column search_result_display_extra_columns = proposal_id, 2. at run time:: import lightkurve as lk lk.conf.search_result_display_extra_columns = ['proposal_id'] 3. for a specific `SearchResult` object instance:: result.display_extra_columns = ['proposal_id'] See :ref:`configuration <api.config>` for more information. """
[docs] def __init__(self, table=None): if table is None: self.table = Table() else: self.table = table if len(table) > 0: self._add_columns() self._sort_table() self.display_extra_columns = conf.search_result_display_extra_columns
def _sort_table(self): """Sort the table of search results by distance, author, and filename. The reason we include "author" in the sort criteria is that Lightkurve v1 only showed data products created by the official pipelines (i.e. author equal to "Kepler", "K2", or "SPOC"). To maintain backwards compatibility, we want to show products from these authors at the top, so that `search.download()` operations tend to download the same product in Lightkurve v1 vs v2. This ordering is not a judgement on the quality of one product vs another, because we love all pipelines! """ sort_priority = {"Kepler": 1, "K2": 1, "SPOC": 1, "TESS-SPOC": 2, "QLP": 3} self.table["sort_order"] = [ sort_priority.get(author, 9) for author in self.table["author"] ] self.table.sort(["distance", "year", "mission", "sort_order", "exptime"]) def _add_columns(self): """Adds a user-friendly index (``#``) column and adds column unit and display format information. """ if "#" not in self.table.columns: self.table["#"] = None self.table["exptime"].unit = "s" self.table["exptime"].format = ".0f" self.table["distance"].unit = "arcsec" # Add the year column from `t_min` or `productFilename` year = np.floor(Time(self.table["t_min"], format="mjd").decimalyear) self.table["year"] = year.astype(int) # `t_min` is incorrect for Kepler products, so we extract year from the filename for those =( for idx in np.where(self.table["author"] == "Kepler")[0]: self.table["year"][idx] = re.findall( r"\d+.(\d{4})\d+", self.table["productFilename"][idx] )[0] def __repr__(self, html=False): def to_tess_gi_url(proposal_id): if re.match("^G0[12].+", proposal_id) is not None: return f"https://heasarc.gsfc.nasa.gov/docs/tess/approved-programs-primary.html#:~:text={proposal_id}" else: return f"https://heasarc.gsfc.nasa.gov/docs/tess/approved-programs.html#:~:text={proposal_id}" out = "SearchResult containing {} data products.".format(len(self.table)) if len(self.table) == 0: return out columns = REPR_COLUMNS_BASE if self.display_extra_columns is not None: columns = REPR_COLUMNS_BASE + self.display_extra_columns # search_tesscut() has fewer columns, ensure we don't try to display columns that do not exist columns = [c for c in columns if c in self.table.colnames] self.table["#"] = [idx for idx in range(len(self.table))] out += "\n\n" + "\n".join(self.table[columns].pformat(max_width=300, html=html)) # Make sure author names show up as clickable links if html: for author, url in AUTHOR_LINKS.items(): out = out.replace(f">{author}<", f"><a href='{url}'>{author}</a><") # special HTML formating for TESS proposal_id tess_table = self.table[self.table["project"] == "TESS"] if "proposal_id" in tess_table.colnames: proposal_id_col = np.unique(tess_table["proposal_id"]) else: proposal_id_col = [] for p_ids in proposal_id_col: # for CDIPS products, proposal_id is a np MaskedConstant, not a string if p_ids == "N/A" or (not isinstance(p_ids, str)): continue # e.g., handle cases with multiple proposals, e.g., G12345_G67890 p_id_links = [f"""\ <a href='{to_tess_gi_url(p_id)}'>{p_id}</a>\ """ for p_id in p_ids.split("_")] out = out.replace(f">{p_ids}<", f">{' , '.join(p_id_links)}<") return out def _repr_html_(self): return self.__repr__(html=True) def __getitem__(self, key): """Implements indexing and slicing, e.g. SearchResult[2:5].""" selection = self.table[key] # Indexing a Table with an integer will return a Row if isinstance(selection, Row): selection = Table(selection) return SearchResult(table=selection) def __len__(self): """Returns the number of products in the SearchResult table.""" return len(self.table) @property def unique_targets(self): """Returns a table of targets and their RA & dec values produced by search""" mask = ["target_name", "s_ra", "s_dec"] return Table.from_pandas( self.table[mask] .to_pandas() .drop_duplicates("target_name") .reset_index(drop=True) ) @property def obsid(self): """MAST observation ID for each data product found.""" return np.asarray(np.unique(self.table["obsid"]), dtype="int64") @property def ra(self): """Right Ascension coordinate for each data product found.""" return self.table["s_ra"].data.data @property def dec(self): """Declination coordinate for each data product found.""" return self.table["s_dec"].data.data @property def mission(self): """Kepler quarter or TESS sector names for each data product found.""" return self.table["mission"].data @property def year(self): """Year the observation was made.""" return self.table["year"].data @property def author(self): """Pipeline name for each data product found.""" return self.table["author"].data @property def target_name(self): """Target name for each data product found.""" return self.table["target_name"].data @property def exptime(self): """Exposure time for each data product found.""" return self.table["exptime"].quantity @property def distance(self): """Distance from the search position for each data product found.""" return self.table["distance"].quantity def _download_one( self, table, quality_bitmask, download_dir, cutout_size, **kwargs ): """Private method used by `download()` and `download_all()` to download exactly one file from the MAST archive. Always returns a `TargetPixelFile` or `LightCurve` object. """ # Make sure astroquery uses the same level of verbosity logging.getLogger("astropy").setLevel(log.getEffectiveLevel()) if download_dir is None: download_dir = self._default_download_dir() # if the SearchResult row is a TESScut entry, then download cutout if "FFI Cutout" in table[0]["description"]: try: log.debug( "Started downloading TESSCut for '{}' sector {}." "".format(table[0]["target_name"], table[0]["sequence_number"]) ) path = self._fetch_tesscut_path( table[0]["target_name"], table[0]["sequence_number"], download_dir, cutout_size, ) except Exception as exc: msg = str(exc) if "504" in msg: # TESSCut will occasionally return a "504 Gateway Timeout # error" when it is overloaded. raise HTTPError( "The TESS FFI cutout service at MAST appears " "to be temporarily unavailable. It returned " "the following error: {}".format(exc) ) else: raise SearchError( "Unable to download FFI cutout. Desired target " "coordinates may be too near the edge of the FFI." "Error: {}".format(exc) ) return read( path, quality_bitmask=quality_bitmask, targetid=table[0]["targetid"] ) else: if cutout_size is not None: warnings.warn( "`cutout_size` can only be specified for TESS " "Full Frame Image cutouts.", LightkurveWarning, ) # Whenever `astroquery.mast.Observations.download_products` is called, # a HTTP request will be sent to determine the length of the file # prior to checking if the file already exists in the local cache. # For performance, we skip this HTTP request and immediately try to # find the file in the cache. The path we check here is consistent # with the one hard-coded inside `astroquery.mast.Observations._download_files()` # in Astroquery v0.4.1. It would be good to submit a PR to astroquery # so we can avoid having to use this hard-coded hack. path = os.path.join( download_dir.rstrip("/"), "mastDownload", table["obs_collection"][0], table["obs_id"][0], table["productFilename"][0], ) if os.path.exists(path): log.debug("File found in local cache.") else: from astroquery.mast import Observations download_url = table[:1]["dataURL"][0] log.debug("Started downloading {}.".format(download_url)) download_response = Observations.download_products( table[:1], mrp_only=False, download_dir=download_dir )[0] if download_response["Status"] != "COMPLETE": raise LightkurveError( f"Download of {download_url} failed. " f"MAST returns {download_response['Status']}: {download_response['Message']}" ) path = download_response["Local Path"] log.debug("Finished downloading.") return read(path, quality_bitmask=quality_bitmask, **kwargs)
[docs] @suppress_stdout def download( self, quality_bitmask="default", download_dir=None, cutout_size=None, **kwargs ): """Download and open the first data product in the search result. If multiple files are present in `SearchResult.table`, only the first will be downloaded. Parameters ---------- quality_bitmask : str or int, optional Bitmask (integer) which identifies the quality flag bitmask that should be used to mask out bad cadences. If a string is passed, it has the following meaning: * "none": no cadences will be ignored * "default": cadences with severe quality issues will be ignored * "hard": more conservative choice of flags to ignore This is known to remove good data. * "hardest": removes all data that has been flagged This mask is not recommended. See the :class:`KeplerQualityFlags <lightkurve.utils.KeplerQualityFlags>` or :class:`TessQualityFlags <lightkurve.utils.TessQualityFlags>` class for details on the bitmasks. download_dir : str, optional Location where the data files will be stored. Defaults to "~/.lightkurve-cache" if `None` is passed. cutout_size : int, float or tuple, optional Side length of cutout in pixels. Tuples should have dimensions (y, x). Default size is (5, 5) flux_column : str, optional The column in the FITS file to be read as `flux`. Defaults to 'pdcsap_flux'. Typically 'pdcsap_flux' or 'sap_flux'. kwargs : dict, optional Extra keyword arguments passed on to the file format reader function. Returns ------- data : `TargetPixelFile` or `LightCurve` object The first entry in the products table. Raises ------ HTTPError If the TESSCut service times out (i.e. returns HTTP status 504). SearchError If any other error occurs. """ if len(self.table) == 0: warnings.warn( "Cannot download from an empty search result.", LightkurveWarning ) return None if len(self.table) != 1: warnings.warn( "Warning: {} files available to download. " "Only the first file has been downloaded. " "Please use `download_all()` or specify additional " "criteria (e.g. quarter, campaign, or sector) " "to limit your search.".format(len(self.table)), LightkurveWarning, ) return self._download_one( table=self.table[:1], quality_bitmask=quality_bitmask, download_dir=download_dir, cutout_size=cutout_size, **kwargs, )
[docs] @suppress_stdout def download_all( self, quality_bitmask="default", download_dir=None, cutout_size=None, **kwargs ): """Download and open all data products in the search result. This method will return a `~lightkurve.TargetPixelFileCollection` or `~lightkurve.LightCurveCollection`. Parameters ---------- quality_bitmask : str or int, optional Bitmask (integer) which identifies the quality flag bitmask that should be used to mask out bad cadences. If a string is passed, it has the following meaning: * "none": no cadences will be ignored * "default": cadences with severe quality issues will be ignored * "hard": more conservative choice of flags to ignore This is known to remove good data. * "hardest": removes all data that has been flagged This mask is not recommended. See the :class:`KeplerQualityFlags <lightkurve.utils.KeplerQualityFlags>` or :class:`TessQualityFlags <lightkurve.utils.TessQualityFlags>` class for details on the bitmasks. download_dir : str, optional Location where the data files will be stored. Defaults to "~/.lightkurve-cache" if `None` is passed. cutout_size : int, float or tuple, optional Side length of cutout in pixels. Tuples should have dimensions (y, x). Default size is (5, 5) flux_column : str, optional The column in the FITS file to be read as `flux`. Defaults to 'pdcsap_flux'. Typically 'pdcsap_flux' or 'sap_flux'. kwargs : dict, optional Extra keyword arguments passed on to the file format reader function. Returns ------- collection : `~lightkurve.collections.Collection` object Returns a `~lightkurve.LightCurveCollection` or `~lightkurve.TargetPixelFileCollection`, containing all entries in the products table Raises ------ HTTPError If the TESSCut service times out (i.e. returns HTTP status 504). SearchError If any other error occurs. """ if len(self.table) == 0: warnings.warn( "Cannot download from an empty search result.", LightkurveWarning ) return None log.debug("{} files will be downloaded.".format(len(self.table))) products = [] for idx in range(len(self.table)): products.append( self._download_one( table=self.table[idx : idx + 1], quality_bitmask=quality_bitmask, download_dir=download_dir, cutout_size=cutout_size, **kwargs, ) ) if isinstance(products[0], TargetPixelFile): return TargetPixelFileCollection(products) else: return LightCurveCollection(products)
def _default_download_dir(self): """Returns the default path to the directory where files will be downloaded. By default, this method will return "~/.lightkurve-cache" and create this directory if it does not exist. If the directory cannot be access or created, then it returns the local directory ("."). Returns ------- download_dir : str Path to location of `mastDownload` folder where data downloaded from MAST are stored """ download_dir = os.path.join(os.path.expanduser("~"), ".lightkurve-cache") if os.path.isdir(download_dir): return download_dir else: # if it doesn't exist, make a new cache directory try: os.mkdir(download_dir) # downloads locally if OS error occurs except OSError: log.warning( "Warning: unable to create {}. " "Downloading MAST files to the current " "working directory instead.".format(download_dir) ) download_dir = "." return download_dir def _fetch_tesscut_path(self, target, sector, download_dir, cutout_size): """Downloads TESS FFI cutout and returns path to local file. Parameters ---------- download_dir : str Path to location of `.lightkurve-cache` directory where downloaded cutouts are stored cutout_size : int, float or tuple Side length of cutout in pixels. Tuples should have dimensions (y, x). Default size is (5, 5) Returns ------- path : str Path to locally downloaded cutout file """ from astroquery.mast import TesscutClass coords = _resolve_object(target) # Set cutout_size defaults if cutout_size is None: cutout_size = 5 # Check existence of `~/.lightkurve-cache/tesscut` tesscut_dir = os.path.join(download_dir, "tesscut") if not os.path.isdir(tesscut_dir): # if it doesn't exist, make a new cache directory try: os.mkdir(tesscut_dir) # downloads into default cache if OSError occurs except OSError: tesscut_dir = download_dir # Resolve SkyCoord of given target coords = _resolve_object(target) # build path string name and check if it exists # this is necessary to ensure cutouts are not downloaded multiple times sec = TesscutClass().get_sectors(coordinates=coords) sector_name = sec[sec["sector"] == sector]["sectorName"][0] if isinstance(cutout_size, int): size_str = str(int(cutout_size)) + "x" + str(int(cutout_size)) elif isinstance(cutout_size, tuple) or isinstance(cutout_size, list): size_str = str(int(cutout_size[1])) + "x" + str(int(cutout_size[0])) # search cache for file with matching ra, dec, and cutout size # ra and dec are searched within 0.001 degrees of input target ra_string = str(coords.ra.value) dec_string = str(coords.dec.value) matchstring = r"{}_{}*_{}*_{}_astrocut.fits".format( sector_name, ra_string[: ra_string.find(".") + 4], dec_string[: dec_string.find(".") + 4], size_str, ) cached_files = glob.glob(os.path.join(tesscut_dir, matchstring)) # if any files exist, return the path to them instead of downloading if len(cached_files) > 0: path = cached_files[0] log.debug("Cached file found.") # otherwise the file will be downloaded else: cutout_path = TesscutClass().download_cutouts( coordinates=coords, size=cutout_size, sector=sector, path=tesscut_dir ) path = cutout_path[0][0] # the cutoutpath already contains testcut_dir log.debug("Finished downloading.") return path
[docs]@cached def search_targetpixelfile( target, radius=None, exptime=None, cadence=None, mission=("Kepler", "K2", "TESS"), author=None, quarter=None, month=None, campaign=None, sector=None, limit=None, ): """Search the `MAST data archive <https://archive.stsci.edu>`_ for target pixel files. This function fetches a data table that lists the Target Pixel Files (TPFs) that fall within a region of sky centered around the position of `target` and within a cone of a given `radius`. If no value is provided for `radius`, only a single target will be returned. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object Target around which to search. Valid inputs include: * The name of the object as a string, e.g. "Kepler-10". * The KIC or EPIC identifier as an integer, e.g. 11904151. * A coordinate string in decimal format, e.g. "285.67942179 +50.24130576". * A coordinate string in sexagesimal format, e.g. "19:02:43.1 +50:14:28.7". * An `astropy.coordinates.SkyCoord` object. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. exptime : 'long', 'short', 'fast', or float 'long' selects 10-min and 30-min cadence products; 'short' selects 1-min and 2-min products; 'fast' selects 20-sec products. Alternatively, you can pass the exact exposure time in seconds as an int or a float, e.g., ``exptime=600`` selects 10-minute cadence. By default, all cadence modes are returned. cadence : 'long', 'short', 'fast', or float Synonym for `exptime`. Will likely be deprecated in the future. mission : str, tuple of str 'Kepler', 'K2', or 'TESS'. By default, all will be returned. author : str, tuple of str, or "any" Author of the data product (`provenance_name` in the MAST API). Official Kepler, K2, and TESS pipeline products have author names 'Kepler', 'K2', and 'SPOC'. By default, all light curves are returned regardless of the author. quarter, campaign, sector : int, list of ints Kepler Quarter, K2 Campaign, or TESS Sector number. By default all quarters/campaigns/sectors will be returned. month : 1, 2, 3, 4 or list of int For Kepler's prime mission, there are three short-cadence TargetPixelFiles for each quarter, each covering one month. Hence, if ``exptime='short'`` you can specify month=1, 2, 3, or 4. By default all months will be returned. limit : int Maximum number of products to return. Returns ------- result : :class:`SearchResult` object Object detailing the data products found. Examples -------- This example demonstrates how to use the `search_targetpixelfile()` function to query and download data. Before instantiating a `~lightkurve.targetpixelfile.KeplerTargetPixelFile` object or downloading any science products, we can identify potential desired targets with `search_targetpixelfile()`:: >>> search_result = search_targetpixelfile('Kepler-10') # doctest: +SKIP >>> print(search_result) # doctest: +SKIP The above code will query mast for Target Pixel Files (TPFs) available for the known planet system Kepler-10, and display a table containing the available science products. Because Kepler-10 was observed during 15 Quarters, the table will have 15 entries. To obtain a `~lightkurve.collections.TargetPixelFileCollection` object containing all 15 observations, use:: >>> search_result.download_all() # doctest: +SKIP or we can download a single product by limiting our search:: >>> tpf = search_targetpixelfile('Kepler-10', quarter=2).download() # doctest: +SKIP The above line of code will only download Quarter 2 and create a single `~lightkurve.targetpixelfile.KeplerTargetPixelFile` object called `tpf`. We can also pass a radius into `search_targetpixelfile` to perform a cone search:: >>> search_targetpixelfile('Kepler-10', radius=100).targets # doctest: +SKIP This will display a table containing all targets within 100 arcseconds of Kepler-10. We can download a `~lightkurve.collections.TargetPixelFileCollection` object containing all available products for these targets in Quarter 4 with:: >>> search_targetpixelfile('Kepler-10', radius=100, quarter=4).download_all() # doctest: +SKIP """ try: return _search_products( target, radius=radius, filetype="Target Pixel", exptime=exptime or cadence, mission=mission, provenance_name=author, quarter=quarter, month=month, campaign=campaign, sector=sector, limit=limit, ) except SearchError as exc: log.error(exc) return SearchResult(None)
@deprecated( "2.0", alternative="search_lightcurve()", warning_type=LightkurveDeprecationWarning ) def search_lightcurvefile(*args, **kwargs): return search_lightcurve(*args, **kwargs)
[docs]@cached def search_lightcurve( target, radius=None, exptime=None, cadence=None, mission=("Kepler", "K2", "TESS"), author=None, quarter=None, month=None, campaign=None, sector=None, limit=None, ): """Search the `MAST data archive <https://archive.stsci.edu>`_ for light curves. This function fetches a data table that lists the Light Curve Files that fall within a region of sky centered around the position of `target` and within a cone of a given `radius`. If no value is provided for `radius`, only a single target will be returned. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object Target around which to search. Valid inputs include: * The name of the object as a string, e.g. "Kepler-10". * The KIC or EPIC identifier as an integer, e.g. 11904151. * A coordinate string in decimal format, e.g. "285.67942179 +50.24130576". * A coordinate string in sexagesimal format, e.g. "19:02:43.1 +50:14:28.7". * An `astropy.coordinates.SkyCoord` object. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. exptime : 'long', 'short', 'fast', or float 'long' selects 10-min and 30-min cadence products; 'short' selects 1-min and 2-min products; 'fast' selects 20-sec products. Alternatively, you can pass the exact exposure time in seconds as an int or a float, e.g., ``exptime=600`` selects 10-minute cadence. By default, all cadence modes are returned. cadence : 'long', 'short', 'fast', or float Synonym for `exptime`. This keyword will likely be deprecated in the future. mission : str, tuple of str 'Kepler', 'K2', or 'TESS'. By default, all will be returned. author : str, tuple of str, or "any" Author of the data product (`provenance_name` in the MAST API). Official Kepler, K2, and TESS pipeline products have author names 'Kepler', 'K2', and 'SPOC'. Community-provided products that are supported include 'K2SFF', 'EVEREST'. By default, all light curves are returned regardless of the author. quarter, campaign, sector : int, list of ints Kepler Quarter, K2 Campaign, or TESS Sector number. By default all quarters/campaigns/sectors will be returned. month : 1, 2, 3, 4 or list of int For Kepler's prime mission, there are three short-cadence TargetPixelFiles for each quarter, each covering one month. Hence, if ``exptime='short'`` you can specify month=1, 2, 3, or 4. By default all months will be returned. limit : int Maximum number of products to return. Returns ------- result : :class:`SearchResult` object Object detailing the data products found. Examples -------- This example demonstrates how to use the `search_lightcurve()` function to query and download data. Before instantiating a `LightCurve` object or downloading any science products, we can identify potential desired targets with `search_lightcurve`:: >>> from lightkurve import search_lightcurve # doctest: +SKIP >>> search_result = search_lightcurve("Kepler-10") # doctest: +SKIP >>> print(search_result) # doctest: +SKIP The above code will query mast for lightcurve files available for the known planet system Kepler-10, and display a table containing the available data products. Because Kepler-10 was observed in multiple quarters and sectors by both Kepler and TESS, the search will return many dozen results. If we want to narrow down the search to only return Kepler light curves in long cadence, we can use:: >>> search_result = search_lightcurve("Kepler-10", author="Kepler", exptime=1800) # doctest: +SKIP >>> print(search_result) # doctest: +SKIP That is better, we now see 15 light curves corresponding to 15 Kepler quarters. If we want to download a `~lightkurve.collections.LightCurveCollection` object containing all 15 observations, use:: >>> search_result.download_all() # doctest: +SKIP or we can specify the downloaded products by selecting a specific row using rectangular brackets, for example:: >>> lc = search_result[2].download() # doctest: +SKIP The above line of code will only search and download Quarter 2 data and create a `LightCurve` object called lc. We can also pass a radius into `search_lightcurve` to perform a cone search:: >>> search_lightcurve('Kepler-10', radius=100, quarter=4, exptime=1800) # doctest: +SKIP This will display a table containing all targets within 100 arcseconds of Kepler-10 and in Quarter 4. We can then download a `~lightkurve.collections.LightCurveFile` containing all these light curves using:: >>> search_lightcurve('Kepler-10', radius=100, quarter=4, exptime=1800).download_all() # doctest: +SKIP """ try: return _search_products( target, radius=radius, filetype="Lightcurve", exptime=exptime or cadence, mission=mission, provenance_name=author, quarter=quarter, month=month, campaign=campaign, sector=sector, limit=limit, ) except SearchError as exc: log.error(exc) return SearchResult(None)
[docs]@cached def search_tesscut(target, sector=None): """Search the `MAST TESSCut service <https://mast.stsci.edu/tesscut/>`_ for a region of sky that is available as a TESS Full Frame Image cutout. This feature uses the `TESScut service <https://mast.stsci.edu/tesscut/>`_ provided by the TESS data archive at MAST. If you use this service in your work, please `cite TESScut <https://ascl.net/code/v/2239>`_ in your publications. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object Target around which to search. Valid inputs include: * The name of the object as a string, e.g. "Kepler-10". * The KIC or EPIC identifier as an integer, e.g. 11904151. * A coordinate string in decimal format, e.g. "285.67942179 +50.24130576". * A coordinate string in sexagesimal format, e.g. "19:02:43.1 +50:14:28.7". * An `astropy.coordinates.SkyCoord` object. sector : int or list TESS Sector number. Default (None) will return all available sectors. A list of desired sectors can also be provided. Returns ------- result : :class:`SearchResult` object Object detailing the data products found. """ try: return _search_products(target, filetype="ffi", mission="TESS", sector=sector) except SearchError as exc: log.error(exc) return SearchResult(None)
def _search_products( target, radius=None, filetype="Lightcurve", mission=("Kepler", "K2", "TESS"), provenance_name=None, exptime=(0, 9999), quarter=None, month=None, campaign=None, sector=None, limit=None, **extra_query_criteria, ): """Helper function which returns a SearchResult object containing MAST products that match several criteria. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object See docstrings above. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. filetype : {'Target pixel', 'Lightcurve', 'FFI'} Type of files queried at MAST. exptime : 'long', 'short', 'fast', or float 'long' selects 10-min and 30-min cadence products; 'short' selects 1-min and 2-min products; 'fast' selects 20-sec products. Alternatively, you can pass the exact exposure time in seconds as an int or a float, e.g., ``exptime=600`` selects 10-minute cadence. By default, all cadence modes are returned. mission : str, list of str 'Kepler', 'K2', or 'TESS'. By default, all will be returned. provenance_name : str, list of str Provenance of the data product. Defaults to official products, i.e. ('Kepler', 'K2', 'SPOC'). Community-provided products such as 'K2SFF' are supported as well. quarter, campaign, sector : int, list of ints Kepler Quarter, K2 Campaign, or TESS Sector number. By default all quarters/campaigns/sectors will be returned. month : 1, 2, 3, 4 or list of int For Kepler's prime mission, there are three short-cadence TargetPixelFiles for each quarter, each covering one month. Hence, if ``exptime='short'`` you can specify month=1, 2, 3, or 4. By default all months will be returned. limit : int Maximum number of products to return Returns ------- SearchResult : :class:`SearchResult` object. """ if isinstance(target, int): if (0 < target) and (target < 13161030): log.warning( "Warning: {} may refer to a different Kepler or TESS target. " "Please add the prefix 'KIC' or 'TIC' to disambiguate." "".format(target) ) elif (0 < 200000000) and (target < 251813739): log.warning( "Warning: {} may refer to a different K2 or TESS target. " "Please add the prefix 'EPIC' or 'TIC' to disambiguate." "".format(target) ) # Specifying quarter, campaign, or quarter should constrain the mission if quarter: mission = "Kepler" if campaign: mission = "K2" if sector: mission = "TESS" # Ensure mission is a list mission = np.atleast_1d(mission).tolist() # Avoid filtering on `provenance_name` if `author` equals "any" or "all" if provenance_name in ("any", "all") or provenance_name is None: provenance_name = None else: provenance_name = np.atleast_1d(provenance_name).tolist() # Speed up by restricting the MAST query if we don't want FFI image data extra_query_criteria = {} if filetype in ["Lightcurve", "Target Pixel"]: # At MAST, non-FFI Kepler pipeline products are known as "cube" products, # and non-FFI TESS pipeline products are listed as "timeseries". extra_query_criteria["dataproduct_type"] = ["cube", "timeseries"] # Make sure `search_tesscut` always performs a cone search (i.e. always # passed a radius value), because strict target name search does not apply. if filetype.lower() == "ffi" and radius is None: radius = 0.0001 * u.arcsec observations = _query_mast( target, radius=radius, project=mission, provenance_name=provenance_name, exptime=exptime, sequence_number=campaign or sector, **extra_query_criteria, ) log.debug( "MAST found {} observations. " "Now querying MAST for the corresponding data products." "".format(len(observations)) ) if len(observations) == 0: raise SearchError('No data found for target "{}".'.format(target)) # Light curves and target pixel files if filetype.lower() != "ffi": from astroquery.mast import Observations products = Observations.get_product_list(observations) result = join( observations, products, keys="obs_id", join_type="right", uniq_col_name="{col_name}{table_name}", table_names=["", "_products"], ) result.sort(["distance", "obs_id"]) # Add the user-friendly 'author' column (synonym for 'provenance_name') result["author"] = result["provenance_name"] # Add the user-friendly 'mission' column result["mission"] = None obs_prefix = {"Kepler": "Quarter", "K2": "Campaign", "TESS": "Sector"} for idx in range(len(result)): obs_project = result["project"][idx] tmp_seqno = result["sequence_number"][idx] obs_seqno = f"{tmp_seqno:02d}" if tmp_seqno else "" # Kepler sequence_number values were not populated at the time of # writing this code, so we parse them from the description field. if obs_project == "Kepler" and result["sequence_number"].mask[idx]: try: tmp_seqno = re.findall(r".*Q(\d+)", result["description"][idx])[0] obs_seqno = f"{int(tmp_seqno):02d}" except IndexError: obs_seqno = "" # K2 campaigns 9, 10, and 11 were split into two sections, which are # listed separately in the table with suffixes "a" and "b" if obs_project == "K2" and result["sequence_number"][idx] in [9, 10, 11]: for half,letter in zip([1,2],['a','b']): if f"c{tmp_seqno}{half}" in result["productFilename"][idx]: obs_seqno = f"{int(tmp_seqno):02d}{letter}" result["mission"][idx] = "{} {} {}".format( obs_project, obs_prefix.get(obs_project, ""), obs_seqno ) masked_result = _filter_products( result, filetype=filetype, campaign=campaign, quarter=quarter, exptime=exptime, project=mission, provenance_name=provenance_name, month=month, sector=sector, limit=limit, ) log.debug("MAST found {} matching data products.".format(len(masked_result))) masked_result["distance"].info.format = ".1f" # display <0.1 arcsec return SearchResult(masked_result) # Full Frame Images else: cutouts = [] for idx in np.where(["TESS FFI" in t for t in observations["target_name"]])[0]: # if target passed in is a SkyCoord object, convert to RA, dec pair if isinstance(target, SkyCoord): target = "{}, {}".format(target.ra.deg, target.dec.deg) # pull sector numbers s = observations["sequence_number"][idx] # if the desired sector is available, add a row if s in np.atleast_1d(sector) or sector is None: cutouts.append( { "description": f"TESS FFI Cutout (sector {s})", "mission": f"TESS Sector {s:02d}", "target_name": str(target), "targetid": str(target), "t_min": observations["t_min"][idx], "exptime": observations["exptime"][idx], "productFilename": "TESScut", "provenance_name": "TESScut", "author": "TESScut", "distance": 0.0, "sequence_number": s, "project": "TESS", "obs_collection": "TESS", } ) if len(cutouts) > 0: log.debug("Found {} matching cutouts.".format(len(cutouts))) masked_result = Table(cutouts) masked_result.sort(["distance", "sequence_number"]) else: masked_result = None return SearchResult(masked_result) def _query_mast( target, radius=None, project=("Kepler", "K2", "TESS"), provenance_name=None, exptime=(0, 9999), sequence_number=None, **extra_query_criteria, ): """Helper function which wraps `astroquery.mast.Observations.query_criteria()` to return a table of all Kepler/K2/TESS observations of a given target. By default only the official data products are returned, but this can be adjusted by adding alternative data product names into `provenance_name`. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object See docstrings above. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. project : str, list of str Mission name. Typically 'Kepler', 'K2', or 'TESS'. This parameter is case-insensitive. provenance_name : str, list of str Provenance of the observation. Common options include 'Kepler', 'K2', 'SPOC', 'K2SFF', 'EVEREST', 'KEPSEISMIC'. This parameter is case-insensitive. exptime : (float, float) tuple Exposure time range in seconds. Common values include `(59, 61)` for Kepler short cadence and `(1799, 1801)` for Kepler long cadence. sequence_number : int, list of int Quarter, Campaign, or Sector number. **extra_query_criteria : kwargs Extra criteria to be passed to `astroquery.mast.Observations.query_criteria`. Returns ------- obs : astropy.Table Table detailing the available observations on MAST. """ # Local astroquery import because the package is not used elsewhere from astroquery.mast import Observations from astroquery.exceptions import ResolverError, NoResultsWarning # If passed a SkyCoord, convert it to an "ra, dec" string for MAST if isinstance(target, SkyCoord): target = "{}, {}".format(target.ra.deg, target.dec.deg) # We pass the following `query_criteria` to MAST regardless of whether # we search by position or target name: query_criteria = {"project": project, **extra_query_criteria} if provenance_name is not None: query_criteria["provenance_name"] = provenance_name if sequence_number is not None: query_criteria["sequence_number"] = sequence_number if exptime is not None: query_criteria["t_exptime"] = exptime # If an exact KIC ID is passed, we will search by the exact `target_name` # under which MAST will know the object to prevent source confusion. # For discussion, see e.g. GitHub issues #148, #718. exact_target_name = None target_lower = str(target).lower() # Was a Kepler target ID passed? kplr_match = re.match("^(kplr|kic) ?(\d+)$", target_lower) if kplr_match: exact_target_name = f"kplr{kplr_match.group(2).zfill(9)}" # Was a K2 target ID passed? ktwo_match = re.match("^(ktwo|epic) ?(\d+)$", target_lower) if ktwo_match: exact_target_name = f"ktwo{ktwo_match.group(2).zfill(9)}" # Was a TESS target ID passed? tess_match = re.match("^(tess|tic) ?(\d+)$", target_lower) if tess_match: exact_target_name = f"{tess_match.group(2).zfill(9)}" if exact_target_name and radius is None: log.debug( "Started querying MAST for observations with the exact " f"target_name='{exact_target_name}'." ) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=NoResultsWarning) warnings.filterwarnings("ignore", message="t_exptime is continuous") obs = Observations.query_criteria( target_name=exact_target_name, **query_criteria ) if len(obs) > 0: # We use `exptime` as an alias for `t_exptime` obs["exptime"] = obs["t_exptime"] # astroquery does not report distance when querying by `target_name`; # we add it here so that the table returned always has this column. obs["distance"] = 0.0 return obs else: log.debug(f"No observations found. Now performing a cone search instead.") # If the above did not return a result, then do a cone search using the MAST name resolver # `radius` defaults to 0.0001 and unit arcsecond if radius is None: radius = 0.0001 * u.arcsec elif not isinstance(radius, u.quantity.Quantity): radius = radius * u.arcsec query_criteria["radius"] = str(radius.to(u.deg)) try: log.debug( "Started querying MAST for observations within " f"{radius.to(u.arcsec)} arcsec of objectname='{target}'." ) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=NoResultsWarning) warnings.filterwarnings("ignore", message="t_exptime is continuous") obs = Observations.query_criteria(objectname=target, **query_criteria) obs.sort("distance") # We use `exptime` as an alias for `t_exptime` obs["exptime"] = obs["t_exptime"] return obs except ResolverError as exc: # MAST failed to resolve the object name to sky coordinates raise SearchError(exc) from exc def _filter_products( products, campaign=None, quarter=None, month=None, sector=None, exptime=None, limit=None, project=("Kepler", "K2", "TESS"), provenance_name=None, filetype="Target Pixel", ): """Helper function which filters a SearchResult's products table by one or more criteria. Parameters ---------- products : `astropy.table.Table` object Astropy table containing data products returned by MAST campaign : int or list Desired campaign of observation for data products quarter : int or list Desired quarter of observation for data products month : int or list Desired month of observation for data products exptime : 'long', 'short', 'fast', or float 'long' selects 10-min and 30-min cadence products; 'short' selects 1-min and 2-min products; 'fast' selects 20-sec products. Alternatively, you can pass the exact exposure time in seconds as an int or a float, e.g., ``exptime=600`` selects 10-minute cadence. By default, all cadence modes are returned. filetype : str Type of files queried at MAST (`Target Pixel` or `Lightcurve`). Returns ------- products : `astropy.table.Table` object Masked astropy table containing desired data products """ if provenance_name is None: # apply all filters provenance_lower = ("kepler", "k2", "spoc") else: provenance_lower = [p.lower() for p in np.atleast_1d(provenance_name)] mask = np.ones(len(products), dtype=bool) # Kepler data needs a special filter for quarter and month mask &= ~np.array( [prov.lower() == "kepler" for prov in products["provenance_name"]] ) if "kepler" in provenance_lower and campaign is None and sector is None: mask |= _mask_kepler_products(products, quarter=quarter, month=month) # HLSP products need to be filtered by extension if filetype.lower() == "lightcurve": mask &= np.array( [uri.lower().endswith("lc.fits") for uri in products["productFilename"]] ) elif filetype.lower() == "target pixel": mask &= np.array( [ uri.lower().endswith(("tp.fits", "targ.fits.gz")) for uri in products["productFilename"] ] ) elif filetype.lower() == "ffi": mask &= np.array(["TESScut" in desc for desc in products["description"]]) # Allow only fits files mask &= np.array( [ uri.lower().endswith("fits") or uri.lower().endswith("fits.gz") for uri in products["productFilename"] ] ) # Filter by cadence mask &= _mask_by_exptime(products, exptime) products = products[mask] products.sort(["distance", "productFilename"]) if limit is not None: return products[0:limit] return products def _mask_kepler_products(products, quarter=None, month=None): """Returns a mask flagging the Kepler products that match the criteria.""" mask = np.array([proj.lower() == "kepler" for proj in products["provenance_name"]]) if mask.sum() == 0: return mask # Identify quarter by the description. # This is necessary because the `sequence_number` field was not populated # for Kepler prime data at the time of writing this function. if quarter is not None: quarter_mask = np.zeros(len(products), dtype=bool) for q in np.atleast_1d(quarter): quarter_mask |= np.array( [ desc.lower().replace("-", "").endswith("q{}".format(q)) for desc in products["description"] ] ) mask &= quarter_mask # For Kepler short cadence data the month can be specified if month is not None: month = np.atleast_1d(month) # Get the short cadence date lookup table. table = ascii.read( os.path.join(PACKAGEDIR, "data", "short_cadence_month_lookup.csv") ) # The following line is needed for systems where the default integer type # is int32 (e.g. Windows/Appveyor), the column will then be interpreted # as string which makes the test fail. table["StartTime"] = table["StartTime"].astype(str) # Grab the dates of each of the short cadence files. # Make sure every entry has the correct month is_shortcadence = mask & np.asarray( ["Short" in desc for desc in products["description"]] ) for idx in np.where(is_shortcadence)[0]: quarter = int( products["description"][idx].split(" - ")[-1][1:].replace("-", "") ) date = products["dataURI"][idx].split("/")[-1].split("-")[1].split("_")[0] permitted_dates = [] for m in month: try: permitted_dates.append( table["StartTime"][ np.where( (table["Month"] == m) & (table["Quarter"] == quarter) )[0][0] ] ) except IndexError: pass if not (date in permitted_dates): mask[idx] = False return mask def _mask_by_exptime(products, exptime): """Helper function to filter by exposure time.""" mask = np.ones(len(products), dtype=bool) if isinstance(exptime, (int, float)): mask &= products["exptime"] == exptime elif isinstance(exptime, str): exptime = exptime.lower() if exptime in ["fast"]: mask &= products["exptime"] < 60 elif exptime in ["short"]: mask &= (products["exptime"] >= 60) & (products["exptime"] < 300) elif exptime in ["long", "ffi"]: mask &= products["exptime"] >= 300 return mask def _resolve_object(target): """Ask MAST to resolve an object string to a set of coordinates.""" from astroquery.mast import MastClass # Note: `_resolve_object` was renamed `resolve_object` in astroquery 0.3.10 (2019) return MastClass().resolve_object(target)