Merge pull request #60 from bsipocz/euclid_swap_pyvo_for_astroquery

ENH: Euclid swap pyvo for astroquery

Author: bsipocz

tutorials/euclid_access/1_Euclid_intro_MER_images.md

@@ -58,7 +58,7 @@ Each MER image is approximately 1.47 GB. Downloading can take some time.
 
 ```{code-cell} ipython3
 # Uncomment the next line to install dependencies if needed.
-# !pip install numpy 'astropy>=5.3' matplotlib pyvo 'sep>=1.4' fsspec pandas
+# !pip install numpy 'astropy>=5.3' matplotlib 'astroquery>=0.4.10' 'sep>=1.4' fsspec
 ```
 
 ```{code-cell} ipython3
@@ -78,7 +78,7 @@ from astropy.visualization import ImageNormalize, PercentileInterval, AsinhStret
 from astropy.wcs import WCS
 from astropy import units as u
 
-import pyvo as vo
+from astroquery.ipac.irsa import Irsa
 import sep
 
 # Copy-on-write is more performant and avoids unexpected modifications of the original DataFrame.
@@ -95,44 +95,23 @@ coord = SkyCoord.from_name('HD 168151')
 
 Use IRSA's Simple Image Access (SIA) API to search for all Euclid MER mosaics that overlap with the search region you have specified. We specify the euclid_DpdMerBksMosaic "collection" because it lists all of the multiwavelength MER mosaics, along with their associated catalogs.
 
-```{code-cell} ipython3
-irsa_service= vo.dal.sia2.SIA2Service('https://irsa.ipac.caltech.edu/SIA')
-
-image_table = irsa_service.search(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic')
-```
-
-Convert the table to pandas dataframe
-
-```{code-cell} ipython3
-df_im_irsa=image_table.to_table().to_pandas().reset_index()
-```
+```{tip}
+The IRSA SIA collections can be listed using using the ``list_collections`` method, we can filter on the ones containing "euclid" in the collection name:
 
-Change the settings so we can see all the columns in the dataframe and the full column width (to see the full long URL)
-
-```{code-cell} ipython3
-pd.set_option('display.max_columns', None)
-pd.set_option('display.max_colwidth', None)
-
-
-## Can use the following lines to reset the max columns and column width of pandas
-# pd.reset_option('display.max_columns')
-# pd.reset_option('display.max_colwidth')
+    Irsa.list_collections(filter='euclid')
 ```
 
 ```{code-cell} ipython3
-df_im_irsa
+image_table = Irsa.query_sia(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic')
 ```
 
-This dataframe contains lots of datasets that have been "Euclidized", which means that they have been put on a common pixel scale chosen for the Euclid mission. Choose "science" as the data product subtype to see all science images of this tile.
+This table lists all MER mosaic images available in this search position. These mosaics include the Euclid VIS, Y, J, H images, as well as ground-based telescopes which have been put on the same pixel scale. For more information, see the [Euclid documentation at IPAC](https://euclid.caltech.edu/page/euclid-faq-tech/).
 
-```{code-cell} ipython3
-df_im_euclid=df_im_irsa[ (df_im_irsa['dataproduct_subtype']=='science')].reset_index()
-
-df_im_euclid
-```
+Note that there are various image types are returned as well, we filter out the `science` images from these:
 
 ```{code-cell} ipython3
-print('There are',len(df_im_euclid),'MER images of this object/MER tile.')
+science_images = image_table[image_table['dataproduct_subtype'] == 'science']
+science_images
 ```
 
 ## 2. Retrieve a Euclid Q1 MER mosaic image in the VIS bandpass
@@ -144,8 +123,8 @@ print('There are',len(df_im_euclid),'MER images of this object/MER tile.')
 Note that 'access_estsize' is in units of kb
 
 ```{code-cell} ipython3
-filename = df_im_euclid[df_im_euclid['energy_bandpassname']=='VIS']['access_url'].to_list()[0]
-filesize = df_im_euclid[df_im_euclid['energy_bandpassname']=='VIS']['access_estsize'].to_list()[0]/1000000
+filename = science_images[science_images['energy_bandpassname']=='VIS']['access_url'][0]
+filesize = science_images[science_images['energy_bandpassname']=='VIS']['access_estsize'][0]/1000000
 
 print(filename)
 
@@ -216,7 +195,7 @@ We'd like to take a look at the multiwavelength images of our object, but the fu
 ```
 
 ```{code-cell} ipython3
-urls = df_im_euclid['access_url'].to_list()
+urls = science_images['access_url']
 
 urls
 ```

tutorials/euclid_access/2_Euclid_intro_MER_catalog.md

@@ -50,27 +50,24 @@ If you have questions about this notebook, please contact the [IRSA helpdesk](ht
 
 ```{code-cell} ipython3
 # Uncomment the next line to install dependencies if needed
-# !pip install numpy matplotlib pyvo
+# !pip install numpy matplotlib 'astroquery>=0.4.10'
 ```
 
 ```{code-cell} ipython3
 import numpy as np
 import matplotlib.pyplot as plt
 
-import pyvo as vo
+from astroquery.ipac.irsa import Irsa
 ```
 
 ## 1. Download MER catalog from IRSA directly to this notebook
 
-```{code-cell} ipython3
-service = vo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP")
-```
++++
+
+First, have a look at what Euclid catalogs are available. With the ``list_catalogs`` functionality, we'll receive a list of the name of the catalogs as well as their brief desciption.
 
 ```{code-cell} ipython3
-tables = service.tables
-for tablename in tables.keys():
-    if "tap_schema" not in tablename and "euclid_q1" in tablename:
-            tables[tablename].describe()
+Irsa.list_catalogs(filter='euclid')
 ```
 
 ### Choose the Euclid MER table
@@ -79,18 +76,17 @@ for tablename in tables.keys():
 table_mer = 'euclid_q1_mer_catalogue'
 ```
 
-### Learn some information about the table:
+### Learn some information about the MER catalog:
 - How many columns are there?
 - List the column names
 
 ```{code-cell} ipython3
-columns = tables[table_mer].columns
-print(len(columns))
+columns_info = Irsa.list_columns(catalog=table_mer)
+print(len(columns_info))
 ```
 
 ```{code-cell} ipython3
-for col in columns:
-    print(f'{f"{col.name}":30s}  {col.unit}  {col.description}')
+columns_info
 ```
 
 ```{tip}
@@ -121,7 +117,7 @@ adql_stars = ("SELECT TOP 10000 mer.object_id, mer.ra, mer.dec, mer.flux_vis_psf
 
 # Run the query
 
-result_stars = service.search(adql_stars)
+result_stars = Irsa.query_tap(adql_stars)
 ```
 
 ```{code-cell} ipython3

tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md

@@ -51,7 +51,7 @@ If you have questions about this notebook, please contact the [IRSA helpdesk](ht
 
 ```{code-cell} ipython3
 # Uncomment the next line to install dependencies if needed.
-# !pip install requests matplotlib pandas 'astropy>=5.3' pyvo fsspec firefly_client
+# !pip install requests matplotlib pandas 'astropy>=5.3' 'astroquery>=0.4.10' fsspec firefly_client
 ```
 
 ```{code-cell} ipython3
@@ -72,7 +72,7 @@ from astropy.visualization import ImageNormalize, PercentileInterval, AsinhStret
 from astropy.wcs import WCS
 
 from firefly_client import FireflyClient
-import pyvo as vo
+from astroquery.ipac.irsa import Irsa
 ```
 
 ## 1. Find the MER Tile ID that corresponds to a given RA and Dec
@@ -92,9 +92,7 @@ coord = SkyCoord(ra, dec, unit='deg', frame='icrs')
 This searches specifically in the euclid_DpdMerBksMosaic "collection" which is the MER images and catalogs.
 
 ```{code-cell} ipython3
-irsa_service= vo.dal.sia2.SIA2Service('https://irsa.ipac.caltech.edu/SIA')
-
-image_table = irsa_service.search(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic').to_table()
+image_table = Irsa.query_sia(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic')
 ```
 
 ```{note}
@@ -123,47 +121,42 @@ print('The MER tile ID for this object is :',tileID)
 
 ## 2. Download PHZ catalog from IRSA
 
-```{code-cell} ipython3
-## Use IRSA to search for catalogs
-
-service = vo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP")
+Use IRSA's TAP to search catalogs
 
-
-## Search for all tables in IRSA labled as euclid_q1
-tables = service.tables
-for tablename in tables.keys():
-    if "tap_schema" not in tablename and "euclid_q1" in tablename:
-            tables[tablename].describe()
+```{code-cell} ipython3
+Irsa.list_catalogs(filter='euclid')
 ```
 
 ```{code-cell} ipython3
-table_mer= 'euclid_q1_mer_catalogue'
-table_phz= 'euclid_q1_phz_photo_z'
-table_1dspectra= 'euclid.objectid_spectrafile_association_q1'
+table_mer = 'euclid_q1_mer_catalogue'
+table_phz = 'euclid_q1_phz_photo_z'
+table_1dspectra = 'euclid.objectid_spectrafile_association_q1'
 ```
 
-### Learn some information about the table:
+### Learn some information about the photo-z catalog:
 
 - How many columns are there?
 - List the column names
 
 ```{code-cell} ipython3
-columns = tables[table_phz].columns
-print(len(columns))
-```
-
-```{code-cell} ipython3
-for col in columns:
-    print(f'{f"{col.name}":30s}  {col.unit}  {col.description}') ## Currently no descriptions
+columns_info = Irsa.list_columns(catalog=table_phz)
+print(len(columns_info))
 ```
 
+```{tip}
 The PHZ catalog contains 67 columns, below are a few highlights:
 
 - object_id
 - flux_vis_unif, flux_y_unif, flux_j_unif, flux_h_unif
 - median redshift (phz_median)
 - phz_classification
 - phz_90_int1,  phz_90_int2 (The phz PDF interval containing 90% of the probability, upper and lower values)
+```
+
+```{code-cell} ipython3
+# Full list of columns and their description
+columns_info
+```
 
 ```{note}
 The phz_catalog on IRSA has more columns than it does on the ESA archive.
@@ -228,8 +221,8 @@ AND phz.phz_median BETWEEN 1.4 AND 1.6 \
 adql
 
 
-## Use TAP with this ADQL string using pyvo
-result = service.search(adql)
+## Use TAP with this ADQL string
+result = Irsa.query_tap(adql)
 
 
 ## Convert table to pandas dataframe
@@ -326,7 +319,7 @@ FROM {table_1dspectra} \
 WHERE objectid = {obj_id}"
 
 ## Pull the data on this particular galaxy
-result2 = service.search(adql_object)
+result2 = Irsa.query_tap(adql_object)
 df2=result2.to_table().to_pandas()
 df2
 ```

tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md

@@ -50,7 +50,7 @@ If you have questions about this notebook, please contact the [IRSA helpdesk](ht
 
 ```{code-cell} ipython3
 # Uncomment the next line to install dependencies if needed
-# !pip install matplotlib pandas astropy pyvo
+# !pip install matplotlib pandas astropy 'astroquery>=0.4.10'
 ```
 
 ```{code-cell} ipython3
@@ -69,7 +69,7 @@ from astropy import units as u
 from astropy.utils.data import download_file
 from astropy.visualization import ImageNormalize, PercentileInterval, AsinhStretch
 
-import pyvo as vo
+from astroquery.ipac.irsa import Irsa
 ```
 
 ## 1. Find the MER Tile ID that corresponds to a given RA and Dec
@@ -86,12 +86,10 @@ coord = SkyCoord.from_name('HD 168151')
 This searches specifically in the euclid_DpdMerBksMosaic "collection" which is the MER images and catalogs.
 
 ```{code-cell} ipython3
-irsa_service= vo.dal.sia2.SIA2Service('https://irsa.ipac.caltech.edu/SIA')
-
-im_table = irsa_service.search(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic')
+im_table = Irsa.query_sia(pos=(coord, search_radius), collection='euclid_DpdMerBksMosaic')
 
 ## Convert the table to pandas dataframe
-df_im_irsa=im_table.to_table().to_pandas()
+df_im_irsa=im_table.to_pandas()
 ```
 
 ```{code-cell} ipython3
@@ -125,45 +123,28 @@ print('The MER tile ID for this object is :',tileID)
 Search for all tables in IRSA labeled as euclid
 
 ```{code-cell} ipython3
-service = vo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP")
-
-tables = service.tables
-for tablename in tables.keys():
-    if "tap_schema" not in tablename and "euclid" in tablename:
-            tables[tablename].describe()
+Irsa.list_catalogs(filter='euclid')
 ```
 
 ```{code-cell} ipython3
-table_mer= 'euclid_q1_mer_catalogue'
-table_galaxy_candidates= 'euclid_q1_spectro_zcatalog_spe_galaxy_candidates'
-table_1dspectra= 'euclid.objectid_spectrafile_association_q1'
-table_lines= 'euclid_q1_spe_lines_line_features'
+table_mer = 'euclid_q1_mer_catalogue'
+table_galaxy_candidates = 'euclid_q1_spectro_zcatalog_spe_galaxy_candidates'
+table_1dspectra = 'euclid.objectid_spectrafile_association_q1'
+table_lines = 'euclid_q1_spe_lines_line_features'
 ```
 
 ### Learn some information about the table:
 - How many columns are there?
 - List the column names
 
 ```{code-cell} ipython3
-columns = tables[table_lines].columns
-print(len(columns))
-```
-
-```{code-cell} ipython3
-for col in columns:
-    print(f'{f"{col.name}":30s}  {col.unit}  {col.description}') ## Currently no descriptions
+columns_info = Irsa.list_columns(catalog=table_lines)
+print(len(columns_info))
 ```
 
 ```{code-cell} ipython3
-## Change the settings so we can see all the columns in the dataframe and the full column width
-## (to see the full long URL)
-pd.set_option('display.max_columns', None)
-pd.set_option('display.max_colwidth', None)
-
-
-## Can use the following lines to reset the max columns and column width of pandas
-# pd.reset_option('display.max_columns')
-# pd.reset_option('display.max_colwidth')
+# Full list of columns and their description
+columns_info
 ```
 
 ## Find some objects with spectra in our tileID
@@ -196,8 +177,8 @@ AND lines.spe_line_flux_gf > 2E-16 \
 ORDER BY lines.spe_line_snr_gf DESC \
 "
 
-# Use TAP with this ADQL string using pyvo
-result = service.search(adql)
+# Use TAP with this ADQL string
+result = Irsa.query_tap(adql)
 
 # Convert table to pandas dataframe and drop duplicates
 result_table = result.to_qtable()
@@ -222,7 +203,7 @@ obj_tab
 ```{code-cell} ipython3
 adql_object = f"SELECT *  FROM {table_1dspectra}  WHERE objectid = {obj_id}"
 
-result2 = service.search(adql_object)
+result2 = Irsa.query_tap(adql_object)
 df2 = result2.to_table().to_pandas()
 df2
 ```