astroquery
astroquery copied to clipboard
astropy
Vizier.find_catalogs doesn't find Gaia DR3
I get UnitsWarnings, and I see those in other issues. But, I also get ValueErrors and TableParseError, with the result that no table is returned although I see on the Vizier website that they do have the table available.
Here is my code (straight from the documentation):
gaia_catalog_list = Vizier.find_catalogs('GAIA DR3') print({k:v.description for k,v in gaia_catalog_list.items()})
The last error I get is TableParseError: Failed to parse VIZIER result! The raw response can be found in self.response, and the error in self.table_parse_error. The attempted parsed result is in self.parsed_result. Exception: 3520:4: not well-formed (invalid token)
And I do see that astroquery.gaia exists, but I want to use xmatch with the gaia catalog, which I think I have to do through vizier since I do not have a gaia login.
@jkrick please post the full tracebacks for errors like this.
This looks like an upstream error:
from astroquery.vizier import Vizier
result = Vizier(catalog='IV/38/tic').query_constraints_async(TIC='1402786096')
Vizier.response.text.split("\n")[-13:]
['<INFO name="QUERY_STATUS" value="ERROR">',
'',
' --POSTGRES MSG db=2',
' Report from: stdb_pg, Status: STDB_ERR',
'ERROR: relation "ssoobservation" does not exist LINE 1: ..._gaia, vy_gaia, vz_g',
'aia, position_angle_scan From SsoObserva...',
' ^',
'ERROR: relation "ssoobservation" does not exist LINE 1: ..._gaia, vy_gaia, vz_g',
'aia, position_angle_scan From SsoObserva...',
' ^',
'</INFO>',
'</VOTABLE>',
'']
ok, full tracebacks below:
`WARNING: UnitsWarning: The unit 'ph' has been deprecated in the VOUnit standard. [astropy.units.format.utils]
/opt/conda/lib/python3.8/site-packages/astroquery/vizier/core.py:769: UserWarning: VOTABLE parsing raised exception: 3520:4: not well-formed (invalid token)
warnings.warn("VOTABLE parsing raised exception: {0}".format(ex))
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/opt/conda/lib/python3.8/site-packages/astroquery/vizier/core.py in parse_vizier_votable(data, verbose, invalid, get_catalog_names)
766 try:
--> 767 vo_tree = votable.parse(tf, verify='warn', invalid='exception')
768 except Exception as ex:
/opt/conda/lib/python3.8/site-packages/astropy/utils/decorators.py in wrapper(*args, **kwargs)
545
--> 546 return function(*args, **kwargs)
547
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/table.py in parse(source, columns, invalid, verify, chunk_size, table_number, table_id, filename, unit_format, datatype_mapping, _debug_python_based_parser)
158 _debug_python_based_parser=_debug_python_based_parser) as iterator:
--> 159 return tree.VOTableFile(
160 config=config, pos=(1, 1)).parse(iterator, config)
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in parse(self, iterator, config)
3615 if start:
-> 3616 tag_mapping.get(tag, self._add_unknown_tag)(
3617 iterator, tag, data, config, pos)
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in _add_resource(self, iterator, tag, data, config, pos)
3494 self.resources.append(resource)
-> 3495 resource.parse(self, iterator, config)
3496
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in parse(self, votable, iterator, config)
3292 if start:
-> 3293 tag_mapping.get(tag, self._add_unknown_tag)(
3294 iterator, tag, data, config, pos)
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in _add_table(self, iterator, tag, data, config, pos)
3238 self.tables.append(table)
-> 3239 table.parse(iterator, config)
3240
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in parse(self, iterator, config)
2468
-> 2469 tag_mapping.get(tag, self._add_unknown_tag)(
2470 iterator, tag, data, config, pos)
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in _add_field(self, iterator, tag, data, config, pos)
2384 self.fields.append(field)
-> 2385 field.parse(iterator, config)
2386
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in parse(self, iterator, config)
1475 def parse(self, iterator, config):
-> 1476 for start, tag, data, pos in iterator:
1477 if start:
ValueError: 3520:4: not well-formed (invalid token)
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
/opt/conda/lib/python3.8/site-packages/astroquery/vizier/core.py in _parse_result(self, response, get_catalog_names, verbose, invalid)
698 try:
--> 699 return parse_vizier_votable(
700 response.content, verbose=verbose, invalid=invalid,
/opt/conda/lib/python3.8/site-packages/astroquery/vizier/core.py in parse_vizier_votable(data, verbose, invalid, get_catalog_names)
769 warnings.warn("VOTABLE parsing raised exception: {0}".format(ex))
--> 770 vo_tree = votable.parse(tf, verify='warn', invalid='mask')
771 elif invalid == 'exception':
/opt/conda/lib/python3.8/site-packages/astropy/utils/decorators.py in wrapper(*args, **kwargs)
545
--> 546 return function(*args, **kwargs)
547
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/table.py in parse(source, columns, invalid, verify, chunk_size, table_number, table_id, filename, unit_format, datatype_mapping, _debug_python_based_parser)
158 _debug_python_based_parser=_debug_python_based_parser) as iterator:
--> 159 return tree.VOTableFile(
160 config=config, pos=(1, 1)).parse(iterator, config)
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in parse(self, iterator, config)
3615 if start:
-> 3616 tag_mapping.get(tag, self._add_unknown_tag)(
3617 iterator, tag, data, config, pos)
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in _add_resource(self, iterator, tag, data, config, pos)
3494 self.resources.append(resource)
-> 3495 resource.parse(self, iterator, config)
3496
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in parse(self, votable, iterator, config)
3292 if start:
-> 3293 tag_mapping.get(tag, self._add_unknown_tag)(
3294 iterator, tag, data, config, pos)
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in _add_table(self, iterator, tag, data, config, pos)
3238 self.tables.append(table)
-> 3239 table.parse(iterator, config)
3240
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in parse(self, iterator, config)
2468
-> 2469 tag_mapping.get(tag, self._add_unknown_tag)(
2470 iterator, tag, data, config, pos)
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in _add_field(self, iterator, tag, data, config, pos)
2384 self.fields.append(field)
-> 2385 field.parse(iterator, config)
2386
/opt/conda/lib/python3.8/site-packages/astropy/io/votable/tree.py in parse(self, iterator, config)
1475 def parse(self, iterator, config):
-> 1476 for start, tag, data, pos in iterator:
1477 if start:
ValueError: 3520:4: not well-formed (invalid token)
During handling of the above exception, another exception occurred:
TableParseError Traceback (most recent call last)
<ipython-input-16-7c9279157fc9> in <module>
----> 1 gaia_catalog_list = Vizier.find_catalogs('GAIA DR3')
2 #print({k:v.description for k,v in gaia_catalog_list.items()})
/opt/conda/lib/python3.8/site-packages/astroquery/vizier/core.py in find_catalogs(self, keywords, include_obsolete, verbose, max_catalogs, return_type)
233 "setting max_catalogs to a large number and"
234 " try again")
--> 235 result = self._parse_result(response, verbose=verbose,
236 get_catalog_names=True)
237
/opt/conda/lib/python3.8/site-packages/astroquery/vizier/core.py in _parse_result(self, response, get_catalog_names, verbose, invalid)
703 self.response = response
704 self.table_parse_error = ex
--> 705 raise TableParseError("Failed to parse VIZIER result! The "
706 "raw response can be found in "
707 "self.response, and the error in "
TableParseError: Failed to parse VIZIER result! The raw response can be found in self.response, and the error in self.table_parse_error. The attempted parsed result is in self.parsed_result.
Exception: 3520:4: not well-formed (invalid token)`
The upstream part has been fixed, at least Adams example now works.
But the actual query and it's resulted VOtable from the OP is still not parsed properly, so I change the labels for this to be a bug.
In [20]: gaia_catalog_list = Vizier.find_catalogs('GAIA DR3')
/Users/bsipocz/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/units/format/utils.py:220: UnitsWarning: The unit 'ph' has been deprecated in the VOUnit standard.
warnings.warn(message, UnitsWarning)
/Users/bsipocz/munka/devel/astroquery/astroquery/vizier/core.py:750: UserWarning: VOTABLE parsing raised exception: 3524:4: not well-formed (invalid token)
warnings.warn("VOTABLE parsing raised exception: {0}".format(ex))
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
File ~/munka/devel/astroquery/astroquery/vizier/core.py:748, in _parse_vizier_votable(data, verbose, invalid, get_catalog_names)
747 try:
--> 748 vo_tree = votable.parse(tf, verify='warn', invalid='exception')
749 except Exception as ex:
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/utils/decorators.py:607, in deprecated_renamed_argument.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
605 warnings.warn(msg, warning_type, stacklevel=2)
--> 607 return function(*args, **kwargs)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/table.py:177, in parse(source, columns, invalid, verify, chunk_size, table_number, table_id, filename, unit_format, datatype_mapping, _debug_python_based_parser)
174 with iterparser.get_xml_iterator(
175 source, _debug_python_based_parser=_debug_python_based_parser
176 ) as iterator:
--> 177 return tree.VOTableFile(config=config, pos=(1, 1)).parse(iterator, config)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:3897, in VOTableFile.parse(self, iterator, config)
3896 if start:
-> 3897 tag_mapping.get(tag, self._add_unknown_tag)(
3898 iterator, tag, data, config, pos
3899 )
3900 elif tag == "DESCRIPTION":
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:3777, in VOTableFile._add_resource(self, iterator, tag, data, config, pos)
3776 self.resources.append(resource)
-> 3777 resource.parse(self, iterator, config)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:3575, in Resource.parse(self, votable, iterator, config)
3574 if start:
-> 3575 tag_mapping.get(tag, self._add_unknown_tag)(
3576 iterator, tag, data, config, pos
3577 )
3578 elif tag == "DESCRIPTION":
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:3521, in Resource._add_table(self, iterator, tag, data, config, pos)
3520 self.tables.append(table)
-> 3521 table.parse(iterator, config)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:2700, in Table.parse(self, iterator, config)
2698 break
-> 2700 tag_mapping.get(tag, self._add_unknown_tag)(
2701 iterator, tag, data, config, pos
2702 )
2703 else:
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:2618, in Table._add_field(self, iterator, tag, data, config, pos)
2617 self.fields.append(field)
-> 2618 field.parse(iterator, config)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:1618, in Field.parse(self, iterator, config)
1617 def parse(self, iterator, config):
-> 1618 for start, tag, data, pos in iterator:
1619 if start:
ValueError: 3524:4: not well-formed (invalid token)
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
File ~/munka/devel/astroquery/astroquery/vizier/core.py:680, in VizierClass._parse_result(self, response, get_catalog_names, verbose, invalid)
679 try:
--> 680 return _parse_vizier_votable(
681 response.content, verbose=verbose, invalid=invalid,
682 get_catalog_names=get_catalog_names)
683 except Exception as ex:
File ~/munka/devel/astroquery/astroquery/vizier/core.py:751, in _parse_vizier_votable(data, verbose, invalid, get_catalog_names)
750 warnings.warn("VOTABLE parsing raised exception: {0}".format(ex))
--> 751 vo_tree = votable.parse(tf, verify='warn', invalid='mask')
752 elif invalid == 'exception':
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/utils/decorators.py:607, in deprecated_renamed_argument.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
605 warnings.warn(msg, warning_type, stacklevel=2)
--> 607 return function(*args, **kwargs)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/table.py:177, in parse(source, columns, invalid, verify, chunk_size, table_number, table_id, filename, unit_format, datatype_mapping, _debug_python_based_parser)
174 with iterparser.get_xml_iterator(
175 source, _debug_python_based_parser=_debug_python_based_parser
176 ) as iterator:
--> 177 return tree.VOTableFile(config=config, pos=(1, 1)).parse(iterator, config)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:3897, in VOTableFile.parse(self, iterator, config)
3896 if start:
-> 3897 tag_mapping.get(tag, self._add_unknown_tag)(
3898 iterator, tag, data, config, pos
3899 )
3900 elif tag == "DESCRIPTION":
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:3777, in VOTableFile._add_resource(self, iterator, tag, data, config, pos)
3776 self.resources.append(resource)
-> 3777 resource.parse(self, iterator, config)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:3575, in Resource.parse(self, votable, iterator, config)
3574 if start:
-> 3575 tag_mapping.get(tag, self._add_unknown_tag)(
3576 iterator, tag, data, config, pos
3577 )
3578 elif tag == "DESCRIPTION":
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:3521, in Resource._add_table(self, iterator, tag, data, config, pos)
3520 self.tables.append(table)
-> 3521 table.parse(iterator, config)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:2700, in Table.parse(self, iterator, config)
2698 break
-> 2700 tag_mapping.get(tag, self._add_unknown_tag)(
2701 iterator, tag, data, config, pos
2702 )
2703 else:
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:2618, in Table._add_field(self, iterator, tag, data, config, pos)
2617 self.fields.append(field)
-> 2618 field.parse(iterator, config)
File ~/.pyenv/versions/3.11.0/lib/python3.11/site-packages/astropy/io/votable/tree.py:1618, in Field.parse(self, iterator, config)
1617 def parse(self, iterator, config):
-> 1618 for start, tag, data, pos in iterator:
1619 if start:
ValueError: 3524:4: not well-formed (invalid token)
During handling of the above exception, another exception occurred:
TableParseError Traceback (most recent call last)
Cell In [20], line 1
----> 1 gaia_catalog_list = Vizier.find_catalogs('GAIA DR3')
File ~/munka/devel/astroquery/astroquery/vizier/core.py:235, in VizierClass.find_catalogs(self, keywords, include_obsolete, verbose, max_catalogs, return_type)
231 if 'STOP, Max. number of RESOURCE reached' in response.text:
232 raise ValueError("Maximum number of catalogs exceeded. Try "
233 "setting max_catalogs to a large number and"
234 " try again")
--> 235 result = self._parse_result(response, verbose=verbose,
236 get_catalog_names=True)
238 # Filter out the obsolete catalogs, unless requested
239 if include_obsolete is False:
File ~/munka/devel/astroquery/astroquery/vizier/core.py:686, in VizierClass._parse_result(self, response, get_catalog_names, verbose, invalid)
684 self.response = response
685 self.table_parse_error = ex
--> 686 raise TableParseError("Failed to parse VIZIER result! The "
687 "raw response can be found in "
688 "self.response, and the error in "
689 "self.table_parse_error. The attempted "
690 "parsed result is in "
691 "self.parsed_result.\n Exception: " +
692 str(self.table_parse_error))
693 elif response.content[:5] == b'#\n# ':
694 return _parse_vizier_tsvfile(response.content, verbose=verbose)
TableParseError: Failed to parse VIZIER result! The raw response can be found in self.response, and the error in self.table_parse_error. The attempted parsed result is in self.parsed_result.
Exception: 3524:4: not well-formed (invalid token)
It's upstream, we're on it.
To get the votable,
from astroquery.vizier import Vizier
Vizier.clear_cache()
data_payload = {'-words': 'GAIA DR3', '-meta.all': 1}
data_payload['-ucd'] = Vizier.ucd
response = Vizier._request(method='POST', url=Vizier._server_to_url(return_type='votable'),data=data_payload, timeout=Vizier.TIMEOUT)
with open("votable.xml", "w") as f:
f.write(response.text)
And the faulty line looks like 3527 where a link is not closed
───────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
│ File: votable.xml
───────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
3520 │ <LINK href="http://axel.u-strasbg.fr/cgi-bin/gaia_dr3_bprp_coeff.cgi?print_header=true&mode[BP][recno]=${BPrecnoS}&
│ mode[BP][nrecords]=${BPNelem}"/>
3521 │ </FIELD>
3522 │ <FIELD name="BPNelem" ucd="meta.number" display="1" datatype="short" width="2"><!-- ucd="NUMBER" -->
3523 │ <DESCRIPTION>[0/55] Number of elements in the separated bp_coeff/e_bp_coeff file (always 55 except for 1 line) (added by CD
│ S) (bp_n_elems)</DESCRIPTION>
3524 │ </FIELD>
3525 │ <FIELD name="BPcorrrecnoS" ucd="meta.number" display="1" datatype="long" width="12"><!-- ucd="NUMBER" -->
3526 │ <DESCRIPTION>Starting record number in the separated bp_coeff_corr file (added by CDS) (bp_corr_start_recno)</DESCRIPTION>
3527 │ <LINK href=" http://axel.u-strasbg.fr/cgi-bin/gaia_dr3_bprp_coeff.cgi?print_header=true&mode[BPCorr][recno]=${BPcorrrec
│ noS}&mode[BPCorr][nrecords]=${BPcorrNelem}+/>
3528 │ </FIELD>
3529 │ <FIELD name="BPcorrNelem" ucd="meta.number" display="1" datatype="short" width="4"><!-- ucd="NUMBER" -->
3530 │ <DESCRIPTION>[0/1485] Number of elements in the separated bp_coeff_corr file (added by CDS) (always 1485 except for 1 line)
│ (bp_corr_n_elems)</DESCRIPTION>
3531 │ </FIELD>
3532 │ <FIELD name="RPrecnoS" ucd="meta.number" display="1" datatype="long" width="11"><!-- ucd="NUMBER" -->
3533 │ <DESCRIPTION>Starting record number in the separated rp_coeff/e_rp_coeff file (added by CDS) (rp_start_recno)</DESCRIPTION>
3534 │ <LINK href="http://axel.u-strasbg.fr/cgi-bin/gaia_dr3_bprp_coeff.cgi?print_header=true&mode[RP][recno]=${RPrecnoS}"/>
3535 │ </FIELD>
3536 │ <FIELD name="RPNelem" ucd="meta.number" display="1" datatype="short" width="2"><!-- ucd="NUMBER" -->
3537 │ <DESCRIPTION>Number of elements in the separated rp_coeff/e_rp_coeff file (always 55) (rp_n_elems)</DESCRIPTION>
3538 │ </FIELD>
3539 │ <FIELD name="RPcorrrecnoS" ucd="meta.number" display="1" datatype="long" width="12"><!-- ucd="NUMBER" -->
3540 │ <DESCRIPTION>Starting record number in the separated rp_coeff_corr file (added by CDS) (rp_corr_start_recno)</DESCRIPTION>
───────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
The investigation is on-going
