""" :mod:`pandas.io.html` is a module containing functionality for dealing with HTML IO. """ from __future__ import annotations from collections import abc import numbers import os import re from typing import ( Pattern, Sequence, ) from pandas._typing import FilePathOrBuffer from pandas.compat._optional import import_optional_dependency from pandas.errors import ( AbstractMethodError, EmptyDataError, ) from pandas.util._decorators import deprecate_nonkeyword_arguments from pandas.core.dtypes.common import is_list_like from pandas.core.construction import create_series_with_explicit_dtype from pandas.core.frame import DataFrame from pandas.io.common import ( is_url, stringify_path, urlopen, validate_header_arg, ) from pandas.io.formats.printing import pprint_thing from pandas.io.parsers import TextParser _IMPORTS = False _HAS_BS4 = False _HAS_LXML = False _HAS_HTML5LIB = False def _importers(): # import things we need # but make this done on a first use basis global _IMPORTS if _IMPORTS: return global _HAS_BS4, _HAS_LXML, _HAS_HTML5LIB bs4 = import_optional_dependency("bs4", errors="ignore") _HAS_BS4 = bs4 is not None lxml = import_optional_dependency("lxml.etree", errors="ignore") _HAS_LXML = lxml is not None html5lib = import_optional_dependency("html5lib", errors="ignore") _HAS_HTML5LIB = html5lib is not None _IMPORTS = True ############# # READ HTML # ############# _RE_WHITESPACE = re.compile(r"[\r\n]+|\s{2,}") def _remove_whitespace(s: str, regex=_RE_WHITESPACE) -> str: """ Replace extra whitespace inside of a string with a single space. Parameters ---------- s : str or unicode The string from which to remove extra whitespace. regex : re.Pattern The regular expression to use to remove extra whitespace. Returns ------- subd : str or unicode `s` with all extra whitespace replaced with a single space. """ return regex.sub(" ", s.strip()) def _get_skiprows(skiprows): """ Get an iterator given an integer, slice or container. Parameters ---------- skiprows : int, slice, container The iterator to use to skip rows; can also be a slice. Raises ------ TypeError * If `skiprows` is not a slice, integer, or Container Returns ------- it : iterable A proper iterator to use to skip rows of a DataFrame. """ if isinstance(skiprows, slice): start, step = skiprows.start or 0, skiprows.step or 1 return list(range(start, skiprows.stop, step)) elif isinstance(skiprows, numbers.Integral) or is_list_like(skiprows): return skiprows elif skiprows is None: return 0 raise TypeError(f"{type(skiprows).__name__} is not a valid type for skipping rows") def _read(obj): """ Try to read from a url, file or string. Parameters ---------- obj : str, unicode, or file-like Returns ------- raw_text : str """ if is_url(obj): with urlopen(obj) as url: text = url.read() elif hasattr(obj, "read"): text = obj.read() elif isinstance(obj, (str, bytes)): text = obj try: if os.path.isfile(text): with open(text, "rb") as f: return f.read() except (TypeError, ValueError): pass else: raise TypeError(f"Cannot read object of type '{type(obj).__name__}'") return text class _HtmlFrameParser: """ Base class for parsers that parse HTML into DataFrames. Parameters ---------- io : str or file-like This can be either a string of raw HTML, a valid URL using the HTTP, FTP, or FILE protocols or a file-like object. match : str or regex The text to match in the document. attrs : dict List of HTML
- Move rows from bottom of body to footer only if all elements inside row are |
"""
header_rows = self._parse_thead_tr(table_html)
body_rows = self._parse_tbody_tr(table_html)
footer_rows = self._parse_tfoot_tr(table_html)
def row_is_all_th(row):
return all(self._equals_tag(t, "th") for t in self._parse_td(row))
if not header_rows:
# The table has no . Move the top all- rows from
# body_rows to header_rows. (This is a common case because many
# tables in the wild have no or | |
while body_rows and row_is_all_th(body_rows[0]):
header_rows.append(body_rows.pop(0))
header = self._expand_colspan_rowspan(header_rows)
body = self._expand_colspan_rowspan(body_rows)
footer = self._expand_colspan_rowspan(footer_rows)
return header, body, footer
def _expand_colspan_rowspan(self, rows):
"""
Given a list of |||
---|---|---|---|---|---|
while remainder and remainder[0][0] <= index: prev_i, prev_text, prev_rowspan = remainder.pop(0) texts.append(prev_text) if prev_rowspan > 1: next_remainder.append((prev_i, prev_text, prev_rowspan - 1)) index += 1 # Append the text from this | , colspan times text = _remove_whitespace(self._text_getter(td)) rowspan = int(self._attr_getter(td, "rowspan") or 1) colspan = int(self._attr_getter(td, "colspan") or 1) for _ in range(colspan): texts.append(text) if rowspan > 1: next_remainder.append((index, text, rowspan - 1)) index += 1 # Append texts from previous rows at the final position for prev_i, prev_text, prev_rowspan in remainder: texts.append(prev_text) if prev_rowspan > 1: next_remainder.append((prev_i, prev_text, prev_rowspan - 1)) all_texts.append(texts) remainder = next_remainder # Append rows that only appear because the previous row had non-1 # rowspan while remainder: next_remainder = [] texts = [] for prev_i, prev_text, prev_rowspan in remainder: texts.append(prev_text) if prev_rowspan > 1: next_remainder.append((prev_i, prev_text, prev_rowspan - 1)) all_texts.append(texts) remainder = next_remainder return all_texts def _handle_hidden_tables(self, tbl_list, attr_name): """ Return list of tables, potentially removing hidden elements Parameters ---------- tbl_list : list of node-like Type of list elements will vary depending upon parser used attr_name : str Name of the accessor for retrieving HTML attributes Returns ------- list of node-like Return type matches `tbl_list` """ if not self.displayed_only: return tbl_list return [ x for x in tbl_list if "display:none" not in getattr(x, attr_name).get("style", "").replace(" ", "") ] class _BeautifulSoupHtml5LibFrameParser(_HtmlFrameParser): """ HTML to DataFrame parser that uses BeautifulSoup under the hood. See Also -------- pandas.io.html._HtmlFrameParser pandas.io.html._LxmlFrameParser Notes ----- Documentation strings for this class are in the base class :class:`pandas.io.html._HtmlFrameParser`. """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) from bs4 import SoupStrainer self._strainer = SoupStrainer("table") def _parse_tables(self, doc, match, attrs): element_name = self._strainer.name tables = doc.find_all(element_name, attrs=attrs) if not tables: raise ValueError("No tables found") result = [] unique_tables = set() tables = self._handle_hidden_tables(tables, "attrs") for table in tables: if self.displayed_only: for elem in table.find_all(style=re.compile(r"display:\s*none")): elem.decompose() if table not in unique_tables and table.find(text=match) is not None: result.append(table) unique_tables.add(table) if not result: raise ValueError(f"No tables found matching pattern {repr(match.pattern)}") return result def _text_getter(self, obj): return obj.text def _equals_tag(self, obj, tag): return obj.name == tag def _parse_td(self, row): return row.find_all(("td", "th"), recursive=False) def _parse_thead_tr(self, table): return table.select("thead tr") def _parse_tbody_tr(self, table): from_tbody = table.select("tbody tr") from_root = table.find_all("tr", recursive=False) # HTML spec: at most one of these lists has content return from_tbody + from_root def _parse_tfoot_tr(self, table): return table.select("tfoot tr") def _setup_build_doc(self): raw_text = _read(self.io) if not raw_text: raise ValueError(f"No text parsed from document: {self.io}") return raw_text def _build_doc(self): from bs4 import BeautifulSoup bdoc = self._setup_build_doc() if isinstance(bdoc, bytes) and self.encoding is not None: udoc = bdoc.decode(self.encoding) from_encoding = None else: udoc = bdoc from_encoding = self.encoding return BeautifulSoup(udoc, features="html5lib", from_encoding=from_encoding) def _build_xpath_expr(attrs) -> str: """ Build an xpath expression to simulate bs4's ability to pass in kwargs to search for attributes when using the lxml parser. Parameters ---------- attrs : dict A dict of HTML attributes. These are NOT checked for validity. Returns ------- expr : unicode An XPath expression that checks for the given HTML attributes. """ # give class attribute as class_ because class is a python keyword if "class_" in attrs: attrs["class"] = attrs.pop("class_") s = " and ".join(f"@{k}={repr(v)}" for k, v in attrs.items()) return f"[{s}]" _re_namespace = {"re": "http://exslt.org/regular-expressions"} _valid_schemes = "http", "file", "ftp" class _LxmlFrameParser(_HtmlFrameParser): """ HTML to DataFrame parser that uses lxml under the hood. Warning ------- This parser can only handle HTTP, FTP, and FILE urls. See Also -------- _HtmlFrameParser _BeautifulSoupLxmlFrameParser Notes ----- Documentation strings for this class are in the base class :class:`_HtmlFrameParser`. """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def _text_getter(self, obj): return obj.text_content() def _parse_td(self, row): # Look for direct children only: the "row" element here may be a # or | foo | bar | . (Missing||
-only rows
if header is None:
if len(head) == 1:
header = 0
else:
# ignore all-empty-text rows
header = [i for i, row in enumerate(head) if any(text for text in row)]
if foot:
body += foot
# fill out elements of body that are "ragged"
_expand_elements(body)
with TextParser(body, header=header, **kwargs) as tp:
return tp.read()
_valid_parsers = {
"lxml": _LxmlFrameParser,
None: _LxmlFrameParser,
"html5lib": _BeautifulSoupHtml5LibFrameParser,
"bs4": _BeautifulSoupHtml5LibFrameParser,
}
def _parser_dispatch(flavor):
"""
Choose the parser based on the input flavor.
Parameters
----------
flavor : str
The type of parser to use. This must be a valid backend.
Returns
-------
cls : _HtmlFrameParser subclass
The parser class based on the requested input flavor.
Raises
------
ValueError
* If `flavor` is not a valid backend.
ImportError
* If you do not have the requested `flavor`
"""
valid_parsers = list(_valid_parsers.keys())
if flavor not in valid_parsers:
raise ValueError(
f"{repr(flavor)} is not a valid flavor, valid flavors are {valid_parsers}"
)
if flavor in ("bs4", "html5lib"):
if not _HAS_HTML5LIB:
raise ImportError("html5lib not found, please install it")
if not _HAS_BS4:
raise ImportError("BeautifulSoup4 (bs4) not found, please install it")
# Although we call this above, we want to raise here right before use.
bs4 = import_optional_dependency("bs4") # noqa:F841
else:
if not _HAS_LXML:
raise ImportError("lxml not found, please install it")
return _valid_parsers[flavor]
def _print_as_set(s) -> str:
arg = ", ".join(pprint_thing(el) for el in s)
return f"{{{arg}}}"
def _validate_flavor(flavor):
if flavor is None:
flavor = "lxml", "bs4"
elif isinstance(flavor, str):
flavor = (flavor,)
elif isinstance(flavor, abc.Iterable):
if not all(isinstance(flav, str) for flav in flavor):
raise TypeError(
f"Object of type {repr(type(flavor).__name__)} "
f"is not an iterable of strings"
)
else:
msg = repr(flavor) if isinstance(flavor, str) else str(flavor)
msg += " is not a valid flavor"
raise ValueError(msg)
flavor = tuple(flavor)
valid_flavors = set(_valid_parsers)
flavor_set = set(flavor)
if not flavor_set & valid_flavors:
raise ValueError(
f"{_print_as_set(flavor_set)} is not a valid set of flavors, valid "
f"flavors are {_print_as_set(valid_flavors)}"
)
return flavor
def _parse(flavor, io, match, attrs, encoding, displayed_only, **kwargs):
flavor = _validate_flavor(flavor)
compiled_match = re.compile(match) # you can pass a compiled regex here
retained = None
for flav in flavor:
parser = _parser_dispatch(flav)
p = parser(io, compiled_match, attrs, encoding, displayed_only)
try:
tables = p.parse_tables()
except ValueError as caught:
# if `io` is an io-like object, check if it's seekable
# and try to rewind it before trying the next parser
if hasattr(io, "seekable") and io.seekable():
io.seek(0)
elif hasattr(io, "seekable") and not io.seekable():
# if we couldn't rewind it, let the user know
raise ValueError(
f"The flavor {flav} failed to parse your input. "
"Since you passed a non-rewindable file "
"object, we can't rewind it to try "
"another parser. Try read_html() with a different flavor."
) from caught
retained = caught
else:
break
else:
assert retained is not None # for mypy
raise retained
ret = []
for table in tables:
try:
ret.append(_data_to_frame(data=table, **kwargs))
except EmptyDataError: # empty table
continue
return ret
@deprecate_nonkeyword_arguments(version="2.0")
def read_html(
io: FilePathOrBuffer,
match: str | Pattern = ".+",
flavor: str | None = None,
header: int | Sequence[int] | None = None,
index_col: int | Sequence[int] | None = None,
skiprows: int | Sequence[int] | slice | None = None,
attrs: dict[str, str] | None = None,
parse_dates: bool = False,
thousands: str | None = ",",
encoding: str | None = None,
decimal: str = ".",
converters: dict | None = None,
na_values=None,
keep_default_na: bool = True,
displayed_only: bool = True,
) -> list[DataFrame]:
r"""
Read HTML tables into a ``list`` of ``DataFrame`` objects.
Parameters
----------
io : str, path object or file-like object
A URL, a file-like object, or a raw string containing HTML. Note that
lxml only accepts the http, ftp and file url protocols. If you have a
URL that starts with ``'https'`` you might try removing the ``'s'``.
match : str or compiled regular expression, optional
The set of tables containing text matching this regex or string will be
returned. Unless the HTML is extremely simple you will probably need to
pass a non-empty string here. Defaults to '.+' (match any non-empty
string). The default value will return all tables contained on a page.
This value is converted to a regular expression so that there is
consistent behavior between Beautiful Soup and lxml.
flavor : str, optional
The parsing engine to use. 'bs4' and 'html5lib' are synonymous with
each other, they are both there for backwards compatibility. The
default of ``None`` tries to use ``lxml`` to parse and if that fails it
falls back on ``bs4`` + ``html5lib``.
header : int or list-like, optional
The row (or list of rows for a :class:`~pandas.MultiIndex`) to use to
make the columns headers.
index_col : int or list-like, optional
The column (or list of columns) to use to create the index.
skiprows : int, list-like or slice, optional
Number of rows to skip after parsing the column integer. 0-based. If a
sequence of integers or a slice is given, will skip the rows indexed by
that sequence. Note that a single element sequence means 'skip the nth
row' whereas an integer means 'skip n rows'.
attrs : dict, optional
This is a dictionary of attributes that you can pass to use to identify
the table in the HTML. These are not checked for validity before being
passed to lxml or Beautiful Soup. However, these attributes must be
valid HTML table attributes to work correctly. For example, ::
attrs = {'id': 'table'}
is a valid attribute dictionary because the 'id' HTML tag attribute is
a valid HTML attribute for *any* HTML tag as per `this document
|