requests-html
requests-html copied to clipboard
Support for passing browser options to pyppeteer
This is really useful, also faced the same issue when I wanted to pass executablePath
argument when running in docker container.
Would love to see this merged.
In my case I decided to monkey patch over the original library:
from requests_html import BaseSession, HTMLSession
import asyncio
import pyppeteer
class MyBaseSession(BaseSession):
def __init__(self, *args, **kwargs):
super(MyBaseSession, self).__init__(*args, **kwargs)
@property
async def browser(self):
if not hasattr(self, "_browser"):
# override browser args to pass default chromium location
self._browser = await pyppeteer.launch(executablePath="/usr/bin/chromium", ignoreHTTPSErrors=not(self.verify), headless=True, args=['--no-sandbox'])
return self._browser
# multiple inheritance: due to how super() works
# property "browser" is searched for in HTMLSession then instead of going to BaseSession.browser as normal
# it goes to MyBaseSession where we have it overriden to pass custom pyppeteer args
class MyHTMLSession(HTMLSession, MyBaseSession):
def __init__(self, **kwargs):
super(MyHTMLSession, self).__init__(**kwargs)