Solving the Issue #3 Yahoo! Datasource stoped to work

[charlesschaefer]

Looks like Yahoo! changed its API and it stopped to work with python requests library (also wget and others way to download files automatically). The solution is to simulate a browser request, changing the User-Agent header.

[sujitpal]

Wanted to mention here that I merged your fix into my fork of auquantoolkit and then tried to use it in the Coursera lab on momentum strategies and it still fails with the same error message as mentioned in Issue #3.

I git clone my fork and install it with pip install -e . . I also had to ln -s the backtester folder under the practice folder because of the way they were calling it. The error message comes from the following snippet:

tf = MyTradingFunctions()
tsParams = MomentumTradingParams(tf)
tradingSystem = TradingSystem(tsParams)

and the error message is:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Cell In[10], line 3
      1 tf = MyTradingFunctions()
      2 tsParams = MomentumTradingParams(tf)
----> 3 tradingSystem = TradingSystem(tsParams)

File ~/training-data-analyst/courses/ai-for-finance/practice/backtester/trading_system.py:35, in TradingSystem.__init__(self, tsParams)
     33 self.executionSystem = None
     34 self.orderPlacer = None
---> 35 self.dataParser = self.tsParams.getDataParser()
     36 self.executionSystem = self.tsParams.getExecutionSystem()
     37 self.orderPlacer = self.tsParams.getOrderPlacer()

File /opt/conda/lib/python3.10/site-packages/qq_training_wheels/momentum_trading.py:31, in MomentumTradingParams.getDataParser(self)
     27 '''
     28 Returns an instance of class DataParser. Source of data for instruments
     29 '''
     30 instrumentIds = self.__tradingFunctions.getSymbolsToTrade()
---> 31 return YahooStockDataSource(
     32     cachedFolderName = 'historicalData/',
     33     dataSetId = self.__dataSetId,
     34     instrumentIds = instrumentIds,
     35     startDateStr = self.__startDate,
     36     endDateStr = self.__endDate,
     37 )

File ~/training-data-analyst/courses/ai-for-finance/practice/backtester/dataSource/yahoo_data_source.py:119, in YahooStockDataSource.__init__(self, cachedFolderName, dataSetId, instrumentIds, startDateStr, endDateStr, event, adjustPrice, downloadId, liveUpdates, pad)
    117 self.event = event
    118 if liveUpdates:
--> 119     self._allTimes, self._groupedInstrumentUpdates = self.getGroupedInstrumentUpdates()
    120     self.processGroupedInstrumentUpdates()
    121     self._bookDataFeatureKeys = self.__bookDataByFeature.keys()

File ~/training-data-analyst/courses/ai-for-finance/practice/backtester/dataSource/data_source.py:67, in DataSource.getGroupedInstrumentUpdates(self)
     65 print('Processing data for stock: %s' % (instrumentId))
     66 fileName = self.getFileName(instrumentId)
---> 67 if not self.downloadAndAdjustData(instrumentId, fileName):
     68     continue
     69 with open(fileName) as f:

File ~/training-data-analyst/courses/ai-for-finance/practice/backtester/dataSource/yahoo_data_source.py:133, in YahooStockDataSource.downloadAndAdjustData(self, instrumentId, fileName)
    131 def downloadAndAdjustData(self, instrumentId, fileName):
    132     if not os.path.isfile(fileName):
--> 133         if not downloadFileFromYahoo(self._startDate, self._endDate, instrumentId, fileName):
    134             logError('Skipping %s:' % (instrumentId))
    135             return False

File ~/training-data-analyst/courses/ai-for-finance/practice/backtester/dataSource/data_source_utils.py:36, in downloadFileFromYahoo(startDate, endDate, instrumentId, fileName, event)
     34 def downloadFileFromYahoo(startDate, endDate, instrumentId, fileName, event='history'):
     35     logInfo('Downloading %s' % fileName)
---> 36     cookie, crumb = getCookieForYahoo(instrumentId)
     37     start = int(mktime(startDate.timetuple()))
     38     end = int(mktime(endDate.timetuple()))

File ~/training-data-analyst/courses/ai-for-finance/practice/backtester/dataSource/data_source_utils.py:23, in getCookieForYahoo(instrumentId)
     21 req = requests.get(url, headers=getHeadersForYahoo())
     22 txt = req.content
---> 23 cookie = req.cookies['B']
     24 pattern = re.compile('.*"CrumbStore":\{"crumb":"(?P<crumb>[^"]+)"\}')
     26 for line in txt.splitlines():

File /opt/conda/lib/python3.10/site-packages/requests/cookies.py:334, in RequestsCookieJar.__getitem__(self, name)
    327 def __getitem__(self, name):
    328     """Dict-like __getitem__() for compatibility with client code. Throws
    329     exception if there are more than one cookie with name. In that case,
    330     use the more explicit get() method instead.
    331 
    332     .. warning:: operation is O(n), not O(1).
    333     """
--> 334     return self._find_no_duplicates(name)

File /opt/conda/lib/python3.10/site-packages/requests/cookies.py:413, in RequestsCookieJar._find_no_duplicates(self, name, domain, path)
    411 if toReturn:
    412     return toReturn
--> 413 raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}")

KeyError: "name='B', domain=None, path=None"

Just noticed thatPR #6 and PR #7 are later than yours, I will try them and and see if it fixes things.