Result of any Strategy execution is not added to data frame after a strategy runs for a ticker/dataframe with insufficient data, all subsequent executions for different tickers also fail to add the columns

[amitach]

Which version are you running? The lastest version is on Github. Pip is for major releases.

>>> import pandas_ta as ta
>>> print(ta.version)
0.2.62b0

The bug Let's say we have a sample strategy defined for calculating the 20 EMA. This strategy will be run on 3 tickers identified by data, data1 and data2 data-frames respectively

• data has enough rows so the strategy runs fine and the 20_EMA column is added to the data frame
• data 1 doesn't have enough rows(1 row), the EMA_20 column is not added, ideally should be added with Nan
• data2 again has enough rows, upon executing the strategy the EMA_20 column isn't added
• In fact anything executed after data1 the EMA column doesn't get added.

To Reproduce

Please follow the comments before every line of code to follow the issue

#import pandas_ta
import pandas_ta as ta

#import pandas
import pandas as pd

#import yahoo finance
import yfinance as yf

# download first ticker which has sufficient data for 20ema
data = yf.download('INDIGOPNTS.NS')

[*********************100%***********************]  1 of 1 completed

# print data
data

	Open	High	Low	Close	Adj Close	Volume
Date
2021-02-02	2607.500000	3129.000000	2436.050049	3117.149902	3117.149902	12652036
2021-02-03	3239.000000	3329.949951	2831.250000	2924.250000	2924.250000	2955005
2021-02-04	2948.850098	2967.949951	2853.000000	2873.750000	2873.750000	476054
2021-02-05	2866.000000	2900.000000	2603.000000	2630.050049	2630.050049	587315
2021-02-08	2590.000000	2736.000000	2551.000000	2677.050049	2677.050049	507665
2021-02-09	2680.000000	2824.899902	2665.000000	2704.850098	2704.850098	478124
2021-02-10	2706.199951	2730.000000	2641.000000	2666.250000	2666.250000	167707
2021-02-11	2666.000000	2684.899902	2641.000000	2649.399902	2649.399902	73280
2021-02-12	2649.300049	2667.850098	2585.000000	2610.750000	2610.750000	88127
2021-02-15	2600.000000	2650.000000	2565.050049	2595.649902	2595.649902	128961
2021-02-16	2600.000000	2617.100098	2557.000000	2564.550049	2564.550049	68502
2021-02-17	2564.000000	2724.000000	2545.949951	2642.500000	2642.500000	230228
2021-02-18	2660.050049	2709.600098	2625.000000	2641.600098	2641.600098	110801
2021-02-19	2636.000000	2660.000000	2563.699951	2572.600098	2572.600098	66948
2021-02-22	2597.949951	2612.949951	2535.000000	2542.699951	2542.699951	50238
2021-02-23	2550.000000	2588.000000	2501.050049	2510.949951	2510.949951	56678
2021-02-24	2524.300049	2533.399902	2480.000000	2487.949951	2487.949951	34049
2021-02-25	2499.000000	2516.000000	2492.300049	2499.699951	2499.699951	66150
2021-02-26	2484.500000	2574.100098	2475.000000	2522.199951	2522.199951	80150
2021-03-01	2544.000000	2546.850098	2490.000000	2506.149902	2506.149902	330580
2021-03-02	2529.000000	2717.699951	2508.000000	2640.500000	2640.500000	201061
2021-03-03	2661.000000	2709.899902	2595.000000	2618.300049	2618.300049	101277
2021-03-04	2609.949951	2653.949951	2583.300049	2589.250000	2589.250000	48424
2021-03-05	2595.000000	2602.149902	2513.050049	2525.500000	2525.500000	129156
2021-03-08	2547.399902	2629.000000	2492.550049	2548.600098	2548.600098	118075
2021-03-09	2548.600098	2556.899902	2475.850098	2484.100098	2484.100098	105018
2021-03-10	2500.000000	2523.949951	2492.000000	2515.600098	2515.600098	51215
2021-03-12	2529.399902	2559.300049	2490.000000	2500.949951	2500.949951	51953
2021-03-15	2543.399902	2543.399902	2455.500000	2464.350098	2464.350098	38580
2021-03-16	2474.000000	2492.850098	2435.000000	2443.899902	2443.899902	71139
2021-03-17	2450.000000	2452.800049	2360.000000	2367.800049	2367.800049	50338
2021-03-18	2379.000000	2396.949951	2294.800049	2305.800049	2305.800049	91519
2021-03-19	2321.000000	2329.000000	2220.250000	2285.449951	2285.449951	117491
2021-03-22	2290.000000	2399.949951	2289.800049	2378.500000	2378.500000	166836
2021-03-23	2390.000000	2395.000000	2334.050049	2367.250000	2367.250000	40074
2021-03-24	2367.250000	2473.350098	2315.000000	2336.399902	2336.399902	143738
2021-03-25	2340.000000	2350.000000	2281.000000	2290.699951	2290.699951	39885
2021-03-26	2321.000000	2340.199951	2265.300049	2298.149902	2298.149902	57676
2021-03-30	2306.000000	2324.949951	2295.649902	2310.399902	2310.399902	28565
2021-03-31	2330.000000	2449.000000	2316.000000	2395.850098	2395.850098	106146
2021-04-01	2400.000000	2414.850098	2350.949951	2384.250000	2384.250000	15817
2021-04-05	2398.800049	2398.800049	2311.850098	2324.449951	2324.449951	26484
2021-04-06	2320.000000	2394.699951	2316.000000	2374.399902	2374.399902	50250
2021-04-07	2395.000000	2444.850098	2380.199951	2412.750000	2412.750000	53960

# Define a simple strategy, can be any strategy
EmaStrategy = ta.Strategy(
        name="20EMA",
        description="Simple 20ema",
        ta=[
            {"kind": "ema", "length": 20}
        ]
    )

#set multiprocessing to 0, otherwise its way too slow
data.ta.cores = 0

# execute the strategy for the first ticker
data.ta.strategy(EmaStrategy, timed=True, verbose=True)

[+] Strategy: 20EMA
[i] Indicator arguments: {'timed': True, 'append': True}
[i] No mulitproccessing (cores = 0).
[i] Total indicators: 1
[i] Columns added: 0
[i] Runtime: 3.0985 ms (0.0031 s)

# One indicator added EMA_20 as it can be seen below
data

	open	high	low	close	adj_close	volume	EMA_20
date
2021-02-02	2607.500000	3129.000000	2436.050049	3117.149902	3117.149902	12652036	NaN
2021-02-03	3239.000000	3329.949951	2831.250000	2924.250000	2924.250000	2955005	NaN
2021-02-04	2948.850098	2967.949951	2853.000000	2873.750000	2873.750000	476054	NaN
2021-02-05	2866.000000	2900.000000	2603.000000	2630.050049	2630.050049	587315	NaN
2021-02-08	2590.000000	2736.000000	2551.000000	2677.050049	2677.050049	507665	NaN
2021-02-09	2680.000000	2824.899902	2665.000000	2704.850098	2704.850098	478124	NaN
2021-02-10	2706.199951	2730.000000	2641.000000	2666.250000	2666.250000	167707	NaN
2021-02-11	2666.000000	2684.899902	2641.000000	2649.399902	2649.399902	73280	NaN
2021-02-12	2649.300049	2667.850098	2585.000000	2610.750000	2610.750000	88127	NaN
2021-02-15	2600.000000	2650.000000	2565.050049	2595.649902	2595.649902	128961	NaN
2021-02-16	2600.000000	2617.100098	2557.000000	2564.550049	2564.550049	68502	NaN
2021-02-17	2564.000000	2724.000000	2545.949951	2642.500000	2642.500000	230228	NaN
2021-02-18	2660.050049	2709.600098	2625.000000	2641.600098	2641.600098	110801	NaN
2021-02-19	2636.000000	2660.000000	2563.699951	2572.600098	2572.600098	66948	NaN
2021-02-22	2597.949951	2612.949951	2535.000000	2542.699951	2542.699951	50238	NaN
2021-02-23	2550.000000	2588.000000	2501.050049	2510.949951	2510.949951	56678	NaN
2021-02-24	2524.300049	2533.399902	2480.000000	2487.949951	2487.949951	34049	NaN
2021-02-25	2499.000000	2516.000000	2492.300049	2499.699951	2499.699951	66150	NaN
2021-02-26	2484.500000	2574.100098	2475.000000	2522.199951	2522.199951	80150	NaN
2021-03-01	2544.000000	2546.850098	2490.000000	2506.149902	2506.149902	330580	2647.002490
2021-03-02	2529.000000	2717.699951	2508.000000	2640.500000	2640.500000	201061	2646.383205
2021-03-03	2661.000000	2709.899902	2595.000000	2618.300049	2618.300049	101277	2643.708619
2021-03-04	2609.949951	2653.949951	2583.300049	2589.250000	2589.250000	48424	2638.522084
2021-03-05	2595.000000	2602.149902	2513.050049	2525.500000	2525.500000	129156	2627.758076
2021-03-08	2547.399902	2629.000000	2492.550049	2548.600098	2548.600098	118075	2620.219221
2021-03-09	2548.600098	2556.899902	2475.850098	2484.100098	2484.100098	105018	2607.255495
2021-03-10	2500.000000	2523.949951	2492.000000	2515.600098	2515.600098	51215	2598.526409
2021-03-12	2529.399902	2559.300049	2490.000000	2500.949951	2500.949951	51953	2589.233413
2021-03-15	2543.399902	2543.399902	2455.500000	2464.350098	2464.350098	38580	2577.339764
2021-03-16	2474.000000	2492.850098	2435.000000	2443.899902	2443.899902	71139	2564.631206
2021-03-17	2450.000000	2452.800049	2360.000000	2367.800049	2367.800049	50338	2545.885382
2021-03-18	2379.000000	2396.949951	2294.800049	2305.800049	2305.800049	91519	2523.020112
2021-03-19	2321.000000	2329.000000	2220.250000	2285.449951	2285.449951	117491	2500.394382
2021-03-22	2290.000000	2399.949951	2289.800049	2378.500000	2378.500000	166836	2488.785393
2021-03-23	2390.000000	2395.000000	2334.050049	2367.250000	2367.250000	40074	2477.210594
2021-03-24	2367.250000	2473.350098	2315.000000	2336.399902	2336.399902	143738	2463.800052
2021-03-25	2340.000000	2350.000000	2281.000000	2290.699951	2290.699951	39885	2447.314328
2021-03-26	2321.000000	2340.199951	2265.300049	2298.149902	2298.149902	57676	2433.108192
2021-03-30	2306.000000	2324.949951	2295.649902	2310.399902	2310.399902	28565	2421.421688
2021-03-31	2330.000000	2449.000000	2316.000000	2395.850098	2395.850098	106146	2418.986299
2021-04-01	2400.000000	2414.850098	2350.949951	2384.250000	2384.250000	15817	2415.678080
2021-04-05	2398.800049	2398.800049	2311.850098	2324.449951	2324.449951	26484	2406.989687
2021-04-06	2320.000000	2394.699951	2316.000000	2374.399902	2374.399902	50250	2403.885898
2021-04-07	2395.000000	2444.850098	2380.199951	2412.750000	2412.750000	53960	2404.730098

# Download second ticker which doesn't have data or has few rows available
data1=yf.download('GUJRAFFIA.NS',period='1y')
data1

[*********************100%***********************]  1 of 1 completed

	Open	High	Low	Close	Adj Close	Volume
Date
2021-04-07	41.5	43.549999	40.0	43.5	43.5	4125

# again set multiprocessing to 0
data1.ta.cores = 0

# only one row available
data1

	open	high	low	close	adj_close	volume
date
2021-04-07	41.5	43.549999	40.0	43.5	43.5	4125

# execute the strategy- No columns are added in this case
data1.ta.strategy(EmaStrategy, timed=True, verbose=True)

[+] Strategy: 20EMA
[i] Indicator arguments: {'timed': True, 'append': True}
[i] No mulitproccessing (cores = 0).
[i] Total indicators: 0
[i] Columns added: 0
[i] Runtime: 0.1025 ms (0.0001 s)

# No columns added, can be seen below
data1

	open	high	low	close	adj_close	volume
date
2021-04-07	41.5	43.549999	40.0	43.5	43.5	4125

# download third ticker which has sufficient data
data2 = yf.download('RELIANCE.NS', period='1y')

[*********************100%***********************]  1 of 1 completed

data2

	Open	High	Low	Close	Adj Close	Volume
Date
2020-04-07	1091.751343	1202.600586	1089.671021	1194.774780	1190.305420	25093419
2020-04-08	1168.919800	1217.459717	1149.107666	1180.955688	1176.537964	23019764
2020-04-09	1202.600586	1221.223999	1181.797729	1208.494629	1203.973877	15244658
2020-04-13	1192.644897	1203.591187	1168.919800	1177.983887	1173.577271	10925447
2020-04-15	1185.809692	1224.393921	1132.267212	1139.052856	1134.791870	16301024
...	...	...	...	...	...	...
2021-03-31	2018.000000	2049.899902	1999.000000	2003.099976	2003.099976	7499740
2021-04-01	2018.000000	2030.000000	2003.150024	2021.849976	2021.849976	5410307
2021-04-05	2024.949951	2025.000000	1962.099976	1992.599976	1992.599976	6864856
2021-04-06	2004.000000	2004.949951	1969.000000	1984.300049	1984.300049	6465241
2021-04-07	2000.000000	2046.900024	1993.300049	2002.849976	2002.849976	11196354

250 rows × 6 columns

# execute the strategy
data2.ta.strategy(EmaStrategy, timed=True, verbose=True)

[+] Strategy: 20EMA
[i] Indicator arguments: {'timed': True, 'append': True}
[i] Multiprocessing: 4 of 4 cores.
[i] Total indicators: 0
[i] Columns added: 0
[i] Runtime: 272.5645 ms (0.2726 s)

# as it can be seen, no column for the EMA_20 added for the strategy once it failed earlier, 
data2

	open	high	low	close	adj_close	volume
date
2020-04-07	1091.751343	1202.600586	1089.671021	1194.774780	1190.305420	25093419
2020-04-08	1168.919800	1217.459717	1149.107666	1180.955688	1176.537964	23019764
2020-04-09	1202.600586	1221.223999	1181.797729	1208.494629	1203.973877	15244658
2020-04-13	1192.644897	1203.591187	1168.919800	1177.983887	1173.577271	10925447
2020-04-15	1185.809692	1224.393921	1132.267212	1139.052856	1134.791870	16301024
...	...	...	...	...	...	...
2021-03-31	2018.000000	2049.899902	1999.000000	2003.099976	2003.099976	7499740
2021-04-01	2018.000000	2030.000000	2003.150024	2021.849976	2021.849976	5410307
2021-04-05	2024.949951	2025.000000	1962.099976	1992.599976	1992.599976	6864856
2021-04-06	2004.000000	2004.949951	1969.000000	1984.300049	1984.300049	6465241
2021-04-07	2000.000000	2046.900024	1993.300049	2002.849976	2002.849976	11196354

250 rows × 6 columns

Expected behavior* The EMA_20 column or any strategy for that matter should bed added for the subsequent execution of the strategy.

Additional context pandas_ta version: 0.2.62b0 python version: 3.9.4

Thanks for making Pandas TA!

[twopirllc]

Hello @amitach,

Thanks for using Pandas TA and providing a detailed analysis for this Issue! I will add it to my list.

Expected behavior* The EMA_20 column or any strategy for that matter should bed added for the subsequent execution of the strategy.

Unfortunately I currently am tied up with other _PR_s and Issues and do not have an estimated time to resolve this. If you would like to help solve this Issue and submit a PR to speed up the progress of this bug, that would be greatly appreciated! 😎

Kind Regards, KJ

[withersjn]

A workaround for subsequent executions is to re-create the strategy before it is used each time. This example pulls a list of ticker data data from Yahoo and outputs it in CSV files. ARKX is missing SMA_50 and SMA_200.

# Update_Stock_Metrics
# Necessary Libraries
import yfinance as yf, pandas as pd, shutil, os, time
import pandas_ta as ta
#
if __name__ == '__main__':  # Bug Fix for multithreading
    #
    # Create Stocks folder.  Obtain Stock History from Yahoo Finance in CSV files by ticker Symbol.
    #  
    tickers = ["AAPL","ABT","AMD","ARKF","ARKG","ARKK","ARKQ","ARKX",
    "CHPT","CRM","F","ICHR","KMX","LHX","MSFT","PBD","PBW","PEP",
    "PH","PHO","PKB","PKB","PLTR","PPA","PPL","PSJ","PWB","PXE","PXI",
    "QCOM","ROKU","SPHD","TSM","VLO","YUM"]
    # Check that the amount of tickers isn't more than 2000
    print("The amount of stocks chosen to observe: " + str(len(tickers)))
    # These two lines remove the Stocks folder and then recreate it in order to remove old stocks. 
    # Make sure you have created a Stocks Folder the first time you run this.
    shutil.rmtree("stocks")
    os.mkdir("stocks")
    # Use this folder for all data files
    os.chdir("stocks")
    # for use with Review_Stock_Metrics
    os.mkdir("graphs")
    #
    # Do not make more than 2,000 calls per hour or 48,000 calls per day or Yahoo Finance may block your IP. 
    # The clause "(Amount_of_API_Calls < 1800)" below will stop the loop from making too many calls to the yfinance API. 
    Stock_Failure = 0
    Stocks_Not_Imported = 0
    Amount_of_API_Calls=0
    #
    # Iterate through list of tickers
    i=0
    while (i < len(tickers)) and (Amount_of_API_Calls < 1800):
        try:
            print("Iteration = " + str(i))
            #
            stock = tickers[i]  # Gets the stock from ticker list
            # Load data
            temp = yf.Ticker(str(stock))
            df = temp.history(period="2y")  # Tells yfinance what kind of data we want about this stock (Use "Max" for all historical data)
            #
            # (1) Create the Strategy
            MyStrategy = ta.Strategy(
                name="Momo and Volatility",
                description="SMA 50,200, BBANDS, RSI, MACD and Volume SMA 20",
                ta=[
                    {"kind": "sma", "close": "close", "length": 50, "prefix": "Close"},
                    {"kind": "sma", "close": "close", "length": 200, "prefix": "Close"},
                    {"kind": "bbands", "length": 20},
                    {"kind": "rsi"},
                    {"kind": "macd", "fast": 8, "slow": 21},
                    {"kind": "sma", "close": "volume", "length": 20, "prefix": "VOLUME"},                
                    ]   
                )
            # 
            # (2) Run the Strategy
            df.ta.strategy(MyStrategy)
            #df.ta.strategy("all")  # This is the kitchen sink strategy
            #
            # Output data to CSV file 
            df.to_csv(stock+".csv")  # Saves the historical data in csv format for further processing later
            time.sleep(1)  # Pauses the loop for 1 second so we don't cause issues with Yahoo Finance's backend operations
            Amount_of_API_Calls += 1 
            Stock_Failure = 0
            i += 1  # Iteration to the next ticker
        except ValueError:
            print("Yahoo Finance Backend Error, Attempting to Fix")  # An error occured on Yahoo Finance's backend. We will attempt to retreive the data again
            if Stock_Failure > 5:  # Move on to the next ticker if the current ticker fails more than 5 times
                i+=1
                Stocks_Not_Imported += 1
            Amount_of_API_Calls += 1
            Stock_Failure += 1
        # Handle SSL error
        except requests.exceptions.SSLError as e:
            print("Yahoo Finance Backend Error, Attempting to Fix SSL")  # An error occured on Yahoo Finance's backend. We will attempt to retreive the data again
            if Stock_Failure > 5:  # Move on to the next ticker if the current ticker fails more than 5 times
                i+=1
                Stocks_Not_Imported += 1
            Amount_of_API_Calls += 1
            Stock_Failure += 1
    print("The amount of stocks we successfully imported: " + str(i - Stocks_Not_Imported))