RuntimeError: Index put requires the source and destination dtypes match, got Float for the destination and Double for the source.

[JosePeeterson]

• PyTorch-Forecasting version: 0.10.2
• PyTorch version:1.12.1
• Python version:3.10.4
• Operating System: windows

Expected behavior

No Error

Actual behavior

The Error is

File c:\Users\josepeeterson.er\Miniconda3\envs\pytorch\lib\site-packages\pytorch_forecasting\models\deepar_init_.py:292, in DeepAR.decode..decode_one(idx, lagged_targets, hidden_state) 286 def decode_one( 287 idx, 288 lagged_targets, 289 hidden_state, 290 ): 291 x = input_vector[:, [idx]] --> 292 x[:, 0, target_pos] = lagged_targets[-1] 293 for lag, lag_positions in lagged_target_positions.items(): 294 if idx > lag:

RuntimeError: Index put requires the source and destination dtypes match, got Float for the destination and Double for the source.

How do I set them to be of same datatype? What is the index here? This is happening internally. I do not have control over this. I am not using any GPUs.

The link to the .csv file with input data is https://github.com/JosePeeterson/Demand_forecasting The data is just sampled from a negative binomila distribution wiht parameters (9,0.5) every 4 hours. the time inbetween is all zero. I just want to see if DeepAR can learn this pattern.

Code to reproduce the problem

from pytorch_forecasting.data.examples import generate_ar_data
import matplotlib.pyplot as plt
import pandas as pd
from pytorch_forecasting.data import TimeSeriesDataSet
from pytorch_forecasting.data import NaNLabelEncoder
from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
import pytorch_lightning as pl
from pytorch_forecasting import NegativeBinomialDistributionLoss, DeepAR
import torch
from pytorch_forecasting.data.encoders import TorchNormalizer

data = [pd.read_csv('1_f_nbinom_train.csv')]

data["date"] = pd.Timestamp("2021-08-24") + pd.to_timedelta(data.time_idx, "H")
data['_hour_of_day'] = str(data["date"].dt.hour)
data['_day_of_week'] = str(data["date"].dt.dayofweek)
data['_day_of_month'] = str(data["date"].dt.day)
data['_day_of_year'] = str(data["date"].dt.dayofyear)
data['_week_of_year'] = str(data["date"].dt.weekofyear)
data['_month_of_year'] = str(data["date"].dt.month)
data['_year'] = str(data["date"].dt.year)

max_encoder_length = 60
max_prediction_length = 20
training_cutoff = data["time_idx"].max() - max_prediction_length

training = TimeSeriesDataSet(
    data.iloc[0:-620],
    time_idx="time_idx",
    target="value",
    categorical_encoders={"series": NaNLabelEncoder(add_nan=True).fit(data.series), "_hour_of_day": NaNLabelEncoder(add_nan=True).fit(data._hour_of_day), \
       "_day_of_week": NaNLabelEncoder(add_nan=True).fit(data._day_of_week), "_day_of_month" : NaNLabelEncoder(add_nan=True).fit(data._day_of_month), "_day_of_year" : NaNLabelEncoder(add_nan=True).fit(data._day_of_year), \
        "_week_of_year": NaNLabelEncoder(add_nan=True).fit(data._week_of_year), "_year": NaNLabelEncoder(add_nan=True).fit(data._year)},
    group_ids=["series"],
    min_encoder_length=max_encoder_length,
    max_encoder_length=max_encoder_length,
    min_prediction_length=max_prediction_length,
    max_prediction_length=max_prediction_length,
    time_varying_unknown_reals=["value"],
    time_varying_known_categoricals=["_hour_of_day","_day_of_week","_day_of_month","_day_of_year","_week_of_year","_year" ],
    time_varying_known_reals=["time_idx"],
    add_relative_time_idx=False,
    randomize_length=None,
    scalers=[],
    target_normalizer=TorchNormalizer(method="identity",center=False,transformation=None )

)

validation = TimeSeriesDataSet.from_dataset(
    training,
    data.iloc[-620:-420],
    # predict=True,
    stop_randomization=True,
)       


batch_size = 64
train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=8)
val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=8)


# save datasets
training.save("training.pkl")
validation.save("validation.pkl")


early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=5, verbose=False, mode="min")
lr_logger = LearningRateMonitor()


trainer = pl.Trainer(
    max_epochs=10,
    gpus=0,
    gradient_clip_val=0.1,
    limit_train_batches=30,
    limit_val_batches=3,
    # fast_dev_run=True,
    # logger=logger,
    # profiler=True,
    callbacks=[lr_logger, early_stop_callback],
)


deepar = DeepAR.from_dataset(
    training,
    learning_rate=0.1,
    hidden_size=32,
    dropout=0.1,
    loss=NegativeBinomialDistributionLoss(),
    log_interval=10,
    log_val_interval=3,
    # reduce_on_plateau_patience=3,
)
print(f"Number of parameters in network: {deepar.size()/1e3:.1f}k")



torch.set_num_threads(10)
trainer.fit(
    deepar,
    train_dataloaders=train_dataloader,
    val_dataloaders=val_dataloader,
)

[JosePeeterson]

Change \Miniconda3\envs\pytorch\Lib\site-packages\pytorch_forecasting\models\deepar_init_.py as follows

        # define function to run at every decoding step
        def decode_one(
            idx,
            lagged_targets,
            hidden_state,
        ):
            x = torch.as_tensor(input_vector[:, [idx]],dtype=torch.float32)
            x[:, 0, target_pos] = torch.as_tensor(lagged_targets[-1],dtype=torch.float32)
            for lag, lag_positions in lagged_target_positions.items():
                if idx > lag:
                    x[:, 0, lag_positions] = lagged_targets[-lag]
            prediction, hidden_state = self.decode_all(x, hidden_state)
            prediction = apply_to_list(prediction, lambda x: x[:, 0])  # select first time step
            return prediction, hidden_state

[mirko-m]

I am also running into this issue when I use the NegativeBinomialDistributionLoss , but things work fine when I use the NormalDistributionLoss instead.

[mirko-m]

In my case I was able to fix the problem by changing the following line

target_normalizer=TorchNormalizer(method="identity",center=False,transformation=None )

to

target_normalizer=EncoderNormalizer(method="identity", center=False, transformation=None)

when defining the TimeSeriesDataSet.

The underlying issue appears to be that when using TorchNormalizer the type of x["target_scale"] is torch.float.64 whereas that of x["decoder_target"] is torch.float32. By x I mean the dictionary of tensors that is fed to the model, i.e.

x = next(iter(val_dataloader))[0]

[JosePeeterson]

Hi @mirko-m,

I am able to use negative binomial distribution with the following training routine. I am able to keep

target_normalizer=TorchNormalizer(method="identity",center=False,transformation=None )

for negbinom. distribution.

#early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=p, verbose=False, mode="min")
lr_logger = LearningRateMonitor()

for neu,lay,bat,lr,enc_len,pred_len,drop,num_ep in product(*[x for x in hparams_grid.values()]):
     
  cat_dict = {"_hour_of_day": NaNLabelEncoder(add_nan=True).fit(df._hour_of_day), \
  "_day_of_week": NaNLabelEncoder(add_nan=True).fit(df._day_of_week), "_day_of_month" : NaNLabelEncoder(add_nan=True).fit(df._day_of_month), "_day_of_year" : NaNLabelEncoder(add_nan=True).fit(df._day_of_year), \
      "_week_of_year": NaNLabelEncoder(add_nan=True).fit(df._week_of_year), "_month_of_year": NaNLabelEncoder(add_nan=True).fit(df._month_of_year) ,"_year": NaNLabelEncoder(add_nan=True).fit(df._year),\
  }#cat_col1: NaNLabelEncoder(add_nan=True).fit(df[cat_col1])}
  cat_list = ["_hour_of_day","_day_of_week","_day_of_month","_day_of_year","_week_of_year","_month_of_year","_year"]#,cat_col1]


  train_dataset = TimeSeriesDataSet(
      new_df_pkl.iloc[0:tr_stop_idx],
      time_idx="time_idx",
      target=Target,
      categorical_encoders=cat_dict,
      group_ids=["group"],
      min_encoder_length=enc_len,
      max_encoder_length=enc_len,
      min_prediction_length=pred_len,
      max_prediction_length=pred_len,
      time_varying_unknown_reals=[Target],
      time_varying_known_reals=num_cols_list,
      time_varying_known_categoricals=cat_list,
      add_relative_time_idx=False,
      randomize_length=False,
      scalers={},
      target_normalizer=TorchNormalizer(method="identity",center=False,transformation=None )

  )

  val_dataset = TimeSeriesDataSet.from_dataset(train_dataset,new_df_pkl.iloc[tr_stop_idx:val_stop_idx], stop_randomization=True, predict=False)
  test_dataset = TimeSeriesDataSet.from_dataset(train_dataset,new_df_pkl.iloc[val_stop_idx:tes_stop_idx], stop_randomization=True)

  train_dataloader = train_dataset.to_dataloader(train=True, batch_size=bat)
  val_dataloader = val_dataset.to_dataloader(train=False, batch_size=bat)
  test_dataloader = test_dataset.to_dataloader(train=False, batch_size=bat)

  """
  Data loading sanity check

  """

  # ####### View the entire data using this code ##################
  # print(train_dataset.data['categoricals'])
  # ####### View the data entire using this code ##################

  # ####### View the data for each batch using this code ##################
  # for x,y in iter(train_dataloader):
  #     print(x)
  #     break
  # ####### View the data for each batch using this code ##################

  trainer = pl.Trainer(
      max_epochs=num_ep,
      gpus=0, #-1,0
      #auto_lr_find=True,
      gradient_clip_val=0.1,
      limit_train_batches=1.0,
      limit_val_batches=1.0,
      #fast_dev_run=fdv_steps,
      logger=True,
      #log_every_n_steps=10,
      # profiler=True,
      callbacks=[lr_logger],
      enable_checkpointing=True)

  #print(f"training routing:\n \n {trainer}")
  deepar = DeepAR.from_dataset(
      train_dataset,
      learning_rate=lr,
      hidden_size=neu,
      rnn_layers=lay,
      dropout=drop,
      loss=Loss,
      log_interval=10,
      log_val_interval=3,
      log_gradient_flow=False,
      # reduce_on_plateau_patience=3,
  )


  #print(f"Number of parameters in network: {deepar.size()/1e3:.1f}k")
  # print(f"Model :\n \n {deepar}")
  torch.set_num_threads(10)
  trainer.fit(
      deepar,
      train_dataloaders=train_dataloader,
      val_dataloaders=val_dataloader,
  )

  ########## Prediction #####################
  test_output = deepar.predict(data=test_dataloader,mode='prediction',return_index=True,num_workers=8,show_progress_bar=True)
  #print('test_output shape',test_output.shape)



  RMSE_list = np.array([])
  #RMSE_preak_pred_list = np.array([])
  days = test_output[1].shape[0]
  for j in range(0,days,pred_len):

    x = new_df_pkl.index[ (new_df_pkl['group'] ==  test_output[1]['group'][j])].tolist()
    y = new_df_pkl.index[ (new_df_pkl['time_idx'] == test_output[1]['time_idx'][j]) ].tolist()
    pred_start_idx = list(set(x).intersection(y))[0]
    #pred_start_idx = int(test_output[1]['time_idx'][j])

    print('pred_start_idx ',pred_start_idx)

    actual = new_df_pkl[Target].iloc[pred_start_idx:pred_start_idx+pred_len].values
    plt.plot(actual)

    pred = np.array(np.rint(test_output[0][j])).astype(int)
    plt.plot(pred)
    plt.show()

    RMSE = np.sqrt(mean_squared_error(actual,pred ))
    print('RMSE : ',RMSE)
    RMSE_list = np.append(RMSE_list,RMSE)

    # peak prediction metric
    # error wrt 5 largest peaks 
    #peak_idx = np.argsort(actual)
    #rmse_peak_pred = np.sqrt(mean_squared_error(actual[peak_idx[-5:]] , pred[peak_idx[-5:]]))
    #print('RMSE_peak_pred : ',rmse_peak_pred)
    #RMSE_preak_pred_list = np.append(RMSE_preak_pred_list,rmse_peak_pred)

  print(f'Average RMSE for {days} days: ',np.mean(RMSE_list))
  #print(f'Average RMSE_peak_pred for {days} days: ',np.mean(RMSE_preak_pred_list))

  print('\n Hyperparameter: param_comb_cnt,neu,lay,bat,lr,enc_len,pred_len,drop,num_ep,df_cov_col1,df_cov_col2\n')
  print(param_comb_cnt,neu,lay,bat,lr,enc_len,pred_len,drop,num_ep,inflow,' \n')
  ########## Prediction #####################

If you would like to discuss further about this you can message me at [email protected]

[mirko-m]

Thanks, Do you happen to know what actually fixed the error? Is it something that is different in the way in which you instantiate the TimeSeriesDataSet?