Different values for MASE in between accuracy and modeltime_accuracy

[mabuimo]

Hi all, I'm not sure if there is a bug or, but think it is worth sharing that I am spotting differences in the MASE results comparing the output of accuracy() and modeltime_accuracy()

A side question I have is if Theil U and R2 make sense for appraising the accuracy of an ETS model. Usually I just look into RMSE and MASE. Many thanks.

library(tidymodels)
library(tidyverse)
library(lubridate)
library(timetk)
library(modeltime)
library(forecast)

# Data
m750 <- m4_monthly %>% filter(id == "M750") %>% select(-id)

# Split Data 80/20
splits <- initial_time_split(m750, prop = 0.8)
# <Analysis/Assess/Total>
# <244/62/306>
train_tbl <- training(splits)
test_tbl  <- testing(splits)

# --- MODELS ---
# Model 1: exp ----
model_fit_ets <- exp_smoothing(
  error  = "auto",
  trend  = "auto",
  season = "auto",
  damping = "auto"
) %>% 
  set_engine("ets") %>% 
  fit(value ~ date, data = training(splits))

# ---- MODELTIME TABLE ----

exp_models <- modeltime_table(
  model_fit_ets
)

# ---- ACCURACY ----

# * Calibration ----
calibration_tbl <- exp_models %>% 
  modeltime_calibrate(test_tbl)

accuracy_tbl <- calibration_tbl %>% modeltime_accuracy()

calibration_tbl %>% 
  modeltime_forecast(
    new_data = test_tbl,
    actual_data = m750
  ) %>% 
  plot_modeltime_forecast()

# ----

# ETS exponential smoothing state space models, from forecast
train_2 <- m750 %>% head(244)
test_2 <- m750 %>% tail(62)

train_2 <- train_2 %>% 
 select(value) %>%
  data.frame() %>%
  ts(start = c(1990, 1), frequency = 12)

test_2 <- test_2 %>% 
  select(value) %>%
  data.frame() %>%
  ts(start = c(2010, 5), frequency = 12)

ets_model<- ets(train_2)
f_ets_model <- fabletools::forecast(ets_model, new_data = test_2, h = 62)

f_ets_model
autoplot(f_ets_model)
fabletools::accuracy(f_ets_model, test_2)

# ME     RMSE      MAE       MPE     MAPE      MASE       ACF1 Theil's U
# Training set   -9.705894 154.1525 108.8509 -0.136685 1.288250 0.3475082 0.09251034        NA
# Test set     -784.869458 834.2611 786.3949 -7.621448 7.635691 2.5105769 0.53315224  1.564927  1.564927

accuracy_tbl

# A tibble: 1 x 9
# .model_id .model_desc .type   mae  mape  mase smape  rmse   rsq
# <int> <chr>       <chr> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>
#   1         1 ETS(M,A,A)  Test   786.  7.64  2.34  7.32  834. 0.799