backtesting.py
backtesting.py copied to clipboard
kernc
error in example doc with scikit-optimize
The example doc "Parameter Heatmap", section "Model-based optimization". https://kernc.github.io/backtesting.py/doc/examples/Parameter%20Heatmap%20&%20Optimization.html
When I run this snippet I get an error.
stats_skopt, heatmap, optimize_result = backtest.optimize(
n1=[10, 100], # Note: For method="skopt", we
n2=[20, 200], # only need interval end-points
n_enter=[10, 40],
n_exit=[10, 30],
constraint=lambda p: p.n_exit < p.n_enter < p.n1 < p.n2,
maximize='Equity Final [$]',
method='skopt',
max_tries=200,
random_state=0,
return_heatmap=True,
return_optimization=True)
---------------------------------------------------------------------------
InvalidParameterError Traceback (most recent call last)
File <timed exec>:1, in <module>
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/backtesting/backtesting.py:1490, in Backtest.optimize(self, maximize, method, max_tries, constraint, return_heatmap, return_optimization, random_state, **kwargs)
1488 output = _optimize_grid()
1489 elif method == 'skopt':
-> 1490 output = _optimize_skopt()
1491 else:
1492 raise ValueError(f"Method should be 'grid' or 'skopt', not {method!r}")
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/backtesting/backtesting.py:1456, in Backtest.optimize.<locals>._optimize_skopt()
1452 with warnings.catch_warnings():
1453 warnings.filterwarnings(
1454 'ignore', 'The objective has been evaluated at this point before.')
-> 1456 res = forest_minimize(
1457 func=objective_function,
1458 dimensions=dimensions,
1459 n_calls=max_tries,
1460 base_estimator=ExtraTreesRegressor(n_estimators=20, min_samples_leaf=2),
1461 acq_func='LCB',
1462 kappa=3,
1463 n_initial_points=min(max_tries, 20 + 3 * len(kwargs)),
1464 initial_point_generator='lhs', # 'sobel' requires n_initial_points ~ 2**N
1465 callback=DeltaXStopper(9e-7),
1466 random_state=random_state)
1468 stats = self.run(**dict(zip(kwargs.keys(), res.x)))
1469 output = [stats]
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/skopt/optimizer/forest.py:186, in forest_minimize(func, dimensions, base_estimator, n_calls, n_random_starts, n_initial_points, acq_func, initial_point_generator, x0, y0, random_state, verbose, callback, n_points, xi, kappa, n_jobs, model_queue_size)
10 def forest_minimize(func, dimensions, base_estimator="ET", n_calls=100,
11 n_random_starts=None, n_initial_points=10, acq_func="EI",
12 initial_point_generator="random",
13 x0=None, y0=None, random_state=None, verbose=False,
14 callback=None, n_points=10000, xi=0.01, kappa=1.96,
15 n_jobs=1, model_queue_size=None):
16 """Sequential optimisation using decision trees.
17
18 A tree based regression model is used to model the expensive to evaluate
(...)
184 :class:`skopt.dummy_minimize`, :class:`skopt.gbrt_minimize`
185 """
--> 186 return base_minimize(func, dimensions, base_estimator,
187 n_calls=n_calls, n_points=n_points,
188 n_random_starts=n_random_starts,
189 n_initial_points=n_initial_points,
190 initial_point_generator=initial_point_generator,
191 x0=x0, y0=y0, random_state=random_state,
192 n_jobs=n_jobs,
193 acq_func=acq_func,
194 xi=xi, kappa=kappa, verbose=verbose,
195 callback=callback, acq_optimizer="sampling",
196 model_queue_size=model_queue_size)
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/skopt/optimizer/base.py:300, in base_minimize(func, dimensions, base_estimator, n_calls, n_random_starts, n_initial_points, initial_point_generator, acq_func, acq_optimizer, x0, y0, random_state, verbose, callback, n_points, n_restarts_optimizer, xi, kappa, n_jobs, model_queue_size)
298 next_x = optimizer.ask()
299 next_y = func(next_x)
--> 300 result = optimizer.tell(next_x, next_y)
301 result.specs = specs
302 if eval_callbacks(callbacks, result):
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/skopt/optimizer/optimizer.py:493, in Optimizer.tell(self, x, y, fit)
490 y = list(y)
491 y[1] = log(y[1])
--> 493 return self._tell(x, y, fit=fit)
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/skopt/optimizer/optimizer.py:536, in Optimizer._tell(self, x, y, fit)
534 with warnings.catch_warnings():
535 warnings.simplefilter("ignore")
--> 536 est.fit(self.space.transform(self.Xi), self.yi)
538 if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
539 self.gains_ -= est.predict(np.vstack(self.next_xs_))
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/sklearn/ensemble/_forest.py:341, in BaseForest.fit(self, X, y, sample_weight)
314 def fit(self, X, y, sample_weight=None):
315 """
316 Build a forest of trees from the training set (X, y).
317
(...)
339 Fitted estimator.
340 """
--> 341 self._validate_params()
343 # Validate or convert input data
344 if issparse(y):
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/sklearn/base.py:570, in BaseEstimator._validate_params(self)
562 def _validate_params(self):
563 """Validate types and values of constructor parameters
564
565 The expected type and values must be defined in the `_parameter_constraints`
(...)
568 accepted constraints.
569 """
--> 570 validate_parameter_constraints(
571 self._parameter_constraints,
572 self.get_params(deep=False),
573 caller_name=self.__class__.__name__,
574 )
File ~/anaconda3/envs/trade/lib/python3.8/site-packages/sklearn/utils/_param_validation.py:97, in validate_parameter_constraints(parameter_constraints, params, caller_name)
91 else:
92 constraints_str = (
93 f"{', '.join([str(c) for c in constraints[:-1]])} or"
94 f" {constraints[-1]}"
95 )
---> 97 raise InvalidParameterError(
98 f"The {param_name!r} parameter of {caller_name} must be"
99 f" {constraints_str}. Got {param_val!r} instead."
100 )
InvalidParameterError: The 'criterion' parameter of ExtraTreesRegressor must be a str among {'squared_error', 'absolute_error', 'poisson', 'friedman_mse'}. Got 'mse' instead.
- Backtesting version: 0.3.3
-
bokeh.__version__: Python 3.11, bokeh 2.4.3; Python 3.8.11, bokeh 3.0.3; - scikit-optimize 0.9.0
- OS: Ubuntu 20.04
Hi there, I am following the tutorial at https://kernc.github.io/backtesting.py/doc/examples/Parameter%20Heatmap%20&%20Optimization.html and I run into the same problem. Here are the details:
Throws on the following code:
stats_skopt, heatmap, optimize_result = backtest.optimize(
n1=[10, 100], # Note: For method="skopt", we
n2=[20, 200], # only need interval end-points
n_enter=[10, 40],
n_exit=[10, 30],
constraint=lambda p: p.n_exit < p.n_enter < p.n1 < p.n2,
maximize='Equity Final [$]',
max_tries=200,
random_state=0,
return_heatmap=True,
method='skopt',
return_optimization=True)
With the following deps on Python 3.11:
Backtesting~=0.3.3
bokeh~=2.4.3
pandas~=1.5.2
seaborn~=0.12.1
scikit-optimize~=0.9.0
numpy~=1.23.0
With the following error:
File "/lib/python3.11/site-packages/sklearn/utils/_param_validation.py", line 97, in validate_parameter_constraints
raise InvalidParameterError(
sklearn.utils._param_validation.InvalidParameterError: The 'criterion' parameter of ExtraTreesRegressor must be a str among {'squared_error', 'poisson', 'friedman_mse', 'absolute_error'}. Got 'mse' instead.
When I upgrade the numpy version to 1.24.0:
Backtesting~=0.3.3
bokeh~=2.4.3
pandas~=1.5.2
seaborn~=0.12.1
scikit-optimize~=0.9.0
numpy~=1.24.0
I see the following error:
File "/lib/python3.11/site-packages/skopt/space/transformers.py", line 275, in inverse_transform
return np.round(X_orig).astype(np.int)
^^^^^^
File "/lib/python3.11/site-packages/numpy/__init__.py", line 284, in __getattr__
raise AttributeError("module {!r} has no attribute "
AttributeError: module 'numpy' has no attribute 'int'. Did you mean: 'inf'?
Please help! Thanks in advance.
Full Python 3.11 stacktrace is here https://pastebin.com/4865192r Please let me know if you need any other info.
This appears to be a problem in scikit-optimize. Hence scikit-learn here https://github.com/kernc/backtesting.py/commit/d5db8bed504165fa395708ead586d57c0e052b92 See https://github.com/scikit-optimize/scikit-optimize/pull/1123 which fixes the bug and https://github.com/scikit-optimize/scikit-optimize/issues/1138
Hi @sshquack , apologies mate, I am getting this error in one of my remote servers...
Can I check how this error is actually fixed?
Is there any particular version of numpy, or scikit-learn, or scikit-optimize we need to install/upgrade/downgrade to?
I read this thread [( https://github.com/scikit-optimize/scikit-optimize/pull/1123)], but I am still not sure where/how the fix is..
@kernc I saw you put in a pr for the skopt but it seems they have gone dormant over there. Is there any anticipated fix for this? Perhaps something we can do based on your pr?
Or can we possibly downgrade numpy to a lower version?
