Initial design fails with GreaterThanCondition or InCondtition above a threshold

[LarsDoorenbos]

See this issue: https://github.com/automl/SMAC3/issues/531 When running the following code:

import numpy as np
import sklearn.datasets
import sklearn.metrics
from sklearn.neural_network import MLPClassifier

from ConfigSpace import (
    UniformIntegerHyperparameter,
    CategoricalHyperparameter,
    GreaterThanCondition,
    ConfigurationSpace,
    UniformFloatHyperparameter,
)

from smac.scenario.scenario import Scenario
from smac.facade.smac_hpo_facade import SMAC4HPO


cs = ConfigurationSpace()
alpha = UniformFloatHyperparameter(
    name='alpha', lower=0.000001, upper=1, default_value=0.0001, log=True,
)
lrate = UniformFloatHyperparameter(
    name='lrate', lower=0.000001, upper=1, default_value=0.001, log=True,
)
momentum = UniformFloatHyperparameter(
    name='momentum', lower=0.5, upper=1, default_value=0.9,
)
n_layers = UniformIntegerHyperparameter(
    name='n_layers', lower=1, upper=3,
)
number_neurons_1 = UniformIntegerHyperparameter(
    name='neurons_1', lower=16, upper=256, default_value=32, log=True,
)
number_neurons_2 = UniformIntegerHyperparameter(
    name='neurons_2', lower=16, upper=256, default_value=32, log=True,
)
number_neurons_3 = UniformIntegerHyperparameter(
    name='neurons_3', lower=16, upper=256, default_value=32, log=True,
)
lrate_schedule = CategoricalHyperparameter(
    name='lrate_schedule', choices=['constant', 'invscaling', 'adaptive'],
)
cs.add_hyperparameters([
    alpha, lrate, momentum, number_neurons_1, number_neurons_2,
    number_neurons_3, n_layers, lrate_schedule,
])
neurons_2_condition = GreaterThanCondition(number_neurons_2, n_layers, 1)
neurons_3_condition = GreaterThanCondition(number_neurons_3, n_layers, 2)
cs.add_conditions([neurons_2_condition, neurons_3_condition])


# Load data
X, y = sklearn.datasets.load_breast_cancer(return_X_y=True)
X_train, X_valid, y_train, y_valid = \
        sklearn.model_selection.train_test_split(X, y, random_state=1)


def eval_mlp(config, seed):
    n_layers = config['n_layers']
    layer_sizes = []
    for i in range(1, n_layers + 1):
        layer_sizes.append(config['neurons_%d' % i])
    layer_sizes = tuple(layer_sizes)

    mlp = MLPClassifier(
        hidden_layer_sizes=layer_sizes,
        activation='relu',
        solver='sgd',
        alpha=config['alpha'],
        learning_rate=config['lrate_schedule'],
        learning_rate_init=config['lrate'],
        momentum=config['momentum'],
        warm_start=False,
        random_state=seed,
    )

    classes = np.unique(y_train)
    mlp.fit(X_train, y_train)

    return 1 - mlp.score(X_valid, y_valid)


scenario = Scenario({
    "run_obj": "quality",   # we optimize quality (alternatively runtime)
    "runcount-limit": 12,  # maximum function evaluations
    "cs": cs,               # configuration space
    "deterministic": "false"
})

smac = SMAC4HPO(
    scenario=scenario,
    rng=np.random.RandomState(42),
    tae_runner=eval_mlp,
    intensifier_kwargs={'maxR': 4},
)

incumbent = smac.optimize()

the program works properly when setting runcount-limit in the Scenario to 11 or lower, but above 12 this error shows:

/usr/local/lib/python3.7/site-packages/pyparsing.py:2703: FutureWarning: Possible set intersection at position 3
  self.re = re.compile( self.reString )
INFO:smac.utils.io.cmd_reader.CMDReader:Output to smac3-output_2020-05-02_15:36:30_965175
INFO:smac.initial_design.sobol_design.SobolDesign:Running initial design for 3 configurations
INFO:smac.facade.smac_hpo_facade.SMAC4HPO:<class 'smac.facade.smac_hpo_facade.SMAC4HPO'>
INFO:smac.optimizer.smbo.SMBO:Running initial design
INFO:smac.stats.stats.Stats:##########################################################
INFO:smac.stats.stats.Stats:Statistics:
INFO:smac.stats.stats.Stats:#Incumbent changed: -1
INFO:smac.stats.stats.Stats:#Target algorithm runs: 0 / 12.0
INFO:smac.stats.stats.Stats:#Configurations: 0
INFO:smac.stats.stats.Stats:Used wallclock time: 0.01 / inf sec 
INFO:smac.stats.stats.Stats:Used target algorithm runtime: 0.00 / inf sec
INFO:smac.stats.stats.Stats:##########################################################
INFO:smac.facade.smac_hpo_facade.SMAC4HPO:Final Incumbent: None
Traceback (most recent call last):
  File "testf.py", line 97, in <module>
    incumbent = smac.optimize()
  File "/usr/local/lib/python3.7/site-packages/smac/facade/smac_ac_facade.py", line 555, in optimize
    incumbent = self.solver.run()
  File "/usr/local/lib/python3.7/site-packages/smac/optimizer/smbo.py", line 173, in run
    self.start()
  File "/usr/local/lib/python3.7/site-packages/smac/optimizer/smbo.py", line 144, in start
    self.initial_design_configs = self.initial_design.select_configurations()
  File "/usr/local/lib/python3.7/site-packages/smac/initial_design/initial_design.py", line 99, in select_configurations
    self.configs = self._select_configurations()
  File "/usr/local/lib/python3.7/site-packages/smac/initial_design/sobol_design.py", line 46, in _select_configurations
    cs=self.cs)
  File "/usr/local/lib/python3.7/site-packages/smac/initial_design/initial_design.py", line 149, in _transform_continuous_designs
    vector=vector)
  File "ConfigSpace/util.pyx", line 392, in ConfigSpace.util.deactivate_inactive_hyperparameters
  File "ConfigSpace/configuration_space.pyx", line 1398, in ConfigSpace.configuration_space.Configuration.__init__
  File "ConfigSpace/configuration_space.pyx", line 1428, in ConfigSpace.configuration_space.Configuration.is_valid_configuration
  File "ConfigSpace/c_util.pyx", line 40, in ConfigSpace.c_util.check_configuration
  File "ConfigSpace/c_util.pyx", line 115, in ConfigSpace.c_util.check_configuration
ValueError: Inactive hyperparameter 'neurons_2' must not be specified, but has the vector value: '0.24699765174986646'.

If I change the conditions from GreaterThan to InCondition as such:

neurons_2_condition = InCondition(number_neurons_2, n_layers, [2, 3])
neurons_3_condition = InCondition(number_neurons_3, n_layers, [2, 3])

The program suddenly works only when runcount-limit is less than 7, otherwise the following error is produced:

  File "testf.py", line 98, in <module>
    incumbent = smac.optimize()
  File "/usr/local/lib/python3.7/site-packages/smac/facade/smac_ac_facade.py", line 555, in optimize
    incumbent = self.solver.run()
  File "/usr/local/lib/python3.7/site-packages/smac/optimizer/smbo.py", line 173, in run
    self.start()
  File "/usr/local/lib/python3.7/site-packages/smac/optimizer/smbo.py", line 144, in start
    self.initial_design_configs = self.initial_design.select_configurations()
  File "/usr/local/lib/python3.7/site-packages/smac/initial_design/initial_design.py", line 99, in select_configurations
    self.configs = self._select_configurations()
  File "/usr/local/lib/python3.7/site-packages/smac/initial_design/sobol_design.py", line 46, in _select_configurations
    cs=self.cs)
  File "/usr/local/lib/python3.7/site-packages/smac/initial_design/initial_design.py", line 149, in _transform_continuous_designs
    vector=vector)
  File "ConfigSpace/util.pyx", line 373, in ConfigSpace.util.deactivate_inactive_hyperparameters
  File "ConfigSpace/configuration_space.pyx", line 1398, in ConfigSpace.configuration_space.Configuration.__init__
  File "ConfigSpace/configuration_space.pyx", line 1428, in ConfigSpace.configuration_space.Configuration.is_valid_configuration
  File "ConfigSpace/c_util.pyx", line 40, in ConfigSpace.c_util.check_configuration
  File "ConfigSpace/c_util.pyx", line 105, in ConfigSpace.c_util.check_configuration
ValueError: Active hyperparameter 'neurons_2' not specified!

The first example only works when using 1 or 2 configurations in the initial design, the second one only when there is 1 configuration.