Use SM kernel in Fully Bayesian GPs show errors

[lk1983823]

Follow the Fully Bayesian GPs example, I change the RBF kernel to SpectralMixture Kernel, as follows:

class ExactGPModel(gpytorch.models.ExactGP):
    def __init__(self, train_x, train_y, likelihood):
        super(ExactGPModel, self).__init__(train_x, train_y, likelihood)
        self.mean_module = gpytorch.means.ConstantMean()
        self.covar_module = gpytorch.kernels.SpectralMixtureKernel(num_mixtures=10)
        self.covar_module.initialize_from_data(train_x, train_y)
    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)

Register the prior to test:

model.mean_module.register_prior("mean_prior", UniformPrior(-1, 1), "constant")
model.covar_module.register_prior("mixture_scales_prior", UniformPrior(1, 2), "mixture_scales")
likelihood.register_prior("noise_prior", UniformPrior(0.05, 0.3), "noise")

Nothing esle changed.

But when I run the code:

model.pyro_load_from_samples(mcmc_run.get_samples())
model.eval()
test_x = torch.linspace(0, 1, 101).unsqueeze(-1)
test_y = torch.sin(test_x * (2 * math.pi))
expanded_test_x = test_x.unsqueeze(0).repeat(num_samples, 1, 1)
print(expanded_test_x.shape)
output = model(expanded_test_x)

it shows:

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/models/exact_gp.py in call(self, *args, **kwargs) 326 # Make the prediction 327 with settings._use_eval_tolerance(): --> 328 predictive_mean, predictive_covar = self.prediction_strategy.exact_prediction(full_mean, full_covar) 329 330 # Reshape predictive mean to match the appropriate event shape

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/models/exact_prediction_strategies.py in exact_prediction(self, joint_mean, joint_covar) 292 # For efficiency - we can make things more efficient 293 if joint_covar.size(-1) <= settings.max_eager_kernel_size.value(): --> 294 test_covar = joint_covar[..., self.num_train :, :].evaluate() 295 test_test_covar = test_covar[..., self.num_train :] 296 test_train_covar = test_covar[..., : self.num_train]

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/utils/memoize.py in g(self, *args, **kwargs) 32 cache_name = name if name is not None else method 33 if not is_in_cache(self, cache_name): ---> 34 add_to_cache(self, cache_name, method(self, *args, **kwargs)) 35 return get_from_cache(self, cache_name) 36

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/lazy/lazy_evaluated_kernel_tensor.py in evaluate(self) 293 @cached 294 def evaluate(self): --> 295 return self.evaluate_kernel().evaluate() 296 297 def repeat(self, *repeats):

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/utils/memoize.py in g(self, *args, **kwargs) 32 cache_name = name if name is not None else method 33 if not is_in_cache(self, cache_name): ---> 34 add_to_cache(self, cache_name, method(self, *args, **kwargs)) 35 return get_from_cache(self, cache_name) 36

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/lazy/lazy_evaluated_kernel_tensor.py in evaluate_kernel(self) 278 temp_active_dims = self.kernel.active_dims 279 self.kernel.active_dims = None --> 280 res = self.kernel(x1, x2, diag=False, last_dim_is_batch=self.last_dim_is_batch, **self.params) 281 self.kernel.active_dims = temp_active_dims 282

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/kernels/kernel.py in call(self, x1, x2, diag, last_dim_is_batch, **params) 394 res = LazyEvaluatedKernelTensor(x1_, x2_, kernel=self, last_dim_is_batch=last_dim_is_batch, **params) 395 else: --> 396 res = lazify(super(Kernel, self).call(x1_, x2_, last_dim_is_batch=last_dim_is_batch, **params)) 397 return res 398

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/module.py in call(self, *inputs, **kwargs) 26 27 def call(self, *inputs, **kwargs): ---> 28 outputs = self.forward(*inputs, **kwargs) 29 if isinstance(outputs, list): 30 return [_validate_module_outputs(output) for output in outputs]

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/kernels/spectral_mixture_kernel.py in forward(self, x1, x2, last_dim_is_batch, **params) 302 mixture_weights = mixture_weights.unsqueeze(-1) 303 --> 304 res = (res * mixture_weights).sum(len(batch_shape)) 305 return res

RuntimeError: The size of tensor a (100) must match the size of tensor b (10) at non-singleton dimension 0

Then, when I set num_samples to 10, which is different with the one using in mcmc.run, it shows:

---

RuntimeError Traceback (most recent call last) in 4 expanded_test_x = test_x.unsqueeze(0).repeat(10, 1, 1) 5 print(expanded_test_x.shape) ----> 6 output = model(expanded_test_x)

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/models/exact_gp.py in call(self, *args, **kwargs) 306 train_input = train_input.expand(*batch_shape, *train_input.shape[-2:]) 307 if batch_shape != input.shape[:-2]: --> 308 batch_shape = _mul_broadcast_shape(batch_shape, input.shape[:-2]) 309 train_input = train_input.expand(*batch_shape, *train_input.shape[-2:]) 310 input = input.expand(*batch_shape, *input.shape[-2:])

~/.pyenv/versions/anaconda3-5.1.0/envs/sgcc/lib/python3.6/site-packages/gpytorch-1.1.1-py3.6.egg/gpytorch/utils/broadcasting.py in _mul_broadcast_shape(error_msg, *shapes) 18 if any(size != non_singleton_sizes[0] for size in non_singleton_sizes): 19 if error_msg is None: ---> 20 raise RuntimeError("Shapes are not broadcastable for mul operation") 21 else: 22 raise RuntimeError(error_msg)

RuntimeError: Shapes are not broadcastable for mul operation Any one can help? Thanks!

[lk1983823]

I find it the setting of num_mixtures in the model should be equal to the num_samples to make it work. I wondered why I have to do this? Thanks

[gpleiss]

I'm guessing this is actually a bug on our end. My guess is that the batch dimension for sampling and the batch dimension for the SM-kernel mixtures are somehow getting combined.

I'm not sure the fully-Bayesian code has been previously tested with SM kernels, so this is a reasonable bug :)

cc/ @jacobrgardner

[lk1983823]

I'm guessing this is actually a bug on our end. My guess is that the batch dimension for sampling and the batch dimension for the SM-kernel mixtures are somehow getting combined.

I'm not sure the fully-Bayesian code has been previously tested with SM kernels, so this is a reasonable bug :)

cc/ @jacobrgardner

Thanks. Btw, please inform me as you get the code fixed.

[iprada]

I am also very interested on this issue. I would love the hear whether there is any progress planned

[jacobrgardner]

@iprada @lk1983823 I'll take a look at this this week and see if I can get an easy fix out.

[iprada]

Great. Looking forward :)

[NeelKanwal]

Hi, @jacobrgardner

Did you fix the this issue, I have similar problem. Somehow the grid interpolation variational strategy looks problematic.

Please let me know if there is any fix.

File "/home/neel/DKLModel/my_functions.py", line 104, in train output = model(data) File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/module.py", line 30, in call outputs = self.forward(*inputs, **kwargs) File "/home/neel/DKLModel/my_functions.py", line 73, in forward res = self.gp_layer(features) File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/models/approximate_gp.py", line 81, in call return self.variational_strategy(inputs, prior=prior, **kwargs) File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/variational/independent_multitask_variational_strategy.py", line 56, in call function_dist = self.base_variational_strategy(x, prior=prior, **kwargs) File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/variational/_variational_strategy.py", line 129, in call **kwargs, File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/module.py", line 30, in call outputs = self.forward(*inputs, **kwargs) File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/variational/grid_interpolation_variational_strategy.py", line 93, in forward predictive_mean = left_interp(interp_indices, interp_values, inducing_values.unsqueeze(-1)) File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/utils/interpolation.py", line 183, in left_interp output_shape = _matmul_broadcast_shape(interp_shape, rhs.shape) File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/utils/broadcasting.py", line 57, in _matmul_broadcast_shape bc_shape = _mul_broadcast_shape(batch_shape_a, batch_shape_b) File "/home/neel/miniconda3/envs/DKL/lib/python3.6/site-packages/gpytorch/utils/broadcasting.py", line 20, in _mul_broadcast_shape raise RuntimeError("Shapes are not broadcastable for mul operation") RuntimeError: Shapes are not broadcastable for mul operation

[gpleiss]

#1808 is unrelated to this bug