botorch
botorch copied to clipboard
pytorch
NumericalWarning: Runtime Error when computing Cholesky decomposition: CUDA out of memory
Issue description
I am trying to run the turbo_1 from the tutorial : https://botorch.org/tutorials/turbo_1
I used 200D Ackley function as the objective function to be optimized bu turbo_1, and I used the exact same code with the official tutorial. Please see the following code
Code for reproducing the issue
import os import math from dataclasses import dataclass
import torch from botorch.acquisition import qExpectedImprovement from botorch.fit import fit_gpytorch_model from botorch.generation import MaxPosteriorSampling from botorch.models import FixedNoiseGP, SingleTaskGP from botorch.optim import optimize_acqf from botorch.test_functions import Ackley from botorch.utils.transforms import unnormalize from torch.quasirandom import SobolEngine
import gpytorch from gpytorch.constraints import Interval from gpytorch.likelihoods import GaussianLikelihood from gpytorch.mlls import ExactMarginalLogLikelihood from gpytorch.priors import HorseshoePrior
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") dtype = torch.double SMOKE_TEST = os.environ.get("SMOKE_TEST")
fun = Ackley(dim=10, negate=True).to(dtype=dtype, device=device) fun.bounds[0, :].fill_(-10) fun.bounds[1, :].fill_(15) dim = fun.dim lb, ub = fun.bounds
def eval_objective(x): """This is a helper function we use to unnormalize and evalaute a point""" return fun(unnormalize(x, fun.bounds))
@dataclass class TurboState: dim: int batch_size: int length: float = 0.8 length_min: float = 0.5 ** 7 length_max: float = 1.6 failure_counter: int = 0 failure_tolerance: int = float("nan") # Note: Post-initialized success_counter: int = 0 success_tolerance: int = 10 # Note: The original paper uses 3 best_value: float = -float("inf") restart_triggered: bool = False
def __post_init__(self):
self.failure_tolerance = math.ceil(
max([4.0 / self.batch_size, float(self.dim) / self.batch_size])
)
def update_state(state, Y_next): if max(Y_next) > state.best_value + 1e-3 * math.fabs(state.best_value): state.success_counter += 1 state.failure_counter = 0 else: state.success_counter = 0 state.failure_counter += 1
if state.success_counter == state.success_tolerance: # Expand trust region
state.length = min(2.0 * state.length, state.length_max)
state.success_counter = 0
elif state.failure_counter == state.failure_tolerance: # Shrink trust region
state.length /= 2.0
state.failure_counter = 0
state.best_value = max(state.best_value, max(Y_next).item())
if state.length < state.length_min:
state.restart_triggered = True
return state
state = TurboState(dim=dim, batch_size=4) print(state)
def get_initial_points(dim, n_pts): sobol = SobolEngine(dimension=dim, scramble=True) X_init = sobol.draw(n=n_pts).to(dtype=dtype, device=device) return X_init
def generate_batch( state, model, # GP model X, # Evaluated points on the domain [0, 1]^d Y, # Function values batch_size, n_candidates=None, # Number of candidates for Thompson sampling num_restarts=10, raw_samples=512, acqf="ts", # "ei" or "ts" ): assert acqf in ("ts", "ei") assert X.min() >= 0.0 and X.max() <= 1.0 and torch.all(torch.isfinite(Y)) if n_candidates is None: n_candidates = min(5000, max(2000, 200 * X.shape[-1]))
# Scale the TR to be proportional to the lengthscales
x_center = X[Y.argmax(), :].clone()
weights = model.covar_module.base_kernel.lengthscale.squeeze().detach()
weights = weights / weights.mean()
weights = weights / torch.prod(weights.pow(1.0 / len(weights)))
tr_lb = torch.clamp(x_center - weights * state.length / 2.0, 0.0, 1.0)
tr_ub = torch.clamp(x_center + weights * state.length / 2.0, 0.0, 1.0)
if acqf == "ts":
dim = X.shape[-1]
sobol = SobolEngine(dim, scramble=True)
pert = sobol.draw(n_candidates).to(dtype=dtype, device=device)
pert = tr_lb + (tr_ub - tr_lb) * pert
# Create a perturbation mask
prob_perturb = min(20.0 / dim, 1.0)
mask = (
torch.rand(n_candidates, dim, dtype=dtype, device=device)
<= prob_perturb
)
ind = torch.where(mask.sum(dim=1) == 0)[0]
mask[ind, torch.randint(0, dim - 1, size=(len(ind),), device=device)] = 1
# Create candidate points from the perturbations and the mask
X_cand = x_center.expand(n_candidates, dim).clone()
X_cand[mask] = pert[mask]
# Sample on the candidate points
thompson_sampling = MaxPosteriorSampling(model=model, replacement=False)
X_next = thompson_sampling(X_cand, num_samples=batch_size)
elif acqf == "ei":
ei = qExpectedImprovement(model, train_Y.max(), maximize=True)
X_next, acq_value = optimize_acqf(
ei,
bounds=torch.stack([tr_lb, tr_ub]),
q=batch_size,
num_restarts=num_restarts,
raw_samples=raw_samples,
)
return X_next
batch_size = 4 n_init = 20 # 2*dim, which corresponds to 5 batches of 4
X_turbo = get_initial_points(dim, n_init) Y_turbo = torch.tensor( [eval_objective(x) for x in X_turbo], dtype=dtype, device=device ).unsqueeze(-1)
state = TurboState(dim, batch_size=batch_size)
NUM_RESTARTS = 10 if not SMOKE_TEST else 2 RAW_SAMPLES = 512 if not SMOKE_TEST else 4 N_CANDIDATES = min(5000, max(2000, 200 * dim)) if not SMOKE_TEST else 4
while not state.restart_triggered: # Run until TuRBO converges # Fit a GP model train_Y = (Y_turbo - Y_turbo.mean()) / Y_turbo.std() likelihood = GaussianLikelihood(noise_constraint=Interval(1e-8, 1e-3)) model = SingleTaskGP(X_turbo, train_Y, likelihood=likelihood) mll = ExactMarginalLogLikelihood(model.likelihood, model) fit_gpytorch_model(mll)
# Create a batch
X_next = generate_batch(
state=state,
model=model,
X=X_turbo,
Y=train_Y,
batch_size=batch_size,
n_candidates=N_CANDIDATES,
num_restarts=NUM_RESTARTS,
raw_samples=RAW_SAMPLES,
acqf="ts",
)
Y_next = torch.tensor(
[eval_objective(x) for x in X_next], dtype=dtype, device=device
).unsqueeze(-1)
# Update state
state = update_state(state=state, Y_next=Y_next)
# Append data
X_turbo = torch.cat((X_turbo, X_next), dim=0)
Y_turbo = torch.cat((Y_turbo, Y_next), dim=0)
# Print current status
print(
f"{len(X_turbo)}) Best value: {state.best_value:.2e}, TR length: {state.length:.2e}"
)
After around 2000 function evaluations, the following errors happened: /.conda/envs/3d-vae/lib/python3.7/site-packages/gpytorch/lazy/lazy_tensor.py:1633: NumericalWarning: Runtime Error when computing Cholesky decomposition: CUDA out of memory. Tried to allocate 96.00 MiB (GPU 0; 15.78 GiB total capacity; 14.26 GiB already allocated; 9.56 MiB free; 14.57 GiB reserved in total by PyTorch). Using RootDecomposition. NumericalWarning,
I am wondering if botorch only allows for small number of function evaluations? Do I missunderstand something? Please help solve my issue. Thanks a lot
System Info
Please provide information about your setup, including
- BoTorch Version 0.4.0
- GPyTorch Version 1.4.1
- PyTorch Version 1.7.1
- Computer OS (Ubuntu 18.04.5 LTS (Bionic Beaver))
If you're using a 200D function (not sure if you are from the code which is inconsistent with that), then it's plausible that there may be quite a bit of memory taken up by intermediate covariances, caches and such. The heuristic n_candidates = min(5000, max(2000, 200 * X.shape[-1])) for limiting the discrete batch size to be used for Thompson Sampling will mean that you're always considering 5000 candidates jointly, which is quite a bit and will be pretty memory heavy (see https://botorch.org/tutorials/thompson_sampling fro some alternatives).
The fact that this failure happens when falling back to RootDecomposition after a Cholesky failure is interesting though, I wonder if something goes wrong there within gpytorch.
The other thing you may want to try is to wrap the line X_next = thompson_sampling(X_cand, num_samples=batch_size) into a with torch.no_grad(): context to make there is no unnecessary compute graph being constructed in the background as we don't intend on differentiating the resulting candidates.
@dme65 may have additional thoughts.
@Balandat Thanks a lot for your detailed solutions. I used the exact same code with the tutorial on botorch site, with 20d-200d change. I also tried torbo-1 with option 'ei', after 1704 iterations, the out of memory error happened:
1704) Best value: -4.44e+00, TR length: 1.00e-01
Traceback (most recent call last):
File "turbo_1.py", line 258, in
I am wondering if this is a bug? I am not very clear where went wrong. Looking forward to your reply.
Hi @yangxiufengsia. It's not surprising that you run memory into issues after 1700+ iterations. With a batch_size=4, after 1700 iterations you have 6800+ training points, which results in a 6800x6800 prior covariance matrix for the training data.
File "/home/yang/anaconda3/envs/bo/lib/python3.7/site-packages/gpytorch/kernels/kernel.py", line 43, in sq_dist res = x1.matmul(x2.transpose(-2, -1))
If I'm not mistaken, in this line, x1 and x2 are 6800 x 200-dim tensors (or larger with possible batch dimensions), so multiplying them (particularly with the gradient computations enabled) requires significant amount of memory. Passing in options={"batch_limit": <some small number>} to optimize_acqf may help but I am not too sure about that. I don't have much experience using BO with such large training data sets (6800 points in this case). To answer your question, this is not a bug, it just results from using exact GP computations with large datasets.
One other thing you could try is to wrap the optimize_acqf call with with gpytorch.settings.fast_computations():. I am not sure if it would help with this particular error but it should reduce the overall memory usage. (This will use some fast approximations, see https://docs.gpytorch.ai/en/stable/settings.html#gpytorch.settings.fast_computations)
Other options include
- Using KeOps to optimize the Kernel operations (https://github.com/cornellius-gp/gpytorch/blob/master/examples/02_Scalable_Exact_GPs/KeOps_GP_Regression.ipynb)
- Using an approximate GP model, e.g. this one: https://github.com/cornellius-gp/gpytorch/blob/master/examples/02_Scalable_Exact_GPs/KISSGP_Regression.ipynb
We don't have either of those models as canned models in botorch currently, but it should be relatively straightforward to set those up.
I just ran the notebook on the 200D Ackley function for a few thousand evaluations on a GPU without any issues. I'm noticing that the GP fitting seems to be failing (the hypers don't move) because of the lengthscale priors, so I think it makes sense to update the notebook to be consistent with the priors used by my original TuRBO implementation. I'll also add a no_grad context when using TS since that will help lowering the memory usage.
Hi, thanks a lot for dealing with this issue and provide such detailed solutions. I will take a look into how to properly use botorch for high-dimensional optimization problem.
I just tried an analytic BO for optimizing a 6-D function (I used the exact same code from here: https://botorch.org/tutorials/compare_mc_analytic_acquisition ). But the GPU memory increases gradually (beginning 624MB)to the maximum (12GB) after several hundred iterations, then suddenly reduce to (around 1400MB). Then increase again. This situation repeatedly several times and finally run out of CUDA memory. Do you know what caused this?
The following is the code for reproduce this issue:
import torch
from botorch.fit import fit_gpytorch_model
from botorch.models import SingleTaskGP
#from botorch.test_functions import neg_hartmann6
from botorch.test_functions import Hartmann
from botorch.optim import optimize_acqf
from gpytorch.mlls import ExactMarginalLogLikelihood
train_x = torch.rand(10, 6)
neg_hartmann6 = Hartmann(negate=True)
train_obj = neg_hartmann6(train_x).unsqueeze(-1)
from botorch.acquisition import ExpectedImprovement
def traingp(x,y):
x = x.to('cuda')
y = y.to('cuda')
neg_hartmann6 = Hartmann(negate=True)
train_obj = neg_hartmann6(x).unsqueeze(-1)
model = SingleTaskGP(train_X=x, train_Y=y)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
fit_gpytorch_model(mll)
del x, y
model.eval()
return mode
for i in range(10000):
model=traingp(train_x, train_obj)
best_value = train_obj.max()
print (best_value)
EI = ExpectedImprovement(model=model, best_f=best_value)
#del model
next_x, _ = optimize_acqf(
acq_function=EI,
bounds=torch.tensor([[0.0] * 6, [1.0] * 6]).to('cuda'),
q=1,
num_restarts=10,
raw_samples=10000,
options={},
)
new_x = next_x.cpu()
#del EI,next_x
new_y = neg_hartmann6(new_x).unsqueeze(-1)
train_obj=torch.cat((train_obj,new_y),0)
train_x=torch.cat((train_x,new_x),0)
Hmm this should not happen. It might be related to a memory leak we've seen in the past, but weren't able to isolate so far: #641.
I don't see any obvious issues in your code with memory accumulating, so it appears that there could be some issue with things (likely the graph) not properly being garbage collected. Let me see if I can repro this.
Could you try running this on botorch 0.5.0 / gpytorch 1.5.0 / pytorch 1.8.1+ also? Just to make sure we're not chasing something that got fixed since then.
Thanks a lot @Balandat . I will run on botorch 0.5.0 / gpytorch 1.5.0 / pytorch 1.8.1 and test it. Then share the result here.
Hi @Balandat , I just tested the above code using botorch 0.5.0, gpytorch 1.5.0, pytorch 1.9.0. The GPU memory increases gradually to the maximum (12GB) after several hundred iterations, then suddenly reduce to (around 1400MB). Then increase again. This situation repeatedly several times and finally run out of CUDA memory. I shared the code to reproduce and GPU memory output here, please help check the reasons. Thanks a lot!
Code to reproduce the memory issue:
import torch from botorch.fit import fit_gpytorch_model from botorch.models import SingleTaskGP #from botorch.test_functions import neg_hartmann6 from botorch.test_functions import Hartmann from botorch.optim import optimize_acqf import os from gpytorch.mlls import ExactMarginalLogLikelihood from botorch.acquisition import ExpectedImprovement import GPUtil import nvidia_smi nvidia_smi.nvmlInit()
os.environ["CUDA_VISIBLE_DEVICES"] = "5" train_x = torch.rand(10, 6) neg_hartmann6 = Hartmann(negate=True) train_obj = neg_hartmann6(train_x).unsqueeze(-1)
def traingp(x,y): x = x.to('cuda:0') y = y.to('cuda:0')
neg_hartmann6 = Hartmann(negate=True)
train_obj = neg_hartmann6(x).unsqueeze(-1)
model = SingleTaskGP(train_X=x, train_Y=y)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
fit_gpytorch_model(mll)
del x, y
model.eval()
return model
for i in range(10000): #print (i,GPUtil.showUtilization()) handle = nvidia_smi.nvmlDeviceGetHandleByIndex(5) info = nvidia_smi.nvmlDeviceGetMemoryInfo(handle) print("bo iteration {}: Memory : ({:.2f}% free): {}(total), {} (free), {} (used)".format(i, 100*info.free/info.total, info.total, info.free, info.used))
model=traingp(train_x, train_obj)
best_value = train_obj.max()
#print (best_value)
EI = ExpectedImprovement(model=model, best_f=best_value)
#del model
next_x, _ = optimize_acqf(
acq_function=EI,
bounds=torch.tensor([[0.0] * 6, [1.0] * 6]).to('cuda:0'),
q=1,
num_restarts=10,
raw_samples=10000,
options={},
)
new_x = next_x.cpu()
#del EI,next_x
new_y = neg_hartmann6(new_x).unsqueeze(-1)
train_obj=torch.cat((train_obj,new_y),0)
train_x=torch.cat((train_x,new_x),0)
nvidia_smi.nvmlShutdown()
CUDA memory usage (only parts are shown, the CUDA usaged memory increase gradually): bo iteration 0: Memory : (99.96% free): 11554717696(total), 11550654464 (free), 4063232 (used) bo iteration 1: Memory : (89.45% free): 11554717696(total), 10335354880 (free), 1219362816 (used) bo iteration 2: Memory : (89.45% free): 11554717696(total), 10335354880 (free), 1219362816 (used) bo iteration 3: Memory : (89.45% free): 11554717696(total), 10335354880 (free), 1219362816 (used) bo iteration 4: Memory : (89.45% free): 11554717696(total), 10335354880 (free), 1219362816 (used) bo iteration 5: Memory : (89.43% free): 11554717696(total), 10333257728 (free), 1221459968 (used) bo iteration 6: Memory : (89.43% free): 11554717696(total), 10333257728 (free), 1221459968 (used) bo iteration 7: Memory : (89.43% free): 11554717696(total), 10333257728 (free), 1221459968 (used) bo iteration 8: Memory : (89.43% free): 11554717696(total), 10333257728 (free), 1221459968 (used) bo iteration 9: Memory : (89.43% free): 11554717696(total), 10333257728 (free), 1221459968 (used) bo iteration 10: Memory : (89.43% free): 11554717696(total), 10333257728 (free), 1221459968 (used) bo iteration 11: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 12: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 13: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 14: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 15: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 16: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 17: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 18: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 19: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 20: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 21: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 22: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 23: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 24: Memory : (89.25% free): 11554717696(total), 10312286208 (free), 1242431488 (used) bo iteration 25: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 26: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 27: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 28: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 29: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 30: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 31: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 32: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 33: Memory : (89.07% free): 11554717696(total), 10291314688 (free), 1263403008 (used) bo iteration 34: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 35: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 36: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 37: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 38: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 39: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 40: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 41: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 42: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 43: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 44: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 45: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 46: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 47: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 48: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 49: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 50: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 51: Memory : (88.83% free): 11554717696(total), 10264051712 (free), 1290665984 (used) bo iteration 52: Memory : (88.50% free): 11554717696(total), 10226302976 (free), 1328414720 (used) bo iteration 53: Memory : (88.50% free): 11554717696(total), 10226302976 (free), 1328414720 (used) bo iteration 54: Memory : (88.50% free): 11554717696(total), 10226302976 (free), 1328414720 (used) bo iteration 55: Memory : (88.50% free): 11554717696(total), 10226302976 (free), 1328414720 (used) bo iteration 56: Memory : (88.30% free): 11554717696(total), 10203234304 (free), 1351483392 (used) bo iteration 57: Memory : (88.30% free): 11554717696(total), 10203234304 (free), 1351483392 (used) bo iteration 58: Memory : (88.30% free): 11554717696(total), 10203234304 (free), 1351483392 (used) bo iteration 59: Memory : (88.30% free): 11554717696(total), 10203234304 (free), 1351483392 (used) bo iteration 60: Memory : (88.30% free): 11554717696(total), 10203234304 (free), 1351483392 (used) bo iteration 61: Memory : (88.30% free): 11554717696(total), 10203234304 (free), 1351483392 (used) bo iteration 62: Memory : (88.30% free): 11554717696(total), 10203234304 (free), 1351483392 (used) bo iteration 63: Memory : (88.09% free): 11554717696(total), 10178068480 (free), 1376649216 (used) bo iteration 64: Memory : (88.09% free): 11554717696(total), 10178068480 (free), 1376649216 (used) bo iteration 65: Memory : (88.09% free): 11554717696(total), 10178068480 (free), 1376649216 (used) bo iteration 66: Memory : (88.09% free): 11554717696(total), 10178068480 (free), 1376649216 (used) bo iteration 67: Memory : (88.09% free): 11554717696(total), 10178068480 (free), 1376649216 (used) bo iteration 68: Memory : (88.09% free): 11554717696(total), 10178068480 (free), 1376649216 (used) bo iteration 69: Memory : (87.85% free): 11554717696(total), 10150805504 (free), 1403912192 (used) bo iteration 70: Memory : (87.85% free): 11554717696(total), 10150805504 (free), 1403912192 (used) bo iteration 71: Memory : (87.85% free): 11554717696(total), 10150805504 (free), 1403912192 (used) bo iteration 72: Memory : (87.85% free): 11554717696(total), 10150805504 (free), 1403912192 (used) bo iteration 73: Memory : (87.85% free): 11554717696(total), 10150805504 (free), 1403912192 (used) bo iteration 74: Memory : (87.85% free): 11554717696(total), 10150805504 (free), 1403912192 (used) bo iteration 75: Memory : (87.85% free): 11554717696(total), 10150805504 (free), 1403912192 (used) bo iteration 76: Memory : (87.60% free): 11554717696(total), 10121445376 (free), 1433272320 (used) bo iteration 77: Memory : (87.60% free): 11554717696(total), 10121445376 (free), 1433272320 (used) bo iteration 78: Memory : (87.60% free): 11554717696(total), 10121445376 (free), 1433272320 (used) bo iteration 79: Memory : (87.60% free): 11554717696(total), 10121445376 (free), 1433272320 (used) bo iteration 80: Memory : (87.60% free): 11554717696(total), 10121445376 (free), 1433272320 (used) bo iteration 81: Memory : (87.60% free): 11554717696(total), 10121445376 (free), 1433272320 (used) bo iteration 82: Memory : (87.32% free): 11554717696(total), 10089988096 (free), 1464729600 (used) bo iteration 83: Memory : (87.32% free): 11554717696(total), 10089988096 (free), 1464729600 (used) bo iteration 84: Memory : (87.32% free): 11554717696(total), 10089988096 (free), 1464729600 (used) bo iteration 85: Memory : (87.32% free): 11554717696(total), 10089988096 (free), 1464729600 (used) bo iteration 86: Memory : (87.32% free): 11554717696(total), 10089988096 (free), 1464729600 (used) bo iteration 87: Memory : (87.32% free): 11554717696(total), 10089988096 (free), 1464729600 (used) bo iteration 88: Memory : (87.32% free): 11554717696(total), 10089988096 (free), 1464729600 (used) bo iteration 89: Memory : (87.03% free): 11554717696(total), 10056433664 (free), 1498284032 (used) bo iteration 90: Memory : (87.03% free): 11554717696(total), 10056433664 (free), 1498284032 (used) bo iteration 91: Memory : (87.03% free): 11554717696(total), 10056433664 (free), 1498284032 (used) bo iteration 92: Memory : (87.03% free): 11554717696(total), 10056433664 (free), 1498284032 (used) bo iteration 93: Memory : (87.03% free): 11554717696(total), 10056433664 (free), 1498284032 (used) bo iteration 94: Memory : (87.03% free): 11554717696(total), 10056433664 (free), 1498284032 (used) bo iteration 95: Memory : (86.72% free): 11554717696(total), 10020782080 (free), 1533935616 (used) bo iteration 96: Memory : (86.72% free): 11554717696(total), 10020782080 (free), 1533935616 (used) bo iteration 97: Memory : (86.72% free): 11554717696(total), 10020782080 (free), 1533935616 (used) bo iteration 98: Memory : (86.72% free): 11554717696(total), 10020782080 (free), 1533935616 (used) bo iteration 99: Memory : (86.72% free): 11554717696(total), 10020782080 (free), 1533935616 (used) bo iteration 100: Memory : (86.72% free): 11554717696(total), 10020782080 (free), 1533935616 (used) bo iteration 101: Memory : (86.72% free): 11554717696(total), 10020782080 (free), 1533935616 (used) bo iteration 102: Memory : (86.40% free): 11554717696(total), 9983033344 (free), 1571684352 (used) bo iteration 103: Memory : (86.40% free): 11554717696(total), 9983033344 (free), 1571684352 (used) bo iteration 104: Memory : (86.40% free): 11554717696(total), 9983033344 (free), 1571684352 (used) bo iteration 105: Memory : (86.40% free): 11554717696(total), 9983033344 (free), 1571684352 (used) bo iteration 106: Memory : (86.40% free): 11554717696(total), 9983033344 (free), 1571684352 (used) bo iteration 107: Memory : (86.40% free): 11554717696(total), 9983033344 (free), 1571684352 (used) bo iteration 108: Memory : (86.05% free): 11554717696(total), 9943187456 (free), 1611530240 (used) bo iteration 109: Memory : (86.05% free): 11554717696(total), 9943187456 (free), 1611530240 (used) bo iteration 110: Memory : (86.05% free): 11554717696(total), 9943187456 (free), 1611530240 (used) bo iteration 111: Memory : (86.03% free): 11554717696(total), 9941090304 (free), 1613627392 (used) bo iteration 112: Memory : (86.03% free): 11554717696(total), 9941090304 (free), 1613627392 (used) bo iteration 113: Memory : (86.03% free): 11554717696(total), 9941090304 (free), 1613627392 (used) bo iteration 114: Memory : (86.03% free): 11554717696(total), 9941090304 (free), 1613627392 (used) bo iteration 115: Memory : (85.67% free): 11554717696(total), 9899147264 (free), 1655570432 (used) bo iteration 116: Memory : (85.67% free): 11554717696(total), 9899147264 (free), 1655570432 (used) bo iteration 117: Memory : (85.67% free): 11554717696(total), 9899147264 (free), 1655570432 (used) bo iteration 118: Memory : (85.67% free): 11554717696(total), 9899147264 (free), 1655570432 (used) bo iteration 119: Memory : (85.67% free): 11554717696(total), 9899147264 (free), 1655570432 (used) bo iteration 120: Memory : (85.67% free): 11554717696(total), 9899147264 (free), 1655570432 (used) bo iteration 121: Memory : (85.67% free): 11554717696(total), 9899147264 (free), 1655570432 (used) bo iteration 122: Memory : (85.29% free): 11554717696(total), 9855107072 (free), 1699610624 (used) bo iteration 123: Memory : (85.29% free): 11554717696(total), 9855107072 (free), 1699610624 (used) bo iteration 124: Memory : (85.29% free): 11554717696(total), 9855107072 (free), 1699610624 (used) bo iteration 125: Memory : (85.29% free): 11554717696(total), 9855107072 (free), 1699610624 (used) bo iteration 126: Memory : (85.29% free): 11554717696(total), 9855107072 (free), 1699610624 (used) bo iteration 127: Memory : (85.29% free): 11554717696(total), 9855107072 (free), 1699610624 (used) bo iteration 128: Memory : (84.89% free): 11554717696(total), 9808969728 (free), 1745747968 (used) bo iteration 129: Memory : (84.89% free): 11554717696(total), 9808969728 (free), 1745747968 (used) bo iteration 130: Memory : (84.89% free): 11554717696(total), 9808969728 (free), 1745747968 (used) bo iteration 131: Memory : (84.89% free): 11554717696(total), 9808969728 (free), 1745747968 (used) bo iteration 132: Memory : (84.89% free): 11554717696(total), 9808969728 (free), 1745747968 (used) bo iteration 133: Memory : (84.89% free): 11554717696(total), 9808969728 (free), 1745747968 (used) bo iteration 134: Memory : (84.89% free): 11554717696(total), 9808969728 (free), 1745747968 (used) bo iteration 135: Memory : (84.46% free): 11554717696(total), 9758638080 (free), 1796079616 (used) bo iteration 136: Memory : (84.46% free): 11554717696(total), 9758638080 (free), 1796079616 (used) bo iteration 137: Memory : (84.46% free): 11554717696(total), 9758638080 (free), 1796079616 (used) bo iteration 138: Memory : (84.46% free): 11554717696(total), 9758638080 (free), 1796079616 (used) bo iteration 139: Memory : (84.44% free): 11554717696(total), 9756540928 (free), 1798176768 (used) bo iteration 140: Memory : (84.44% free): 11554717696(total), 9756540928 (free), 1798176768 (used) bo iteration 141: Memory : (84.00% free): 11554717696(total), 9706209280 (free), 1848508416 (used) bo iteration 142: Memory : (84.00% free): 11554717696(total), 9706209280 (free), 1848508416 (used) bo iteration 143: Memory : (84.00% free): 11554717696(total), 9706209280 (free), 1848508416 (used) bo iteration 144: Memory : (84.00% free): 11554717696(total), 9706209280 (free), 1848508416 (used) bo iteration 145: Memory : (84.00% free): 11554717696(total), 9706209280 (free), 1848508416 (used) bo iteration 146: Memory : (84.00% free): 11554717696(total), 9706209280 (free), 1848508416 (used) bo iteration 147: Memory : (84.00% free): 11554717696(total), 9706209280 (free), 1848508416 (used) bo iteration 148: Memory : (83.55% free): 11554717696(total), 9653780480 (free), 1900937216 (used) bo iteration 149: Memory : (83.55% free): 11554717696(total), 9653780480 (free), 1900937216 (used) bo iteration 150: Memory : (83.55% free): 11554717696(total), 9653780480 (free), 1900937216 (used) bo iteration 151: Memory : (83.55% free): 11554717696(total), 9653780480 (free), 1900937216 (used) bo iteration 152: Memory : (83.55% free): 11554717696(total), 9653780480 (free), 1900937216 (used) bo iteration 153: Memory : (83.55% free): 11554717696(total), 9653780480 (free), 1900937216 (used) bo iteration 154: Memory : (83.08% free): 11554717696(total), 9599254528 (free), 1955463168 (used) bo iteration 155: Memory : (83.08% free): 11554717696(total), 9599254528 (free), 1955463168 (used) bo iteration 156: Memory : (83.08% free): 11554717696(total), 9599254528 (free), 1955463168 (used) bo iteration 157: Memory : (83.08% free): 11554717696(total), 9599254528 (free), 1955463168 (used) bo iteration 158: Memory : (83.08% free): 11554717696(total), 9599254528 (free), 1955463168 (used) bo iteration 159: Memory : (83.08% free): 11554717696(total), 9599254528 (free), 1955463168 (used) bo iteration 160: Memory : (83.06% free): 11554717696(total), 9597157376 (free), 1957560320 (used) bo iteration 161: Memory : (82.57% free): 11554717696(total), 9540534272 (free), 2014183424 (used) bo iteration 162: Memory : (82.57% free): 11554717696(total), 9540534272 (free), 2014183424 (used) bo iteration 163: Memory : (82.55% free): 11554717696(total), 9538437120 (free), 2016280576 (used) bo iteration 164: Memory : (82.55% free): 11554717696(total), 9538437120 (free), 2016280576 (used) bo iteration 165: Memory : (82.55% free): 11554717696(total), 9538437120 (free), 2016280576 (used) bo iteration 166: Memory : (82.55% free): 11554717696(total), 9538437120 (free), 2016280576 (used) bo iteration 167: Memory : (82.04% free): 11554717696(total), 9479716864 (free), 2075000832 (used) bo iteration 168: Memory : (82.04% free): 11554717696(total), 9479716864 (free), 2075000832 (used) bo iteration 169: Memory : (82.04% free): 11554717696(total), 9479716864 (free), 2075000832 (used) bo iteration 170: Memory : (82.04% free): 11554717696(total), 9479716864 (free), 2075000832 (used) bo iteration 171: Memory : (82.04% free): 11554717696(total), 9479716864 (free), 2075000832 (used) bo iteration 172: Memory : (82.04% free): 11554717696(total), 9479716864 (free), 2075000832 (used) bo iteration 173: Memory : (82.04% free): 11554717696(total), 9479716864 (free), 2075000832 (used) bo iteration 174: Memory : (81.52% free): 11554717696(total), 9418899456 (free), 2135818240 (used) bo iteration 175: Memory : (81.52% free): 11554717696(total), 9418899456 (free), 2135818240 (used) bo iteration 176: Memory : (81.52% free): 11554717696(total), 9418899456 (free), 2135818240 (used) bo iteration 177: Memory : (81.52% free): 11554717696(total), 9418899456 (free), 2135818240 (used) bo iteration 178: Memory : (81.52% free): 11554717696(total), 9418899456 (free), 2135818240 (used) bo iteration 179: Memory : (81.52% free): 11554717696(total), 9418899456 (free), 2135818240 (used) bo iteration 180: Memory : (81.52% free): 11554717696(total), 9418899456 (free), 2135818240 (used) bo iteration 181: Memory : (80.97% free): 11554717696(total), 9355984896 (free), 2198732800 (used) bo iteration 182: Memory : (80.97% free): 11554717696(total), 9355984896 (free), 2198732800 (used) bo iteration 183: Memory : (80.97% free): 11554717696(total), 9355984896 (free), 2198732800 (used) bo iteration 184: Memory : (80.95% free): 11554717696(total), 9353887744 (free), 2200829952 (used) bo iteration 185: Memory : (80.95% free): 11554717696(total), 9353887744 (free), 2200829952 (used) bo iteration 186: Memory : (80.95% free): 11554717696(total), 9353887744 (free), 2200829952 (used) bo iteration 187: Memory : (80.37% free): 11554717696(total), 9286778880 (free), 2267938816 (used) bo iteration 188: Memory : (80.37% free): 11554717696(total), 9286778880 (free), 2267938816 (used) bo iteration 189: Memory : (80.37% free): 11554717696(total), 9286778880 (free), 2267938816 (used) bo iteration 190: Memory : (80.37% free): 11554717696(total), 9286778880 (free), 2267938816 (used) bo iteration 191: Memory : (80.37% free): 11554717696(total), 9286778880 (free), 2267938816 (used) bo iteration 192: Memory : (80.37% free): 11554717696(total), 9286778880 (free), 2267938816 (used) bo iteration 193: Memory : (80.37% free): 11554717696(total), 9286778880 (free), 2267938816 (used) bo iteration 194: Memory : (79.79% free): 11554717696(total), 9219670016 (free), 2335047680 (used) bo iteration 195: Memory : (79.79% free): 11554717696(total), 9219670016 (free), 2335047680 (used) bo iteration 196: Memory : (79.79% free): 11554717696(total), 9219670016 (free), 2335047680 (used) bo iteration 197: Memory : (79.79% free): 11554717696(total), 9219670016 (free), 2335047680 (used) bo iteration 198: Memory : (79.79% free): 11554717696(total), 9219670016 (free), 2335047680 (used) bo iteration 199: Memory : (79.79% free): 11554717696(total), 9219670016 (free), 2335047680 (used) bo iteration 200: Memory : (79.19% free): 11554717696(total), 9150464000 (free), 2404253696 (used) bo iteration 201: Memory : (79.19% free): 11554717696(total), 9150464000 (free), 2404253696 (used) bo iteration 202: Memory : (79.19% free): 11554717696(total), 9150464000 (free), 2404253696 (used) bo iteration 203: Memory : (79.19% free): 11554717696(total), 9150464000 (free), 2404253696 (used) bo iteration 204: Memory : (79.19% free): 11554717696(total), 9150464000 (free), 2404253696 (used) bo iteration 205: Memory : (79.19% free): 11554717696(total), 9150464000 (free), 2404253696 (used) bo iteration 206: Memory : (79.19% free): 11554717696(total), 9150464000 (free), 2404253696 (used) bo iteration 207: Memory : (78.56% free): 11554717696(total), 9077063680 (free), 2477654016 (used) bo iteration 208: Memory : (78.56% free): 11554717696(total), 9077063680 (free), 2477654016 (used) bo iteration 209: Memory : (78.54% free): 11554717696(total), 9074966528 (free), 2479751168 (used) bo iteration 210: Memory : (78.54% free): 11554717696(total), 9074966528 (free), 2479751168 (used) bo iteration 211: Memory : (78.52% free): 11554717696(total), 9072869376 (free), 2481848320 (used) bo iteration 212: Memory : (78.52% free): 11554717696(total), 9072869376 (free), 2481848320 (used) bo iteration 213: Memory : (77.87% free): 11554717696(total), 8997371904 (free), 2557345792 (used) bo iteration 214: Memory : (77.87% free): 11554717696(total), 8997371904 (free), 2557345792 (used) bo iteration 215: Memory : (77.87% free): 11554717696(total), 8997371904 (free), 2557345792 (used) bo iteration 216: Memory : (77.87% free): 11554717696(total), 8997371904 (free), 2557345792 (used) bo iteration 217: Memory : (77.87% free): 11554717696(total), 8997371904 (free), 2557345792 (used) bo iteration 218: Memory : (77.87% free): 11554717696(total), 8997371904 (free), 2557345792 (used) bo iteration 219: Memory : (77.87% free): 11554717696(total), 8997371904 (free), 2557345792 (used) bo iteration 220: Memory : (77.21% free): 11554717696(total), 8921874432 (free), 2632843264 (used) bo iteration 221: Memory : (77.21% free): 11554717696(total), 8921874432 (free), 2632843264 (used) bo iteration 222: Memory : (77.21% free): 11554717696(total), 8921874432 (free), 2632843264 (used) bo iteration 223: Memory : (77.21% free): 11554717696(total), 8921874432 (free), 2632843264 (used) bo iteration 224: Memory : (77.21% free): 11554717696(total), 8921874432 (free), 2632843264 (used) bo iteration 225: Memory : (77.21% free): 11554717696(total), 8921874432 (free), 2632843264 (used) bo iteration 226: Memory : (76.54% free): 11554717696(total), 8844279808 (free), 2710437888 (used) bo iteration 227: Memory : (76.54% free): 11554717696(total), 8844279808 (free), 2710437888 (used) bo iteration 228: Memory : (76.54% free): 11554717696(total), 8844279808 (free), 2710437888 (used) bo iteration 229: Memory : (76.54% free): 11554717696(total), 8844279808 (free), 2710437888 (used) bo iteration 230: Memory : (76.54% free): 11554717696(total), 8844279808 (free), 2710437888 (used) bo iteration 231: Memory : (76.54% free): 11554717696(total), 8844279808 (free), 2710437888 (used) bo iteration 232: Memory : (76.54% free): 11554717696(total), 8844279808 (free), 2710437888 (used) bo iteration 233: Memory : (75.83% free): 11554717696(total), 8762490880 (free), 2792226816 (used) bo iteration 234: Memory : (75.83% free): 11554717696(total), 8762490880 (free), 2792226816 (used) bo iteration 235: Memory : (75.82% free): 11554717696(total), 8760393728 (free), 2794323968 (used) bo iteration 236: Memory : (75.82% free): 11554717696(total), 8760393728 (free), 2794323968 (used) bo iteration 237: Memory : (75.80% free): 11554717696(total), 8758296576 (free), 2796421120 (used) bo iteration 238: Memory : (75.80% free): 11554717696(total), 8758296576 (free), 2796421120 (used) bo iteration 239: Memory : (75.80% free): 11554717696(total), 8758296576 (free), 2796421120 (used) bo iteration 240: Memory : (75.09% free): 11554717696(total), 8676507648 (free), 2878210048 (used) bo iteration 241: Memory : (75.09% free): 11554717696(total), 8676507648 (free), 2878210048 (used) bo iteration 242: Memory : (75.09% free): 11554717696(total), 8676507648 (free), 2878210048 (used) bo iteration 243: Memory : (75.09% free): 11554717696(total), 8676507648 (free), 2878210048 (used) bo iteration 244: Memory : (75.09% free): 11554717696(total), 8676507648 (free), 2878210048 (used) bo iteration 245: Memory : (75.09% free): 11554717696(total), 8676507648 (free), 2878210048 (used) bo iteration 246: Memory : (74.36% free): 11554717696(total), 8592621568 (free), 2962096128 (used) bo iteration 247: Memory : (74.36% free): 11554717696(total), 8592621568 (free), 2962096128 (used) bo iteration 248: Memory : (74.36% free): 11554717696(total), 8592621568 (free), 2962096128 (used) bo iteration 249: Memory : (74.36% free): 11554717696(total), 8592621568 (free), 2962096128 (used) bo iteration 250: Memory : (74.36% free): 11554717696(total), 8592621568 (free), 2962096128 (used) bo iteration 251: Memory : (74.36% free): 11554717696(total), 8592621568 (free), 2962096128 (used) bo iteration 252: Memory : (74.36% free): 11554717696(total), 8592621568 (free), 2962096128 (used) bo iteration 253: Memory : (73.62% free): 11554717696(total), 8506638336 (free), 3048079360 (used) bo iteration 254: Memory : (73.62% free): 11554717696(total), 8506638336 (free), 3048079360 (used) bo iteration 255: Memory : (73.62% free): 11554717696(total), 8506638336 (free), 3048079360 (used) bo iteration 256: Memory : (73.60% free): 11554717696(total), 8504541184 (free), 3050176512 (used) bo iteration 257: Memory : (73.58% free): 11554717696(total), 8502444032 (free), 3052273664 (used) bo iteration 258: Memory : (73.58% free): 11554717696(total), 8502444032 (free), 3052273664 (used) bo iteration 259: Memory : (72.80% free): 11554717696(total), 8412266496 (free), 3142451200 (used) bo iteration 260: Memory : (72.79% free): 11554717696(total), 8410169344 (free), 3144548352 (used)
bo iteration 261: Memory : (72.79% free): 11554717696(total), 8410169344 (free), 3144548352 (used) bo iteration 262: Memory : (72.79% free): 11554717696(total), 8410169344 (free), 3144548352 (used) bo iteration 263: Memory : (72.79% free): 11554717696(total), 8410169344 (free), 3144548352 (used) bo iteration 264: Memory : (72.79% free): 11554717696(total), 8410169344 (free), 3144548352 (used) bo iteration 265: Memory : (72.79% free): 11554717696(total), 8410169344 (free), 3144548352 (used) bo iteration 266: Memory : (72.01% free): 11554717696(total), 8319991808 (free), 3234725888 (used) bo iteration 267: Memory : (72.01% free): 11554717696(total), 8319991808 (free), 3234725888 (used) bo iteration 268: Memory : (72.01% free): 11554717696(total), 8319991808 (free), 3234725888 (used) bo iteration 269: Memory : (72.01% free): 11554717696(total), 8319991808 (free), 3234725888 (used) bo iteration 270: Memory : (72.01% free): 11554717696(total), 8319991808 (free), 3234725888 (used) bo iteration 271: Memory : (72.01% free): 11554717696(total), 8319991808 (free), 3234725888 (used) bo iteration 272: Memory : (71.21% free): 11554717696(total), 8227717120 (free), 3327000576 (used) bo iteration 273: Memory : (71.21% free): 11554717696(total), 8227717120 (free), 3327000576 (used) bo iteration 274: Memory : (71.21% free): 11554717696(total), 8227717120 (free), 3327000576 (used) bo iteration 275: Memory : (71.21% free): 11554717696(total), 8227717120 (free), 3327000576 (used) bo iteration 276: Memory : (71.21% free): 11554717696(total), 8227717120 (free), 3327000576 (used) bo iteration 277: Memory : (71.21% free): 11554717696(total), 8227717120 (free), 3327000576 (used) bo iteration 278: Memory : (71.21% free): 11554717696(total), 8227717120 (free), 3327000576 (used) bo iteration 279: Memory : (70.39% free): 11554717696(total), 8133345280 (free), 3421372416 (used) bo iteration 280: Memory : (70.39% free): 11554717696(total), 8133345280 (free), 3421372416 (used) bo iteration 281: Memory : (70.39% free): 11554717696(total), 8133345280 (free), 3421372416 (used) bo iteration 282: Memory : (70.37% free): 11554717696(total), 8131248128 (free), 3423469568 (used) bo iteration 283: Memory : (70.37% free): 11554717696(total), 8131248128 (free), 3423469568 (used) bo iteration 284: Memory : (70.35% free): 11554717696(total), 8129150976 (free), 3425566720 (used) bo iteration 285: Memory : (69.50% free): 11554717696(total), 8030584832 (free), 3524132864 (used) bo iteration 286: Memory : (69.50% free): 11554717696(total), 8030584832 (free), 3524132864 (used) bo iteration 287: Memory : (69.50% free): 11554717696(total), 8030584832 (free), 3524132864 (used) bo iteration 288: Memory : (69.50% free): 11554717696(total), 8030584832 (free), 3524132864 (used) bo iteration 289: Memory : (69.50% free): 11554717696(total), 8030584832 (free), 3524132864 (used) bo iteration 290: Memory : (69.50% free): 11554717696(total), 8030584832 (free), 3524132864 (used) bo iteration 291: Memory : (69.50% free): 11554717696(total), 8030584832 (free), 3524132864 (used) bo iteration 292: Memory : (68.65% free): 11554717696(total), 7932018688 (free), 3622699008 (used) bo iteration 293: Memory : (68.65% free): 11554717696(total), 7932018688 (free), 3622699008 (used) bo iteration 294: Memory : (68.65% free): 11554717696(total), 7932018688 (free), 3622699008 (used) bo iteration 295: Memory : (68.65% free): 11554717696(total), 7932018688 (free), 3622699008 (used) bo iteration 296: Memory : (68.65% free): 11554717696(total), 7932018688 (free), 3622699008 (used) bo iteration 297: Memory : (68.65% free): 11554717696(total), 7932018688 (free), 3622699008 (used) bo iteration 298: Memory : (68.65% free): 11554717696(total), 7932018688 (free), 3622699008 (used) bo iteration 299: Memory : (67.78% free): 11554717696(total), 7831355392 (free), 3723362304 (used) bo iteration 300: Memory : (67.78% free): 11554717696(total), 7831355392 (free), 3723362304 (used) bo iteration 301: Memory : (67.78% free): 11554717696(total), 7831355392 (free), 3723362304 (used) bo iteration 302: Memory : (67.78% free): 11554717696(total), 7831355392 (free), 3723362304 (used) bo iteration 303: Memory : (67.78% free): 11554717696(total), 7831355392 (free), 3723362304 (used) bo iteration 304: Memory : (67.76% free): 11554717696(total), 7829258240 (free), 3725459456 (used) bo iteration 305: Memory : (66.85% free): 11554717696(total), 7724400640 (free), 3830317056 (used) bo iteration 306: Memory : (66.83% free): 11554717696(total), 7722303488 (free), 3832414208 (used) bo iteration 307: Memory : (66.81% free): 11554717696(total), 7720206336 (free), 3834511360 (used) bo iteration 308: Memory : (66.80% free): 11554717696(total), 7718109184 (free), 3836608512 (used) bo iteration 309: Memory : (66.78% free): 11554717696(total), 7716012032 (free), 3838705664 (used) bo iteration 310: Memory : (66.78% free): 11554717696(total), 7716012032 (free), 3838705664 (used) bo iteration 311: Memory : (66.78% free): 11554717696(total), 7716012032 (free), 3838705664 (used) bo iteration 312: Memory : (65.87% free): 11554717696(total), 7611154432 (free), 3943563264 (used) bo iteration 313: Memory : (65.87% free): 11554717696(total), 7611154432 (free), 3943563264 (used) bo iteration 314: Memory : (65.87% free): 11554717696(total), 7611154432 (free), 3943563264 (used) bo iteration 315: Memory : (65.87% free): 11554717696(total), 7611154432 (free), 3943563264 (used) bo iteration 316: Memory : (65.87% free): 11554717696(total), 7611154432 (free), 3943563264 (used) bo iteration 317: Memory : (65.87% free): 11554717696(total), 7611154432 (free), 3943563264 (used) bo iteration 318: Memory : (64.94% free): 11554717696(total), 7504199680 (free), 4050518016 (used) bo iteration 319: Memory : (64.94% free): 11554717696(total), 7504199680 (free), 4050518016 (used) bo iteration 320: Memory : (64.94% free): 11554717696(total), 7504199680 (free), 4050518016 (used) bo iteration 321: Memory : (64.94% free): 11554717696(total), 7504199680 (free), 4050518016 (used) bo iteration 322: Memory : (64.94% free): 11554717696(total), 7504199680 (free), 4050518016 (used) bo iteration 323: Memory : (64.94% free): 11554717696(total), 7504199680 (free), 4050518016 (used) bo iteration 324: Memory : (64.94% free): 11554717696(total), 7504199680 (free), 4050518016 (used) bo iteration 325: Memory : (64.00% free): 11554717696(total), 7395147776 (free), 4159569920 (used) bo iteration 326: Memory : (64.00% free): 11554717696(total), 7395147776 (free), 4159569920 (used) bo iteration 327: Memory : (64.00% free): 11554717696(total), 7395147776 (free), 4159569920 (used) bo iteration 328: Memory : (64.00% free): 11554717696(total), 7395147776 (free), 4159569920 (used) bo iteration 329: Memory : (64.00% free): 11554717696(total), 7395147776 (free), 4159569920 (used) bo iteration 330: Memory : (64.00% free): 11554717696(total), 7395147776 (free), 4159569920 (used) bo iteration 331: Memory : (63.02% free): 11554717696(total), 7281901568 (free), 4272816128 (used) bo iteration 332: Memory : (63.00% free): 11554717696(total), 7279804416 (free), 4274913280 (used) bo iteration 333: Memory : (62.98% free): 11554717696(total), 7277707264 (free), 4277010432 (used) bo iteration 334: Memory : (62.98% free): 11554717696(total), 7277707264 (free), 4277010432 (used) bo iteration 335: Memory : (62.98% free): 11554717696(total), 7277707264 (free), 4277010432 (used) bo iteration 336: Memory : (62.98% free): 11554717696(total), 7277707264 (free), 4277010432 (used) bo iteration 337: Memory : (62.98% free): 11554717696(total), 7277707264 (free), 4277010432 (used) bo iteration 338: Memory : (62.00% free): 11554717696(total), 7164461056 (free), 4390256640 (used) bo iteration 339: Memory : (62.00% free): 11554717696(total), 7164461056 (free), 4390256640 (used) bo iteration 340: Memory : (62.00% free): 11554717696(total), 7164461056 (free), 4390256640 (used) bo iteration 341: Memory : (62.00% free): 11554717696(total), 7164461056 (free), 4390256640 (used) bo iteration 342: Memory : (62.00% free): 11554717696(total), 7164461056 (free), 4390256640 (used) bo iteration 343: Memory : (62.00% free): 11554717696(total), 7164461056 (free), 4390256640 (used) bo iteration 344: Memory : (61.01% free): 11554717696(total), 7049117696 (free), 4505600000 (used) bo iteration 345: Memory : (61.01% free): 11554717696(total), 7049117696 (free), 4505600000 (used) bo iteration 346: Memory : (61.01% free): 11554717696(total), 7049117696 (free), 4505600000 (used) bo iteration 347: Memory : (61.01% free): 11554717696(total), 7049117696 (free), 4505600000 (used) bo iteration 348: Memory : (61.01% free): 11554717696(total), 7049117696 (free), 4505600000 (used) bo iteration 349: Memory : (61.01% free): 11554717696(total), 7049117696 (free), 4505600000 (used) bo iteration 350: Memory : (61.01% free): 11554717696(total), 7049117696 (free), 4505600000 (used) bo iteration 351: Memory : (59.99% free): 11554717696(total), 6931677184 (free), 4623040512 (used) bo iteration 352: Memory : (59.99% free): 11554717696(total), 6931677184 (free), 4623040512 (used) bo iteration 353: Memory : (59.99% free): 11554717696(total), 6931677184 (free), 4623040512 (used) bo iteration 354: Memory : (59.99% free): 11554717696(total), 6931677184 (free), 4623040512 (used) bo iteration 355: Memory : (59.95% free): 11554717696(total), 6927482880 (free), 4627234816 (used) bo iteration 356: Memory : (59.94% free): 11554717696(total), 6925385728 (free), 4629331968 (used) bo iteration 357: Memory : (59.92% free): 11554717696(total), 6923288576 (free), 4631429120 (used) bo iteration 358: Memory : (58.88% free): 11554717696(total), 6803750912 (free), 4750966784 (used) bo iteration 359: Memory : (58.88% free): 11554717696(total), 6803750912 (free), 4750966784 (used) bo iteration 360: Memory : (58.88% free): 11554717696(total), 6803750912 (free), 4750966784 (used) bo iteration 361: Memory : (58.88% free): 11554717696(total), 6803750912 (free), 4750966784 (used) bo iteration 362: Memory : (58.88% free): 11554717696(total), 6803750912 (free), 4750966784 (used) bo iteration 363: Memory : (58.88% free): 11554717696(total), 6803750912 (free), 4750966784 (used) bo iteration 364: Memory : (57.83% free): 11554717696(total), 6682116096 (free), 4872601600 (used) bo iteration 365: Memory : (57.83% free): 11554717696(total), 6682116096 (free), 4872601600 (used) bo iteration 366: Memory : (57.83% free): 11554717696(total), 6682116096 (free), 4872601600 (used) bo iteration 367: Memory : (57.83% free): 11554717696(total), 6682116096 (free), 4872601600 (used) bo iteration 368: Memory : (57.83% free): 11554717696(total), 6682116096 (free), 4872601600 (used) bo iteration 369: Memory : (57.83% free): 11554717696(total), 6682116096 (free), 4872601600 (used) bo iteration 370: Memory : (57.83% free): 11554717696(total), 6682116096 (free), 4872601600 (used) bo iteration 371: Memory : (56.76% free): 11554717696(total), 6558384128 (free), 4996333568 (used) bo iteration 372: Memory : (56.76% free): 11554717696(total), 6558384128 (free), 4996333568 (used) bo iteration 373: Memory : (56.76% free): 11554717696(total), 6558384128 (free), 4996333568 (used) bo iteration 374: Memory : (56.76% free): 11554717696(total), 6558384128 (free), 4996333568 (used) bo iteration 375: Memory : (56.76% free): 11554717696(total), 6558384128 (free), 4996333568 (used) bo iteration 376: Memory : (56.76% free): 11554717696(total), 6558384128 (free), 4996333568 (used) bo iteration 377: Memory : (55.67% free): 11554717696(total), 6432555008 (free), 5122162688 (used) bo iteration 378: Memory : (55.67% free): 11554717696(total), 6432555008 (free), 5122162688 (used) bo iteration 379: Memory : (55.67% free): 11554717696(total), 6432555008 (free), 5122162688 (used) bo iteration 380: Memory : (55.65% free): 11554717696(total), 6430457856 (free), 5124259840 (used) bo iteration 381: Memory : (55.63% free): 11554717696(total), 6428360704 (free), 5126356992 (used) bo iteration 382: Memory : (55.63% free): 11554717696(total), 6428360704 (free), 5126356992 (used) bo iteration 383: Memory : (55.63% free): 11554717696(total), 6428360704 (free), 5126356992 (used) bo iteration 384: Memory : (54.53% free): 11554717696(total), 6300434432 (free), 5254283264 (used) bo iteration 385: Memory : (54.53% free): 11554717696(total), 6300434432 (free), 5254283264 (used) bo iteration 386: Memory : (54.53% free): 11554717696(total), 6300434432 (free), 5254283264 (used) bo iteration 387: Memory : (54.53% free): 11554717696(total), 6300434432 (free), 5254283264 (used) bo iteration 388: Memory : (54.53% free): 11554717696(total), 6300434432 (free), 5254283264 (used) bo iteration 389: Memory : (54.53% free): 11554717696(total), 6300434432 (free), 5254283264 (used) bo iteration 390: Memory : (53.40% free): 11554717696(total), 6170411008 (free), 5384306688 (used) bo iteration 391: Memory : (53.40% free): 11554717696(total), 6170411008 (free), 5384306688 (used) bo iteration 392: Memory : (53.40% free): 11554717696(total), 6170411008 (free), 5384306688 (used) bo iteration 393: Memory : (53.40% free): 11554717696(total), 6170411008 (free), 5384306688 (used) bo iteration 394: Memory : (53.40% free): 11554717696(total), 6170411008 (free), 5384306688 (used) bo iteration 395: Memory : (53.40% free): 11554717696(total), 6170411008 (free), 5384306688 (used) bo iteration 396: Memory : (53.40% free): 11554717696(total), 6170411008 (free), 5384306688 (used) bo iteration 397: Memory : (52.26% free): 11554717696(total), 6038290432 (free), 5516427264 (used) bo iteration 398: Memory : (52.26% free): 11554717696(total), 6038290432 (free), 5516427264 (used) bo iteration 399: Memory : (52.24% free): 11554717696(total), 6036193280 (free), 5518524416 (used) bo iteration 400: Memory : (52.22% free): 11554717696(total), 6034096128 (free), 5520621568 (used) bo iteration 401: Memory : (52.20% free): 11554717696(total), 6031998976 (free), 5522718720 (used) bo iteration 402: Memory : (52.19% free): 11554717696(total), 6029901824 (free), 5524815872 (used) bo iteration 403: Memory : (51.01% free): 11554717696(total), 5893586944 (free), 5661130752 (used) bo iteration 404: Memory : (50.99% free): 11554717696(total), 5891489792 (free), 5663227904 (used) bo iteration 405: Memory : (50.97% free): 11554717696(total), 5889392640 (free), 5665325056 (used) bo iteration 406: Memory : (50.97% free): 11554717696(total), 5889392640 (free), 5665325056 (used) bo iteration 407: Memory : (50.97% free): 11554717696(total), 5889392640 (free), 5665325056 (used) bo iteration 408: Memory : (50.97% free): 11554717696(total), 5889392640 (free), 5665325056 (used) bo iteration 409: Memory : (50.97% free): 11554717696(total), 5889392640 (free), 5665325056 (used) bo iteration 410: Memory : (49.79% free): 11554717696(total), 5753077760 (free), 5801639936 (used) bo iteration 411: Memory : (49.79% free): 11554717696(total), 5753077760 (free), 5801639936 (used) bo iteration 412: Memory : (49.79% free): 11554717696(total), 5753077760 (free), 5801639936 (used) bo iteration 413: Memory : (49.79% free): 11554717696(total), 5753077760 (free), 5801639936 (used) bo iteration 414: Memory : (49.79% free): 11554717696(total), 5753077760 (free), 5801639936 (used) bo iteration 415: Memory : (49.79% free): 11554717696(total), 5753077760 (free), 5801639936 (used) bo iteration 416: Memory : (48.59% free): 11554717696(total), 5614665728 (free), 5940051968 (used) bo iteration 417: Memory : (48.59% free): 11554717696(total), 5614665728 (free), 5940051968 (used) bo iteration 418: Memory : (48.59% free): 11554717696(total), 5614665728 (free), 5940051968 (used) bo iteration 419: Memory : (48.57% free): 11554717696(total), 5612568576 (free), 5942149120 (used) bo iteration 420: Memory : (48.54% free): 11554717696(total), 5608374272 (free), 5946343424 (used) bo iteration 421: Memory : (48.50% free): 11554717696(total), 5604179968 (free), 5950537728 (used) bo iteration 422: Memory : (48.46% free): 11554717696(total), 5599985664 (free), 5954732032 (used) bo iteration 423: Memory : (47.21% free): 11554717696(total), 5455282176 (free), 6099435520 (used) bo iteration 424: Memory : (47.18% free): 11554717696(total), 5451087872 (free), 6103629824 (used) bo iteration 425: Memory : (47.14% free): 11554717696(total), 5446893568 (free), 6107824128 (used) bo iteration 426: Memory : (47.10% free): 11554717696(total), 5442699264 (free), 6112018432 (used) bo iteration 427: Memory : (47.07% free): 11554717696(total), 5438504960 (free), 6116212736 (used) bo iteration 428: Memory : (47.03% free): 11554717696(total), 5434310656 (free), 6120407040 (used) bo iteration 429: Memory : (46.99% free): 11554717696(total), 5430116352 (free), 6124601344 (used) bo iteration 430: Memory : (45.76% free): 11554717696(total), 5287510016 (free), 6267207680 (used) bo iteration 431: Memory : (45.76% free): 11554717696(total), 5287510016 (free), 6267207680 (used) bo iteration 432: Memory : (45.76% free): 11554717696(total), 5287510016 (free), 6267207680 (used) bo iteration 433: Memory : (45.76% free): 11554717696(total), 5287510016 (free), 6267207680 (used) bo iteration 434: Memory : (45.76% free): 11554717696(total), 5287510016 (free), 6267207680 (used) bo iteration 435: Memory : (45.76% free): 11554717696(total), 5287510016 (free), 6267207680 (used) bo iteration 436: Memory : (44.51% free): 11554717696(total), 5142806528 (free), 6411911168 (used) bo iteration 437: Memory : (44.51% free): 11554717696(total), 5142806528 (free), 6411911168 (used) bo iteration 438: Memory : (44.51% free): 11554717696(total), 5142806528 (free), 6411911168 (used) bo iteration 439: Memory : (44.51% free): 11554717696(total), 5142806528 (free), 6411911168 (used) bo iteration 440: Memory : (44.51% free): 11554717696(total), 5142806528 (free), 6411911168 (used) bo iteration 441: Memory : (44.51% free): 11554717696(total), 5142806528 (free), 6411911168 (used) bo iteration 442: Memory : (44.51% free): 11554717696(total), 5142806528 (free), 6411911168 (used) bo iteration 443: Memory : (43.24% free): 11554717696(total), 4996005888 (free), 6558711808 (used) bo iteration 444: Memory : (43.24% free): 11554717696(total), 4996005888 (free), 6558711808 (used) bo iteration 445: Memory : (43.24% free): 11554717696(total), 4996005888 (free), 6558711808 (used) bo iteration 446: Memory : (43.24% free): 11554717696(total), 4996005888 (free), 6558711808 (used) bo iteration 447: Memory : (43.24% free): 11554717696(total), 4996005888 (free), 6558711808 (used) bo iteration 448: Memory : (43.24% free): 11554717696(total), 4996005888 (free), 6558711808 (used) bo iteration 449: Memory : (41.95% free): 11554717696(total), 4847108096 (free), 6707609600 (used) bo iteration 450: Memory : (41.95% free): 11554717696(total), 4847108096 (free), 6707609600 (used) bo iteration 451: Memory : (41.95% free): 11554717696(total), 4847108096 (free), 6707609600 (used) bo iteration 452: Memory : (41.91% free): 11554717696(total), 4842913792 (free), 6711803904 (used) bo iteration 453: Memory : (41.88% free): 11554717696(total), 4838719488 (free), 6715998208 (used) bo iteration 454: Memory : (41.88% free): 11554717696(total), 4838719488 (free), 6715998208 (used) bo iteration 455: Memory : (41.88% free): 11554717696(total), 4838719488 (free), 6715998208 (used) bo iteration 456: Memory : (40.57% free): 11554717696(total), 4687724544 (free), 6866993152 (used) bo iteration 457: Memory : (40.57% free): 11554717696(total), 4687724544 (free), 6866993152 (used) bo iteration 458: Memory : (40.57% free): 11554717696(total), 4687724544 (free), 6866993152 (used) bo iteration 459: Memory : (40.57% free): 11554717696(total), 4687724544 (free), 6866993152 (used) bo iteration 460: Memory : (40.57% free): 11554717696(total), 4687724544 (free), 6866993152 (used) bo iteration 461: Memory : (40.57% free): 11554717696(total), 4687724544 (free), 6866993152 (used) bo iteration 462: Memory : (39.24% free): 11554717696(total), 4534632448 (free), 7020085248 (used) bo iteration 463: Memory : (39.24% free): 11554717696(total), 4534632448 (free), 7020085248 (used) bo iteration 464: Memory : (39.24% free): 11554717696(total), 4534632448 (free), 7020085248 (used) bo iteration 465: Memory : (39.24% free): 11554717696(total), 4534632448 (free), 7020085248 (used) bo iteration 466: Memory : (39.24% free): 11554717696(total), 4534632448 (free), 7020085248 (used) bo iteration 467: Memory : (39.24% free): 11554717696(total), 4534632448 (free), 7020085248 (used) bo iteration 468: Memory : (39.24% free): 11554717696(total), 4534632448 (free), 7020085248 (used) bo iteration 469: Memory : (37.90% free): 11554717696(total), 4379443200 (free), 7175274496 (used) bo iteration 470: Memory : (37.90% free): 11554717696(total), 4379443200 (free), 7175274496 (used) bo iteration 471: Memory : (37.90% free): 11554717696(total), 4379443200 (free), 7175274496 (used) bo iteration 472: Memory : (37.90% free): 11554717696(total), 4379443200 (free), 7175274496 (used) bo iteration 473: Memory : (37.90% free): 11554717696(total), 4379443200 (free), 7175274496 (used) bo iteration 474: Memory : (37.90% free): 11554717696(total), 4379443200 (free), 7175274496 (used) bo iteration 475: Memory : (36.54% free): 11554717696(total), 4222156800 (free), 7332560896 (used) bo iteration 476: Memory : (36.50% free): 11554717696(total), 4217962496 (free), 7336755200 (used) bo iteration 477: Memory : (36.47% free): 11554717696(total), 4213768192 (free), 7340949504 (used) bo iteration 478: Memory : (36.47% free): 11554717696(total), 4213768192 (free), 7340949504 (used) bo iteration 479: Memory : (36.47% free): 11554717696(total), 4213768192 (free), 7340949504 (used) bo iteration 480: Memory : (36.47% free): 11554717696(total), 4213768192 (free), 7340949504 (used) bo iteration 481: Memory : (36.47% free): 11554717696(total), 4213768192 (free), 7340949504 (used) bo iteration 482: Memory : (35.09% free): 11554717696(total), 4054384640 (free), 7500333056 (used)
CUDA memory usage reached maximum around 627 bo iteration:
bo iteration 610: Memory : (5.36% free): 11554717696(total), 619249664 (free), 10935468032 (used) bo iteration 611: Memory : (5.36% free): 11554717696(total), 619249664 (free), 10935468032 (used) bo iteration 612: Memory : (5.36% free): 11554717696(total), 619249664 (free), 10935468032 (used) bo iteration 613: Memory : (5.36% free): 11554717696(total), 619249664 (free), 10935468032 (used) bo iteration 614: Memory : (3.62% free): 11554717696(total), 417923072 (free), 11136794624 (used) bo iteration 615: Memory : (3.62% free): 11554717696(total), 417923072 (free), 11136794624 (used) bo iteration 616: Memory : (3.62% free): 11554717696(total), 417923072 (free), 11136794624 (used) bo iteration 617: Memory : (3.62% free): 11554717696(total), 417923072 (free), 11136794624 (used) bo iteration 618: Memory : (3.62% free): 11554717696(total), 417923072 (free), 11136794624 (used) bo iteration 619: Memory : (3.62% free): 11554717696(total), 417923072 (free), 11136794624 (used) bo iteration 620: Memory : (3.62% free): 11554717696(total), 417923072 (free), 11136794624 (used) bo iteration 621: Memory : (1.86% free): 11554717696(total), 214499328 (free), 11340218368 (used) bo iteration 622: Memory : (1.86% free): 11554717696(total), 214499328 (free), 11340218368 (used) bo iteration 623: Memory : (1.86% free): 11554717696(total), 214499328 (free), 11340218368 (used) bo iteration 624: Memory : (1.86% free): 11554717696(total), 214499328 (free), 11340218368 (used) bo iteration 625: Memory : (1.86% free): 11554717696(total), 214499328 (free), 11340218368 (used) bo iteration 626: Memory : (1.86% free): 11554717696(total), 214499328 (free), 11340218368 (used) bo iteration 627: Memory : (0.08% free): 11554717696(total), 8978432 (free), 11545739264 (used) bo iteration 628: Memory : (0.08% free): 11554717696(total), 8978432 (free), 11545739264 (used) bo iteration 629: Memory : (0.08% free): 11554717696(total), 8978432 (free), 11545739264 (used) bo iteration 630: Memory : (0.08% free): 11554717696(total), 8978432 (free), 11545739264 (used) bo iteration 631: Memory : (0.08% free): 11554717696(total), 8978432 (free), 11545739264 (used) bo iteration 632: Memory : (0.08% free): 11554717696(total), 8978432 (free), 11545739264 (used) bo iteration 633: Memory : (0.08% free): 11554717696(total), 8978432 (free), 11545739264 (used) bo iteration 634: Memory : (73.57% free): 11554717696(total), 8500346880 (free), 3054370816 (used) bo iteration 635: Memory : (73.57% free): 11554717696(total), 8500346880 (free), 3054370816 (used) bo iteration 636: Memory : (70.43% free): 11554717696(total), 8137539584 (free), 3417178112 (used) bo iteration 637: Memory : (70.43% free): 11554717696(total), 8137539584 (free), 3417178112 (used) bo iteration 638: Memory : (70.43% free): 11554717696(total), 8137539584 (free), 3417178112 (used) bo iteration 639: Memory : (67.27% free): 11554717696(total), 7772635136 (free), 3782082560 (used) bo iteration 640: Memory : (65.45% free): 11554717696(total), 7562919936 (free), 3991797760 (used) bo iteration 641: Memory : (65.45% free): 11554717696(total), 7562919936 (free), 3991797760 (used) bo iteration 642: Memory : (65.45% free): 11554717696(total), 7562919936 (free), 3991797760 (used) bo iteration 643: Memory : (65.45% free): 11554717696(total), 7562919936 (free), 3991797760 (used) bo iteration 644: Memory : (65.45% free): 11554717696(total), 7562919936 (free), 3991797760 (used) bo iteration 645: Memory : (65.45% free): 11554717696(total), 7562919936 (free), 3991797760 (used) bo iteration 646: Memory : (65.45% free): 11554717696(total), 7562919936 (free), 3991797760 (used) bo iteration 647: Memory : (63.62% free): 11554717696(total), 7351107584 (free), 4203610112 (used) bo iteration 648: Memory : (63.62% free): 11554717696(total), 7351107584 (free), 4203610112 (used) bo iteration 649: Memory : (63.62% free): 11554717696(total), 7351107584 (free), 4203610112 (used) bo iteration 650: Memory : (63.62% free): 11554717696(total), 7351107584 (free), 4203610112 (used) bo iteration 651: Memory : (63.62% free): 11554717696(total), 7351107584 (free), 4203610112 (used) bo iteration 652: Memory : (63.62% free): 11554717696(total), 7351107584 (free), 4203610112 (used) bo iteration 653: Memory : (61.77% free): 11554717696(total), 7137198080 (free), 4417519616 (used) bo iteration 654: Memory : (61.77% free): 11554717696(total), 7137198080 (free), 4417519616 (used) bo iteration 655: Memory : (61.77% free): 11554717696(total), 7137198080 (free), 4417519616 (used) bo iteration 656: Memory : (61.77% free): 11554717696(total), 7137198080 (free), 4417519616 (used) bo iteration 657: Memory : (61.77% free): 11554717696(total), 7137198080 (free), 4417519616 (used)
bo iteration 792: Memory : (18.70% free): 11554717696(total), 2160656384 (free), 9394061312 (used) bo iteration 793: Memory : (18.70% free): 11554717696(total), 2160656384 (free), 9394061312 (used) bo iteration 794: Memory : (18.70% free): 11554717696(total), 2160656384 (free), 9394061312 (used) bo iteration 795: Memory : (18.70% free): 11554717696(total), 2160656384 (free), 9394061312 (used) bo iteration 796: Memory : (18.70% free): 11554717696(total), 2160656384 (free), 9394061312 (used) bo iteration 797: Memory : (18.70% free): 11554717696(total), 2160656384 (free), 9394061312 (used) bo iteration 798: Memory : (16.45% free): 11554717696(total), 1900609536 (free), 9654108160 (used) bo iteration 799: Memory : (16.45% free): 11554717696(total), 1900609536 (free), 9654108160 (used) bo iteration 800: Memory : (16.45% free): 11554717696(total), 1900609536 (free), 9654108160 (used) bo iteration 801: Memory : (16.45% free): 11554717696(total), 1900609536 (free), 9654108160 (used) bo iteration 802: Memory : (16.45% free): 11554717696(total), 1900609536 (free), 9654108160 (used) bo iteration 803: Memory : (16.45% free): 11554717696(total), 1900609536 (free), 9654108160 (used) bo iteration 804: Memory : (14.18% free): 11554717696(total), 1638465536 (free), 9916252160 (used) bo iteration 805: Memory : (14.18% free): 11554717696(total), 1638465536 (free), 9916252160 (used) bo iteration 806: Memory : (14.18% free): 11554717696(total), 1638465536 (free), 9916252160 (used) bo iteration 807: Memory : (14.18% free): 11554717696(total), 1638465536 (free), 9916252160 (used) bo iteration 808: Memory : (14.18% free): 11554717696(total), 1638465536 (free), 9916252160 (used) bo iteration 809: Memory : (14.18% free): 11554717696(total), 1638465536 (free), 9916252160 (used) bo iteration 810: Memory : (14.18% free): 11554717696(total), 1638465536 (free), 9916252160 (used) bo iteration 811: Memory : (11.89% free): 11554717696(total), 1374224384 (free), 10180493312 (used) bo iteration 812: Memory : (11.89% free): 11554717696(total), 1374224384 (free), 10180493312 (used) bo iteration 813: Memory : (11.89% free): 11554717696(total), 1374224384 (free), 10180493312 (used) bo iteration 814: Memory : (11.89% free): 11554717696(total), 1374224384 (free), 10180493312 (used) bo iteration 815: Memory : (11.89% free): 11554717696(total), 1374224384 (free), 10180493312 (used) bo iteration 816: Memory : (11.89% free): 11554717696(total), 1374224384 (free), 10180493312 (used) bo iteration 817: Memory : (9.59% free): 11554717696(total), 1107886080 (free), 10446831616 (used) bo iteration 818: Memory : (9.59% free): 11554717696(total), 1107886080 (free), 10446831616 (used) bo iteration 819: Memory : (9.59% free): 11554717696(total), 1107886080 (free), 10446831616 (used) bo iteration 820: Memory : (9.59% free): 11554717696(total), 1107886080 (free), 10446831616 (used) bo iteration 821: Memory : (9.59% free): 11554717696(total), 1107886080 (free), 10446831616 (used) bo iteration 822: Memory : (9.59% free): 11554717696(total), 1107886080 (free), 10446831616 (used) bo iteration 823: Memory : (9.59% free): 11554717696(total), 1107886080 (free), 10446831616 (used) bo iteration 824: Memory : (7.27% free): 11554717696(total), 839450624 (free), 10715267072 (used) bo iteration 825: Memory : (7.27% free): 11554717696(total), 839450624 (free), 10715267072 (used) bo iteration 826: Memory : (7.27% free): 11554717696(total), 839450624 (free), 10715267072 (used) bo iteration 827: Memory : (7.27% free): 11554717696(total), 839450624 (free), 10715267072 (used) bo iteration 828: Memory : (7.27% free): 11554717696(total), 839450624 (free), 10715267072 (used) bo iteration 829: Memory : (7.27% free): 11554717696(total), 839450624 (free), 10715267072 (used) bo iteration 830: Memory : (4.92% free): 11554717696(total), 568918016 (free), 10985799680 (used) bo iteration 831: Memory : (4.92% free): 11554717696(total), 568918016 (free), 10985799680 (used)
bo iteration 832: Memory : (4.92% free): 11554717696(total), 568918016 (free), 10985799680 (used) bo iteration 833: Memory : (4.92% free): 11554717696(total), 568918016 (free), 10985799680 (used) bo iteration 834: Memory : (4.92% free): 11554717696(total), 568918016 (free), 10985799680 (used) bo iteration 835: Memory : (4.92% free): 11554717696(total), 568918016 (free), 10985799680 (used) bo iteration 836: Memory : (4.92% free): 11554717696(total), 568918016 (free), 10985799680 (used) bo iteration 837: Memory : (2.56% free): 11554717696(total), 296288256 (free), 11258429440 (used) bo iteration 838: Memory : (2.56% free): 11554717696(total), 296288256 (free), 11258429440 (used) bo iteration 839: Memory : (2.56% free): 11554717696(total), 296288256 (free), 11258429440 (used) bo iteration 840: Memory : (2.56% free): 11554717696(total), 296288256 (free), 11258429440 (used) bo iteration 841: Memory : (2.56% free): 11554717696(total), 296288256 (free), 11258429440 (used) bo iteration 842: Memory : (2.56% free): 11554717696(total), 296288256 (free), 11258429440 (used) bo iteration 843: Memory : (0.19% free): 11554717696(total), 21561344 (free), 11533156352 (used) bo iteration 844: Memory : (0.19% free): 11554717696(total), 21561344 (free), 11533156352 (used) bo iteration 845: Memory : (0.19% free): 11554717696(total), 21561344 (free), 11533156352 (used) bo iteration 846: Memory : (0.19% free): 11554717696(total), 21561344 (free), 11533156352 (used) bo iteration 847: Memory : (0.19% free): 11554717696(total), 21561344 (free), 11533156352 (used) bo iteration 848: Memory : (0.19% free): 11554717696(total), 21561344 (free), 11533156352 (used) bo iteration 849: Memory : (0.19% free): 11554717696(total), 21561344 (free), 11533156352 (used) bo iteration 850: Memory : (58.32% free): 11554717696(total), 6738739200 (free), 4815978496 (used) bo iteration 851: Memory : (58.32% free): 11554717696(total), 6738739200 (free), 4815978496 (used) bo iteration 852: Memory : (58.32% free): 11554717696(total), 6738739200 (free), 4815978496 (used) bo iteration 853: Memory : (54.13% free): 11554717696(total), 6254297088 (free), 5300420608 (used) bo iteration 854: Memory : (54.13% free): 11554717696(total), 6254297088 (free), 5300420608 (used) bo iteration 855: Memory : (54.13% free): 11554717696(total), 6254297088 (free), 5300420608 (used) bo iteration 856: Memory : (54.13% free): 11554717696(total), 6254297088 (free), 5300420608 (used) bo iteration 857: Memory : (49.90% free): 11554717696(total), 5765660672 (free), 5789057024 (used) bo iteration 858: Memory : (49.90% free): 11554717696(total), 5765660672 (free), 5789057024 (used) bo iteration 859: Memory : (49.90% free): 11554717696(total), 5765660672 (free), 5789057024 (used) bo iteration 860: Memory : (49.90% free): 11554717696(total), 5765660672 (free), 5789057024 (used)
CUDA out of memory error happened at 2419 iteration. I am wondering if botorch allows few thousands of bo iterations. bo iteration 2400: Memory : (6.34% free): 11554717696(total), 732495872 (free), 10822221824 (used) bo iteration 2401: Memory : (6.34% free): 11554717696(total), 732495872 (free), 10822221824 (used) bo iteration 2402: Memory : (6.34% free): 11554717696(total), 732495872 (free), 10822221824 (used) bo iteration 2403: Memory : (6.32% free): 11554717696(total), 730398720 (free), 10824318976 (used) bo iteration 2404: Memory : (6.32% free): 11554717696(total), 730398720 (free), 10824318976 (used) bo iteration 2405: Memory : (6.32% free): 11554717696(total), 730398720 (free), 10824318976 (used) bo iteration 2406: Memory : (6.32% free): 11554717696(total), 730398720 (free), 10824318976 (used) bo iteration 2407: Memory : (6.32% free): 11554717696(total), 730398720 (free), 10824318976 (used) bo iteration 2408: Memory : (6.32% free): 11554717696(total), 730398720 (free), 10824318976 (used) bo iteration 2409: Memory : (6.32% free): 11554717696(total), 730398720 (free), 10824318976 (used)
bo iteration 2410: Memory : (6.30% free): 11554717696(total), 728301568 (free), 10826416128 (used) bo iteration 2411: Memory : (6.30% free): 11554717696(total), 728301568 (free), 10826416128 (used) bo iteration 2412: Memory : (6.30% free): 11554717696(total), 728301568 (free), 10826416128 (used) bo iteration 2413: Memory : (6.30% free): 11554717696(total), 728301568 (free), 10826416128 (used) bo iteration 2414: Memory : (6.30% free): 11554717696(total), 728301568 (free), 10826416128 (used) bo iteration 2415: Memory : (6.30% free): 11554717696(total), 728301568 (free), 10826416128 (used) bo iteration 2416: Memory : (6.25% free): 11554717696(total), 722010112 (free), 10832707584 (used) bo iteration 2417: Memory : (6.23% free): 11554717696(total), 719912960 (free), 10834804736 (used) bo iteration 2418: Memory : (6.21% free): 11554717696(total), 717815808 (free), 10836901888 (used) bo iteration 2419: Memory : (6.21% free): 11554717696(total), 717815808 (free), 10836901888 (used) x2_ = torch.cat([x2, x2_pad, x2_norm], dim=-1) RuntimeError: CUDA out of memory. Tried to allocate 742.00 MiB (GPU 0; 10.76 GiB total capacity; 2.76 GiB already allocated; 688.56 MiB free; 8.98 GiB reserved in total by PyTorch)
If i changed to CPU, the following error happened at iteration 147. You can reproduce the following error by change cuda to cpu.
L = _psd_safe_cholesky(A, out=out, jitter=jitter, max_tries=max_tries) File "/home/yangx94/anaconda3/envs/botorch/lib/python3.7/site-packages/gpytorch/utils/cholesky.py", line 48, in _psd_safe_cholesky raise NotPSDError(f"Matrix not positive definite after repeatedly adding jitter up to {jitter_new:.1e}.") gpytorch.utils.errors.NotPSDError: Matrix not positive definite after repeatedly adding jitter up to 1.0e-04.
Sorry for the late response.
Re cuda memory increasing: PyTorch has a habit of reserving the memory and not freeing until it is asked to do so. See here for details on this: https://pytorch.org/docs/stable/notes/cuda.html#memory-management. TL;DR is that it reserves the memory from CUDA and keeps it around (for use in other PyTorch operations) even if it is not being actively used to avoid the overhead of CUDA memory allocation. This is why the nvidia-smi can show weird memory patterns. You can call torch.cuda.empty_cache() if you want PyTorch to release all non-allocated memory. I found this (coupled with gc.collect()) to help with some GPU OOM issues.
Re NotPSDError: This suggests that you have observations clustered together in the search space / repeated observations. This is likely to lead to numerical issues, which typically surface as a NotPSDError. If you're not already doing so, using dtype=torch.double will help with this. Another thing you can do is to wrap the part of the code that raises this error in gpytorch.settings.cholesky_max_tries(value) where larger values will add larger jitters to try to avoid numerical singularity. This comes at the expense of numerical accuracy (you're computing Cholesky factor of a different matrix than you originally had) but sometimes that is necessary. The default value is 3, and we typically go up to 6.
