linopy
linopy copied to clipboard
PyPSA
`io_api="mps"` leads to memory overhead
Hi,
I was looking into memory consumption a little, and it seems that that passing an LP to the solver (Gurobi in this case) using the .mps format has a significant memory overhead. Here using mps:
Filename: /home/koen/linopy/linopy/solvers.py
Line # Mem usage Increment Occurrences Line Contents
=============================================================
568 758.0 MiB 758.0 MiB 1 @profile
569 def run_gurobi(
570 model,
571 io_api=None,
572 problem_fn=None,
573 solution_fn=None,
574 log_fn=None,
575 warmstart_fn=None,
576 basis_fn=None,
577 keep_files=False,
578 env=None,
579 **solver_options,
580 ):
581 """
582 Solve a linear problem using the gurobi solver.
583
584 This function communicates with gurobi using the gurubipy package.
585 """
586 # see https://www.gurobi.com/documentation/10.0/refman/optimization_status_codes.html
587 758.0 MiB 0.0 MiB 18 CONDITION_MAP = {
588 758.0 MiB 0.0 MiB 1 1: "unknown",
589 758.0 MiB 0.0 MiB 1 2: "optimal",
590 758.0 MiB 0.0 MiB 1 3: "infeasible",
591 758.0 MiB 0.0 MiB 1 4: "infeasible_or_unbounded",
592 758.0 MiB 0.0 MiB 1 5: "unbounded",
593 758.0 MiB 0.0 MiB 1 6: "other",
594 758.0 MiB 0.0 MiB 1 7: "iteration_limit",
595 758.0 MiB 0.0 MiB 1 8: "terminated_by_limit",
596 758.0 MiB 0.0 MiB 1 9: "time_limit",
597 758.0 MiB 0.0 MiB 1 10: "optimal",
598 758.0 MiB 0.0 MiB 1 11: "user_interrupt",
599 758.0 MiB 0.0 MiB 1 12: "other",
600 758.0 MiB 0.0 MiB 1 13: "suboptimal",
601 758.0 MiB 0.0 MiB 1 14: "unknown",
602 758.0 MiB 0.0 MiB 1 15: "terminated_by_limit",
603 758.0 MiB 0.0 MiB 1 16: "internal_solver_error",
604 758.0 MiB 0.0 MiB 1 17: "internal_solver_error",
605 }
606
607 758.0 MiB 0.0 MiB 1 log_fn = maybe_convert_path(log_fn)
608 758.0 MiB 0.0 MiB 1 warmstart_fn = maybe_convert_path(warmstart_fn)
609 758.0 MiB 0.0 MiB 1 basis_fn = maybe_convert_path(basis_fn)
610
611 1509.5 MiB 0.0 MiB 2 with contextlib.ExitStack() as stack:
612 758.0 MiB 0.0 MiB 1 if env is None:
613 758.9 MiB 0.9 MiB 1 env = stack.enter_context(gurobipy.Env())
614
615 758.9 MiB 0.0 MiB 1 if io_api is None or io_api in ["lp", "mps"]:
616 1197.0 MiB 438.0 MiB 1 problem_fn = model.to_file(problem_fn)
617 1197.0 MiB 0.0 MiB 1 problem_fn = maybe_convert_path(problem_fn)
618 1222.5 MiB 25.5 MiB 1 m = gurobipy.read(problem_fn, env=env)
619 elif io_api == "direct":
620 problem_fn = None
621 m = model.to_gurobipy(env=env)
622 else:
623 raise ValueError(
624 "Keyword argument `io_api` has to be one of `lp`, `mps`, `direct` or None"
625 )
626
627 1222.5 MiB 0.0 MiB 1 if solver_options is not None:
628 1222.5 MiB 0.0 MiB 10 for key, value in solver_options.items():
629 1222.5 MiB 0.0 MiB 9 m.setParam(key, value)
630 1222.5 MiB 0.0 MiB 1 if log_fn is not None:
631 1222.5 MiB 0.0 MiB 1 m.setParam("logfile", log_fn)
632
633 1222.5 MiB 0.0 MiB 1 if warmstart_fn:
634 m.read(warmstart_fn)
635 1509.5 MiB 287.0 MiB 1 m.optimize()
636
637 1509.5 MiB 0.0 MiB 1 if basis_fn:
638 try:
639 m.write(basis_fn)
640 except gurobipy.GurobiError as err:
641 logger.info("No model basis stored. Raised error: %s", err)
642
643 1509.5 MiB 0.0 MiB 1 condition = m.status
644 1509.5 MiB 0.0 MiB 1 termination_condition = CONDITION_MAP.get(condition, condition)
645 1509.5 MiB 0.0 MiB 1 status = Status.from_termination_condition(termination_condition)
646 1509.5 MiB 0.0 MiB 1 status.legacy_status = condition
647
648 1509.5 MiB 0.0 MiB 2 def get_solver_solution() -> Solution:
649 1509.5 MiB 0.0 MiB 1 objective = m.ObjVal
650
651 1771.6 MiB 262.1 MiB 447055 sol = pd.Series({v.VarName: v.x for v in m.getVars()}, dtype=float)
652 1774.6 MiB 3.0 MiB 1 sol = set_int_index(sol)
653
654 1774.6 MiB 0.0 MiB 1 try:
655 1828.0 MiB 0.0 MiB 2 dual = pd.Series(
656 1828.0 MiB 53.4 MiB 915807 {c.ConstrName: c.Pi for c in m.getConstrs()}, dtype=float
657 )
658 1830.2 MiB 2.3 MiB 1 dual = set_int_index(dual)
659 except AttributeError:
660 logger.warning("Dual values of MILP couldn't be parsed")
661 dual = pd.Series(dtype=float)
662
663 1830.2 MiB 0.0 MiB 1 return Solution(sol, dual, objective)
664
665 1830.2 MiB 0.0 MiB 1 solution = safe_get_solution(status, get_solver_solution)
666 1830.2 MiB 0.0 MiB 1 maybe_adjust_objective_sign(solution, model.objective.sense, io_api)
667
668 1830.2 MiB 0.0 MiB 1 return Result(status, solution, m)
Compared with the .lp format:
Filename: /home/koen/linopy/linopy/solvers.py
Line # Mem usage Increment Occurrences Line Contents
=============================================================
568 756.6 MiB 756.6 MiB 1 @profile
569 def run_gurobi(
570 model,
571 io_api=None,
572 problem_fn=None,
573 solution_fn=None,
574 log_fn=None,
575 warmstart_fn=None,
576 basis_fn=None,
577 keep_files=False,
578 env=None,
579 **solver_options,
580 ):
581 """
582 Solve a linear problem using the gurobi solver.
583
584 This function communicates with gurobi using the gurubipy package.
585 """
586 # see https://www.gurobi.com/documentation/10.0/refman/optimization_status_codes.html
587 756.6 MiB 0.0 MiB 18 CONDITION_MAP = {
588 756.6 MiB 0.0 MiB 1 1: "unknown",
589 756.6 MiB 0.0 MiB 1 2: "optimal",
590 756.6 MiB 0.0 MiB 1 3: "infeasible",
591 756.6 MiB 0.0 MiB 1 4: "infeasible_or_unbounded",
592 756.6 MiB 0.0 MiB 1 5: "unbounded",
593 756.6 MiB 0.0 MiB 1 6: "other",
594 756.6 MiB 0.0 MiB 1 7: "iteration_limit",
595 756.6 MiB 0.0 MiB 1 8: "terminated_by_limit",
596 756.6 MiB 0.0 MiB 1 9: "time_limit",
597 756.6 MiB 0.0 MiB 1 10: "optimal",
598 756.6 MiB 0.0 MiB 1 11: "user_interrupt",
599 756.6 MiB 0.0 MiB 1 12: "other",
600 756.6 MiB 0.0 MiB 1 13: "suboptimal",
601 756.6 MiB 0.0 MiB 1 14: "unknown",
602 756.6 MiB 0.0 MiB 1 15: "terminated_by_limit",
603 756.6 MiB 0.0 MiB 1 16: "internal_solver_error",
604 756.6 MiB 0.0 MiB 1 17: "internal_solver_error",
605 }
606
607 756.6 MiB 0.0 MiB 1 log_fn = maybe_convert_path(log_fn)
608 756.6 MiB 0.0 MiB 1 warmstart_fn = maybe_convert_path(warmstart_fn)
609 756.6 MiB 0.0 MiB 1 basis_fn = maybe_convert_path(basis_fn)
610
611 1189.5 MiB 0.0 MiB 2 with contextlib.ExitStack() as stack:
612 756.6 MiB 0.0 MiB 1 if env is None:
613 757.5 MiB 0.9 MiB 1 env = stack.enter_context(gurobipy.Env())
614
615 757.5 MiB 0.0 MiB 1 if io_api is None or io_api in ["lp", "mps"]:
616 746.6 MiB -10.9 MiB 1 problem_fn = model.to_file(problem_fn)
617 746.6 MiB 0.0 MiB 1 problem_fn = maybe_convert_path(problem_fn)
618 999.4 MiB 252.8 MiB 1 m = gurobipy.read(problem_fn, env=env)
619 elif io_api == "direct":
620 problem_fn = None
621 m = model.to_gurobipy(env=env)
622 else:
623 raise ValueError(
624 "Keyword argument `io_api` has to be one of `lp`, `mps`, `direct` or None"
625 )
626
627 999.4 MiB 0.0 MiB 1 if solver_options is not None:
628 1000.0 MiB 0.0 MiB 10 for key, value in solver_options.items():
629 1000.0 MiB 0.6 MiB 9 m.setParam(key, value)
630 1000.0 MiB 0.0 MiB 1 if log_fn is not None:
631 1000.0 MiB 0.0 MiB 1 m.setParam("logfile", log_fn)
632
633 1000.0 MiB 0.0 MiB 1 if warmstart_fn:
634 m.read(warmstart_fn)
635 1189.5 MiB 189.5 MiB 1 m.optimize()
636
637 1189.5 MiB 0.0 MiB 1 if basis_fn:
638 try:
639 m.write(basis_fn)
640 except gurobipy.GurobiError as err:
641 logger.info("No model basis stored. Raised error: %s", err)
642
643 1189.5 MiB 0.0 MiB 1 condition = m.status
644 1189.5 MiB 0.0 MiB 1 termination_condition = CONDITION_MAP.get(condition, condition)
645 1189.5 MiB 0.0 MiB 1 status = Status.from_termination_condition(termination_condition)
646 1189.5 MiB 0.0 MiB 1 status.legacy_status = condition
647
648 1189.5 MiB 0.0 MiB 2 def get_solver_solution() -> Solution:
649 1189.5 MiB 0.0 MiB 1 objective = m.ObjVal
650
651 1440.2 MiB 250.7 MiB 447055 sol = pd.Series({v.VarName: v.x for v in m.getVars()}, dtype=float)
652 1442.3 MiB 2.1 MiB 1 sol = set_int_index(sol)
653
654 1442.3 MiB 0.0 MiB 1 try:
655 1496.7 MiB 0.0 MiB 2 dual = pd.Series(
656 1496.7 MiB 54.4 MiB 915807 {c.ConstrName: c.Pi for c in m.getConstrs()}, dtype=float
657 )
658 1498.9 MiB 2.2 MiB 1 dual = set_int_index(dual)
659 except AttributeError:
660 logger.warning("Dual values of MILP couldn't be parsed")
661 dual = pd.Series(dtype=float)
662
663 1498.9 MiB 0.0 MiB 1 return Solution(sol, dual, objective)
664
665 1498.9 MiB 0.0 MiB 1 solution = safe_get_solution(status, get_solver_solution)
666 1498.9 MiB 0.0 MiB 1 maybe_adjust_objective_sign(solution, model.objective.sense, io_api)
667
668 1498.9 MiB 0.0 MiB 1 return Result(status, solution, m)
I'm using pypsa-eur to create and solve these models, and "from the outside", pypsa-eur reports a maximum total memory usage of 2531MB in the first case (mps) and 2215MB in the second case (lp).
It's a little strange to me since I would think that the highspy model that's allocated for the mps export would be deallocated immediately after the file export (when it's not needed anymore), but I guess that's not the case. A naive attempt at adding inserting del h at line 302 in io.py doesn't make any difference whatsoever:
h = m.to_highspy()
h.writeModel(str(fn))
del h
I guess I'm just not very good at understanding how memory works in Python.
For completeness, the direct api interface for Gurobi is the worst of the three; the relevant section of the memory profile looks like this
607 754.9 MiB 0.0 MiB 1 log_fn = maybe_convert_path(log_fn)
608 754.9 MiB 0.0 MiB 1 warmstart_fn = maybe_convert_path(warmstart_fn)
609 754.9 MiB 0.0 MiB 1 basis_fn = maybe_convert_path(basis_fn)
610
611 1701.5 MiB 0.0 MiB 2 with contextlib.ExitStack() as stack:
612 754.9 MiB 0.0 MiB 1 if env is None:
613 755.8 MiB 0.9 MiB 1 env = stack.enter_context(gurobipy.Env())
614
615 755.8 MiB 0.0 MiB 1 if io_api is None or io_api in ["lp", "mps"]:
616 problem_fn = model.to_file(problem_fn)
617 problem_fn = maybe_convert_path(problem_fn)
618 m = gurobipy.read(problem_fn, env=env)
619 755.8 MiB 0.0 MiB 1 elif io_api == "direct":
620 755.8 MiB 0.0 MiB 1 problem_fn = None
621 1422.2 MiB 666.4 MiB 1 m = model.to_gurobipy(env=env)
622 else:
623 raise ValueError(
624 "Keyword argument `io_api` has to be one of `lp`, `mps`, `direct` or None"
625 )
626
627 1422.2 MiB 0.0 MiB 1 if solver_options is not None:
628 1422.2 MiB 0.0 MiB 10 for key, value in solver_options.items():
629 1422.2 MiB 0.0 MiB 9 m.setParam(key, value)
630 1422.2 MiB 0.0 MiB 1 if log_fn is not None:
631 1422.2 MiB 0.0 MiB 1 m.setParam("logfile", log_fn)
632
633 1422.2 MiB 0.0 MiB 1 if warmstart_fn:
634 m.read(warmstart_fn)
635 1701.5 MiB 279.3 MiB 1 m.optimize()
636
637 1701.5 MiB 0.0 MiB 1 if basis_fn:
638 try:
639 m.write(basis_fn)
640 except gurobipy.GurobiError as err:
641 logger.info("No model basis stored. Raised error: %s", err)
642
But pypsa-eur reports a maximum memory usage of 2722MB.
Memory usage of gurobipy seems to be somewhat of a knows issue, see https://support.gurobi.com/hc/en-us/community/posts/12387779995025/comments/12416251123217
Until anyone has any bright ideas for improvements, I figured this issue could at least serve as a reference; the short take-away is that you should use the .lp io interface for the moment if you care about memory.
This is somewhat expected, MPS files are larger in size than LP files. Maybe I'm missing something? It looks like the largest differences are from handling a larger file?
