Speed up Batch inference

[cliffordkleinsr]

Hello there, what is the most efficient way to carry out batch inference on multiple image sequences, current inference time is 5 seconds per image

[weizmann]

I speed up the inference by add more concurrency -- adding more processes and more gpu instances But I think this is not an elegant way. Need help too...

[cliffordkleinsr]

Could you explain how, :)

[aliagha1997]

inference is running frame by frame, is there a way to speed this up?

[hungtooc]

@cliffordkleinsr @aliagha1997 @weizmann @cliffordkleinsr. We can modify the code to batch inference, here is my simple code with batch_size = 2:

from PIL import Image
import time
import cv2
from basicsr.utils.registry import ARCH_REGISTRY
from basicsr.utils.download_util import load_file_from_url
from basicsr.utils.misc import gpu_is_available, get_device
import torch
from basicsr.utils import imwrite, img2tensor, tensor2img
from torchvision.transforms.functional import normalize
import numpy as np
pretrain_model_url = {
    'restoration': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth',
}


if __name__ == "__main__":
    device = get_device()
    w = 0.5
    net = ARCH_REGISTRY.get('CodeFormer')(dim_embd=512, codebook_size=1024, n_head=8, n_layers=9,
                                          connect_list=['32', '64', '128', '256']).to(device)

    # ckpt_path = 'weights/CodeFormer/codeformer.pth'
    ckpt_path = load_file_from_url(url=pretrain_model_url['restoration'],
                                   model_dir='weights/CodeFormer', progress=True, file_name=None)
    checkpoint = torch.load(ckpt_path)['params_ema']
    net.load_state_dict(checkpoint)
    net.eval()

    images = [
        cv2.imread(
        "inputs/cropped_faces/0143.png"), 
        cv2.imread(
        "inputs/cropped_faces/0240.png")
        ]
    image_ts = []
    for image in images:
        image_t = img2tensor(image / 255., bgr2rgb=True, float32=True)
        normalize(image_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)
        image_ts.append(image_t.unsqueeze(0))
    
    input_batch = torch.cat(image_ts)
    input_batch = input_batch.to(device)

    
    restored_faces = []
    t0 = time.time()
    with torch.no_grad():
        output = net(input_batch, w=w, adain=True)[0]
        for iter in output:
            print(iter.shape)
            restored_face = tensor2img(iter, rgb2bgr=True, min_max=(-1, 1)).astype('uint8')
            restored_faces.append(restored_face)
    del output
    torch.cuda.empty_cache()

    t1 = time.time()
    print(t1 - t0)
    print(restored_faces)
    # show result
    for restored_face in restored_faces:
        Image.fromarray(restored_face[:,:,::-1]).show()

[raheem-imcapsule]

Ho to apply to the exisiting inference code

@cliffordkleinsr @aliagha1997 @weizmann @cliffordkleinsr. We can modify the code to batch inference, here is my simple code with batch_size = 2:

from PIL import Image
import time
import cv2
from basicsr.utils.registry import ARCH_REGISTRY
from basicsr.utils.download_util import load_file_from_url
from basicsr.utils.misc import gpu_is_available, get_device
import torch
from basicsr.utils import imwrite, img2tensor, tensor2img
from torchvision.transforms.functional import normalize
import numpy as np
pretrain_model_url = {
    'restoration': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth',
}


if __name__ == "__main__":
    device = get_device()
    w = 0.5
    net = ARCH_REGISTRY.get('CodeFormer')(dim_embd=512, codebook_size=1024, n_head=8, n_layers=9,
                                          connect_list=['32', '64', '128', '256']).to(device)

    # ckpt_path = 'weights/CodeFormer/codeformer.pth'
    ckpt_path = load_file_from_url(url=pretrain_model_url['restoration'],
                                   model_dir='weights/CodeFormer', progress=True, file_name=None)
    checkpoint = torch.load(ckpt_path)['params_ema']
    net.load_state_dict(checkpoint)
    net.eval()

    images = [
        cv2.imread(
        "inputs/cropped_faces/0143.png"), 
        cv2.imread(
        "inputs/cropped_faces/0240.png")
        ]
    image_ts = []
    for image in images:
        image_t = img2tensor(image / 255., bgr2rgb=True, float32=True)
        normalize(image_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)
        image_ts.append(image_t.unsqueeze(0))
    
    input_batch = torch.cat(image_ts)
    input_batch = input_batch.to(device)

    
    restored_faces = []
    t0 = time.time()
    with torch.no_grad():
        output = net(input_batch, w=w, adain=True)[0]
        for iter in output:
            print(iter.shape)
            restored_face = tensor2img(iter, rgb2bgr=True, min_max=(-1, 1)).astype('uint8')
            restored_faces.append(restored_face)
    del output
    torch.cuda.empty_cache()

    t1 = time.time()
    print(t1 - t0)
    print(restored_faces)
    # show result
    for restored_face in restored_faces:
        Image.fromarray(restored_face[:,:,::-1]).show()

How to apply to existing code former inference file https://github.com/sczhou/CodeFormer/blob/master/inference_codeformer.py

[TasfiqAhmed9]

Hi I can't understanding about the project code algorithm & model can u say about this. I want work on this.

On Sat, 30 Sep 2023, 5:42 pm Raheemuddin Syed, @.***> wrote:

Ho to apply to the exisiting inference code

@cliffordkleinsr https://github.com/cliffordkleinsr @aliagha1997 https://github.com/aliagha1997 @weizmann https://github.com/weizmann @cliffordkleinsr https://github.com/cliffordkleinsr. We can modify the code to batch inference, here is my simple code with batch_size = 2:

from PIL import Imageimport timeimport cv2from basicsr.utils.registry import ARCH_REGISTRYfrom basicsr.utils.download_util import load_file_from_urlfrom basicsr.utils.misc import gpu_is_available, get_deviceimport torchfrom basicsr.utils import imwrite, img2tensor, tensor2imgfrom torchvision.transforms.functional import normalizeimport numpy as nppretrain_model_url = { 'restoration': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth', }

if name == "main": device = get_device() w = 0.5 net = ARCH_REGISTRY.get('CodeFormer')(dim_embd=512, codebook_size=1024, n_head=8, n_layers=9, connect_list=['32', '64', '128', '256']).to(device)

# ckpt_path = 'weights/CodeFormer/codeformer.pth'
ckpt_path = load_file_from_url(url=pretrain_model_url['restoration'],
                               model_dir='weights/CodeFormer', progress=True, file_name=None)
checkpoint = torch.load(ckpt_path)['params_ema']
net.load_state_dict(checkpoint)
net.eval()

images = [
    cv2.imread(
    "inputs/cropped_faces/0143.png"),
    cv2.imread(
    "inputs/cropped_faces/0240.png")
    ]
image_ts = []
for image in images:
    image_t = img2tensor(image / 255., bgr2rgb=True, float32=True)
    normalize(image_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)
    image_ts.append(image_t.unsqueeze(0))

input_batch = torch.cat(image_ts)
input_batch = input_batch.to(device)


restored_faces = []
t0 = time.time()
with torch.no_grad():
    output = net(input_batch, w=w, adain=True)[0]
    for iter in output:
        print(iter.shape)
        restored_face = tensor2img(iter, rgb2bgr=True, min_max=(-1, 1)).astype('uint8')
        restored_faces.append(restored_face)
del output
torch.cuda.empty_cache()

t1 = time.time()
print(t1 - t0)
print(restored_faces)
# show result
for restored_face in restored_faces:
    Image.fromarray(restored_face[:,:,::-1]).show()

How to apply to existing code former inference file https://github.com/sczhou/CodeFormer/blob/master/inference_codeformer.py

— Reply to this email directly, view it on GitHub https://github.com/sczhou/CodeFormer/issues/157#issuecomment-1741747472, or unsubscribe https://github.com/notifications/unsubscribe-auth/AQKRBHSHMXF5BGBGX5ZWTUDX5AATXANCNFSM6AAAAAAVJQGEHY . You are receiving this because you are subscribed to this thread.Message ID: @.***>