UV-Volumes icon indicating copy to clipboard operation
UV-Volumes copied to clipboard

About DensePose

Open leviome opened this issue 1 year ago • 5 comments

Thanks for sharing the nice work!

I'm doing experiments with customed data. But I'm not clear with "UV unwrap" which is handled by DensePose.

├── densepose // Store the pre-defined UV unwrap. The directory name must be the same as the densepose directory name in the config file. ├── Camera_B1 ├── 00000_IUV.png

Could you please give me a sample of 00000_IUV.png?

leviome avatar Mar 23 '23 07:03 leviome

Hi @leviome , I am extracting the IUV map as well with detectron 2, the images look like this in my case: CoreView_313_Camera_(1)_0001_2019-08-23_16-08-50 592_IUV

With detectron 2 I created a script to extract the IUV maps from the .pkl files that detectron2 generates, here's the code, you can adapt it to your needs:

import numpy as np
import torch
from PIL import Image
import pickle
from os.path import join
import os
from tqdm import tqdm

"""
Process densepose extracted data of IUV maps (in .pkl files) into IUV images in the ZJUMoCap folder structure.
 - DensePose: https://github.com/facebookresearch/detectron2/blob/main/projects/DensePose/README.md
 - ZJUMoCap structure: https://github.com/fanegg/UV-Volumes/blob/main/INSTALL.md#get-the-pre-defined-uv-unwrap-in-densepose
"""

seq_name = "CoreView_313"
path_to_seq = f"/home/sergio/data/human/zjumocap/{seq_name}/densepose" # 
path_to_densepose_data = "/home/sergio/data/densepose_raw/" # Path to pkl files
cam_idxs = list(np.arange(1,20)) + [22, 23]

for idx in cam_idxs:
    seq_cam_name = "Camera (" + str(idx) + ")" # Structure from sequences 313, 315, ... (There are others with different ones)
    cam_densepose_path = join(path_to_seq, seq_cam_name)
    os.makedirs(cam_densepose_path, exist_ok=True)
    densepose_data = torch.load(join(path_to_densepose_data, f'cam_{idx}.pkl'), map_location="cpu")

    for img_idx in tqdm(range(len(densepose_data)), desc=f"{seq_name}: Extracting cam. {idx}"):
        # Extract IUV map from pkl
        dp_item = densepose_data[img_idx]
        i = dp_item['pred_densepose'][0].labels.cpu().numpy()
        uv = dp_item['pred_densepose'][0].uv.cpu().numpy()
        iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
        iuv = np.transpose(iuv, (1,2,0))
        iuv_img = Image.fromarray(np.uint8(iuv*255), "RGB")
        
        # iuv_img.show()

        # Show person IUV with original image size, at its corresponding position and save to 
        # corresponding file in the folder structure
        box = dp_item["pred_boxes_XYXY"][0]
        box[2] = box[2] - box[0]
        box[3] = box[3] - box[1]
        x, y, w, h = [int(v) for v in box]

        img_filename = dp_item["file_name"]
        img = Image.open(img_filename)
        img_w, img_h = img.size
        
        bg=np.zeros((img_h, img_w, 3))
        bg[y:y+h, x:x+w, :]=iuv
        bg_img = Image.fromarray(np.uint8(bg*255), "RGB")
        name_iuv_file = img_filename.split("/")[-1]
        name_iuv_file = name_iuv_file[:-4]+"_IUV.jpg"
        path_to_save_IUV = join(cam_densepose_path, name_iuv_file)
        bg_img.save(path_to_save_IUV)

smontode24 avatar Mar 23 '23 09:03 smontode24

Hi @leviome, you can download the sample from here.

fanegg avatar Mar 23 '23 11:03 fanegg

Hi @leviome , I am extracting the IUV map as well with detectron 2, the images look like this in my case: CoreView_313_Camera_(1)_0001_2019-08-23_16-08-50 592_IUV

With detectron 2 I created a script to extract the IUV maps from the .pkl files that detectron2 generates, here's the code, you can adapt it to your needs:

import numpy as np
import torch
from PIL import Image
import pickle
from os.path import join
import os
from tqdm import tqdm

"""
Process densepose extracted data of IUV maps (in .pkl files) into IUV images in the ZJUMoCap folder structure.
 - DensePose: https://github.com/facebookresearch/detectron2/blob/main/projects/DensePose/README.md
 - ZJUMoCap structure: https://github.com/fanegg/UV-Volumes/blob/main/INSTALL.md#get-the-pre-defined-uv-unwrap-in-densepose
"""

seq_name = "CoreView_313"
path_to_seq = f"/home/sergio/data/human/zjumocap/{seq_name}/densepose" # 
path_to_densepose_data = "/home/sergio/data/densepose_raw/" # Path to pkl files
cam_idxs = list(np.arange(1,20)) + [22, 23]

for idx in cam_idxs:
    seq_cam_name = "Camera (" + str(idx) + ")" # Structure from sequences 313, 315, ... (There are others with different ones)
    cam_densepose_path = join(path_to_seq, seq_cam_name)
    os.makedirs(cam_densepose_path, exist_ok=True)
    densepose_data = torch.load(join(path_to_densepose_data, f'cam_{idx}.pkl'), map_location="cpu")

    for img_idx in tqdm(range(len(densepose_data)), desc=f"{seq_name}: Extracting cam. {idx}"):
        # Extract IUV map from pkl
        dp_item = densepose_data[img_idx]
        i = dp_item['pred_densepose'][0].labels.cpu().numpy()
        uv = dp_item['pred_densepose'][0].uv.cpu().numpy()
        iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
        iuv = np.transpose(iuv, (1,2,0))
        iuv_img = Image.fromarray(np.uint8(iuv*255), "RGB")
        
        # iuv_img.show()

        # Show person IUV with original image size, at its corresponding position and save to 
        # corresponding file in the folder structure
        box = dp_item["pred_boxes_XYXY"][0]
        box[2] = box[2] - box[0]
        box[3] = box[3] - box[1]
        x, y, w, h = [int(v) for v in box]

        img_filename = dp_item["file_name"]
        img = Image.open(img_filename)
        img_w, img_h = img.size
        
        bg=np.zeros((img_h, img_w, 3))
        bg[y:y+h, x:x+w, :]=iuv
        bg_img = Image.fromarray(np.uint8(bg*255), "RGB")
        name_iuv_file = img_filename.split("/")[-1]
        name_iuv_file = name_iuv_file[:-4]+"_IUV.jpg"
        path_to_save_IUV = join(cam_densepose_path, name_iuv_file)
        bg_img.save(path_to_save_IUV)

Excellent! Thank you for your sharing. I think it would save others a lot of time!

fanegg avatar Mar 23 '23 11:03 fanegg

Hi @leviome , I am extracting the IUV map as well with detectron 2, the images look like this in my case: CoreView_313_Camera_(1)_0001_2019-08-23_16-08-50 592_IUV

With detectron 2 I created a script to extract the IUV maps from the .pkl files that detectron2 generates, here's the code, you can adapt it to your needs:

import numpy as np
import torch
from PIL import Image
import pickle
from os.path import join
import os
from tqdm import tqdm

"""
Process densepose extracted data of IUV maps (in .pkl files) into IUV images in the ZJUMoCap folder structure.
 - DensePose: https://github.com/facebookresearch/detectron2/blob/main/projects/DensePose/README.md
 - ZJUMoCap structure: https://github.com/fanegg/UV-Volumes/blob/main/INSTALL.md#get-the-pre-defined-uv-unwrap-in-densepose
"""

seq_name = "CoreView_313"
path_to_seq = f"/home/sergio/data/human/zjumocap/{seq_name}/densepose" # 
path_to_densepose_data = "/home/sergio/data/densepose_raw/" # Path to pkl files
cam_idxs = list(np.arange(1,20)) + [22, 23]

for idx in cam_idxs:
    seq_cam_name = "Camera (" + str(idx) + ")" # Structure from sequences 313, 315, ... (There are others with different ones)
    cam_densepose_path = join(path_to_seq, seq_cam_name)
    os.makedirs(cam_densepose_path, exist_ok=True)
    densepose_data = torch.load(join(path_to_densepose_data, f'cam_{idx}.pkl'), map_location="cpu")

    for img_idx in tqdm(range(len(densepose_data)), desc=f"{seq_name}: Extracting cam. {idx}"):
        # Extract IUV map from pkl
        dp_item = densepose_data[img_idx]
        i = dp_item['pred_densepose'][0].labels.cpu().numpy()
        uv = dp_item['pred_densepose'][0].uv.cpu().numpy()
        iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
        iuv = np.transpose(iuv, (1,2,0))
        iuv_img = Image.fromarray(np.uint8(iuv*255), "RGB")
        
        # iuv_img.show()

        # Show person IUV with original image size, at its corresponding position and save to 
        # corresponding file in the folder structure
        box = dp_item["pred_boxes_XYXY"][0]
        box[2] = box[2] - box[0]
        box[3] = box[3] - box[1]
        x, y, w, h = [int(v) for v in box]

        img_filename = dp_item["file_name"]
        img = Image.open(img_filename)
        img_w, img_h = img.size
        
        bg=np.zeros((img_h, img_w, 3))
        bg[y:y+h, x:x+w, :]=iuv
        bg_img = Image.fromarray(np.uint8(bg*255), "RGB")
        name_iuv_file = img_filename.split("/")[-1]
        name_iuv_file = name_iuv_file[:-4]+"_IUV.jpg"
        path_to_save_IUV = join(cam_densepose_path, name_iuv_file)
        bg_img.save(path_to_save_IUV)

I am confused about the Code snippets:

iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
	...
bg=np.zeros((img_h, img_w, 3))
bg[y:y+h, x:x+w, :]=iuv
bg_img = Image.fromarray(np.uint8(bg*255), "RGB")

The datatype of block index I is Integer, which will be erased when saving *_IUV.jpg file? I'm not sure if I misunderstood ? Or need another processing when loading to Dataset ?

fanzhongyi avatar May 04 '23 03:05 fanzhongyi

Thank you for your code @smontode24.

Then, If we use that code as is, we need additional pre-process with images. So, I modified the code a bit.

If you use this code right away, it works according to the code without any problems.


import numpy as np
import torch
from PIL import Image
import pickle
from os.path import join
import os
from tqdm import tqdm
import cv2

"""
Process densepose extracted data of IUV maps (in .pkl files) into IUV images in the ZJUMoCap folder structure.
 - DensePose: https://github.com/facebookresearch/detectron2/blob/main/projects/DensePose/README.md
 - ZJUMoCap structure: https://github.com/fanegg/UV-Volumes/blob/main/INSTALL.md#get-the-pre-defined-uv-unwrap-in-densepose
"""

seq_name = "CoreView_313"
path_to_seq = f"/EasyMocap/{seq_name}/densepose" # 
path_to_densepose_data = "/EasyMocap/CoreView_313/densepose/" # Path to pkl files
cam_idxs = list(np.arange(1,20)) + [22, 23]

for idx in cam_idxs:
    seq_cam_name = "Camera (" + str(idx) + ")" # Structure from sequences 313, 315, ... (There are others with different ones)
    cam_densepose_path = join(path_to_seq, seq_cam_name)
    os.makedirs(cam_densepose_path, exist_ok=True)
    densepose_data = torch.load(join(path_to_densepose_data, f'cam_{idx}.pkl'), map_location="cpu")

    for img_idx in tqdm(range(len(densepose_data)), desc=f"{seq_name}: Extracting cam. {idx}"):
        # Extract IUV map from pkl
        dp_item = densepose_data[img_idx]
        dp_item['file_name'] = "/EasyMocap/"+'/'.join(dp_item['file_name'].split('/')[-4:])
        i = dp_item['pred_densepose'][0].labels.cpu().numpy()
        uv = dp_item['pred_densepose'][0].uv.cpu().numpy()
        iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
        iuv = np.transpose(iuv, (1,2,0))
        iuv_img = Image.fromarray(np.uint8(iuv*255), "RGB")
        
        # iuv_img.show()

        # Show person IUV with original image size, at its corresponding position and save to 
        # corresponding file in the folder structure
        box = dp_item["pred_boxes_XYXY"][0]
        box[2] = box[2] - box[0]
        box[3] = box[3] - box[1]
        x, y, w, h = [int(v) for v in box]

        img_filename = dp_item["file_name"]
        img = Image.open(img_filename)
        img_w, img_h = img.size
        
        bg=np.zeros((img_h, img_w, 3))
        bg[y:y+h, x:x+w, :]=iuv
        bg[:,:,:2] = bg[:,:,:2] * 255
        bg = np.uint8(bg)[:,:,::-1]

        name_iuv_file = img_filename.split("/")[-1]
        name_iuv_file = name_iuv_file[:-4]+"_IUV.png"
        path_to_save_IUV = join(cam_densepose_path, name_iuv_file)

        cv2.imwrite(path_to_save_IUV, bg)

😀 Result IUV Image : test

swanilee avatar May 25 '23 05:05 swanilee