JSI-GAN
JSI-GAN copied to clipboard
questions about how to get dataset ready for testing or training on my own
hi, thanks for your fancy work!
I am now trying to put the testset on my own
I download the testset as you say in 4kmedia.org
For HDR video:
- extract frames from the 4k video:
ffmpeg -i movie_name -color_primaries bt2020 ./4k/LG_Daylight_4K_Demo_BG/%08d.png
- read and wirte them to the mat file for inference evaluation
files = os.listdir('./4k/LG_Daylight_4K_Demo_BG/')
imgs = []
files.sort()
for i, fi in enumerate(files):
` img = cv2.imread(os.path.join('./4k/LG_Daylight_4K_Demo_BGR/',fi), cv2.IMREAD_UNCHANGED)`
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
#print(img.shape)
H_Y, H_u, H_v = rgb2yuv_hdr(np.array(img))
H_Y = H_Y[:,:,np.newaxis]
H_u = H_u[:,:,np.newaxis]
H_v = H_v[:,:,np.newaxis]
img_array = np.concatenate((H_Y, H_u, H_v), axis=2)
img_array = np.array(img)
img_array = np.transpose(img_array, (2, 1, 0))
imgs.append(img_array)
imgs_mat = np.array(imgs)
file = h5py.File('LG_DayLight_HDR.mat','w')
file.create_dataset('HDR_YUV', data = imgs_mat)
for the rgb2yuv_hdr() function, I refer the issue 19 and write them in python:
def rgb2yuv_hdr(linRGB):
"""
return: H_y, H_u, H_v
"""
Bitdepth=10
hdri=linRGB.astype(float)
hdri = np.clip(hdri,0,10000)
r,c,_= hdri.shape
if np.mod(r,2) == 1:
hdri = hdri[:r-1,:,:]
if np.mod(c,2) == 1:
hdri = hdri[:,1:c-1,:]
# Coding TF
print(np.max(hdri))
print(np.min(hdri))
hdri_divide = hdri / 10000
print(np.max(hdri_divide))
print(np.min(hdri_divide))
hdri_divide[np.where(hdri_divide > 1)] = 1.0
print(np.max(hdri_divide))
print(np.min(hdri_divide))
hdri_divide[np.where(hdri_divide < 0)] = 0.0
print(np.max(hdri_divide))
print(np.min(hdri_divide))
Clip_hdri = hdri_divide
#Clip_hdri = np.max(0,np.min(hdri/10000,1))
# comment to keep linear values
# m1=(2610/4096)*0.25;
# m2=(2523/4096)*128;
# c1=3424/4096;
# c2=(2413/4096)*32;
# c3=(2392/4096)*32;
# PQTF_hdri=((c1+c2*(Clip_hdri.^m1))./(1+c3*(Clip_hdri.^m1))).^m2;
PQTF_hdri = Clip_hdri
# R'G'B to Y'CbCr
Y = 0.2627*PQTF_hdri[:,:,0] + 0.6780*PQTF_hdri[:,:,2] + 0.0593*PQTF_hdri[:,:,2]
Cb = (PQTF_hdri[:,:,2]-Y)/1.8814
Cr = (PQTF_hdri[:,:,0]-Y)/1.4746
# Quant 10b
toren = np.power(2, Bitdepth - 8)
toren_1 = np.power(2, Bitdepth)
D_Y=np.clip(np.round(toren * (219*Y + 16)), 0, toren_1-1)
D_Cb=np.clip(np.round(toren * (224*Cb + 128)), 0, toren_1-1)
D_Cr=np.clip(np.round(toren * (224*Cr + 128)), 0, toren_1-1)
# 4:4:4 to 4:4:4
D_Cb_h, D_Cb_w = D_Cb.shape
D_Cb_Hor = 64*D_Cb
D_Cb_Hor=D_Cb_Hor + 384*D_Cb
D_Cb_Hor=D_Cb_Hor + 64*D_Cb
D_Cr_h, D_Cr_w = D_Cr.shape
D_Cr_Hor=64*D_Cr
D_Cr_Hor=D_Cr_Hor + 384*D_Cr
D_Cr_Hor=D_Cr_Hor + 64*D_Cr
D_Cb_Hor_h, D_Cb_Hor_w = D_Cb_Hor.shape
D_Cb_ver=0*D_Cb_Hor
D_Cb_ver=D_Cb_ver + 256*D_Cb_Hor
D_Cb_ver=D_Cb_ver + 256*D_Cb_Hor
D_Cr_Hor_h, D_Cr_Hor_w = D_Cr_Hor.shape
D_Cr_ver=0*D_Cr_Hor
D_Cr_ver=D_Cr_ver + 256*D_Cr_Hor
D_Cr_ver=D_Cr_ver + 256*D_Cr_Hor
"""
# 4:4:4 to 4:2:0
D_Cb_h, D_Cb_w = D_Cb.shape
D_Cb_padd = np.concatenate((D_Cb[:,0:1], D_Cb), axis=1)
D_Cb_Hor = 64*D_Cb_padd[:,0:D_Cb_w:2]
D_Cb_Hor=D_Cb_Hor + 384*D_Cb[:,0:D_Cb_w:2]
D_Cb_Hor=D_Cb_Hor + 64*D_Cb[:,1:D_Cb_w:2]
D_Cr_h, D_Cr_w = D_Cr.shape
D_Cr_padd = np.concatenate((D_Cr[:,0:1], D_Cr), axis=1)
D_Cr_Hor=64*D_Cr_padd[:,0:D_Cr_w:2]
D_Cr_Hor=D_Cr_Hor + 384*D_Cr[:,0:D_Cr_w:2]
D_Cr_Hor=D_Cr_Hor + 64*D_Cr[:,1:D_Cr_w:2]
D_Cb_Hor_h, D_Cb_Hor_w = D_Cb_Hor.shape
D_Cb_Hor_padd = np.concatenate((D_Cb_Hor[0:1, :], D_Cb_Hor), axis=0)
D_Cb_ver=0*D_Cb_Hor_padd[0:D_Cb_Hor_h:2,:]
D_Cb_ver=D_Cb_ver + 256*D_Cb_Hor[0:D_Cb_Hor_h:2,:]
D_Cb_ver=D_Cb_ver + 256*D_Cb_Hor[1:D_Cb_Hor_h:2,:]
D_Cr_Hor_h, D_Cr_Hor_w = D_Cr_Hor.shape
D_Cr_Hor_padd = np.concatenate((D_Cr_Hor[0:1, :], D_Cr_Hor), axis=0)
D_Cr_ver=0*D_Cr_Hor_padd[0:D_Cr_Hor_h:2,:]
D_Cr_ver=D_Cr_ver + 256*D_Cr_Hor[0:D_Cr_Hor_h:2,:]
D_Cr_ver=D_Cr_ver + 256*D_Cr_Hor[1:D_Cr_Hor_h:2,:]
"""
toren_2 = np.power(0.5, 18)
D_Cb = (D_Cb_ver+131072.0) * toren_2
D_Cr = (D_Cr_ver+131072.0) * toren_2
H_Y=D_Y
H_u=D_Cb
H_v=D_Cr
return H_Y, H_u, H_v
questions I happened is below:
1. if I use 4:4:4 -> 4:2:0, the YUV channel will have the different shape, while I read your mat file, the Y,U,V channel are all the shape 3840x2160. 2. so I decide to use 4:4:4, while I try to save mat and try to read them use code below, the image looks something wrong:
datafile = './LG_DayLight_HDR.mat'
mat = h5py.File(datafile)
imgs = mat['HDR_YUV']
def yuv_to_bgr(img, Bitdepth=10):
height, width = img.shape[:2]
D_Y = img[:,:,0]
D_u = img[:,:,1]
D_v = img[:,:,2]
#Inverse quant 10b
toren = np.power(2, Bitdepth - 8)
Y = np.clip((D_Y/toren - 16)/219, 0, 1)
D_Cb = np.clip((D_u/toren - 128)/224, -0.5, 0.5)
D_Cr = np.clip((D_v/toren - 128)/224, -0.5, 0.5)
# Y'CbCr to R'G'B'
# BT 2020,
A = [[1,0.00000000000000,1.47460000000000],[1,-0.16455312684366,-0.57135312684366],[1,1.88140000000000,0.00000000000000]]
A = np.array(A)
RGB = np.zeros([height, width, 3])
RGB[:,:,0] = np.clip(Y + A[0,2]*D_Cr, 0, 1)
RGB[:,:,1] = np.clip(Y + A[1,1]*D_Cb + A[1,2]*D_Cr, 0, 1)
RGB[:,:,2] = np.clip(Y + A[2,1]*D_Cb, 0, 1)
RGB = (RGB * 65535).astype(np.uint16)
BGR = cv2.cvtColor(RGB, cv2.COLOR_RGB2BGR)
return BGR
import numpy as np
for i, img in enumerate(imgs):
img = np.transpose(img, (2, 1, 0))
print(img.shape)
BGR = yuv_to_bgr(img, 10)
cv2.imwrite('./hdr_{}.tiff'.format(i), BGR)
Could you please help me out of that problem? How to write to a mat file from a video type?
I just want to re-implement the results you did. But the released test set are so rare. I want to test more dataset and get the quantitative results.
Looking forward to your reply soooo much! @sooyekim @JihyongOh
Hi @qinhaifangpku , Thanks for your continued interest.
What I did was:
(i) Convert .mp4 file to .yuv using ffmpeg: ffmpeg -i <input_video.mp4> <output_video.yuv>
(ii) Read YUV frames from .yuv videos - like this, there's no need to convert from RGB to YUV. You can refer to Deep-SR-ITM/utils/load_yuv.m for loading YUV frames and this issue for more details on the function.
(iii) Save as .mat file.
As for your question, load_yuv.m basically repeats values in a 4:2:0 frame to create a full 3840x2160 frame. For (ii) and (iii), please refer to the below Matlab code:
file_SDR = '<path_to_SDR_file>'
file_HDR = '<path_to_HDR_file>'
format = '420';
[w_factor, h_factor] = yuv_factor(format);
height = 2160;
width = 3840;
for fr = 1:num_frames
% read data
SDR_YUV(:, :, :, fr) = imresize(uint8(load_yuv(file_SDR , fr, height, width, h_factor, w_factor, 'SDR')), 0.25); % read SDR frame and resize
HDR_YUV(:, :, :, fr) = uint16(loadYUV(file_HDR, fr, height, width, h_factor, w_factor ,'HDR')); % read HDR frame
end
save('./data/test_SDR_x4.mat', 'SDR_YUV', '-v7.3');
save('./data/test_HDR.mat', 'HDR_YUV', '-v7.3');
Hope this helps, and let me know if you have any other issues!
Soo Ye