ahmedfgad
Issue while using multiprocessing
While using multiprocessing for my task my program gets the exception after some generations. Sometimes it works fine for all generations and stops working abruptly and generates "NoneType object is unsubscriptable" or Pool halt errors. I go through the documentation but there is no detail provided about how multiprocessing is working and what to do and what not to do while using this. Please guide me in this regard to what can be the possible issue. My code is
def fitness_func(solution, solution_idx): #cal ISP for each gamma in population => ehanced image conf = Config_file_handling.YAMLfunctions() config = conf.read_yaml('Default_isp_conf.yml') config['gac']['gamma'] = float(solution[0]) config['nlm']['h'] = int(solution[1]) conf.write_yaml('Default_isp_conf.yml', config) config = conf.read_yaml('config.yml') IQM = ISP(config) iqm = mean(IQM) return round(iqm['PSNR'],4)
Create the GA instance
ga_instance = pygad.GA(num_generations=10, num_parents_mating=2, sol_per_pop=20, num_genes=2, gene_type=gene_type, gene_space=gene_space, fitness_func=fitness_func, on_generation=on_generation, parent_selection_type="rank", keep_parents=1, crossover_type='single_point', crossover_probability=0.8, mutation_type='random', mutation_probability=0.2, allow_duplicate_genes=False, #stop_criteria="saturate_5", save_best_solutions=True, save_solutions=True, parallel_processing=['process', 5] )
t1 = time.time()
Run the GA
ga_instance.run() t2 = time.time() print("Time is", t2-t1)
Can you share a full code to test on my end?
Some code is missing from the posted example.
import pygad
import time
def fitness_func(ga_instance, solution, solution_idx):
#cal ISP for each gamma in population => ehanced image
conf = Config_file_handling.YAMLfunctions()
config = conf.read_yaml('Default_isp_conf.yml')
config['gac']['gamma'] = float(solution[0])
config['nlm']['h'] = int(solution[1])
conf.write_yaml('Default_isp_conf.yml', config)
config = conf.read_yaml('config.yml')
IQM = ISP(config)
iqm = mean(IQM)
return round(iqm['PSNR'],4)
ga_instance = pygad.GA(num_generations=10,
num_parents_mating=2,
sol_per_pop=20,
num_genes=2,
gene_type=gene_type,
gene_space=gene_space,
fitness_func=fitness_func,
on_generation=on_generation,
parent_selection_type="rank",
keep_parents=1,
crossover_type='single_point',
crossover_probability=0.8,
mutation_type='random',
mutation_probability=0.2,
allow_duplicate_genes=False,
#stop_criteria="saturate_5",
save_best_solutions=True,
save_solutions=True,
parallel_processing=['process', 5])
t1 = time.time()
ga_instance.run()
t2 = time.time()
print("Time is", t2-t1)
import pygad
import time
def mean(IQM):
mean_dict = {}
for key in IQM[0].keys():
mean_dict[key] = sum(d[key] for d in IQM) / len(IQM)
return mean_dict
def fitness_func(ga_instance, solution, solution_idx):
#cal ISP for each gamma in population => ehanced image
conf = Config_file_handling.YAMLfunctions()
config = conf.read_yaml('Default_isp_conf.yml')
config['gac']['gamma'] = float(solution[0])
config['nlm']['h'] = int(solution[1])
conf.write_yaml('Default_isp_conf.yml', config)
config = conf.read_yaml('config.yml')
IQM = ISP(config)
iqm = mean(IQM)
return round(iqm['PSNR'],4)
ga_instance = pygad.GA(num_generations=10,
num_parents_mating=2,
sol_per_pop=20,
num_genes=2,
gene_type=gene_type,
gene_space=gene_space,
fitness_func=fitness_func,
on_generation=on_generation,
parent_selection_type="rank",
keep_parents=1,
crossover_type='single_point',
crossover_probability=0.8,
mutation_type='random',
mutation_probability=0.2,
allow_duplicate_genes=False,
#stop_criteria="saturate_5",
save_best_solutions=True,
save_solutions=True,
parallel_processing=['process', 5])
t1 = time.time()
ga_instance.run()
t2 = time.time()
print("Time is", t2-t1)
Main:
`def image_quality_matrices(original_img, modified_img) -> dict: """This functions handles the IQM in main"""
iqm = IQM.ImageQualityMatrices(original_img)
iqm_dict = {'PSNR': iqm.PSNR(modified_img), 'SSIM': iqm.SSIM(modified_img), 'UQI': iqm.uqi(P=modified_img)}
#print(iqm_dict)
return iqm_dict
def ISP(conf): """ Executes the Fast Open ISP in main """ list_iqms = [] # Create Log File directory if not available os.makedirs(conf['log_file_directory'], exist_ok=True)
# Checks the input Directory
if not os.path.isdir(conf['images_input_directory']):
print("Please specify the correct input path")
sys.exit()
else:
for image in glob.glob(conf['images_input_directory'] + '*'):
log_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S,')
isp_conf = Config_file_handling.YAMLfunctions()
isp_params = isp_conf.read_yaml(conf['configuration_file_path'])
file_name = image.split('\\')[-1].split('.')[0]
print(file_name)
try:
# Parameters Optimization
# optimize = Parameters_optimization.Optimization(image)
# optimize.param_optimize(conf['configuration_file_path'])
# Getting the image size from metadata
Image_metadata.metadata_conf(image, conf['configuration_file_path'])
# Running the ISP
isp = isp_run.RunISPs(image_file=image, output_dir=conf['images_output_directory'], add_fileTime=conf[
'add_time_along_with_FileName'])
ISPOut_dict = isp.runISP(conf['configuration_file_path'], isp_name=conf['available_ISPs'][conf[
"ISP_to_use"]-1])
if conf['Perform IQM Checks']['status']:
# Getting the Path of Ground Truth Image
file_name = image.split('\\')[-1]
ext = file_name.split('.')[-1]
o_image_name = file_name.replace(ext, 'jpg')
o_image_path = f'{conf["Perform IQM Checks"]["ground_truth_directory"]}\\{o_image_name}'
# Performing Image Quality Checks
iqms = image_quality_matrices(o_image_path, ISPOut_dict['OutputImage'])
#up_iqms = {"Image": file_name}
#up_iqms.update(iqms)
list_iqms.append(iqms)
print(list_iqms)
# Writing in log file
with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file:
log_file.write(f'{log_time} INFO, {ISPOut_dict["FileName"]}, {iqms}, {isp_params}\n')
else:
with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file:
log_file.write(f'{log_time} INFO, {ISPOut_dict["FileName"]}, {isp_params}\n')
except Exception as error:
exc_type, exc_obj, exc_tb = sys.exc_info()
fName = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
template = "{0}, File {1}, Line {2}\n{3}"
message = template.format(exc_type, fName, exc_tb.tb_lineno, error.args).replace('\n', ' ')
with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file:
log_file.write(f'{log_time} {file_name}, {message}, {isp_params}\n')
return list_iqms`
Thanks for sharing the code.
Please provide the following to run the code.
Config_file_handlingisp_runImage_metadatagene_typegene_spaceon_generation