generate_tfrecord.py

[Harsh188]

tf.app.flags no longer works with TensorFlow 2.1

[adam-buckley]

According to the tensorflow upgrade guide you can use either a compat tf lib for flags or use Abseil. I had much better luck with the latter when following their guide.

[12343954]

tensorflow upgrade guide is not completely useful.

TF 2.1.0 can run below. i upgrade the generate_tfrecord.py.

ps:

1. i cann't get csv_input value, so i change it to csv_path
2. the key point is "image/format" v2 type is JPEG, v1 is JPG

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import os, sys, traceback
import io
import pandas as pd
import tensorflow as tf

from PIL import Image
from object_detection.utils import dataset_util
from collections import namedtuple, OrderedDict

from absl import app, flags
from absl.flags import FLAGS

# flags = tf.compat.v1.app.flags
# flags.DEFINE_string("csv_input", "", "Path to the CSV input")
flags.DEFINE_string("image_dir", "", "Path to the image directory")
flags.DEFINE_string("output_path", "", "Path to output TFRecord")
flags.DEFINE_string("csv_path", "", "Path to the CSV input")
# FLAGS = flags.FLAGS


# TO-DO replace this with label map
def class_text_to_int(row_label):
    if row_label == "nine":
        return 1
    elif row_label == "ten":
        return 2
    elif row_label == "jack":
        return 3
    elif row_label == "queen":
        return 4
    elif row_label == "king":
        return 5
    elif row_label == "ace":
        return 6
    else:
        None


def split(df, group):
    data = namedtuple("data", ["filename", "object"])
    gb = df.groupby(group)
    return [
        data(filename, gb.get_group(x))
        for filename, x in zip(gb.groups.keys(), gb.groups)
    ]


def create_tf_example(group, path):
    with tf.io.gfile.GFile(
        os.path.join(path, "{}".format(group.filename)), "rb"
    ) as fid:
        encoded_jpg = fid.read()

    encoded_jpg_io = io.BytesIO(encoded_jpg)
    image = Image.open(encoded_jpg_io)
    width, height = image.size

    filename = group.filename.encode("utf8")
    # image_format = "jpg"
    xmins = []
    xmaxs = []
    ymins = []
    ymaxs = []
    classes_text = []
    classes = []

    for index, row in group.object.iterrows():
        xmins.append(row["xmin"] / width)
        xmaxs.append(row["xmax"] / width)
        ymins.append(row["ymin"] / height)
        ymaxs.append(row["ymax"] / height)
        classes_text.append(row["class"].encode("utf8"))
        classes.append(class_text_to_int(row["class"]))

    tf_example = tf.train.Example(
        features=tf.train.Features(
            feature={
                "image/height": dataset_util.int64_feature(height),
                "image/width": dataset_util.int64_feature(width),
                "image/filename": dataset_util.bytes_feature(filename),
                "image/source_id": dataset_util.bytes_feature(filename),
                "image/encoded": dataset_util.bytes_feature(encoded_jpg),
                "image/format": tf.train.Feature(
                    bytes_list=tf.train.BytesList(value=["jpeg".encode("utf8")])
                ),
                # "image/format": dataset_util.bytes_feature(image_format),
                "image/object/bbox/xmin": dataset_util.float_list_feature(xmins),
                "image/object/bbox/xmax": dataset_util.float_list_feature(xmaxs),
                "image/object/bbox/ymin": dataset_util.float_list_feature(ymins),
                "image/object/bbox/ymax": dataset_util.float_list_feature(ymaxs),
                "image/object/class/text": dataset_util.bytes_list_feature(
                    classes_text
                ),
                "image/object/class/label": dataset_util.int64_list_feature(classes),
            }
        )
    )
    return tf_example


def main(_argv):
    writer = tf.io.TFRecordWriter(FLAGS.output_path)
    path = os.path.join(os.getcwd(), FLAGS.image_dir)

    examples = pd.read_csv(os.path.join(os.getcwd(), FLAGS.csv_path))

    grouped = split(examples, "filename")
    for group in grouped:
        print(group)
        tf_example = create_tf_example(group, path)
        writer.write(tf_example.SerializeToString())

    writer.close()
    output_path = os.path.join(os.getcwd(), FLAGS.output_path)
    print("Successfully created the TFRecords: {}".format(output_path))


if __name__ == "__main__":
    tf.compat.v1.app.run()

[ghost]

Hi,

I tried your code and I was getting this error message:

2020-08-17 18:41:08.312730: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'cudart64_101.dll'; dlerror: cudart64_101.dll not found
2020-08-17 18:41:08.315636: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
Traceback (most recent call last):
  File "generate_tfrecord.py", line 118, in <module>
    tf.compat.v1.app.run()
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\platform\app.py", line 40, in run
    _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\absl\app.py", line 299, in run
    _run_main(main, args)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\absl\app.py", line 250, in _run_main
    sys.exit(main(argv))
  File "generate_tfrecord.py", line 104, in main
    examples = pd.read_csv(os.path.join(os.getcwd(), FLAGS.csv_path))
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 685, in parser_f
    return _read(filepath_or_buffer, kwds)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 457, in _read
    parser = TextFileReader(fp_or_buf, **kwds)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 895, in __init__
    self._make_engine(self.engine)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 1135, in _make_engine
    self._engine = CParserWrapper(self.f, **self.options)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 1906, in __init__
    self._reader = parsers.TextReader(src, **kwds)
  File "pandas\_libs\parsers.pyx", line 380, in pandas._libs.parsers.TextReader.__cinit__
  File "pandas\_libs\parsers.pyx", line 691, in pandas._libs.parsers.TextReader._setup_parser_source
OSError: Initializing from file failed

I used this command: python generate_tfrecord.py --csv_input=images\train_labels.csv --image_dir=images\train --output_path=train.record

I really don't have any idea what to do, I'd really appreciate some help!

[PelinSuK]

Hi,

I tried your code and I was getting this error message:

2020-08-17 18:41:08.312730: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'cudart64_101.dll'; dlerror: cudart64_101.dll not found
2020-08-17 18:41:08.315636: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
Traceback (most recent call last):
  File "generate_tfrecord.py", line 118, in <module>
    tf.compat.v1.app.run()
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\platform\app.py", line 40, in run
    _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\absl\app.py", line 299, in run
    _run_main(main, args)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\absl\app.py", line 250, in _run_main
    sys.exit(main(argv))
  File "generate_tfrecord.py", line 104, in main
    examples = pd.read_csv(os.path.join(os.getcwd(), FLAGS.csv_path))
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 685, in parser_f
    return _read(filepath_or_buffer, kwds)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 457, in _read
    parser = TextFileReader(fp_or_buf, **kwds)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 895, in __init__
    self._make_engine(self.engine)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 1135, in _make_engine
    self._engine = CParserWrapper(self.f, **self.options)
  File "C:\Users\frede\AppData\Local\Programs\Python\Python36\lib\site-packages\pandas\io\parsers.py", line 1906, in __init__
    self._reader = parsers.TextReader(src, **kwds)
  File "pandas\_libs\parsers.pyx", line 380, in pandas._libs.parsers.TextReader.__cinit__
  File "pandas\_libs\parsers.pyx", line 691, in pandas._libs.parsers.TextReader._setup_parser_source
OSError: Initializing from file failed

I used this command: python generate_tfrecord.py --csv_input=images\train_labels.csv --image_dir=images\train --output_path=train.record

I really don't have any idea what to do, I'd really appreciate some help!

did you find a solution ?