Problem with lfs_stat on new files in a dir

[keck-in-space]

Hello,

I'm trying to use LFS to create a directory in which log files will be stored. To do this I'm creating a directory on startup and adding files one at a time as they exceed the desired maximum file size.

The first file works fine, and then I create the next file, which is created fine.

However, when lfs_stat is called on the path of the second file, it always returns LFS_ERR_NOENT, even though writing to that exact same path works fine.

For testing, I'm formatting on each boot.

I've attached the trace below

00> lfs_trace:3522: lfs_format(2001A068, 200004D8 {.context=00000000, .read=08022879, .prog=080228B9, .erase=080228FD, .sync=0802292F, .read_size=16, .prog_size=16, .block_size=262144, .block_count=256, .block_cycles=500, .cache_size=512, .lookahead_size=512, .read_buffer=2001B258, .prog_buffer=2001B058, .lookahead_buffer=2001B458, .name_max=0, .file_max=0, .attr_max=0})
00> lfs_trace:3583: lfs_format -> 0
00> lfs_trace:3603: lfs_mount(2001A068, 200004D8 {.context=00000000, .read=08022879, .prog=080228B9, .erase=080228FD, .sync=0802292F, .read_size=16, .prog_size=16, .block_size=262144, .block_count=256, .block_cycles=500, .cache_size=512, .lookahead_size=512, .read_buffer=2001B258, .prog_buffer=2001B058, .lookahead_buffer=2001B458, .name_max=0, .file_max=0, .attr_max=0})
00> lfs_trace:3717: lfs_mount -> 0
00> lfs_trace:1859: lfs_mkdir(2001A068, "hk-dir")
00> lfs_trace:3738: lfs_fs_traverse(2001A068, 080025E1, 2001A068)
00> lfs_trace:3821: lfs_fs_traverse -> 0
00> lfs_trace:1941: lfs_mkdir -> 0
00> lfs_trace:1946: lfs_dir_open(2001A068, 20010750, "hk-dir")
00> lfs_trace:1992: lfs_dir_open -> 0
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010750, 20010578)
00> lfs_trace:2020: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010750, 20010578)
00> lfs_trace:2026: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010750, 20010578)
00> lfs_trace:2033: lfs_dir_read -> 0
00> lfs_trace:1997: lfs_dir_close(2001A068, 20010750)
00> lfs_trace:2006: lfs_dir_close -> 0
00> lfs_trace:2330: lfs_file_opencfg(2001A068, 2001A594, "hk-dir/0", 903, 2001A588 {.buffer=2001A388, .attrs=00000000, .attr_count=0})
00> lfs_trace:2467: lfs_file_opencfg -> 0
00> lfs_trace:3088: lfs_stat(2001A068, "hk-dir/0", 20010D00)
00> lfs_trace:3097: lfs_stat -> 0
00> current size: 0
00> lfs_trace:2824: lfs_file_write(2001A068, 2001A594, 20010E28, 321)
00> lfs_trace:2948: lfs_file_write -> 321
00> lfs_trace:2676: lfs_file_sync(2001A068, 2001A594)
00> lfs_trace:2727: lfs_file_sync -> 0
00> lfs_trace:3088: lfs_stat(2001A068, "hk-dir/0", 20010D00)
00> lfs_trace:3097: lfs_stat -> 0
00> current size: 321
00> lfs_trace:2824: lfs_file_write(2001A068, 2001A594, 20010E28, 321)
00> lfs_trace:2948: lfs_file_write -> 321
00> lfs_trace:2676: lfs_file_sync(2001A068, 2001A594)
00> lfs_trace:2727: lfs_file_sync -> 0
00> lfs_trace:3088: lfs_stat(2001A068, "hk-dir/0", 20010D00)
00> lfs_trace:3097: lfs_stat -> 0
00> current size: 642
00> lfs_trace:2824: lfs_file_write(2001A068, 2001A594, 20010E28, 321)
00> lfs_trace:2948: lfs_file_write -> 321
00> lfs_trace:2676: lfs_file_sync(2001A068, 2001A594)
00> lfs_trace:2727: lfs_file_sync -> 0

-- snip--

00> lfs_trace:3088: lfs_stat(2001A068, "hk-dir/0", 20010D00)
00> lfs_trace:3097: lfs_stat -> 0
00> current size: 3210
00> lfs_trace:2824: lfs_file_write(2001A068, 2001A594, 20010E28, 321)
00> lfs_trace:2948: lfs_file_write -> 321
00> lfs_trace:2676: lfs_file_sync(2001A068, 2001A594)
00> lfs_trace:2727: lfs_file_sync -> 0
00> lfs_trace:3088: lfs_stat(2001A068, "hk-dir/0", 20010D00)
00> lfs_trace:3097: lfs_stat -> 0
00> current size: 3531
00> lfs_trace:2824: lfs_file_write(2001A068, 2001A594, 20010E28, 321)
00> lfs_trace:2948: lfs_file_write -> 321
00> lfs_trace:2676: lfs_file_sync(2001A068, 2001A594)
00> lfs_trace:2727: lfs_file_sync -> 0
00> lfs_trace:3088: lfs_stat(2001A068, "hk-dir/0", 20010D00)
00> lfs_trace:3097: lfs_stat -> 0
00> current size: 3852
00> lfs_trace:2489: lfs_file_close(2001A068, 2001A594)
00> lfs_trace:2676: lfs_file_sync(2001A068, 2001A594)
00> lfs_trace:2727: lfs_file_sync -> 0
00> lfs_trace:2508: lfs_file_close -> 0
00> lfs_trace:1946: lfs_dir_open(2001A068, 20010610, "hk-dir")
00> lfs_trace:1992: lfs_dir_open -> 0
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2020: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2026: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2059: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2033: lfs_dir_read -> 0
00> lfs_trace:1997: lfs_dir_close(2001A068, 20010610)
00> lfs_trace:2006: lfs_dir_close -> 0
00> lfs_trace:2330: lfs_file_opencfg(2001A068, 2001A594, "hk-dir/1", 903, 2001A588 {.buffer=2001A388, .attrs=00000000, .attr_count=0})
00> lfs_trace:2467: lfs_file_opencfg -> 0
00> lfs_trace:1946: lfs_dir_open(2001A068, 20010610, "hk-dir")
00> lfs_trace:1992: lfs_dir_open -> 0
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2020: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2026: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2059: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2059: lfs_dir_read -> 1
00> lfs_trace:2012: lfs_dir_read(2001A068, 20010610, 20010438)
00> lfs_trace:2033: lfs_dir_read -> 0
00> lfs_trace:1997: lfs_dir_close(2001A068, 20010610)
00> lfs_trace:2006: lfs_dir_close -> 0
00> lfs_trace:2824: lfs_file_write(2001A068, 2001A594, 20010E28, 321)
00> lfs_trace:2948: lfs_file_write -> 321
00> lfs_trace:2676: lfs_file_sync(2001A068, 2001A594)
00> lfs_trace:2727: lfs_file_sync -> 0
00> lfs_trace:3088: lfs_stat(2001A068, "hk-dir/1", 20010D00)
00> lfs_trace:3092: lfs_stat -> -2
00> current size: 0
00> lfs_trace:2824: lfs_file_write(2001A068, 2001A594, 20010E28, 321)
00> lfs_trace:2948: lfs_file_write -> 321
00> lfs_trace:2676: lfs_file_sync(2001A068, 2001A594)
00> lfs_trace:2727: lfs_file_sync -> 0

[geky]

Hi @keck-in-space, thanks for creating an issue. This doesn't look like any bug I've seen before, so I'm not sure what could be causing the issue. I'm currently prioritizing a couple other bugs so it will be a bit before I can look into this.

[keck-in-space]

Thanks for the reply, @geky. I'm going to keep looking into this and I'll update this issue if I find a solution. Right now it's looking like it is something I'm doing wrong. I'm reusing all the structures used by the first file to create the next file and I'm wondering if that's causing the issue.

[keck-in-space]

A few more details:

• When the file is created, walking the directory shows that it exists.
• I am able to call lfs_stat on the new file successfully until I call lfs_file_write
• The lfs_file_write call seems to corrupt the entire directory as the previously written file (file "0") now has the wrong size when lfs_stat is called on it. Correct size is 3852, but after the write call on the second file, the size is 1605.
• After the write call on the second file if I walk the directory, the file no longer exists.
• Note this is a static memory file open using lfs_file_opencfg

[keck-in-space]

I'm pretty sure this is a memory over-write problem. I'm not sure why this changed things, but if the LFS_CACHE_SIZE is 64 instead of 512, I don't see this error.

The LFS_CACHE_SIZE is used to define the sizes of the lookahead_size, cache_size, and the read_buffer, prog_buffer, and lookahead_buffer. Also, it defines the size of the buffer used by the lfs_file_config in the call to lfs_file_open_cfg.

My current cofiguration is below.

#define LFS_CACHE_SIZE 64U

uint8_t readBuffer[LFS_CACHE_SIZE];
uint8_t writeBuffer[LFS_CACHE_SIZE];
uint8_t lookBuffer[LFS_CACHE_SIZE] __attribute__((aligned(32)));

int lfs_read(const struct lfs_config *c, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size);
int lfs_write(const struct lfs_config *c, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size);
int lfs_erase(const struct lfs_config *c, lfs_block_t block);
int lfs_sync(const struct lfs_config *c);

struct lfs_config cfg = {
    // block device operations
    .read = &lfs_read,
    .prog = &lfs_write,
    .erase = &lfs_erase,
    .sync = &lfs_sync,

    // block device configuration
    .read_size = 16,
    .prog_size = 16,
    .block_size = 256 * 1024,
    .block_count = (64 * 1024 * 1024) / (256 * 1024),
    .block_cycles = 500,
    .cache_size = LFS_CACHE_SIZE,
    .lookahead_size = LFS_CACHE_SIZE,

    .read_buffer = readBuffer,
    .prog_buffer = writeBuffer,
    .lookahead_buffer = lookBuffer,
};

uint8_t fileBuffer[LFS_CACHE_SIZE];
struct lfs_file_config fileConfig;
fileConfig.buffer = fileBuffer;
fileConfig.attr_count = 0;

lfs_file_t file;

char dirPath[16] = "hk-dir\0"

lfs_mkdir(lfs, dirPath);

char filePath[16] = "hk-dir/0\0"

lfs_file_opencfg(lfs, &file, filePath, LFS_O_RDWR | LFS_O_CREAT | LFS_O_APPEND, &fileConfig);

If the LFS_CACHE_SIZE is changed to 512 then the error returns.

[keck-in-space]

Possibly related to #335

(Not trying to flood you with notifications, geky... just documenting this issue for reference.)

[keck-in-space]

I think I have a configuration that works now.

Note I am using S25FL256L chip now which has 4KB sectors.

Pasting config here in case it helps anyone else

#define LFS_CACHE_SIZE 256U

uint8_t readBuffer[LFS_CACHE_SIZE];
uint8_t writeBuffer[LFS_CACHE_SIZE];
uint8_t lookBuffer[LFS_CACHE_SIZE] __attribute__((aligned(32)));

int lfs_read(const struct lfs_config *c, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size);
int lfs_write(const struct lfs_config *c, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size);
int lfs_erase(const struct lfs_config *c, lfs_block_t block);
int lfs_sync(const struct lfs_config *c);

struct lfs_config cfg = {
    // block device operations
    .read = &lfs_read,
    .prog = &lfs_write,
    .erase = &lfs_erase,
    .sync = &lfs_sync,

    // block device configuration
    .read_size = LFS_CACHE_SIZE,
    .prog_size = LFS_CACHE_SIZE,
    .block_size = 4096,
    .block_count = 8192,
    .block_cycles = 500,
    .cache_size = LFS_CACHE_SIZE,
    .lookahead_size = LFS_CACHE_SIZE,

    .read_buffer = readBuffer,
    .prog_buffer = writeBuffer,
    .lookahead_buffer = lookBuffer,
};

[michaelkollmann]

I am facing a very similar / identical problem as well. After creating, deleting and recreating some small files a call to lfs_stat returns LFS_ERR_NOENT.

The problem occurs to me when I have set the config as following:

#define READ_SIZE 8
#define WRITE_SIZE 32
#define CACHE_SIZE 128
#define BLOCK_SIZE 4096
#define BLOCK_COUNT 4096
#define LOOKAHEAD_SIZE 4096
#define METADATA_SIZE 512
#define BLOCK_CYCLES 10000

const struct lfs_config cfg = {
    // block device operations
    .read = user_provided_block_device_read,
    .prog = user_provided_block_device_prog,
    .erase = user_provided_block_device_erase,
    .sync = user_provided_block_device_sync,

    // block device configuration
    .read_size = READ_SIZE,
    .prog_size = WRITE_SIZE,
    .block_size = BLOCK_SIZE,
    .block_count = BLOCK_COUNT,
    .cache_size = CACHE_SIZE,
    .lookahead_size = LOOKAHEAD_SIZE,
    .block_cycles = BLOCK_CYCLES,
    .metadata_max = METADATA_SIZE
};

It does not occur when I increase the WRITE_SIZE to 128.

I'd prefer to have the WRITE_SIZE to be set to 32 as it improves the performance just enough to allow my device to write data every second to memory including sync with no delay whatsoever.

Any Idea what might be causing this issue? Should I open a now Issue here on Github for this problem I'm facing here?