The 25 items issue

[dmitry-grinko]

I'm getting this error. Member must have length less than or equal to 25, Member must have length greater than or equal to 1. Any idea how to handle it? =)

[lmammino]

Hello @dm-grinko, allow me to ask you a few questions to help me understand what's happening:

• where do you get this error?
• what command are you using?
• what input are you passing?

Thanks

[dmitry-grinko]

• command line
• aws dynamodb batch-write-item --request-items file://5000.json
• a huge file of 5000 items

[lmammino]

Thanks @dm-grinko.

It seems that there's a limit on the number of items that you can send in one request (25 top: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html)

You should split your source file into multiple files with 25 objects (max) each.

It might be nice to have this feature supported in the tool, would you have some time to contribute with a PR?

[dmitry-grinko]

I have found another solution. I have created nodejs app where I am using put-item

const fs = require("fs"), shell = require('shelljs'),
    table = "dictionary", filename = "./500.json";

function exec() {
    fs.readFile(filename, "utf8", function (err, data) {
        const items = JSON.parse(data);
        items.forEach(item => {
            const query = `
                aws dynamodb put-item \
                --table-name ${table}  \
                --item '{"Rank": {"S": "${item.Rank}"},"Word": {"S": "${item.Word}"},"Part of speech": {"S": "${item['Part of speech']}"},"Frequency": {"S": "${item.Frequency}"},"Dispersion": {"S": "${item.Dispersion}"}}' \
                --return-consumed-capacity TOTAL
            `;
            shell.exec(query)
        });
    });
}

exec();

[lmammino]

Thanks @dm-grinko, I'll re-open this one for now. Just in case someone is looking for something similar and as a reminder that we might want to have a more integrated solution.

[mjza]

Could you please add an option to this library to break down the output files to some set of files that contains each 25 items?

[mjza]

I improved @dm-grinko solution as follow:

const fs = require("fs"),
    shell = require("shelljs"),
    table = "tableName",
    filename = "./data.json";

function exec() {
    fs.readFile(filename, "utf8", function (err, data) {
        const items = JSON.parse(data);
        var counter = 0, req = {};
        req[table] = [];
        items.forEach(item => {
            counter++;
            var obj = {}, str = '';
            for (var key in item) {
                if (!isNaN(item[key]) && ("" + item[key]).trim().length > 0) {
                    obj[key] = {
                        "N": item[key]
                    };
                } else {
                    obj[key] = {
                        "S": item[key]
                    };
                }
            }
            req[table].push({
                PutRequest: {
                    Item: obj
                }
            });
            if (counter % 25 === 0) {
                str = JSON.stringify(req);
                const query = `aws dynamodb batch-write-item --request-items '${str}'`;
                console.log(`Write ${counter / 25}th 25 items.`);
                shell.exec(query);
                req[table] = [];
            }
        });
        if (counter % 25 > 0) {
            str = JSON.stringify(req);
            console.log(`Write ${counter % 25} rest items.`);
            const query = `aws dynamodb batch-write-item --request-items '${str}'`;
            shell.exec(query);
            req[table] = [];
        }
    });
}

exec();

[dayvsonsales]

I did some modifications to your code @mjza

Copy the code below and save it as dynamo-chunk.js

Now you just need to pass the file converted by this library (json-dynamo-putrequest) and pass the table name. It'll create the chunk files and automatically upload to dynamodb into your aws account.

const fs = require("fs");
const shell = require("shelljs");

function processFile(filename, table) {
  fs.readFile(filename, "utf8", (err, data) => {
    if (err) {
      console.log("Cannot parse", err);
      process.exit(1);
    }

    const items = JSON.parse(data);

    let counter = 0;
    let req = {};
    let str = "";

    req[table] = [];

    items[table].forEach((item) => {
      counter++;
      req[table].push(item);

      if (counter % 25 === 0) {
        str = JSON.stringify(req);
        fs.writeFileSync("./tmp.json", str);

        console.log(`Write ${counter / 25}th 25 items.`);

        const exec =
          "aws dynamodb batch-write-item --request-items file://tmp.json";
        shell.exec(exec);

        req[table] = [];
      }
    });

    if (counter % 25 > 0) {
      str = JSON.stringify(req);
      fs.writeFileSync("./tmp.json", str);

      console.log(`Write ${counter % 25} rest items.`);

      const exec =
        "aws dynamodb batch-write-item --request-items file://tmp.json";
      shell.exec(exec);

      req[table] = [];
    }
  });
}

if (!process.argv[2] || !process.argv[3]) {
  console.log("Missing arguments");
  console.log("Usage: node dynamo-chunk.js file tableName");

  process.exit(1);
}

processFile(process.argv[2], process.argv[3]);