Error: Argument "data" is not a valid Document

[ErwinAI]

Expected behavior

Commandline export and programmatic import does not cause an error.

Actual behavior

It does cause an error.

Steps to reproduce the behavior

1. Make sure there is a Timestamp in the database.

2. run firestore-export --accountCredentials path/to/credentials/file.json --backupFile /backups/myDatabase.json

3. programmatically import by the example in the readme.md:

const {firestoreImport} = require('node-firestore-import-export');
const FirestoreData = require("./backups/myDatabase.json");
const admin = require('firestore-admin');

let serviceAccount = require('service-account.json');
admin.initializeApp({
  credential: admin.credential.cert(serviceAccount),
  databaseURL: "db-url"
});

firestore = admin.firestore();
const settings = {timestampsInSnapshots: true};
firestore.settings(settings);

firestoreImport(FirestoreData, firestore);

4. Error is thrown as follows:

(node:8088) UnhandledPromiseRejectionWarning: Error: Argument "data" is not a valid Document. Detected an object of type "Timestamp" that doesn't match the expected instance. Please ensure that the Firestore types you are using are from the same NPM package.
    at Validator.(anonymous function).values [as isDocument] (C:\Users\me\projects\maklr-backend\tools\node_modules\@google-cloud\firestore\build\src\validate.js:99:27)
    at WriteBatch.set (C:\Users\me\projects\maklr-backend\tools\node_modules\@google-cloud\firestore\build\src\write-batch.js:232:25)
    at C:\Users\me\projects\maklr-backend\tools\node_modules\node-firestore-import-export\dist\lib\import.js:68:19
    at Array.map (<anonymous>)
    at C:\Users\me\projects\maklr-backend\tools\node_modules\node-firestore-import-export\dist\lib\import.js:57:22
    at Array.map (<anonymous>)
    at setDocuments (C:\Users\me\projects\maklr-backend\tools\node_modules\node-firestore-import-export\dist\lib\import.js:55:32)
    at importData (C:\Users\me\projects\maklr-backend\tools\node_modules\node-firestore-import-export\dist\lib\import.js:28:43)
    at importAuthUsers.then (C:\Users\me\projects\maklr-backend\tools\restoreEnvironment.js:85:20)
    at <anonymous>
(node:8088) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:8088) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

I've seen issue #36 , which seems to be similar to my difficulties. However, the issue mentions that the issue is fixed. I am using the most recent version (0.13.2) so I presume it's either not entirely fixed, or it's the same error but with a different cause.

[ErwinAI]

One addition, I managed to trace it back to two specific documents that both have three fields, all built like:

        "projectStart": {
          "__datatype__": "timestamp",
          "value": {
            "_seconds": 1549076400,
            "_nanoseconds": 0
          }
        }

[ErwinAI]

Using the firestore instance normally used for a web application (so not the firestore instance from the admin package) crashes at the same action it seems, but the error is a bit more revealing. It seems the custom object is not converted properly?

(node:10804) DeprecationWarning: grpc.load: Use the @grpc/proto-loader module with grpc.loadPackageDefinition instead
(node:10804) UnhandledPromiseRejectionWarning: FirebaseError: Function WriteBatch.set() called with invalid data. Unsupported field value: a custom object (found in field dateCreated)
    at new FirestoreError (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:353:28)
    at ParseContext.createError (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:19022:16)
    at validatePlainObject (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:19402:27)
    at UserDataConverter.parseData (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:19208:13)
    at C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:19251:41
    at forEach (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:455:13)
    at UserDataConverter.parseObject (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:19250:13)
    at UserDataConverter.parseData (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:19209:25)
    at UserDataConverter.parseMergeData (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:19082:31)
    at WriteBatch.set (C:\Users\me\projects\maklr-backend\tools\node_modules\@firebase\firestore\dist\index.node.cjs.js:19834:46)
(node:10804) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:10804) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

[jloosli]

Do you run into the same errors when trying to import and export from the command line?

[jay8t6]

+1

I am running into the same issue as well

Update: Im trying to import using a cloud function.

[jay8t6]

@jloosli any help is greatly appreciated

[tafm]

same here

[jay8t6]

@tafm I ended up using this script, it worked great for me: https://gist.github.com/sturmenta/cbbe898227cb1eaca7f85d0191eaec7e

[joelclimbsthings]

+1, came across the same issue when trying to import anything with timestamps:

      "activatedAt": {
         "__datatype__": "timestamp",
         "value": {
            "_seconds": 1609184426,
            "_nanoseconds": 734000000
         }
      },

[beelalamin]

I just put this structure data and it worked.

{
  "test": {
    "first-key": {
      "website": "dalenguyen.me",
      "date": {
        "_seconds": 1534046400,
        "_nanoseconds": 0
      },
      "schedule": {
        "time": {
          "_seconds": 1534046400,
          "_nanoseconds": 0
        }
      },
      "three": {
        "level": {
          "time": {
            "_seconds": 1534046400,
            "_nanoseconds": 0
          }
        }
      },
      "custom": {
        "lastName": "Nguyen",
        "firstName": "Dale"
      },
      "location": {
        "_latitude": 49.290683,
        "_longitude": -123.133956
      },
      "locationNested": {
        "geopoint": {
          "_latitude": 49.290683,
          "_longitude": -123.133956
        }
      },
      "locations": [
        {
          "_latitude": 50.290683,
          "_longitude": -123.133956
        },
        {
          "_latitude": 51.290683,
          "_longitude": -123.133956
        }
      ],
      "email": "[email protected]",
      "secondRef": "test/second-key",
      "arrayRef": ["test/second-key", "test/second-key"],
      "nestedRef": {
        "secondRef": "test/second-key"
      },
      "subCollection": {
        "test/first-key/details": {
          "33J2A10u5902CXagoBP6": {
            "dogId": "2",
            "dogName": "hello"
          },
          "MSZTWEP7Lewx0Qr1Mu5s": {
            "dogName": "lala",
            "dogId": "2"
          }
        },
        "test/first-key/contacts": {
          "33J2A10u5902CXagoBP6": {
            "contactId": "1",
            "name": "Dale Nguyen"
          },
          "MSZTWEP7Lewx0Qr1Mu5s": {
            "contactId": "2",
            "name": "Yen Nguyen"
          }
        }
      }
    },
    "second-key": {
      "website": "google.com",
      "date": {
        "_seconds": 1534262435,
        "_nanoseconds": 0
      },
      "custom": {
        "lastName": "Potter",
        "firstName": "Harry"
      },
      "location": {
        "_latitude": 49.290683,
        "_longitude": -123.133956
      },
      "email": "[email protected]"
    }
  }
}