gax-nodejs
gax-nodejs copied to clipboard
Autopagination does not throw backend errors appropriately
Investigating: https://github.com/googleapis/nodejs-datacatalog/issues/296
It seems like, when autoPagination is on, and you throw an error, it does not stop getting resources. To reproduce:
- call
searchAssets.jsin datacatalog (datacatalog has a lot of resources, we're trying to get a large response):
'use strict';
async function main(projectId) {
// [START data_catalog_search_assets]
// Import the Google Cloud client library.
const {DataCatalogClient} = require('@google-cloud/datacatalog').v1;
const datacatalog = new DataCatalogClient();
async function searchAssets(projectId) {
// Search data assets.
/**
* TODO(developer): Uncomment the following lines before running the sample.
*/
// const projectId = 'my_project'; // Google Cloud Platform project
// Create request.
const scope = {
includeProjectIds: [projectId],
// Alternatively, search using Google Cloud Organization scopes.
//includeOrgIds: ["google.com"],
};
const request = {
scope: scope,
pageSize: 50,
includeGcpPublicDatasets: true,
orderBy: 'relevance',
};
const [result] = await datacatalog.searchCatalog(request);
console.log(`Found ${result.length}`);
console.log('Datasets:');
result.forEach(dataset => {
console.log(dataset.relativeResourceName);
});
}
searchAssets();
// [END data_catalog_search_assets]
}
main(...process.argv.slice(2));
Then, in gax (in pagedApiCaller.ts, https://github.com/googleapis/gax-nodejs/blob/2217633d95b49279a3d00fa7b720443d89ab3086/src/paginationCalls/pagedApiCaller.ts#L91):
Add a global counter, and these statements:
counter++;
callback(err, resources, nextPageRequest, response);
if (counter === 5) { const err = new googleError_1.GoogleError('error'); err.code = 8; /* resource exhausted */ callback(err); return; }
It will continue grabbing pages indefinitely.
In addition to fixing this bug, we should add unit tests here.