Search is now consistently failing to return a result

[chriskinsman]

• Operating System: MacOS 12.4 and Amazon Linux 2
• Node version: 16.13.1
• google-play-scraper version: 8.0.4 and 8.1.0

Description:

Search is now consistently returning no results. Was hit and miss over last couple weeks. 100% failure now. Guessing google play store change?

Example code:

gplay
    .search({
        term: 'facebook',
    })
    .then(function (data) {
        if (data.length === 0) {
            logger.error('Error searching Google Play store. Query returned no results');
            process.exit(1);
        } else {
            logger.info('Received valid response from Play Store');
            process.exit(0);
        }
    })
    .catch(function (error) {
        logger.error('Error searching Android store: Error ' + error);
        process.exit(1);
    });

Error message:

None

[adtimizer]

@chriskinsman the following fixed the search issue for me in package.json replace the google-play-scraper 8.1.0 with the following: "google-play-scraper": "git+https://github.com/facundoolano/google-play-scraper#pull/547/head",

[NathanManning]

@adtimizer It doesn't fix results that are 'clustered', i.e. have 'Similar apps' and 'More results' carousels. search.js used to skipClusterPage(), but now that's broken.

@chriskinsman the following fixed the search issue for me in package.json replace the google-play-scraper 8.1.0 with the following: "google-play-scraper": "git+https://github.com/facundoolano/google-play-scraper#pull/547/head",

[mnikitas]

Hi, any update on releasing a working version?

[mnikitas]

Is there any plan to resolve this?

[manuel-dev]

Is there any plan to resolve this?

The situation is difficult. If it's finally resolved, don't expect it to work like it used to. Google is now returning very limited results, when it returns any... Before this, in a regular search, where you got hundreds of results on a search term, now it might return 9... or 12... or 25...

[NathanManning]

@manuel-dev I would take 9, 12, or 15 right now. Is that change on a pull request somewhere that I'm missing?

[adtimizer]

Hello All, I was able to fix the search on my end. (code to follow- not sure about my qualities to perform an actual PR) The fix checks if there is an extra result (the search return the most suitable result) and if that case it changes the mapping to get all other results. //TODO - add the most suitable result to the list.

const INITIAL_MAPPINGS1 = {
    apps: ['ds:4', 0, 1, 0, 22, 0],
    token: ['ds:4', 1]
  };
  const INITIAL_MAPPINGS2 = {
    apps: ['ds:4', 0, 1, 2, 22, 0],
    token: ['ds:4', 1]
  };
  var mapping=INITIAL_MAPPINGS1;  

  const url = `${BASE_URL}/store/search?c=apps&q=${opts.term}&hl=${opts.lang}&gl=${opts.country}&price=${opts.price}`;

  return request(Object.assign({ url }, opts.requestOptions), opts.throttle)
    .then(skipClusterPage)
    .then(scriptData.parse)
    // comment next line to get raw data
    .then((parsedData) => {
        const isV2 = R.path(['ds:4', 0, 1], parsedData).length==4;
        mapping = isV2 ? INITIAL_MAPPINGS2 : INITIAL_MAPPINGS1;
        console.log(mapping.apps);

      return parsedData;
      })
    .then((html) => processPages(html, opts, [], mapping));
}

[dave-filion]

Hey @adtimizer ! Do you have the fix for search on a branch or fork? Trying to get it working as well.

[adtimizer]

@dave-filion this is the search class (with my modifications) under lib folder (i'm away so can't fork/branch) replace yours with this code and search will work for you.

'use strict';

const request = require('./utils/request');
const { BASE_URL } = require('./constants');
const processPages = require('./utils/processPages');
const scriptData = require('./utils/scriptData');
const R = require('ramda');

/*
 * Make the first search request as in the browser and call `checkfinished` to
 * process the next pages.
 */
function initialRequest (opts) {
  // sometimes the first result page is a cluster of subsections,
  // need to skip to the full results page
  function skipClusterPage (html) {
    const match = html.match(/href="\/store\/apps\/collection\/search_collection_more_results_cluster?(.*?)"/);
    if (match) {
      const innerUrl = BASE_URL + match[0].split(/"/)[1];
      return request(Object.assign({
        url: innerUrl
      }, opts.requestOptions), opts.throttle);
    }
    return html;
  }

  const INITIAL_MAPPINGS1 = {
    apps: ['ds:4', 0, 1, 0, 22, 0],
    // apps: ['ds:4', 0, 1, 1, 21, 0],
    token: ['ds:4', 1]
  };
  const INITIAL_MAPPINGS2 = {
    // apps: ['ds:4', 0, 1, 2, 22, 0],
    apps: ['ds:4', 0, 1, 2, 22, 0],
    token: ['ds:4', 1]
  };
  var mapping=INITIAL_MAPPINGS1;  

  const url = `${BASE_URL}/store/search?c=apps&q=${opts.term}&hl=${opts.lang}&gl=${opts.country}&price=${opts.price}&fpr=false`;

  return request(Object.assign({ url }, opts.requestOptions), opts.throttle)
    .then(skipClusterPage)
    .then(scriptData.parse)
    // comment next line to get raw data
    .then((parsedData) => {
        const isV2 = R.path(['ds:4', 0, 1], parsedData).length==4;
        mapping = isV2 ? INITIAL_MAPPINGS2 : INITIAL_MAPPINGS1;
        // console.log(mapping.apps);

      return parsedData;
      })
    .then((html) => processPages(html, opts, [], mapping));
}

function getPriceGoogleValue (value) {
  switch (value.toLowerCase()) {
    case 'free':
      return 1;
    case 'paid':
      return 2;
    case 'all':
    default:
      return 0;
  }
}

function search (appData, opts) {
  return new Promise(function (resolve, reject) {
    if (!opts || !opts.term) {
      throw Error('Search term missing');
    }

    if (opts.num && opts.num > 250) {
      throw Error("The number of results can't exceed 250");
    }

    opts = {
      term: encodeURIComponent(opts.term),
      lang: opts.lang || 'en',
      country: opts.country || 'us',
      num: opts.num || 20,
      fullDetail: opts.fullDetail,
      price: opts.price ? getPriceGoogleValue(opts.price) : 0,
      throttle: opts.throttle,
      cache: opts.cache,
      requestOptions: opts.requestOptions
    };

    initialRequest(opts)
      .then(resolve)
      .catch(reject);
  }).then((results) => {
    if (opts.fullDetail) {
      // if full detail is wanted get it from the app module
      return Promise.all(results.map((app) => appData({ ...opts, appId: app.appId })));
    }
    return results;
  });
}

module.exports = search;

[mmoksh]

@dave-filion Try this: https://github.com/facundoolano/google-play-scraper/pull/557

[chriskinsman]

@mmoksh #557 worked for a bit but looks to now be failing again today

[mmoksh]

@chriskinsman What query did you use to make it fail? I just tried different queries and it looks like it's working well

[mmoksh]

@chriskinsman It might cache issue. Please try to delete the package-lock.json file and node_modules folder then run npm i and check if that works

[facundoolano]

can you check if this still happens with the latest version?