Test result is not constant

[juhi123]

I am not sure why it's happening , I compared the baseline and results screenshot , both looks same to me , but somehow test case gets failed and its totally random , sometimes fail, sometime pass. Can anyone help me to solve this issue? What could be the reason? Diff directory has diff images with some red color.

[lksv]

Could you please provide the images and snippet of code you are using.

[juhi123]

Thanks for looking into this. Yeah sure I can provide you code , I think images does not matter. It works same for all.

Assertion/compareScreenshot.js

// assertions/compareScreenshot.js
var resemble = require('node-resemble-js'),
    fs = require('fs');

exports.assertion = function(filename,expected) {
    var screenshotPath = './screenshots/',
        baselinePath = screenshotPath + 'baseline/' + filename,
        resultPath = screenshotPath + 'results/' + filename,
        diffPath = screenshotPath + 'diffs/' + filename;

    this.message = 'Unexpected compareScreenshot error.';
    this.expected = expected || 0;   // misMatchPercentage tolerance default 0%

    this.command = function(callback) {
        // create new baseline photo if none exists
        if (!fs.existsSync(baselinePath)) {
            console.log('WARNING: Baseline Photo does NOT exist.');
            // console.log(baselinePath +' baselinepath');
            console.log('Creating Baseline Photo from Result: ' + baselinePath);
            fs.writeFileSync(baselinePath, fs.readFileSync(resultPath));
        }


            resemble(baselinePath)
            .compareTo(resultPath)
              .onComplete(callback);  // calls this.value with the result

        return this;
    };

    this.value = function(result) {
console.log(result);
     result.getDiffImage().pack().pipe(fs.createWriteStream(diffPath));
// var diff = new Buffer(result.getImageDataUrl().replace(/data:image\/png;base64,/,''), 'base64');
// fs.writeFileSync(diffPath, diff);


        return parseFloat(result.misMatchPercentage, 10);  // value this.pass is called with


    };

    this.pass = function(value) {
        var pass = value <= this.expected;
        if (pass) {
            this.message = 'Screenshots Matched for ' + filename +
                ' with a tolerance of ' + this.expected + '%.';
        } else {
            this.message = 'Screenshots Match Failed for ' + filename +
                ' with a tolerance of ' + this.expected + '%.\n' +
                '   Screenshots at:\n' +
                '    Baseline: ' + baselinePath + '\n' +
                '    Result: ' + resultPath + '\n' +
                '    Diff: ' + diffPath + '\n' +
                '   Open ' + diffPath + ' to see how the screenshot has changed.\n' +
                '   If the Result Screenshot is correct you can use it to update the Baseline Screenshot and re-run your test:\n' +
                '    cp ' + resultPath + ' ' + baselinePath;
        }
        return pass;
    };
};

commands/commands/compareScreenshot.js

// commands/compareScreenshot.js
exports.command = function(filename, expected, callback) {
    var self = this,
        screenshotPath = './screenshots/',
        resultPath = screenshotPath + 'results/' + filename;

    self.saveScreenshot(resultPath, function(response) {
        self.assert.compareScreenshot(filename, expected,function(result) {
            if (typeof callback === 'function') {
                callback.call(self, result);
            }
        });
    });

    return this; // allows the command to be chained.
};

Calling in test case like this lostPassword.js

module.exports = {

    'Lost Password test': function(browser) {
      var data =browser.globals;
        browser
        .url(data.liveUrl+'account/lost-password')
        .windowSize('current',1920,1080)
        .waitForElementVisible('body',1000)
        .compareScreenshot('lost-password.png')
        .end();
    }
};

[juhi123]

@lksv

Can you tell me? If I am doing something wrong.

[lksv]

@juhi123 Now this described issue happens to me too. I am trying to fix it. Unfortunately for now I don't know the reason for this non-deterministic behaviour.

[ShobanaV]

@lksv Any updates on this issue?

[lksv]

@juhi123, @ShobanaV Can you test it on master now?

[ShobanaV]

@lksv, When I compare screenshots taken in one PC with similar screenshots taken in another PC, the image shows slight difference.

The resolution of the PCs are the same. And the images look exactly alike.