Dropping with filtering is not working properly.

[IlyaShkurenko]

I'm trying to use filtering to index only needed documents to Elasticsearch with dropping script to drop documents after they were updated to wrong status. Filter is working correctly but after I update product to not active status it's not dropping and not reacting on updates until be moved to active back. Here is my code script, will appreciate for help.

[[filter]]
namespace = "MPowerDev.products"
script = """
module.exports = function(doc, ns, updateDesc) {
    return doc.status === 'active' && (doc.categories && doc.categories[0]);
}
"""

[[script]]
namespace = "MPowerDev.products"
script = """
module.exports = function(doc) {
    if (doc.status !== 'active' || !(doc.categories && doc.categories[0])) {
        return false;
    }
    if (doc.imageDefault) {
        doc.imageDefault = findId(doc.imageDefault, {
          database: "MPowerDev",
          collection: "productMedia",
          select: {
            url: 1,
            urlSmall: 1
          }
        });
    };

    var product = _.pick(doc,
      "_id",
      "productId",
      "categories",
      "facets",
      "facetGroups",
      "imageDefault",
      "imagePrimary",
      "defaultImgUrl",
      "defaultImgUrlSmall",
      "inventoryStatus",
      "slug",
      "decorationProducts",
      "description",
      "productName",
      "status",
      "colorsCount",
      "isCloseout",
      "noPricing",
      "priceMin",
      "priceMax",
      "minQuantity",
      "hasOnlyOnePrice",
      "total_sales"
    );
    product.mongoId = product._id;
    return product;
}
"""

[IlyaShkurenko]

full configuration file

direct-read-namespaces = [
  "MPowerDev.products"
]
change-stream-namespaces = [
  "MPowerDev.products"
]
elasticsearch-max-conns = 1
direct-read-split-max = -1
direct-read-concur = 1
direct-read-bounded = true
direct-read-no-timeout = true
index-as-update = true
replay = false
resume = true
verbose = true
# this sends stats to Elasticsearch that you can create Kibana dashboards for
stats = true
index-stats = true
# this starts a web server that exposes some endpoints
enable-http-server = true
http-server-addr = ":80"

[gtm-settings]
channel-size = 1

[[filter]]
namespace = "MPowerDev.products"
script = """
module.exports = function(doc, ns, updateDesc) {
    return doc.status === 'active' && (doc.categories && doc.categories[0]);
}
"""

[[script]]
namespace = "MPowerDev.products"
script = """
module.exports = function(doc) {
    if (doc.status !== 'active' || !(doc.categories && doc.categories[0])) {
        return false;
    }
    if (doc.imageDefault) {
        doc.imageDefault = findId(doc.imageDefault, {
          database: "MPowerDev",
          collection: "productMedia",
          select: {
            url: 1,
            urlSmall: 1
          }
        });
    };

    var product = _.pick(doc,
      "_id",
      "productId",
      "categories",
      "facets",
      "facetGroups",
      "imageDefault",
      "imagePrimary",
      "defaultImgUrl",
      "defaultImgUrlSmall",
      "inventoryStatus",
      "slug",
      "decorationProducts",
      "description",
      "productName",
      "status",
      "colorsCount",
      "isCloseout",
      "noPricing",
      "priceMin",
      "priceMax",
      "minQuantity",
      "hasOnlyOnePrice",
      "total_sales"
    );
    product.mongoId = product._id;
    return product;
}
"""

[rwynn]

If your filter function returns false the change event (insert, update, delete) will be dropped entirely and never reach your script. You may want to try without the filter.