[BUG] `OpenSearchISMPolicy` doesn't work

[vchirikov]

I created a OpenSearchISMPolicy according to the current spec, but it isn't created in opensearch (error in logs is the object has been modified; please apply your changes to the latest version and try again). Looks like this kind doesn't work at all (check also https://github.com/opensearch-project/opensearch-k8s-operator/discussions/696 this)

apiVersion: opensearch.opster.io/v1
kind: OpenSearchISMPolicy
metadata:
  name: rollover
  namespace: infra
spec:
  opensearchCluster:
    name: opensearch
  policyId: default
  description: Rollover ISM policy
  ismTemplate:
    priority: 100
    indexPatterns:
      - "apps*"
      - "kube*"
      - "node*"
      - "ingress*"
  errorNotification:
    channel: slack
    destination:
      slack:
        url: "${SLACK_ALERTS_WEBHOOK_URL}"
    messageTemplate:
      source: "The index {{ctx.index}} failed during ISM policy execution"
  defaultState: hot
  states:
    - name: hot
      actions:
        - rollover:
            minIndexAge: 10d
            minPrimaryShardSize: 10gb
        - indexPriority:
            priority: 100
      transitions:
        - stateName: warm
          conditions:
            minIndexAge: 7d
            minRolloverAge: 7d
    - name: warm
      actions:
        - indexPriority:
            priority: 50
      transitions:
        - stateName: delete
          conditions:
            minIndexAge: 20d
    - name: delete
      transitions: []
      actions:
        - delete: {}

logs:

{"level":"info","ts":"2024-02-20T13:18:33.780Z","msg":"Reconciling OpenSearchCluster","controller":"opensearchcluster","controllerGroup":"opensearch.opster.io","controllerKind":"OpenSearchCluster","OpenSearchCluster":{"name":"opensearch","namespace":"infra"},"namespace":"infra","name":"opensearch","reconcileID":"7e721d68-7a74-4aa9-ae84-add870efee35","cluster":{"name":"opensearch","namespace":"infra"}}
{"level":"info","ts":"2024-02-20T13:18:33.799Z","msg":"Generating certificates","controller":"opensearchcluster","controllerGroup":"opensearch.opster.io","controllerKind":"OpenSearchCluster","OpenSearchCluster":{"name":"opensearch","namespace":"infra"},"namespace":"infra","name":"opensearch","reconcileID":"7e721d68-7a74-4aa9-ae84-add870efee35","interface":"transport"}
{"level":"info","ts":"2024-02-20T13:18:33.799Z","msg":"Generating certificates","controller":"opensearchcluster","controllerGroup":"opensearch.opster.io","controllerKind":"OpenSearchCluster","OpenSearchCluster":{"name":"opensearch","namespace":"infra"},"namespace":"infra","name":"opensearch","reconcileID":"7e721d68-7a74-4aa9-ae84-add870efee35","interface":"http"}
{"level":"info","ts":"2024-02-20T13:18:57.218Z","msg":"Reconciling OpensearchISMPolicy","controller":"opensearchismpolicy","controllerGroup":"opensearch.opster.io","controllerKind":"OpenSearchISMPolicy","OpenSearchISMPolicy":{"name":"rollover","namespace":"infra"},"namespace":"infra","name":"rollover","reconcileID":"cd48d62d-3287-4063-811f-54a910d41b1f","tenant":{"name":"rollover","namespace":"infra"}}
{"level":"info","ts":"2024-02-20T13:18:57.227Z","msg":"Reconciling OpensearchIndexTemplate","controller":"opensearchindextemplate","controllerGroup":"opensearch.opster.io","controllerKind":"OpensearchIndexTemplate","OpensearchIndexTemplate":{"name":"all","namespace":"infra"},"namespace":"infra","name":"all","reconcileID":"67058e28-82d7-4951-ac6f-0c8235a3d7b9","indextemplate":{"name":"all","namespace":"infra"}}
{"level":"info","ts":"2024-02-20T13:18:57.350Z","msg":"Reconciling OpensearchISMPolicy","controller":"opensearchismpolicy","controllerGroup":"opensearch.opster.io","controllerKind":"OpenSearchISMPolicy","OpenSearchISMPolicy":{"name":"rollover","namespace":"infra"},"namespace":"infra","name":"rollover","reconcileID":"a627ee69-6d8e-4aec-b9e0-53e4cc0a09b7","tenant":{"name":"rollover","namespace":"infra"}}
{"level":"error","ts":"2024-02-20T13:18:57.363Z","msg":"Reconciler error","controller":"opensearchismpolicy","controllerGroup":"opensearch.opster.io","controllerKind":"OpenSearchISMPolicy","OpenSearchISMPolicy":{"name":"rollover","namespace":"infra"},"namespace":"infra","name":"rollover","reconcileID":"a627ee69-6d8e-4aec-b9e0-53e4cc0a09b7","error":"Operation cannot be fulfilled on opensearchismpolicies.opensearch.opster.io \"rollover\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:226"}
{"level":"info","ts":"2024-02-20T13:18:57.363Z","msg":"Reconciling OpensearchISMPolicy","controller":"opensearchismpolicy","controllerGroup":"opensearch.opster.io","controllerKind":"OpenSearchISMPolicy","OpenSearchISMPolicy":{"name":"rollover","namespace":"infra"},"namespace":"infra","name":"rollover","reconcileID":"2deb7560-cf4d-4e6c-a7fa-76143ec10945","tenant":{"name":"rollover","namespace":"infra"}}
{"level":"info","ts":"2024-02-20T13:18:57.422Z","msg":"Reconciling OpensearchIndexTemplate","controller":"opensearchindextemplate","controllerGroup":"opensearch.opster.io","controllerKind":"OpensearchIndexTemplate","OpensearchIndexTemplate":{"name":"all","namespace":"infra"},"namespace":"infra","name":"all","reconcileID":"3b6836b9-10f8-41a9-9ec0-63f0b41450bb","indextemplate":{"name":"all","namespace":"infra"}}
{"level":"info","ts":"2024-02-20T13:18:57.440Z","msg":"Reconciling OpensearchISMPolicy","controller":"opensearchismpolicy","controllerGroup":"opensearch.opster.io","controllerKind":"OpenSearchISMPolicy","OpenSearchISMPolicy":{"name":"rollover","namespace":"infra"},"namespace":"infra","name":"rollover","reconcileID":"cdf2e629-0b1a-4ff8-a8df-fc4613491081","tenant":{"name":"rollover","namespace":"infra"}}

docker image: opensearchproject/opensearch-operator:2.5.1