f5-declarative-onboarding icon indicating copy to clipboard operation
f5-declarative-onboarding copied to clipboard

Published 20 hours ago • verified publisher icon F5Networks

F5 Cluster stucked Node2 Trust

Open rdegoix opened this issue 6 months ago • 0 comments

Dear community, Thanks for your help and time in order to review my issue ;)

Environment

  • Declarative Onboarding Version: v1.35
  • BIG-IP Version: BIG-IP 17.1.0 Build 0.0.16 Final

Summary

Trying to DO a cluster fail-over (with already IPv6 Management configured previously by another automation mechanism). HA Self IP will be used for HA and for Trust. Connectivity has been confirmed between HA interfaces : [root@cloudprov304:Active:Standalone] config # telnet 100.100.1.5 443 Trying 100.100.1.5... Connected to 100.100.1.5. Escape character is '^]'.

[root@cloudprov303:Active:Standalone] config # telnet 100.100.1.6 443 Trying 100.100.1.6... Connected to 100.100.1.6. Escape character is '^]'.

No connectivity between IPv6 Management due to micro-segmentation restriction.

Steps To Reproduce

Steps to reproduce the behavior:

  1. Submit the following declaration:
{
    "schemaVersion": "1.0.0",
    "class": "Device",
    "async": true,
    "controls": {
        "trace": true,
        "traceResponse": true
    },
    "Common": {
        "class": "Tenant",
        "hostname": "cloudprov303.tlabs.online",
        "ha": {
            "class": "VLAN",
            "mtu": 1500,
            "interfaces": [
                {
                    "name": "1.3",
                    "tagged": false
                }
            ]
        },
        "ha-self": {
            "class": "SelfIp",
            "address": "100.100.1.5/30",
            "vlan": "ha",
            "allowService": "default",
            "trafficGroup": "traffic-group-local-only"
        },
        "configsync": {
            "class": "ConfigSync",
            "configsyncIp": "/Common/ha-self/address"
        },
        "failoverAddress": {
            "class": "FailoverUnicast",
            "address": "/Common/ha-self/address"
        },
        "failoverGroup": {
            "class": "DeviceGroup",
            "type": "sync-failover",
            "members": [
                "100.100.1.5",
                "100.100.1.6"
            ],
            "owner": "/Common/failoverGroup/members/0",
            "autoSync": true,
            "saveOnAutoSync": false,
            "networkFailover": true,
            "fullLoadOnSync": false,
            "asmSync": false
        },
        "trust": {
            "class": "DeviceTrust",
            "localUsername": "admin",
            "localPassword": "XXXX",
            "remoteHost": "100.100.1.5",
            "remoteUsername": "admin",
            "remotePassword": "XXXX"
        }
    }
}
  1. Observe the following error response:
{
  "id": "9543a470-a380-4bee-a874-c3fc755f700f",
  "selfLink": "https://localhost/mgmt/shared/declarative-onboarding/task/9543a470-a380-4bee-a874-c3fc755f700f",
  "result": {
    "class": "Result",
    "code": 200,
    "status": "OK",
    "dryRun": false,
    "message": "success",
    "warnings": [
      "The default value for 'allowService' on a 'SelfIp' will change from 'default' to 'none' in f5-declarative-onboarding version 1.35.0."
    ]
  },
  "declaration": {
    "schemaVersion": "1.0.0",
    "class": "Device",
    "async": true,
    "controls": {
      "trace": true,
      "traceResponse": true,
      "dryRun": false
    },
    "Common": {
      "class": "Tenant",
      "hostname": "cloudprov303.tlabs.online",
      "ha": {
        "class": "VLAN",
        "mtu": 1500,
        "interfaces": [
          {
            "name": "1.3",
            "tagged": false
          }
        ],
        "autoLastHop": "default",
        "cmpHash": "default",
        "failsafeEnabled": false,
        "failsafeAction": "failover-restart-tm",
        "failsafeTimeout": 90
      },
      "ha-self": {
        "class": "SelfIp",
        "address": "100.100.1.5/30",
        "vlan": "ha",
        "allowService": "default",
        "trafficGroup": "traffic-group-local-only"
      },
      "configsync": {
        "class": "ConfigSync",
        "configsyncIp": "/Common/ha-self/address"
      },
      "failoverAddress": {
        "class": "FailoverUnicast",
        "address": "/Common/ha-self/address",
        "port": 1026
      },
      "failoverGroup": {
        "class": "DeviceGroup",
        "type": "sync-failover",
        "members": [
          "100.100.1.5",
          "100.100.1.6"
        ],
        "owner": "/Common/failoverGroup/members/0",
        "autoSync": true,
        "saveOnAutoSync": false,
        "networkFailover": true,
        "fullLoadOnSync": false,
        "asmSync": false
      },
      "trust": {
        "class": "DeviceTrust",
        "localUsername": "admin",
        "remoteHost": "100.100.1.5",
        "remoteUsername": "admin"
      }
    }
  },

```json Node 2: 
{
  "id": "66557e4b-331f-4899-a6f4-d00a4105ec2c",
  "selfLink": "https://localhost/mgmt/shared/declarative-onboarding/task/66557e4b-331f-4899-a6f4-d00a4105ec2c",
  "result": {
    "class": "Result",
    "code": 202,
    "status": "RUNNING",
    "dryRun": false,
    "message": "processing"
  },
  "declaration": {
    "schemaVersion": "1.0.0",
    "class": "Device",
    "async": true,
    "controls": {
      "trace": true,
      "traceResponse": true,
      "dryRun": false
    },
    "Common": {
      "class": "Tenant",
      "hostname": "cloudprov304.tlabs.online",
      "ha": {
        "class": "VLAN",
        "mtu": 1500,
        "interfaces": [
          {
            "name": "1.3",
            "tagged": false
          }
        ],
        "autoLastHop": "default",
        "cmpHash": "default",
        "failsafeEnabled": false,
        "failsafeAction": "failover-restart-tm",
        "failsafeTimeout": 90
      },
      "ha-self": {
        "class": "SelfIp",
        "address": "100.100.1.6/30",
        "vlan": "ha",
        "allowService": "default",
        "trafficGroup": "traffic-group-local-only"
      },
      "configsync": {
        "class": "ConfigSync",
        "configsyncIp": "/Common/ha-self/address"
      },
      "failoverAddress": {
        "class": "FailoverUnicast",
        "address": "/Common/ha-self/address",
        "port": 1026
      },
      "failoverGroup": {
        "class": "DeviceGroup",
        "type": "sync-failover",
        "members": [
          "100.100.1.5",
          "100.100.1.6"
        ],
        "owner": "/Common/failoverGroup/members/0",
        "autoSync": true,
        "saveOnAutoSync": false,
        "networkFailover": true,
        "fullLoadOnSync": false,
        "asmSync": false
      },
      "trust": {
        "class": "DeviceTrust",
        "localUsername": "admin",
        "remoteHost": "100.100.1.5",
        "remoteUsername": "admin"
      }
    }
  },

Expected Behavior

Get F5 clustered and Trust negociation should happen.

Actual Behavior

F5 node 2 keep stuck trying to establish Trust :

Fri, 28 Mar 2025 09:33:08 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] {} Fri, 28 Mar 2025 09:33:08 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] list 100.100.1.5 /tm/sys/mcp-state Fri, 28 Mar 2025 09:33:08 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] {"kind":"tm:sys:mcp-state:mcp-statestats","selfLink":"https://localhost/mgmt/tm/sys/mcp-state?ver=17.1.0","entries":{"https://localhost/mgmt/tm/sys/mcp-state/0":{"nestedStats":{"entries":{"endPlatformIdReceived":{"description":"true"},"lastLoad":{"description":"full-config-load-succeed"},"phase":{"description":"running"}}}}}} Fri, 28 Mar 2025 09:33:08 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] list 100.100.1.5 /shared/identified-devices/config/device-info Fri, 28 Mar 2025 09:33:09 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] {"baseMac":"00:50:56:96:41:47","hostMac":"00:50:56:96:41:47","halUuid":"42164fc4-2ccf-b86d-e0df-cb50f07a68d0","chassisSerialNumber":"42164fc4-2ccf-b86d-cb50f07a68d0","slots":[{"volume":"HD1.1","product":"BIG-IP","version":"17.1.0","build":"0.0.16","isActive":true}],"license":{"licenseEndDateTime":"2025-04-28T00:00:00-07:00","registrationKey":"MUYSI-MIYHZ-QYMPR-OPRYL-ONAOCKQ","activeModules":["BIG-IP, VE Trial|FHJCOTZ-GSTAKRB|Rate Shaping|External Interface and Network HSM, VE|SDN Services, VE|SSL, Forward Proxy, VE|BIG-IP VE, Multicast Routing|APM, Limited|SSL, VE|DNS (1K QPS), VE|Routing Bundle, VE|ASM, VE|Crytpo Offload, VE, Tier 1 (25M - 200M)|Max Compression, VE|Advanced Web Application Firewall, VE|AFM, VE|DNSSEC|Anti-Virus Checks|Base Endpoint Security Checks|Firewall Checks|Network Access|Secure Virtual Keyboard|APM, Web Application|Machine Certificate Checks|Protected Workspace|Remote Desktop|App Tunnel|PSM, VE|VE, Carrier Grade NAT (AFM ONLY)"],"generation":0,"lastUpdateMicros":1743148556605609},"interfaces":["1.1","1.2","1.3","mgmt"],"isIControlRestSupported":true,"icrdPort":8100,"time":1743154389128,"physicalMemory":4096,"platform":"Z100","cpu":"Intel(R) Xeon(R) Gold 6240R CPU @ 2.40GHz","machineId":"28a65e48-45e1-4d9a-b053-ca671eb809ee","address":"100.100.1.5","hostname":"cloudprov303.tlabs.online","version":"17.1.0","product":"BIG-IP","platformMarketingName":"BIG-IP Virtual Edition","edition":"Final","build":"0.0.16","restFrameworkVersion":"17.1.0-0.0.16","managementAddress":"2a00:da9:107:2001::6","mcpDeviceName":"/Common/bigip1","isClustered":false,"isVirtual":true,"hypervisorType":"0","generation":0,"lastUpdateMicros":0,"kind":"shared:resolver:device-groups:deviceinfostate","selfLink":"https://localhost/mgmt/shared/identified-devices/config/device-info"} Fri, 28 Mar 2025 09:33:09 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] list 100.100.1.5 /tm/sys/ready Fri, 28 Mar 2025 09:33:09 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] {"kind":"tm:sys:ready:readystats","selfLink":"https://localhost/mgmt/tm/sys/ready?ver=17.1.0","entries":{"https://localhost/mgmt/tm/sys/ready/0":{"nestedStats":{"entries":{"configReady":{"description":"yes"},"licenseReady":{"description":"yes"},"provisionReady":{"description":"yes"}}}}}} Fri, 28 Mar 2025 09:33:09 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] list 100.100.1.5 /tm/cm/trust-domain/Root Fri, 28 Mar 2025 09:33:09 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] {"kind":"tm:cm:trust-domain:trust-domainstate","name":"Root","fullPath":"Root","generation":1,"selfLink":"https://localhost/mgmt/tm/cm/trust-domain/Root?ver=17.1.0","caCert":"/Common/dtca.crt","caCertReference":{"link":"https://localhost/mgmt/tm/cm/cert/~Common~dtca.crt?ver=17.1.0"},"caCertBundle":"/Common/dtca-bundle.crt","caCertBundleReference":{"link":"https://localhost/mgmt/tm/cm/cert/~Common~dtca-bundle.crt?ver=17.1.0"},"caDevices":["/Common/cloudprov303.tlabs.online"],"caDevicesReference":[{"link":"https://localhost/mgmt/tm/cm/device/~Common~cloudprov303.tlabs.online?ver=17.1.0"}],"caKey":"/Common/dtca.key","caKeyReference":{"link":"https://localhost/mgmt/tm/cm/key/~Common~dtca.key?ver=17.1.0"},"status":"standalone","trustGroup":"/Common/device_trust_group","trustGroupReference":{"link":"https://localhost/mgmt/tm/cm/device-group/~Common~device_trust_group?ver=17.1.0"}} Fri, 28 Mar 2025 09:33:09 GMT - fine: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] create 100.100.1.5 /tm/cm/add-to-trust {"command":"run","name":"Root","caDevice":true,"device":"2a00:da9:107:2001::7","username":"admin","password":"********","deviceName":"cloudprov304.tlabs.online"} Fri, 28 Mar 2025 09:34:27 GMT - finest: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] tryUntil: got error {"code":400} Fri, 28 Mar 2025 09:34:27 GMT - finest: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] typeof err object Fri, 28 Mar 2025 09:34:27 GMT - finer: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] tryUntil error: remoteSender:100.100.1.6, method:POST tries left: 0 Fri, 28 Mar 2025 09:34:27 GMT - finest: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] tryUntil: retryOrReject: numRemaining: 0 , code: 400 , message: remoteSender:100.100.1.6, method:POST Fri, 28 Mar 2025 09:34:27 GMT - finer: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] Unrecoverable error from HTTP request. Not retrying. Fri, 28 Mar 2025 09:34:27 GMT - info: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] Add to trust failed: remoteSender:100.100.1.6, method:POST Fri, 28 Mar 2025 09:34:27 GMT - finest: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] tryUntil: got error {"code":400} Fri, 28 Mar 2025 09:34:27 GMT - finest: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] typeof err object Fri, 28 Mar 2025 09:34:27 GMT - finer: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] tryUntil error: remoteSender:100.100.1.6, method:POST tries left: 72 Fri, 28 Mar 2025 09:34:27 GMT - finest: [f5-declarative-onboarding: restWorker.js | 66557e4b-331f-4899-a6f4-d00a4105ec2c] tryUntil: retryOrReject: numRemaining: 72 , code: 400 , message: remoteSender:100.100.1.6, method:POST

Regards,

Robin.

rdegoix avatar Mar 28 '25 09:03 rdegoix