RagsRachamadugu
Contributor
Contributor

Tier-0 BGP session with ToR leaf not established

I'm running NSX-T 2.4.2 and have a tier-0 router that has 2 segments directly connected to it, without any tier-1 router. The t0 router has one uplink that is running on an edge VM that is hosted on ESXi transport node. I configured BGP neighbor, route redistribution etc but I see the BGP session is not established. The neighbor IP is pingable from edge VM. When I login to the edge VM, I see the configuration correctly reflecting what I did on the GUI but there are no BGP sessions showing up. Below is an excerpt from nsxcli running on the edge vm.

NSX CLI running on NSX-T edge

NSX CLI (Edge 2.4.2.0.0.14269554). Press ? for command list or enter: help

hqnsxtedge03> get service router config

ROUTING CONFIGURATION:

======================

{

    "redist": {

        "bgp_enabled": true

    },

    "redistri_rules": [

        {

            "rule": [

                {

                    "to_proto": 1,

                    "name": {

                        "string": "rags-redist-criteria"

                    },

                    "seq_id": 0,

                    "connected_prefix_list": [

                        {

                            "prefix_list_type": 2,

                            "prefix": [

                                {

                                    "action": 1,

                                    "index": 2,

                                    "network": {

                                        "ipv4": "192.168.1.0",

                                        "prefix_length": 24

                                    },

                                    "seq_id": 1

                                },

                                {

                                    "action": 1,

                                    "index": 5,

                                    "network": {

                                        "ipv4": "192.168.10.0",

                                        "prefix_length": 24

                                    },

                                    "seq_id": 2

                                }

                            ],

                            "name": "T0_DOWNLINK",

                            "prefix_list_uuid_name": "2023e5f9-2267-4a4e-96d9-704795786c8d"

                        },

                        {

                            "prefix_list_type": 3,

                            "prefix": [

                                {

                                    "action": 1,

                                    "index": 1025,

                                    "network": {

                                        "ipv4": "172.16.10.0",

                                        "prefix_length": 24

                                    },

                                    "seq_id": 1

                                }

                            ],

                            "name": "T0_UPLINK",

                            "prefix_list_uuid_name": "6931c5fe-c464-4ab6-9fb2-88c9c3c329e3"

                        }

                    ],

                    "flags": 16354,

                    "redist_flags": [

                        "T1_NAT",

                        "T1_DNS_FORWARDER",

                        "STATIC",

                        "T1_CSP",

                        "T1_LB_VIP",

                        "T1_LB_SNAT",

                        "T0_CONNECTED",

                        "T0_NAT",

                        "T1_DOWNLINK",

                        "T1_STATIC"

                    ],

                    "description": {

                        "string": ""

                    }

                }

            ]

        }

    ],

    "bgp_config": {

        "inter_sr_ibgp": true,

        "graceful_restart": true,

        "local_as": 65000,

        "enabled": true,

        "ecmp": true,

        "neighbor": [

            {

                "enable": true,

                "remote_as": 64515,

                "name": {

                    "string": "fac9415b-e515-4501-858f-3c4370ece75a"

                },

                "hold_down_timer": 180,

                "max_hop_limit": 1,

                "keep_alive_timer": 60,

                "src_ip_address": {

                    "ipv4": "172.16.10.100"

                },

                "bgp_neighbor_uuid_name": "00005000-0000-0406-0000-000000000801",

                "ip_address": {

                    "ipv4": "172.16.10.1"

                },

                "enable_bfd": false,

                "address_family": [

                    {

                        "enabled": true,

                        "allow_as_in": false,

                        "type": "IPv4_UNICAST"

                    }

                ]

            }

        ],

        "multipath_relax": true

    },

    "routing_global": {

        "router_id": {

            "ipv4": "172.16.10.100"

        },

        "op_state_up": false,

        "role": 1,

        "forwarding_up_timer": 0

    },

    "prefix_lists": [

        {

            "prefix_list_type": 2,

            "prefix": [

                {

                    "action": 1,

                    "index": 2,

                    "network": {

                        "ipv4": "192.168.1.0",

                        "prefix_length": 24

                    },

                    "seq_id": 1

                },

                {

                    "action": 1,

                    "index": 5,

                    "network": {

                        "ipv4": "192.168.10.0",

                        "prefix_length": 24

                    },

                    "seq_id": 2

                }

            ],

            "name": "T0_DOWNLINK",

            "prefix_list_uuid_name": "2023e5f9-2267-4a4e-96d9-704795786c8d"

        },

        {

            "prefix_list_type": 3,

            "prefix": [

                {

                    "action": 1,

                    "index": 1025,

                    "network": {

                        "ipv4": "172.16.10.0",

                        "prefix_length": 24

                    },

                    "seq_id": 1

                }

            ],

            "name": "T0_UPLINK",

            "prefix_list_uuid_name": "6931c5fe-c464-4ab6-9fb2-88c9c3c329e3"

        }

    ]

}

hqnsxtedge03> get logical-router

Logical Router

UUID                                   VRF    LR-ID  Name                              Type                        Ports

736a80e3-23f6-5a2d-81d6-bbefb2786666   0      0                                        TUNNEL                      3     

277d9a9a-bc1e-4d24-bb16-4ad468064e83   3      2049   SR-t0gw1-adv                      SERVICE_ROUTER_TIER0        6     

51f7b743-1b9e-4033-b595-d9a1941bde3b   4      1028   DR-t0gw1-adv                      DISTRIBUTED_ROUTER_TIER0    5

hqnsxtedge03> vrf 3

hqnsxtedge03(tier0_sr)> get bgp

hqnsxtedge03(tier0_sr)> get bgp neighbor

hqnsxtedge03(tier0_sr)> exit

hqnsxtedge03> get managers

- 10.5.12.25       Connected

hqnsxtedge03> get controllers

Controller IP    Port     SSL         Status       Is Physical Master   Session State  Controller FQDN

   10.5.12.25     1235   enabled     connected             true               up               NA

hqnsxtedge03(tier0_sr)> ping 172.16.10.1

PING 172.16.10.1 (172.16.10.1): 56 data bytes

64 bytes from 172.16.10.1: icmp_seq=0 ttl=255 time=3.803 ms

64 bytes from 172.16.10.1: icmp_seq=1 ttl=255 time=1.569 ms

64 bytes from 172.16.10.1: icmp_seq=2 ttl=255 time=2.051 ms

64 bytes from 172.16.10.1: icmp_seq=3 ttl=255 time=1.459 ms

64 bytes from 172.16.10.1: icmp_seq=4 ttl=255 time=2.249 ms

Any assistance is much appreciated..

0 Kudos
3 Replies
RagsRachamadugu
Contributor
Contributor

For reference the interfaces on edge VM.

NSX CLI on Edge VM hosting tier-0 router

hqnsxtedge03(tier0_sr)> get interfaces

Logical Router

UUID                                   VRF    LR-ID  Name                              Type                      

51f7b743-1b9e-4033-b595-d9a1941bde3b   4      1028   DR-t0gw1-adv                      DISTRIBUTED_ROUTER_TIER0  

Interfaces

    Interface     : 3d76fb96-758a-5531-827b-fe2537e9de64

    Ifuid         : 309

    Mode          : cpu

    Interface     : 77cda89e-582a-4471-a677-4566a35e145f

    Ifuid         : 296

    Name          : benefitswebtierport

    Internal name : downlink-296

    Mode          : lif

    IP/Mask       : 192.168.1.1/24

    MAC           : 02:50:56:56:44:52

    VNI           : 65536

    LS port       : 21f5603a-5019-4f32-8372-5e16cc259197

    Urpf-mode     : STRICT_MODE

    Admin         : up

    Op_state      : up

    MTU           : 1500

    Interface     : d2f7b262-f959-4e7b-808c-9ea6126612b9

    Ifuid         : 300

    Name          : benefitswebtieradvport

    Internal name : downlink-300

    Mode          : lif

    IP/Mask       : 192.168.10.1/24

    MAC           : 02:50:56:56:44:52

    VNI           : 65540

    LS port       : d94c5cf7-1b89-4647-adc3-aaede9260054

    Urpf-mode     : NONE

    Admin         : up

    Op_state      : up

    MTU           : 1500

    Interface     : 666d0b0c-b9ac-5e43-93f7-85a1a9e06f1b

    Ifuid         : 310

    Mode          : blackhole

    Interface     : 7e0e2414-9672-4b1a-8187-c1057a1107a1

    Ifuid         : 304

    Name          : bp-dr-port

    Mode          : lif

    IP/Mask       : 169.254.0.1/25;fe80::50:56ff:fe56:4452/64

    MAC           : 02:50:56:56:44:52

    VNI           : 65538

    LS port       : ea87b8cb-a000-438b-bb2c-2be30bb2e9b0

    Urpf-mode     : PORT_CHECK

    Admin         : up

    Op_state      : up

    MTU           : 1500

Logical Router

UUID                                   VRF    LR-ID  Name                              Type                      

277d9a9a-bc1e-4d24-bb16-4ad468064e83   3      2049   SR-t0gw1-adv                      SERVICE_ROUTER_TIER0      

Interfaces

    Interface     : 2115025f-a7fc-48a9-924d-519daf943bd7

    Ifuid         : 302

    Name          : sr0-internal-routing-port

    Internal name : inter-sr-302

    Mode          : lif

    IP/Mask       : 169.254.0.130/25;fe80::50:56ff:fe56:5200/64

    MAC           : 02:50:56:56:52:00

    VNI           : 65539

    LS port       : 9e95cc45-9e69-4cf6-97b4-06a59acd1bc1

    Urpf-mode     : PORT_CHECK

    Admin         : up

    Op_state      : up

    MTU           : 1500

    Interface     : 5c0219c3-5fc6-4a72-a785-ea2c1108a6ce

    Ifuid         : 312

    Mode          : loopback

    IP/Mask       : 127.0.0.1/8;::1/128

    Interface     : 531c51a6-1222-4a01-9435-f2c0fc7907af

    Ifuid         : 306

    Name          : newext1

    Internal name : uplink-306

    Mode          : lif

    IP/Mask       : 172.16.10.100/24

    MAC           : 00:50:56:ad:ae:51

    LS port       : e83200a7-e231-487d-8d88-4ea195780226

    Urpf-mode     : STRICT_MODE

    Admin         : up

    Op_state      : up

    MTU           : 1500

    Interface     : 5017a4b2-e085-4e00-ab74-f18b4b304dd7

    Ifuid         : 298

    Name          : bp-sr0-port

    Mode          : lif

    IP/Mask       :

    MAC           : 02:50:56:56:53:00

    VNI           : 65538

    LS port       : 4958c8bf-37a8-4411-9195-2eb8b23163f2

    Urpf-mode     : NONE

    Admin         : up

    Op_state      : down

    MTU           : 1500

    Interface     : de5acbc9-ce25-5386-ac06-3bf72522c80f

    Ifuid         : 308

    Mode          : blackhole

    Interface     : 05b5d6f8-7678-5872-9cb5-0348f3dab76a

    Ifuid         : 307

    Mode          : cpu

hqnsxtedge03(tier0_sr)>

0 Kudos
mauricioamorim
VMware Employee
VMware Employee

Global routing seems disabled:

    "routing_global": {

        "router_id": {

            "ipv4": "172.16.10.100"

        },

        "op_state_up": false,

        "role": 1,

        "forwarding_up_timer": 0

"op_state_up" should be true.

I'm just not sure what was done to achieve this. "get bgp neighbor" should show the configured neighbors and it shows nothing.

0 Kudos
RagsRachamadugu
Contributor
Contributor

Thank you for your reply mauricioamorim​. The problem was a missing VLAN on the VTEP network, which was not immediately obvious in terms of it's affect on BGP neighbors, which is happening on a different network. Once I fixed that, things are working as expected and you are right that op_state_up now shows as true!

Thanks Rags

0 Kudos