This is on ESX 4.0.0 Update 1.
When I issue a command such as "vim-cmd /hostsvc/hostsummary | grep inMaintenanceMode" or vimsh –n –e /hostsvc/maintenance_mode_enter” / “vimsh –n –e
/hostsvc/maintenance_mode_exit” all returned the same error message “failed to connect: cannot connect to host localhost: connection refused”.
I've checked the checked the resolv.conf & hosts file.
I'm able to ssh to the server as well as use WinSCP without any problems.
I can connect with the VI Client also without a problem.
Any Ideas?
The problem has been resolved.
The fix:
=====
# vimsh -n -e /hostsvc/hostsummary | grep inMaintenanceMode
Using system libcrypto, version 90809F
inMaintenanceMode = false,
# vim-cmd /hostsvc/hostsummary | grep inMaintenanceMode
inMaintenanceMode = false,
** The reason for the problem is these commands (vim-cmf or vimsh) were unable to communicate due to lack of xinetd process.
# ps -ef|grep -i xinet
root 11527 25978 0 04:30 pts/0 00:00:00 grep -i xinet
# service xinetd restart
#
# service xinetd status
#
# service xinetd start
# service xinetd status
# ls /etc/init.d/xinetd
/etc/init.d/xinetd
/* However NETWORK seems to be normal on this server */
# cat /etc/resolv.conf
nameserver 172.26.6.21
nameserver 172.25.6.51
# ifconfig
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:6053620 errors:0 dropped:0 overruns:0 frame:0
TX packets:6053620 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:2021352939 (1.8 GiB) TX bytes:2021352939 (1.8 GiB)
ppp0 Link encap:Point-to-Point Protocol
inet addr:164.152.87.72 P-t-P:164.152.87.64 Mask:255.255.255.255
UP POINTOPOINT RUNNING NOARP MULTICAST MTU:1500 Metric:1
RX packets:208 errors:2 dropped:0 overruns:0 frame:0
TX packets:205 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:3
RX bytes:18165 (17.7 KiB) TX bytes:22314 (21.7 KiB)
vmnic100200 Link encap:Ethernet HWaddr 00:1B:21:6A:C9:C4
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:1383869 errors:0 dropped:0 overruns:0 frame:0
TX packets:1382815 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:90351869 (86.1 MiB) TX bytes:92106912 (87.8 MiB)
Interrupt:97
vmnic100201 Link encap:Ethernet HWaddr 00:1B:21:6A:C9:C5
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:24239456 errors:0 dropped:0 overruns:0 frame:0
TX packets:23905751 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:2078777971 (1.9 GiB) TX bytes:2226266024 (2.0 GiB)
Interrupt:113
vmnic100600 Link encap:Ethernet HWaddr 00:25:5C:2C:75:30
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:13076343 errors:0 dropped:0 overruns:0 frame:0
TX packets:5226061 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1292112365 (1.2 GiB) TX bytes:1224021381 (1.1 GiB)
Interrupt:129
vmnic100601 Link encap:Ethernet HWaddr 00:25:5C:2C:75:31
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:819469 errors:0 dropped:0 overruns:0 frame:0
TX packets:770101 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:56287946 (53.6 MiB) TX bytes:52499576 (50.0 MiB)
Interrupt:137
vmnic110200 Link encap:Ethernet HWaddr 00:1B:21:6A:CA:44
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:1774795 errors:0 dropped:0 overruns:0 frame:0
TX packets:1681079 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:142435399 (135.8 MiB) TX bytes:135079293 (128.8 MiB)
Interrupt:145
vmnic110201 Link encap:Ethernet HWaddr 00:1B:21:6A:CA:45
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:21079616 errors:0 dropped:0 overruns:0 frame:0
TX packets:20688606 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1859632344 (1.7 GiB) TX bytes:1852761835 (1.7 GiB)
Interrupt:161
vmnic110600 Link encap:Ethernet HWaddr 00:25:5C:2C:75:32
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:39205375 errors:0 dropped:0 overruns:0 frame:0
TX packets:21339531 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:32420064542 (30.1 GiB) TX bytes:3709579479 (3.4 GiB)
Interrupt:177
vmnic110601 Link encap:Ethernet HWaddr 00:25:5C:2C:75:33
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:2922150 errors:0 dropped:0 overruns:0 frame:0
TX packets:2110634 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:398811808 (380.3 MiB) TX bytes:245899848 (234.5 MiB)
Interrupt:185
vswif0 Link encap:Ethernet HWaddr 00:50:56:41:76:EB
inet addr:172.26.6.13 Bcast:172.26.6.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:109779 errors:0 dropped:0 overruns:0 frame:0
TX packets:23715 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:10137604 (9.6 MiB) TX bytes:21716347 (20.7 MiB)
vswif1 Link encap:Ethernet HWaddr 00:50:56:40:30:F7
inet addr:172.26.7.11 Bcast:172.26.7.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:36515 errors:0 dropped:0 overruns:0 frame:0
TX packets:4 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:3702390 (3.5 MiB) TX bytes:168 (168.0 b)
vswif2 Link encap:Ethernet HWaddr 00:50:56:4A:DF:1B
inet addr:172.26.8.11 Bcast:172.26.8.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:20525 errors:0 dropped:0 overruns:0 frame:0
TX packets:6 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:2742990 (2.6 MiB) TX bytes:252 (252.0 b)
vswif3 Link encap:Ethernet HWaddr 00:50:56:4E:1E:24
inet addr:172.26.9.12 Bcast:172.26.9.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:26408 errors:0 dropped:0 overruns:0 frame:0
TX packets:4 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:3095970 (2.9 MiB) TX bytes:168 (168.0 b)
# /opt/ft/bin/ftsmaint lsVnd
Virtual Network Device (VND) Groups
===================================
Group Name Status
--------------------------
vSwitch0 ONLINE
vSwitch1 ONLINE
vSwitch2 ONLINE
vSwitch3 ONLINE
VND Group Members
=================
Member Group Name Status Interface Link State Link Speed
--------------------------------------------------------------------------
vmnic100200 vSwitch2 DUPLEX UP LINK 1000Mb/s-FD
vmnic100201 vSwitch3 DUPLEX UP LINK 1000Mb/s-FD
vmnic100600 vSwitch0 DUPLEX UP LINK 1000Mb/s-FD
vmnic100601 vSwitch1 DUPLEX UP LINK 1000Mb/s-FD
vmnic110200 vSwitch2 DUPLEX UP LINK 1000Mb/s-FD
vmnic110201 vSwitch3 DUPLEX UP LINK 1000Mb/s-FD
vmnic110600 vSwitch0 DUPLEX UP LINK 1000Mb/s-FD
vmnic110601 vSwitch1 DUPLEX UP LINK
1000Mb/s-FD
/* Further review on XINETD startup issue */
Further review shows that the xinetd fails to start since it believed NETWORKING not available.
The xinetd startup uses the /etc/sysconfig/network file contents where there was an incorrect entry "NETWORKINcesv=yes". We corrected this entry and restarted xinetd process.
After this correction, xinetd restart, the system seems to be normal.
# cat /etc/sysconfig/network
NETWORKINcesv=yes
HOSTNAME=server.test.net
GATEWAY=172.26.6.1
IPV6_AUTOCONF=no
NETWORKING_IPV6=no
#
# cat /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=server.test.net
GATEWAY=172.26.6.1
IPV6_AUTOCONF=no
NETWORKING_IPV6=no
# ps -ef|grep -i xinet
root 27977 25978 0 04:41 pts/0 00:00:00 grep -i xinet
# service xinetd start
Starting xinetd: OK ]
# ps -ef|grep -i xinet
root 28119 1 0 04:41 ? 00:00:00 xinetd -stayalive -pidfile /var/run/xinetd.pid
root 28239 25978 0 04:42 pts/0 00:00:00 grep -i xinet
#
# service xinetd restart
Stopping xinetd: OK ]
Starting xinetd: OK ]
# vim-cmd /hostsvc/hostsummary | grep inMaintenanceMode
inMaintenanceMode = false,
# vimsh -n -e /hostsvc/hostsummary | grep inMaintenanceMode
Using system libcrypto, version 90809F
inMaintenanceMode = false,
#
It seems your hostd has crashed. Can you check on the hostd service and the hostd logs to concur the same.
I've checked hosted and it not reporting anything strange or any errors with regard to the commands. Also, so far everything else works.
I can connect with the VI Client also without a problem.
My bad, I might have missed the above point.
just to make sure that your hosts file is having the below requirement.?
Ensure that the /etc/hosts file is using 127.0.0.1 pointing to localhost.localdomain localhost.
Confirmed that hosts file is correct and resolv.conf has relevant entries in.
To isolate hostd and firewall inclusions, can you confirm if you are getting similar errors on commands other than hostsvc. like vim-cmd vmsvc/getallvms?
Also, firewall has been disable.
TThis is a stand alone esx server with no virtual server..
It feels like someone has made done some security hardening With regard to whatever the vim-cmd and vimsh uses.
I Will have to check this tomorrow as I am not currently at work. I have a suspicion that the problem will occur with this as well.
i Am going to be at work very early, so anything you can think of for me to try, please let me know.
i'll be honest and say this is the first time I've come across such and error. (Since ESX 2)
a reboot Aldo did not resolve the problem.
Have a look @http://kb.vmware.com/selfservice/microsites/search.do?language=en_US&cmd=displayKC&externalId=100284...
-SatyS
The problem has been resolved.
The fix:
=====
# vimsh -n -e /hostsvc/hostsummary | grep inMaintenanceMode
Using system libcrypto, version 90809F
inMaintenanceMode = false,
# vim-cmd /hostsvc/hostsummary | grep inMaintenanceMode
inMaintenanceMode = false,
** The reason for the problem is these commands (vim-cmf or vimsh) were unable to communicate due to lack of xinetd process.
# ps -ef|grep -i xinet
root 11527 25978 0 04:30 pts/0 00:00:00 grep -i xinet
# service xinetd restart
#
# service xinetd status
#
# service xinetd start
# service xinetd status
# ls /etc/init.d/xinetd
/etc/init.d/xinetd
/* However NETWORK seems to be normal on this server */
# cat /etc/resolv.conf
nameserver 172.26.6.21
nameserver 172.25.6.51
# ifconfig
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:6053620 errors:0 dropped:0 overruns:0 frame:0
TX packets:6053620 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:2021352939 (1.8 GiB) TX bytes:2021352939 (1.8 GiB)
ppp0 Link encap:Point-to-Point Protocol
inet addr:164.152.87.72 P-t-P:164.152.87.64 Mask:255.255.255.255
UP POINTOPOINT RUNNING NOARP MULTICAST MTU:1500 Metric:1
RX packets:208 errors:2 dropped:0 overruns:0 frame:0
TX packets:205 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:3
RX bytes:18165 (17.7 KiB) TX bytes:22314 (21.7 KiB)
vmnic100200 Link encap:Ethernet HWaddr 00:1B:21:6A:C9:C4
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:1383869 errors:0 dropped:0 overruns:0 frame:0
TX packets:1382815 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:90351869 (86.1 MiB) TX bytes:92106912 (87.8 MiB)
Interrupt:97
vmnic100201 Link encap:Ethernet HWaddr 00:1B:21:6A:C9:C5
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:24239456 errors:0 dropped:0 overruns:0 frame:0
TX packets:23905751 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:2078777971 (1.9 GiB) TX bytes:2226266024 (2.0 GiB)
Interrupt:113
vmnic100600 Link encap:Ethernet HWaddr 00:25:5C:2C:75:30
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:13076343 errors:0 dropped:0 overruns:0 frame:0
TX packets:5226061 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1292112365 (1.2 GiB) TX bytes:1224021381 (1.1 GiB)
Interrupt:129
vmnic100601 Link encap:Ethernet HWaddr 00:25:5C:2C:75:31
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:819469 errors:0 dropped:0 overruns:0 frame:0
TX packets:770101 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:56287946 (53.6 MiB) TX bytes:52499576 (50.0 MiB)
Interrupt:137
vmnic110200 Link encap:Ethernet HWaddr 00:1B:21:6A:CA:44
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:1774795 errors:0 dropped:0 overruns:0 frame:0
TX packets:1681079 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:142435399 (135.8 MiB) TX bytes:135079293 (128.8 MiB)
Interrupt:145
vmnic110201 Link encap:Ethernet HWaddr 00:1B:21:6A:CA:45
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:21079616 errors:0 dropped:0 overruns:0 frame:0
TX packets:20688606 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1859632344 (1.7 GiB) TX bytes:1852761835 (1.7 GiB)
Interrupt:161
vmnic110600 Link encap:Ethernet HWaddr 00:25:5C:2C:75:32
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:39205375 errors:0 dropped:0 overruns:0 frame:0
TX packets:21339531 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:32420064542 (30.1 GiB) TX bytes:3709579479 (3.4 GiB)
Interrupt:177
vmnic110601 Link encap:Ethernet HWaddr 00:25:5C:2C:75:33
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:2922150 errors:0 dropped:0 overruns:0 frame:0
TX packets:2110634 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:398811808 (380.3 MiB) TX bytes:245899848 (234.5 MiB)
Interrupt:185
vswif0 Link encap:Ethernet HWaddr 00:50:56:41:76:EB
inet addr:172.26.6.13 Bcast:172.26.6.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:109779 errors:0 dropped:0 overruns:0 frame:0
TX packets:23715 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:10137604 (9.6 MiB) TX bytes:21716347 (20.7 MiB)
vswif1 Link encap:Ethernet HWaddr 00:50:56:40:30:F7
inet addr:172.26.7.11 Bcast:172.26.7.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:36515 errors:0 dropped:0 overruns:0 frame:0
TX packets:4 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:3702390 (3.5 MiB) TX bytes:168 (168.0 b)
vswif2 Link encap:Ethernet HWaddr 00:50:56:4A:DF:1B
inet addr:172.26.8.11 Bcast:172.26.8.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:20525 errors:0 dropped:0 overruns:0 frame:0
TX packets:6 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:2742990 (2.6 MiB) TX bytes:252 (252.0 b)
vswif3 Link encap:Ethernet HWaddr 00:50:56:4E:1E:24
inet addr:172.26.9.12 Bcast:172.26.9.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:26408 errors:0 dropped:0 overruns:0 frame:0
TX packets:4 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:3095970 (2.9 MiB) TX bytes:168 (168.0 b)
# /opt/ft/bin/ftsmaint lsVnd
Virtual Network Device (VND) Groups
===================================
Group Name Status
--------------------------
vSwitch0 ONLINE
vSwitch1 ONLINE
vSwitch2 ONLINE
vSwitch3 ONLINE
VND Group Members
=================
Member Group Name Status Interface Link State Link Speed
--------------------------------------------------------------------------
vmnic100200 vSwitch2 DUPLEX UP LINK 1000Mb/s-FD
vmnic100201 vSwitch3 DUPLEX UP LINK 1000Mb/s-FD
vmnic100600 vSwitch0 DUPLEX UP LINK 1000Mb/s-FD
vmnic100601 vSwitch1 DUPLEX UP LINK 1000Mb/s-FD
vmnic110200 vSwitch2 DUPLEX UP LINK 1000Mb/s-FD
vmnic110201 vSwitch3 DUPLEX UP LINK 1000Mb/s-FD
vmnic110600 vSwitch0 DUPLEX UP LINK 1000Mb/s-FD
vmnic110601 vSwitch1 DUPLEX UP LINK
1000Mb/s-FD
/* Further review on XINETD startup issue */
Further review shows that the xinetd fails to start since it believed NETWORKING not available.
The xinetd startup uses the /etc/sysconfig/network file contents where there was an incorrect entry "NETWORKINcesv=yes". We corrected this entry and restarted xinetd process.
After this correction, xinetd restart, the system seems to be normal.
# cat /etc/sysconfig/network
NETWORKINcesv=yes
HOSTNAME=server.test.net
GATEWAY=172.26.6.1
IPV6_AUTOCONF=no
NETWORKING_IPV6=no
#
# cat /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=server.test.net
GATEWAY=172.26.6.1
IPV6_AUTOCONF=no
NETWORKING_IPV6=no
# ps -ef|grep -i xinet
root 27977 25978 0 04:41 pts/0 00:00:00 grep -i xinet
# service xinetd start
Starting xinetd: OK ]
# ps -ef|grep -i xinet
root 28119 1 0 04:41 ? 00:00:00 xinetd -stayalive -pidfile /var/run/xinetd.pid
root 28239 25978 0 04:42 pts/0 00:00:00 grep -i xinet
#
# service xinetd restart
Stopping xinetd: OK ]
Starting xinetd: OK ]
# vim-cmd /hostsvc/hostsummary | grep inMaintenanceMode
inMaintenanceMode = false,
# vimsh -n -e /hostsvc/hostsummary | grep inMaintenanceMode
Using system libcrypto, version 90809F
inMaintenanceMode = false,
#