Hi Enigma,
I have iterated on the above script with the suggestions from LucD, the newer version currently runs 15 vmtools/hardware upgrades asynchronously. I have tested this in my lab 8 times successfully. hopefully it works for you too.
Make sure to put in your directories in parameter blocks for both script files.
vmtoolsHWMain_v7
###########################################
### vmtools & HW upgrade wrapper Script ###
###########################################
### Setup parameters/variables for the script ###
[cmdletbinding()]
param(
### Setup Credentials parameter which will be stored in a secure way
[parameter(ValueFromPipelineByPropertyName=$true,ValueFromPipeline=$true,Mandatory=$false)]
[PSCredential]$UserCredential,
### Create Array variables ###
[object] $VMarray,
[object] $jobs,
### Create String variables ###
[string] $vms,
[string] $VMName,
[string] $form_date,
### Change this to the relevant vCenter you are connecting to ###
[object] $VCs = @('********************'), #put vCenter addresses here
### Directory of where the script and files are located ###
[string] $dir = "**********",
### Job sub script path ###
[string] $Subdir = "*********",
### Initialze the log file location ###
[string] $LogFile = "*********" + (Get-Date).tostring("dd-MM-yyyy") + ".txt",
### Set the maximum number of background jobs that the script will run, The max allowed is 10 ###
[int] $MaxJobs = 15,
### Set Hardware version you want to upgrade to ###
[string] $VMHW = "vmx-19"
)
### clear error stream ###
$Error.Clear()
#$DebugPreference="SilentlyContinue"
### Setup environment for script ###
Import-Module -Name VMware.PowerCLI -ErrorAction SilentlyContinue | Out-Null
Import-Module -Name VMware.VimAutomation.Core -ErrorAction SilentlyContinue | Out-Null
Import-Module -Name PoshRSJob -ErrorAction SilentlyContinue | Out-Null
set-powercliconfiguration -InvalidCertificateAction Ignore -defaultviservermode multiple -Confirm:$false
#set-powercliconfiguration -InvalidCertificateAction Ignore -defaultviservermode Single -Confirm:$false
### Import Array ###
$VMarray = Import-Csv "C:\Scripts\vmtoolsHW\vm_upgrades.csv" ## Change this to the location of your csv file
$VMNames = Import-Csv "C:\Scripts\vmtoolsHW\vm_upgrades.csv" | Select -ExpandProperty "Name" # create array variable without a header
### Script starting time ###
$form_date = Get-Date -Format "HH_mm_MM_dd_yyyy"
### Get VMTools, Hardware & Network status of Array VM's ###
function UpgradeCheck
{
### Setup mandatory parameters to be passed to function ###
param(
[Parameter(Mandatory=$true, Position=0)]
[object] $VMarray
)
Write-Output "$(Get-Date) - Collecting VM's in Array information on $VCs" | Tee-Object $LogFile -Append
$vms = @() # Create the $vms Hashtable
### For Loop - Get VMs details ###
ForEach ($item in $VMarray)
{
$VMName = $item.Name
### Variable getvm is required, for some reason the $vm cannot be used to query the host and the IP-address ###
$getvm = Get-VM $VMName
$vmview = Get-VM $VMName | Get-View
### Get required info from all the VM's ###
$vmnic = Get-NetworkAdapter -VM $VMName
$nicmac = Get-NetworkAdapter -VM $VMName | ForEach-Object {$_.MacAddress}
$nictype = Get-NetworkAdapter -VM $VMName | ForEach-Object {$_.Type}
$nicname = Get-NetworkAdapter -VM $VMName | ForEach-Object {$_.NetworkName}
$VMInfo = "" | Select VMName,NICCount,IPAddress,MacAddress,NICType,NetworkName,GuestRunningOS,PowerState,ToolsVersion,ToolsStatus,ToolsRunningStatus,HWLevel,VMHost # Create all the headers for the columns in the Hashtable
$VMInfo.VMName = $vmview.Name
$VMInfo.NICCount = $vmview.Guest.Net.Count
$VMInfo.IPAddress = [String]$getvm.Guest.IPAddress
$VMInfo.MacAddress = [String]$nicmac
$VMInfo.NICType = [String]$nictype
$VMInfo.NetworkName = [String]$nicname
$VMInfo.GuestRunningOS = $vmview.Guest.GuestFullname
$VMInfo.PowerState = $getvm.PowerState
$VMInfo.ToolsVersion = $vmview.Guest.ToolsVersion
$VMInfo.ToolsStatus = $vmview.Guest.ToolsStatus
$VMInfo.ToolsRunningStatus = $vmview.Guest.ToolsRunningStatus
$VMInfo.HWLevel = $vmview.Config.Version
$VMInfo.VMHost = $getvm.VMHost
$vms += $VMInfo # Add all info gathered on VM to $vms Hashtable
}
### Get time & Date for export ###
$form_date = Get-Date -Format "HH_mm_MM_dd_yyyy"
### Format table and export to csv ###
$dir_new = $dir
try{New-Item -Path $dir_new -ItemType Directory -ErrorAction Ignore -WarningAction Ignore | Out-Null}catch{} # ensure export path is available
$csv = $dir_new + "\VMUpgradeReport-" + $form_date + ".csv" # create file name for exported report
$vms | Export-Csv -Path $csv # export $vms array containing all VM info to $csv path
Write-Output "$(Get-Date) - vmtools, Hardware and Network of VMs Report exported to: $csv" | Tee-Object -FilePath $LogFile -Append
}
############################################### Main Code ###################################################################
### Start Log File ###
Write-Output "$(Get-Date) =============== vmtools & HW upgrade Script =======================" | Tee-Object -FilePath $LogFile -Append
Write-Output $VMarray | Out-File $LogFile -Append
### Define form_date variable ###
$form_date = Get-Date -Format "HH_mm_MM_dd_yyyy"
### ensure no prior vCenter connections exist ###
Disconnect-VIServer -Server * -Confirm:$false -Force | Out-Null
### Get login credentials to vCenter ###
$UserCredential = Get-Credential -message "Enter credentials for VCenter in format domain\username"
#Connect-VIServer -Server $VCs -Credential $UserCredential -AllLinked -WarningAction SilentlyContinue | out-null
Connect-VIServer -Server $VCs -Credential $UserCredential -WarningAction SilentlyContinue | out-null # Lab connection only
Write-Output "$(Get-Date) =============== connecting to $VCs =================================" | Tee-Object -FilePath $LogFile -Append
### create HW list of VM's pre upgrade ###
Write-Output "$(Get-Date) - Pre Upgrade - Getting VM configuration and Status info for $VMarray `n" | Tee-Object -FilePath $LogFile -Append
UpgradeCheck $VMarray
### code to kick off the VMtools & HW job for each vm ###
Write-Output "$(Get-Date) - starting upgrade Jobs now `n" | Tee-Object -FilePath $LogFile -Append
$VMNames | ForEach-Object {Start-RSJob -Name $_ -FilePath $Subdir -ArgumentList @($_, $UserCredential, $VCs, $VMHW) -Throttle $MaxJobs -batch "upgrades_$($_.name)" } | Wait-RSJob -ShowProgress # A Job will be queued up for each vm in the csv file we imported, the number of jobs that will run at once is based on $MaxJobs. A Progress bar will show until all Jobs have finished.
Write-Output "$(Get-Date) - All Jobs completed `n`n" | Tee-Object -FilePath $LogFile -Append
### Add all the Job results to the Log file ###
Get-RSJob |Tee-Object -FilePath $LogFile -Append
sleep 30
###create HW list of VM's post upgrade
Write-Output "$(Get-Date) - Post Upgrade - Getting VM configuration and Status info for $VMarray `n" | Tee-Object -FilePath $LogFile -Append
UpgradeCheck $VMarray
Sleep 5
### remove RSJobs & disconnect-vCenter ###
Get-RSJob | Remove-RSJob -Force | Tee-Object -FilePath $LogFile -Append
Disconnect-VIServer -Server * -Confirm:$false -Force | Out-Null
### End Log File ###
Write-Output "$(Get-Date) =============== End of vmtools & HW upgrade Script =======================" | Tee-Object -FilePath $LogFile -Append
### Clear all variables ###
Remove-Variable * -ErrorAction SilentlyContinue
vmtoolsHWsubv3_5
#######################################
### vmtools & HW upgrade Sub Script ###
#######################################
#######################
### Version changes ###
#######################
# v3.1 - added guestoperationsready check to the Power on and Reboot functions
# v3.2 - starting using debug stream to capture steps, added if loop in each function to exit job if Event error still present or timer = on any of the VM commands, modified the do until loop to use $Error instead
# v3.3 - Added individual time variable for Task error checking within Do until loops in functions, changed error message write to include $VM
# v3.4 - Clean up script, remove debug outputs, increase sleep during reboot command loop,
# v3.5 - Changed operator of command do until loop to -contains and removed * from condition
### define required parameters for Sub script ###
[cmdletbinding()]
param(
[string] $VM,
[PSCredential]$UserCredential,
[object] $VCs,
[string] $VMHW
)
### Global variables ###
[string] $Logfile = "********-"+ $VM +"-" + (Get-Date).tostring("dd-MM-yyyy") + ".txt" # Logfile path
[object] $VMView # VM view info
[int] $timer = 1 # timer for loop monitoring and breaks
#$DebugPreference="Continue"
### Power on VM ###
Function PowerOn-VM
{
### define required parameters for Power On function ###
param(
[Parameter(Mandatory=$true, Position=0)]
[object] $VM,
[Parameter(Mandatory=$true, Position=1)]
[object] $VMView,
[Parameter(Mandatory=$true, Position=2)]
[int] $timer,
### Function individual variables ###
[Parameter(Mandatory=$false)]
[object] $EventInfo,
[Parameter(Mandatory=$false)]
[object] $LoopErrorInfo,
[Parameter(Mandatory=$false)]
[string] $ToolsStatus,
[Parameter(Mandatory=$false)]
[string] $ToolsRunningStatus,
[Parameter(Mandatory=$false)]
[string] $PowerState,
[Parameter(Mandatory=$false)]
[string] $GuestOperations,
[Parameter(Mandatory=$false)]
[string] $PowerOnTime,
[Parameter(Mandatory=$false)]
[string] $PowerOnErrorTime
)
### Function start time - will be used to check task has been Implemented ###
$PowerOnTime = Get-Date -UFormat "%d/%m/%Y %R"
$timer = 1
### Issue Start command to $VM - Do Until loop will ensure the command has run ###
do{
$PowerOnErrorTime = Get-Date -UFormat "%d/%m/%Y %R"
$LoopErrorInfo = $null
write-Output "$(Get-Date) - PowerOn-VM Function Step 1 - Checking Power on task ran against $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
$EventInfo = Get-VIEvent -Entity $VM -Types Info -start $PowerOnTime | Where-Object {$_.FullFormattedMessage -Like "Task: Power On virtual machine"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Types Error -start $PowerOnErrorTime
If(($EventInfo -eq $null) -or ($LoopErrorInfo -ne $null))
{
Start-VM -VM $VM -Confirm:$false -RunAsync | Wait-Task
}
Start-Sleep -s 20
$EventInfo = Get-VIEvent -Entity $VM -Types Info -start $PowerOnTime | Where-Object {$_.FullFormattedMessage -Like "Task: Power On virtual machine"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Types Error -start $PowerOnErrorTime
write-output "$(Get-Date) - $VM - Power on Error: `r`n" | Tee-Object -FilePath $Logfile -Append
write-Output $LoopErrorInfo | Tee-Object -FilePath $Logfile -Append
Write-Output "$(Get-Date) - $VM - Power on task = Created Time : $($EventInfo.CreatedTime) , $($EventInfo.FullFormattedMessage). Power On command attempt = $timer `r`n" | Tee-Object -FilePath $Logfile -Append
$timer = ++$timer # Add 1 to the timer variable
Start-Sleep 2
}until((($EventInfo.FullFormattedMessage -contains "Task: Power On virtual machine") -and ($LoopErrorInfo -eq $null)) -or ($timer -eq 5))
If(($LoopErrorInfo -ne $null) -or ($timer -eq 5))
{
Write-Output "$(Get-Date) - PowerOn-VM Function Step 1.1 - Power on VM command failed on $VM, exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
$StepMapArray += "$(Get-Date) - Power On VM Command run - Status = False" | Out-Null
Write-Error "VMHW Error on $VM = True"
}
Write-Output "$(Get-Date) - PowerOn-VM Function Step 2 - Power on VM command issued on $VM, $VM is starting. `r`n" | Tee-Object -FilePath $Logfile -Append
$timer = 1 # Reset timer variable and set to 0, it will be used as an escape from the loop checking VM is powered on
### Do until loop after Power On command has been issued, to wait until VM has Power Status of PowerOn & Tools Status is appearing, will keep checking VM until VM is powered on or timer gets to 30 ###
do
{
$VMView.UpdateViewData("Summary.Runtime.PowerState") # update specfic VM info - PowerState
$Powerstate = $VMView.Summary.Runtime.PowerState # Set $Powerstate variable to new value from $VMView
$VMView.UpdateViewData("Summary.Guest.ToolsStatus") # update specfic VM info - ToolsStatus
$VMView.UpdateViewData("Summary.Guest.ToolsRunningStatus") # update specfic VM info - ToolsStatus
$ToolsStatus = $VMView.Summary.Guest.ToolsStatus # Set $ToolsStatus variable to new value from $VMView
$ToolsRunningStatus = $VMView.Summary.Guest.ToolsRunningStatus # Set $ToolsRunningStatus variable to new value from $VMView
$VMView.UpdateViewData("Guest.GuestOperationsReady") # update specfic VM info - Guest Operations Ready
$GuestOperations = $VMView.Guest.GuestOperationsReady # Set $Powerstate variable to new value from $VMView
$timer = ++$timer # Add 1 to the timer variable
Write-Output "$(Get-Date) - PowerOn-VM Function Step 2 - $VM is starting, powerstate is $Powerstate and toolsstatus is $ToolsStatus. `r`n" | Tee-Object -FilePath $Logfile -Append
Start-Sleep -s 10 # Sleep loop for 10 seconds
}until(($Powerstate -match "PoweredOn") -and ((($ToolsStatus -match "toolsOk") -and ($ToolsRunningStatus -match "guestToolsRunning") -and ($GuestOperations -match "True")) -or (($ToolsStatus -match "toolsOld") -and ($ToolsRunningStatus -match "guestToolsRunning") -and ($GuestOperations -match "True")) -or ($ToolsStatus -match "toolsNotInstalled")) -or ($timer -eq 61))
### Check Power status after checking loop, If powered ON and Tools return normal to log file, If VM is not powered off exit script ###
if (($Powerstate -match "PoweredOn") -and (($ToolsStatus -match "toolsOk") -or ($ToolsStatus -match "toolsOld") -or ($ToolsStatus -match "toolsNotInstalled"))) # Loop will run until conditions are met.
{
Write-Output "$(Get-Date) - PowerOn-VM Function Step 3 - $VM is started and has ToolsStatus - $ToolsStatus. `r`n" | Tee-Object -FilePath $Logfile -Append
}
### Else Loop - If VM didn't power on show Error message and exit script ###
else
{
Write-Output "$(Get-Date) - PowerOn-VM Function Step 4 - PowerOn Error detected on $VM, exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
### Result variables to $null so function is starting from scratch on next call of function ###
$PowerOnTime = $null; $timer = $null; $EventInfo = $null; $LoopErrorInfo = $null
}
### Shutdown VM ###
function poweroff-VM
{
### define required parameters for Power Off function ##
param(
[Parameter(Mandatory=$true, Position=0)]
[object] $VM,
[Parameter(Mandatory=$true, Position=1)]
[object] $VMView,
[Parameter(Mandatory=$true, Position=2)]
[int] $timer,
### Function individual variables ###
[Parameter(Mandatory=$false)]
[object] $EventInfo,
[Parameter(Mandatory=$false)]
[object] $LoopErrorInfo,
[Parameter(Mandatory=$false)]
[string] $ToolsStatus,
[Parameter(Mandatory=$false)]
[string] $ToolsRunningStatus,
[Parameter(Mandatory=$false)]
[string] $PowerState,
[Parameter(Mandatory=$false)]
[string] $PowerOffTime,
[Parameter(Mandatory=$false)]
[string] $PowerOffErrorTime
)
### Function start time - will be used to check task has been Implemented ###
$PowerOffTime = Get-Date -UFormat "%d/%m/%Y %R"
$timer = 1
### Issue Guest OS Shutdown command to $VM - Do Until loop will ensure the command has run ###
do
{
$PowerOffErrorTime = Get-Date -UFormat "%d/%m/%Y %R"
$LoopErrorInfo = $null
Write-Output "$(Get-Date) - PowerOff-VM Function Step 1 - Checking Power Off task ran against $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
$EventInfo = Get-VIEvent -Entity $VM -Types Info -start $PowerOffTime | Where-Object {$_.FullFormattedMessage -like "*Task: Initiate guest OS shutdown*"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Types Error -start $PowerOffErrorTime
If(($EventInfo-eq $null) -or ($LoopErrorInfo -ne $null))
{
Shutdown-VMGuest -VM $VM -Confirm:$false
}
Start-Sleep -s 20 # Sleep loop for 20 seconds
$EventInfo = Get-VIEvent -Entity $VM -Types Info -start $PowerOffTime | Where-Object {$_.FullFormattedMessage -like "*Task: Initiate guest OS shutdown*"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Types Error -start $PowerOffErrorTime
Write-Output "$(Get-Date) - $VM - Power off task = Created Time : $($EventInfo.CreatedTime) , $($EventInfo.FullFormattedMessage). Power Off command attempt = $timer `r`n" | Tee-Object -FilePath $Logfile -Append
write-output "$(Get-Date) - $VM - Power Off Error: `r`n" | Tee-Object -FilePath $Logfile -Append
write-Output $LoopErrorInfo | Tee-Object -FilePath $Logfile -Append
$timer = ++$timer # Add 1 to the timer variable
Start-Sleep 2
}until((($EventInfo.FullFormattedMessage -contains "Task: Initiate guest OS shutdown") -and ($LoopErrorInfo -eq $null)) -or ($timer -eq 5))
If(($LoopErrorInfo -ne $null) -or ($timer -eq 5))
{
Write-Output "$(Get-Date) - PowerOff-VM Function Step 1.1 - Power off VM command failed on $VM, exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
Write-Output "$(Get-Date) - PowerOff-VM Function Step 2 - Shutdown command issued to $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
$timer = 0 # Setup timer variable and set to 0, it will be used as an escape from the loop checking VM is powered down
### 5 minute loop after shutdown Guest OS command has been issued, will keep checking VM until VM is powered off or timer gets to 30 ###
do
{
$VMView.UpdateViewData("Summary.Runtime.PowerState") # update specfic VM info - PowerState
$Powerstate = $VMView.Summary.Runtime.PowerState # Set $Powerstate variable to new value from $VMView
$VMView.UpdateViewData("Summary.Guest.ToolsRunningStatus") # update specfic VM info - ToolsStatus
$ToolsRunningStatus = $VMView.Summary.Guest.ToolsRunningStatus # Set $ToolsStatus variable to new value from $VMView
$timer = ++$timer # Add 1 to the timer variable
Write-Output "$(Get-Date) - PowerOff-VM Function Step 3 - Loop checking $VM is stopping: powerstate = $Powerstate , toolsRunningStatus = $ToolsRunningStatus. `r`n" | Tee-Object -FilePath $Logfile -Append
Start-Sleep -s 10 # Sleep loop for 10 seconds
}until(($Powerstate -match "PoweredOff") -or ($timer -eq 60)) # Loop will run until either condition is met.
### Check Power status after checking loop, If powered off return normal to log file, If VM is not powered off exit script ###
if (($Powerstate -match "PoweredOff") -and (($ToolsRunningStatus -match "toolsNotRunning") -or ($ToolsStatus -match "toolsNotInstalled")))
{
Write-Output "$(Get-Date) - PowerOff-VM Function Step 4 - $VM is powered-off. `r`n" | Tee-Object -FilePath $Logfile -Append
}
### Else Loop - If VM didn't power down show Error message and exit script ###
else
{
Write-Output "$(Get-Date) - PowerOff-VM Function Step 5 - PowerOff Error detected on $VM. `r`n"| Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
### Result variables to $null so function is starting from scratch on next call of function ###
$PowerOffTime = $null; $timer = $null; $EventInfo = $null; $LoopErrorInfo = $null
}
function Reboot-VM
{
### define required parameters for Reboot VM function ###
param(
[Parameter(Mandatory=$true, Position=0)]
[object] $VM,
[Parameter(Mandatory=$true, Position=1)]
[object] $VMView,
[Parameter(Mandatory=$true, Position=2)]
[int] $timer,
### Function individual variables ###
[Parameter(Mandatory=$false)]
[object] $EventInfo,
[Parameter(Mandatory=$false)]
[object] $LoopErrorInfo,
[Parameter(Mandatory=$false)]
[string] $ToolsStatus,
[Parameter(Mandatory=$false)]
[string] $ToolsRunningStatus,
[Parameter(Mandatory=$false)]
[string] $PowerState,
[Parameter(Mandatory=$false)]
[string] $GuestOperations,
[Parameter(Mandatory=$false)]
[string] $RebootTime,
[Parameter(Mandatory=$false)]
[string] $RebootErrorTime
)
$Powerstate = $VMView.Summary.Runtime.PowerState # Set $Powerstate variable to new value from $VMView
### Function start time - will be used to check task has been Implemented ###
$RebootTime = Get-Date -UFormat "%d/%m/%Y %R"
$timer = 1
### Initial If loop to ensure VM is in PoweredOn state before anything is done on VM ###
If($Powerstate -ne "PoweredOn")
{
Write-Output "$(Get-Date) - Step 1.0 - $VM has power state - $Powerstate `r`n Upgrade of VMTools and Hardware will not be attempted. `r`n Exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
### Else Loop - If VM is PoweredOn start reboot ###
else
{
Write-Output "$(Get-Date) - Reboot-VM Function Step 1 - Issuing a Reboot Guest OS command to $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
### Issue restart VM guest OS command to $VM - Do Until loop will ensure the command has run ###
do
{
$RebootErrorTime = Get-Date -UFormat "%d/%m/%Y %R"
$LoopErrorInfo = $null
Write-Output "$(Get-Date) - Reboot-VM Function Step 2 - Checking Reboot Guest OS task ran against $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
$EventInfo = Get-VIEvent -Entity $VM -start $RebootTime | Where-Object {$_.FullFormattedMessage -like "*Task: Initiate guest OS reboot*"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Types Error -start $RebootErrorTime
If(($EventInfo -eq $null) -or ($LoopErrorInfo -ne $null))
{
Get-VM $VM | Restart-VMGuest -Confirm:$false
}
Start-Sleep -s 45 # Sleep loop for 45 seconds
$EventInfo = Get-VIEvent -Entity $VM -start $RebootTime | Where-Object {$_.FullFormattedMessage -contains "Task: Initiate guest OS reboot"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Types Error -start $RebootErrorTime
write-output "$(Get-Date) - $VM - Reboot Error: `r`n" | Tee-Object -FilePath $Logfile -Append
write-Output $LoopErrorInfo | Tee-Object -FilePath $Logfile -Append
Write-Output "$(Get-Date) - $VM - Reboot task = Created Time : $($EventInfo.CreatedTime) , $($EventInfo.FullFormattedMessage). Reboot command attempt = $timer `r`n" | Tee-Object -FilePath $Logfile -Append
$timer = ++$timer # Add 1 to the timer variable
Start-Sleep 2
}until((($EventInfo.FullFormattedMessage -contains "Task: Initiate guest OS reboot") -and ($LoopErrorInfo -eq $null)) -or ($timer -eq 5))
If(($LoopErrorInfo -ne $null) -or ($timer -eq 5))
{
Write-Output "$(Get-Date) - Reboot-VM Function Step 2.1 - Reboot VM command failed on $VM, exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
Start-Sleep -s 30 # pause the script for 5 seconds to allow time for the reboot guest OS command to commence
$timer = 0 # Reset timer variable to 0, it will be used as an escape from the loop checking VM Rebooted
### 20 minute loop after Reboot Guest OS command has been issued, will keep checking VM until VM is powered on or timer gets to 120 ###
do
{
Start-Sleep -s 10 # Sleep loop for 10 seconds
$VMView.UpdateViewData("Summary.Runtime.PowerState") # update specfic VM info - PowerState
$Powerstate = $VMView.Summary.Runtime.PowerState # Set $Powerstate variable to new value from $VMView
$VMView.UpdateViewData("Summary.Guest.ToolsStatus") # update specfic VM info - ToolsStatus
$ToolsStatus = $VMView.Summary.Guest.ToolsStatus # Set $ToolsStatus variable to new value from $VMView
$VMView.UpdateViewData("Summary.Guest.ToolsRunningStatus") # update specfic VM info - ToolsStatus
$ToolsRunningStatus = $VMView.Summary.Guest.ToolsRunningStatus # Set $ToolsRunningStatus variable to new value from $VMView
$VMView.UpdateViewData("Guest.GuestOperationsReady") # update specfic VM info - Guest Operations Ready
$GuestOperations = $VMView.Guest.GuestOperationsReady # Set $Powerstate variable to new value from $VMView
$timer = ++$timer # Add 1 to the timer variable
Write-Output "$(Get-Date) - Reboot-VM Function Step 3 - Loop checking $VM is rebooting: powerstate = $Powerstate , toolsStatus = $ToolsStatus. `r`n" | Tee-Object -FilePath $Logfile -Append
}until((($ToolsStatus -match "toolsOk") -and ($ToolsRunningStatus -match "guestToolsRunning") -and ($GuestOperations -match "True")) -or (($ToolsStatus -match "toolsOld") -and ($ToolsRunningStatus -match "guestToolsRunning") -and ($GuestOperations -match "True")) -or (($ToolsStatus -match "toolsnotinstalled") -and ($Powerstate -match "PoweredOn"))-or ($timer -eq 121)) # Loop will run until a condition is met.
### Check Power status after check loop, If powered ON and Tools return normal to log file, If VM is not rebooted exit script ###
if (($Powerstate -match "PoweredOn") -and (($ToolsStatus -match "toolsOk") -or ($ToolsStatus -match "toolsOld") -or ($ToolsStatus -match "toolsNotInstalled")))
{
Write-Output "$(Get-Date) - Reboot-VM Function Step 4 - $VM is started and has ToolsStatus $ToolsStatus. `r`n" | Tee-Object -FilePath $Logfile -Append
$StepMapArray += "$(Get-Date) - Reboot command successful - Status = True" | Out-Null
}
### Else Loop - If VM didn't Reboot show Error message and exit script ###
else
{
Write-Output "$(Get-Date) - Reboot-VM Function Step 5 - PowerOn Error detected on $VM, exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
}
Start-Sleep -s 20
### Result variables to $null so function is starting from scratch on next call of function ###
$RebootTime = $null; $timer = $null; $EventInfo = $null; $LoopErrorInfo = $null
}
### Take a Snapshot ###
function Make-Snapshot
{
### define required parameters for Make-Snapshot function ###
param(
[Parameter(Mandatory=$true, Position=0)]
[object] $VM,
[Parameter(Mandatory=$true, Position=1)]
[int] $timer,
### function specific parameters ###
[Parameter(Mandatory=$false)]
[string] $SnapTime,
[Parameter(Mandatory=$false)]
[string] $SnapErrorTime
)
### Function Variable values ###
$SnapName = "VMTools&HWPreUpgradeSnap"
$SnapTime = Get-Date -UFormat "%d/%m/%Y %R"
$timer = 1
### Create Snapshot on $VM with below -Name and -Description ###
Write-Output "$(Get-Date) - Make-Snapshot Function Step 1 - Taking Snapshot on $VM pre upgrade work. `r`n" | Tee-Object -FilePath $Logfile -Append
### perform Hardware version upgrade ###
do
{
$SnapErrorTime = Get-Date -UFormat "%d/%m/%Y %R"
$LoopErrorInfo = $null
Write-Output "$(Get-Date) - Make-Snapshot Function Step 2 - Checking Snapshot command runs on $VM. SnapTime = $SnapTime `r`n" | Tee-Object -FilePath $Logfile -Append
$EventInfo = Get-VIEvent -Entity $VM -Type Info -start $SnapTime | Where-Object {$_.FullFormattedMessage -Like "*Task: Create virtual machine snapshot*"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Types Error -start $SnapErrorTime
If(($EventInfo -eq $null) -or ($LoopErrorInfo -ne $null))
{
$SnapshotVM = New-Snapshot -VM $VM -Name $SnapName -Description "Snapshot taken before Tools and Hardware was updated" -Memory:$False -Quiesce:$False -Confirm:$false -RunAsync
}
Start-Sleep -s 20 # Sleep loop for 20 seconds
$EventInfo = Get-VIEvent -Entity $VM -Type Info -start $SnapTime | Where-Object {$_.FullFormattedMessage -Like "*Task: Create virtual machine snapshot*"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Types Error -start $SnapErrorTime
write-output "$(Get-Date) - $VM - Snap Error: `r`n" | Tee-Object -FilePath $Logfile -Append
write-Output $LoopErrorInfo | Tee-Object -FilePath $Logfile -Append
Write-Output "$(Get-Date) - $VM - Snap task = Created Time : $($EventInfo.CreatedTime) , $($EventInfo.FullFormattedMessage). Snapshot command attempt = $timer `r`n" | Tee-Object -FilePath $Logfile -Append
$timer = ++$timer # Add 1 to the timer variable
Start-Sleep 2
}until((($EventInfo.FullFormattedMessage -contains "Task: Create virtual machine snapshot") -and ($LoopErrorInfo -eq $null)) -or ($timer -eq 5))
If(($LoopErrorInfo -ne $null) -or ($timer -eq 5))
{
Write-Output "$(Get-Date) - Snap Function Step 2.1 - Snapshot of VM command failed on $VM, exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
Write-Output " $(Get-Date) - Make-Snapshot Function Step 3 - Snapshot command issued on $VM"| Tee-Object -FilePath $Logfile -Append
Start-Sleep -s 30
### Ensure Snapshot is present on VM before proceeding, exit script if snap not taken
$SnapCheck = Get-Snapshot -VM $VM -Name $SnapName
if($Snapcheck.Name -ne $SnapName)
{
Write-Output "$(Get-Date) - Make-Snapshot Function Step 4 - There was a problem taking a Snapshot on $VM, exiting script.`r`n" | Tee-Object -FilePath $Logfile -Append
PowerOn-VM $VM
Write-Error "VMHW Error on $VM = True"
}
Write-Output " $(Get-Date) - Make-Snapshot Function Step 3 - Snapshot successful on $VM"| Tee-Object -FilePath $Logfile -Append
### Result variables to $null so function is starting from scratch on next call of function ###
$SnapTime = $null; $timer = $null; $EventInfo = $null; $LoopErrorInfo = $null
}
### upgrade VMTools of VM ###
function Upgrade-vmtools-version
{
### define required parameters for Upgrade-vmtools-version function ###
param(
[Parameter(Mandatory=$true, Position=0)]
[object] $VM,
[Parameter(Mandatory=$true, Position=1)]
[object] $VMView,
[Parameter(Mandatory=$true, Position=2)]
[int] $timer,
### Function individual variables ###
[Parameter(Mandatory=$false)]
[object] $EventInfo,
[Parameter(Mandatory=$false)]
[object] $LoopErrorInfo,
[Parameter(Mandatory=$false)]
[string] $ToolsStatus,
[Parameter(Mandatory=$false)]
[string] $ToolsRunningStatus,
[Parameter(Mandatory=$false)]
[string] $PowerState,
[Parameter(Mandatory=$false)]
[string] $family,
[Parameter(Mandatory=$false)]
[string] $ToolsTime,
[Parameter(Mandatory=$false)]
[string] $ToolsErrorTime
)
### Function variable values ###
$ToolsTime = Get-Date -UFormat "%d/%m/%Y %R"
$timer = 1
$family = $VMView.Guest.GuestFamily # Set $family variable to the Guest OS Family of $VM from the $VMView Hashtable
$VMView.UpdateViewData("Summary.Guest.ToolsStatus") # update specfic VM info - ToolsStatus
$ToolsStatus = $VMView.Summary.Guest.ToolsStatus # Set $VMToolsStatus variable to new value from $VMView
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 1 - Starting VMTools Check/Upgrade on $VM , VMTools Status - $ToolsStatus & Guest OS - $family. `r`n" | Tee-Object -FilePath $Logfile -Append
### Inital If loop will check VMTools status, if the Tools are up to date it will skip the upgrade. ###
if($ToolsStatus -eq "toolsOK")
{
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 2 - The VM tools up to date - $ToolsStatus on $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
}
### ElseIF Loop - Will perform a VMTools upgrade on $VM ###
elseif(($family -eq "windowsGuest") -and ($ToolsStatus -ne "toolsNotInstalled") -and ($ToolsStatus -ne "toolsNotRunning"))
{
### Perform VMTools upgrade with a surpressed reboot ###
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 3 - The VM tools are $ToolsStatus on $VM. Starting update/install now, This will take at few minutes.`r`n" | Tee-Object -FilePath $Logfile -Append
Start-Sleep 1 # Pause for 1 second
### Issue VMTools Upgrade to $VM - Do Until loop will ensure the command has run ###
do
{
$ToolsErrorTime = Get-Date -UFormat "%d/%m/%Y %R"
$LoopErrorInfo = $null
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 4 - Checking VMTools upgrade command runs $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
$EventInfo = Get-VIEvent -Entity $VM -Type Info -start $ToolsTime | Where-Object {$_.FullFormattedMessage -Like "*Task: Initiated VMware Tools install or upgrade*"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Type Error -start $ToolsErrorTime
If(($EventInfo -eq $null) -or ($LoopErrorInfo -ne $null))
{
$UpdateToolsVM = Get-VMGuest $VM | Update-Tools -NoReboot -RunAsync | Wait-Task
}
Start-Sleep -s 20 # Sleep loop for 10 seconds
$EventInfo = Get-VIEvent -Entity $VM -Type Info -start $ToolsTime | Where-Object {$_.FullFormattedMessage -Like "Task: Initiated VMware Tools install or upgrade"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Type Error -start $ToolsErrorTime
write-Output "$(Get-Date) - $VM - Tools ugrade Error: `r`n" | Tee-Object -FilePath $Logfile -Append
write-Output $LoopErrorInfo | Tee-Object -FilePath $Logfile -Append
Write-Output "$(Get-Date) - $VM - Tools Upgrade task = Created Time : $($EventInfo.CreatedTime) , $($EventInfo.FullFormattedMessage). VMtools upgrade attempt = $timer `r`n" | Tee-Object -FilePath $Logfile -Append
$timer = ++$timer # Add 1 to the timer variable
Start-Sleep 2
}until((($EventInfo.FullFormattedMessage -contains "Task: Initiated VMware Tools install or upgrade") -and ($LoopErrorInfo -eq $null)) -or ($timer -eq 5))
If(($LoopErrorInfo -ne $null) -or ($timer -eq 5))
{
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 4.1 - attempt to upgrade vmtools failed on $VM, exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
$timer = 0 # reset timer for upgrade status loop
### Loop to check Tools upgrade status ###
do
{
Start-Sleep 10 # Sleep loop for 10 seconds
$VMView.UpdateViewData("Summary.Guest.ToolsStatus") # update specfic VM info - ToolsStatus
$ToolsStatus = $VMView.Summary.Guest.ToolsStatus # Set $VMToolsStatus variable to new value from $VMView
$timer = ++$timer # Add 1 to the timer variable
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 5 - Checking VMTools status on $VM now - $ToolsStatus. `r`n" | Tee-Object -FilePath $Logfile -Append
}until(($ToolsStatus -eq "toolsOK") -or ($timer -eq 61)) # Loop will run until a condition is met.
### If Loop - If Tools upgrade was successful and $VMToolsStatus is now toolsOK reboot the $VM ###
If($ToolsStatus -eq "toolsOK")
{
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 6 - Issuing reboot Guest OS command to $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
Reboot-VM $VM $VMView $timer # Call Reboot-VM function and pass $VM & $VMView parameters
}
### Else loop - If the $VMToolsStatus didn't come back as toolsOK display error and stop script ###
else
{
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 7 - There was a problem while trying to run the vm tools upgrade on $VM. `r`n Tools status - $VMToolsStatus `r`n Guest OS - $family `r`n no upgrade will be performed. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
}
### Else loop - If the $VMToolsStatus is ToolsNotrunning display error and stop script ###
else
{
Write-Output "$(Get-Date) - Upgrade-vmtools-version Function Step 8 - There was a problem while trying to run the vm tools upgrade on $VM. `r`n Tools status - $ToolsStatus `r`n Guest OS - $family `r`n no upgrade will be performed. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
### Result variables to $null so function is starting from scratch on next call of function ###
$ToolsTime = $null; $timer = $null; $EventInfo = $null; $LoopErrorInfo = $null
}
### Check & upgrade Hardware version of VM ###
function Upgrade-hardware-version
{
### define required parameters for Upgrade-vmtools-version function ###
param(
[Parameter(Mandatory=$true, Position=0)]
[object] $VM,
[Parameter(Mandatory=$true, Position=1)]
[object] $VMView,
[Parameter(Mandatory=$true, Position=2)]
[int] $timer,
[Parameter(Mandatory=$true, Position=3)]
[string] $VMHW,
### Function individual variables ###
[Parameter(Mandatory=$false)]
[object] $EventInfo,
[Parameter(Mandatory=$false)]
[object] $LoopErrorInfo,
[Parameter(Mandatory=$false)]
[string] $ToolsStatus,
[Parameter(Mandatory=$false)]
[string] $ToolsRunningStatus,
[Parameter(Mandatory=$false)]
[string] $PowerState,
[Parameter(Mandatory=$false)]
[string] $HWTime,
[Parameter(Mandatory=$false)]
[string] $HWErrorTime
)
### Function variable values set ###
$HWTime = Get-Date -UFormat "%d/%m/%Y %R"
$VMHardware = $VMView.Config.Version # Set $VMHardware to the current Hardware version of the VM
$VMView.UpdateViewData("Summary.Guest.ToolsStatus") # update specfic VM info - ToolsStatus
$ToolsStatus = $VMView.Summary.Guest.ToolsStatus # Set $VMToolsStatus variable to new value from $VMView
write-output "$(Get-Date) - $VM - HW ugrade function starting, Tools status = $ToolsStatus & Hardware version = $VMHardware `r`n" | Tee-Object -FilePath $Logfile -Append
sleep 1
### Inital If loop will check VMTools status & Hardware version, if the Tools are old or Hardware version is up to date it will skip the upgrade. ###
if(($VMHardware -eq "vmx-19") -or ($ToolsStatus -eq "toolsOld"))
{
Write-Output "$(Get-Date) - Upgrade-hardware-version Function Step 1 - The VMTools version is $ToolsStatus on $VM & The hardware version is $VMHardware on $VM. an Hardware upgrade will not be attempted `r`n" | Tee-Object -FilePath $Logfile -Append
}
### ElseIF Loop - Will perform a Hardware upgrade on $VM ###
elseif(($ToolsStatus -eq "toolsOK") -and ($VMHardware -ne "vmx-19"))
{
PowerOff-VM $VM $VMView $timer
$timer = 1
$VMView.UpdateViewData("Summary.Runtime.PowerState") # update specfic VM info - PowerState
$Powerstate = $VMView.Summary.Runtime.PowerState # Set $Powerstate variable to new value from $VMView
### If loop - If VM is powered down perform Hardware upgrade and power on VM ###
if($Powerstate -eq "PoweredOff")
{
Write-Output "$(Get-Date) - Upgrade-hardware-version Function Step 2 - Starting upgrade of hardware version on $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
### perform Hardware version upgrade ###
do
{
$HWErrorTime = Get-Date -UFormat "%d/%m/%Y %R"
$LoopErrorInfo = $null
Write-Output "$(Get-Date) - Upgrade-hardware-version Function Step 3 - Checking HW upgrade command runs $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
$EventInfo = Get-VIEvent -Entity $VM -Type Info -start $HWTime | Where-Object {$_.FullFormattedMessage -Like "*Task: Upgrade VM compatibility*"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Type Error -start $HWErrorTime
If(($EventInfo -eq $null) -or ($LoopErrorInfo -ne $null))
{
Get-View ($VMView.UpgradeVM_Task($VMHW))
}
Start-Sleep -s 20 # Sleep loop for 20 seconds
$EventInfo = Get-VIEvent -Entity $VM -Type Info -start $HWTime | Where-Object {$_.FullFormattedMessage -Like "*Task: Upgrade VM compatibility*"}
$LoopErrorInfo = Get-VIEvent -Entity $VM -Type Error -start $HWErrorTime
write-output "$(Get-Date) - $VM - HW ugrade Error: `r`n" | Tee-Object -FilePath $Logfile -Append
write-Output $LoopErrorInfo | Tee-Object -FilePath $Logfile -Append
Write-Output "$(Get-Date) - $VM - HW Upgrade task = Created Time : $($EventInfo.CreatedTime) , $($EventInfo.FullFormattedMessage). HW upgrade command attempt = $timer `r`n" | Tee-Object -FilePath $Logfile -Append
$timer = ++$timer # Add 1 to the timer variable
Start-Sleep 2
}until((($EventInfo.FullFormattedMessage -contains "Task: Upgrade VM compatibility") -and ($LoopErrorInfo -eq $null)) -or ($timer -eq 5))
If(($LoopErrorInfo -ne $null) -or ($timer -eq 5))
{
Write-Output "$(Get-Date) - upgrade-hardware-function Step 3.1 - Attempt to issue upgrade command failed on $VM, exiting Job. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
sleep 15 # Sleep for 15 seconds to allow the Hardware upgrade time to complete
$VMView.UpdateViewData("Config.Version") # update specfic VM info - Version
$VMHardware = $VMView.Config.Version # Set $VMHardware to the current Hardware version of the VM
PowerOn-VM $VM $VMView $timer # Run PowerOn-VM function
Write-Output "$(Get-Date) - Upgrade-hardware-version Function Step 4 - $VM Hardware has been upgraded to $VMHardware. `r`n" | Tee-Object -FilePath $Logfile -Append
}
### Else loop - If VM doesn't power down display error and stop script ###
else
{
Write-Output "$(Get-Date) - Upgrade-hardware-version Function Step 5 - There is something wrong with the hardware level or the tools of $VM. Skipping $VM. `r`n" | Tee-Object -FilePath $Logfile -Append
Write-Error "VMHW Error on $VM = True"
}
}
Write-Error "VMHW upgrades completed on $VM = True"
### Result variables to $null so function is starting from scratch on next call of function ###
$HWTime = $null; $timer = $null; $EventInfo = $null; $LoopErrorInfo = $null
}
############################################### Main Code ###################################################################
### Start Log File ###
Write-Output "$(Get-Date) =============== vmtools & HW upgrade starting on $VM ======================= `r`n" | Tee-Object -FilePath $Logfile -Append
### Random wait to unsequence jobs ###
$RandomSleep = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 | Get-Random
Write-Output "$(Get-Date) Random Sleep value = $RandomSleep `r`n" | Tee-Object -FilePath $Logfile -Append
Start-Sleep -s $RandomSleep
### Connecting to vCenters ###
Write-Output "$(Get-Date) ===== connecting to all listed vCenters - $VCs ===== `r`n" | Tee-Object $Logfile -Append
Connect-VIServer -Server $VCs -Credential $UserCredential -WarningAction SilentlyContinue #| Tee-Object $Logfile -Append # Login to all vCenters listed in the $VCs array
### Get initial view of $VM ###
$VMView = get-view -ViewType VirtualMachine -filter @{"Name"=$VM} # Get View of VM which contains all Status, parameters and configuration settings. Set the output to the $VMView Hashtable
### Do Until Loop - run each step of the upgrade process, break out of process if an error occurs ###
Do
{
Reboot-VM $VM $VMView $timer # Run Reboot-VM function - perform a Reboot guest OS
Poweroff-VM $VM $VMView $timer # Run Poweroff-VM function - perform a guest OS shutdown
Make-Snapshot $VM $timer # Run Make-Snapshot function - Create snapshot on $VM
PowerOn-VM $VM $VMView $timer # Run PowerOn-VM function - Post snapshot power on of VM pre upgrades
Upgrade-vmtools-version $VM $VMView $timer # Run Upgrade-VMtools-version function - this will check the VMTools version on the VM and upgrade if needed
Upgrade-hardware-version $VM $VMView $timer $VMHW # Run Upgrade-hardware-version function - this will check the Hardware version on the VM and upgrade if needed
}Until (( $Error -like "*VMHW Error on $VM = True*") -or ($Error -like "*VMHW upgrades completed on $VM = True*")) #Loop will exit if $Error contains either constraint
### Write all Job errors to VM Log file ###
Write-Output "$(Get-Date) ===== $VM Job Errors ===== `r`n" | Tee-Object -FilePath $Logfile -Append
$Error | Tee-Object -FilePath $Logfile -Append
### End Log File ###
Write-Output "$(Get-Date) ===== vmtools & HW upgrade Completed on $VM ===== `r`n" | Tee-Object -FilePath $Logfile -Append
