210 lines
9.2 KiB
PowerShell
210 lines
9.2 KiB
PowerShell
$ProfileName = "temp-prod"
|
|
|
|
Get-CachedInstances -Profile $ProfileName -IncludeDR -App
|
|
|
|
|
|
$stoppedInstancesOverflowApp = Get-CachedInstances -Profile $ProfileName -IncludeDR -App | Where-Object { ($_.Tags."alk:overflow" -eq $true) -and ($_.CaptureState.Value -eq 'Stopped') }
|
|
$runningInstancesOverflowApp = Get-CachedInstances -Profile $ProfileName -IncludeDR -App | Where-Object { ($_.Tags."alk:overflow" -eq $true) -and ($_.CaptureState.Value -eq 'Running') }
|
|
$stoppedInstancesNotOverflowApp = Get-CachedInstances -Profile $ProfileName -IncludeDR -App | Where-Object { ($_.Tags."alk:overflow" -ne $true) -and ($_.CaptureState.Value -eq 'Stopped') }
|
|
$runningInstancesNotOverflowApp = Get-CachedInstances -Profile $ProfileName -IncludeDR -App | Where-Object { ($_.Tags."alk:overflow" -ne $true) -and ($_.CaptureState.Value -eq 'Running') }
|
|
|
|
if (($stoppedInstancesOverflowApp.Count + $stoppedInstancesNotOverflowApp.Count + $runningInstancesOverflowApp.Count + $runningInstancesNotOverflowApp.Count) -ne ((Get-CachedInstances -Profile temp-prod -IncludeDR -App).Count)) {
|
|
throw "the server counts don't match"
|
|
}
|
|
|
|
$runningInstancesNotOverflowApp = $runningInstancesNotOverflowApp.Where({ $_.Hostname -notmatch 'view' })
|
|
|
|
$stoppedOverflowProcessed = Get-Content -Path Z:\stuff\stoppedOverflow.txt | ConvertFrom-Json
|
|
$stoppedNotOverflowProcessed = Get-Content -Path Z:\stuff\stoppedNotOverflow.txt | ConvertFrom-Json
|
|
$runningInstancesNotOverflowProcessed = Get-Content -Path Z:\stuff\runningNotOverflow.txt | ConvertFrom-Json
|
|
$runningInstancesOverflowProcessed = Get-Content -Path Z:\stuff\runningOverflow.txt | ConvertFrom-Json
|
|
|
|
$processed = $stoppedOverflowProcessed
|
|
$processed += $stoppedNotOverflowProcessed
|
|
$processed += $runningInstancesNotOverflowProcessed
|
|
$processed += $runningInstancesOverflowProcessed
|
|
|
|
$stoppedInstancesOverflowApp = $stoppedInstancesOverflowApp | Where-Object { $_.Hostname -notin $processed}
|
|
$runningInstancesOverflowApp = $runningInstancesOverflowApp | Where-Object { $_.Hostname -notin $processed }
|
|
$stoppedInstancesNotOverflowApp = $stoppedInstancesNotOverflowApp | Where-Object { $_.Hostname -notin $processed }
|
|
$runningInstancesNotOverflowApp = $runningInstancesNotOverflowApp | Where-Object { $_.Hostname -notin $processed }
|
|
|
|
|
|
return
|
|
|
|
throw 'no'
|
|
|
|
Start-Transcript -Path Z:\stuff\transcript.txt -Append
|
|
|
|
|
|
# $stoppedInstancesOverflowApp
|
|
# $stoppedInstancesNotOverflowApp
|
|
# $runningInstancesOverflowApp
|
|
# $runningInstancesNotOverflowApp
|
|
|
|
# process for offline servers:
|
|
# start the server
|
|
# Sleep one minute
|
|
# Check to see if it is online by Get-Services
|
|
# If it is still not online, sleep again till we get services
|
|
# Now that it is online, report the nag service name, radium service name, hostname, "not overflow"
|
|
# Turn the server back off again
|
|
|
|
Write-Host "`n`tStarting overflow app servers`n"
|
|
$stoppedInstancesOverflowServices = @()
|
|
foreach ($instance in $stoppedInstancesOverflowApp) {
|
|
if (Test-StringIsNullOrEmpty $instance.Hostname) {
|
|
Write-Warning "$($instance.InstanceID) did not have a hostname - This will confuse restarts"
|
|
continue
|
|
}
|
|
if ($instance.LiveState -ne 'Stopped') {
|
|
Write-Warning "$($instance.InstanceID) was not stopped but should have been based on capture state. Will not action this instance."
|
|
continue
|
|
}
|
|
|
|
# start the server
|
|
Write-Host "Starting instance [$($instance.InstanceID)]"
|
|
Start-EC2Instance -InstanceId $instance.InstanceID -Region $instance.Region -ProfileName $ProfileName
|
|
|
|
try {
|
|
$testService = $null
|
|
while ($null -eq $testService) {
|
|
# Sleep one minute
|
|
# Check to see if it is online by Get-Services
|
|
# If it is still not online, sleep again till we get services
|
|
|
|
Write-Host "Sleeping for 60s"
|
|
Start-Sleep -Seconds 60
|
|
$testService = Invoke-Command -ComputerName $instance.Hostname -ScriptBlock { (Get-Service)[0] } -ErrorAction SilentlyContinue
|
|
}
|
|
Write-Host "Got a service back on [$($instance.InstanceID)]"
|
|
|
|
# Now that it is online, report the nag service name, radium service name, hostname, "overflow"
|
|
$serviceInfo = Invoke-Command -ComputerName $instance.Hostname -ScriptBlock {
|
|
return @{
|
|
Hostname = (Get-FullyQualifiedServerName)
|
|
NagServiceName = (Get-ServiceInfoByCIMFragment "C:\Orb\Nag").Name
|
|
RadiumServiceName = (Get-ServiceInfoByCIMFragment "C:\Orb\Radium").Name
|
|
Indicator = "stopped overflow"
|
|
}
|
|
}
|
|
$serviceInfo
|
|
$stoppedInstancesOverflowServices += $serviceInfo
|
|
} catch {
|
|
Write-Host $_.Exception.Message
|
|
Write-Host "Instance failed on data collection, will proceed to shutdown process"
|
|
}
|
|
|
|
try {
|
|
# Turn the server back off again
|
|
Write-Host "Stopping instance [$($instance.InstanceID)]"
|
|
Stop-Computer -ComputerName $instance.Hostname
|
|
} catch {
|
|
try {
|
|
Write-Host "Stopping instance [$($instance.InstanceID)]"
|
|
Stop-Computer -ComputerName $instance.Hostname
|
|
} catch {
|
|
Write-Host "Stop-Computer -ComputerName $($instance.Hostname)"
|
|
}
|
|
}
|
|
Write-Host "----------"
|
|
Write-Host ""
|
|
}
|
|
ConvertTo-Json $stoppedInstancesOverflowServices -Depth 10 | Set-Content -Path Z:\stuff\stoppedOverflow.txt
|
|
|
|
|
|
Write-Host "`n`tStarting not-overflow app servers`n"
|
|
|
|
$stoppedInstancesNotOverflowServices = @()
|
|
foreach ($instance in $stoppedInstancesNotOverflowApp) {
|
|
if (Test-StringIsNullOrEmpty $instance.Hostname) {
|
|
Write-Warning "$($instance.InstanceID) did not have a hostname - This will confuse restarts"
|
|
continue
|
|
}
|
|
if ($instance.LiveState -ne 'Stopped') {
|
|
Write-Warning "$($instance.InstanceID) was not stopped but should have been based on capture state. Will not action this instance."
|
|
continue
|
|
}
|
|
|
|
# start the server
|
|
Write-Host "Starting instance [$($instance.InstanceID)]"
|
|
Start-EC2Instance -InstanceId $instance.InstanceID -Region $instance.Region -ProfileName $ProfileName
|
|
|
|
try {
|
|
$testService = $null
|
|
while ($null -eq $testService) {
|
|
# Sleep one minute
|
|
# Check to see if it is online by Get-Services
|
|
# If it is still not online, sleep again till we get services
|
|
|
|
Write-Host "Sleeping for 60s"
|
|
Start-Sleep -Seconds 60
|
|
$testService = Invoke-Command -ComputerName $instance.Hostname -ScriptBlock { (Get-Service)[0] } -ErrorAction SilentlyContinue
|
|
}
|
|
Write-Host "Got a service back on [$($instance.InstanceID)]"
|
|
|
|
# Now that it is online, report the nag service name, radium service name, hostname, "overflow"
|
|
$serviceInfo = Invoke-Command -ComputerName $instance.Hostname -ScriptBlock {
|
|
return @{
|
|
Hostname = (Get-FullyQualifiedServerName)
|
|
NagServiceName = (Get-ServiceInfoByCIMFragment "C:\Orb\Nag").Name
|
|
RadiumServiceName = (Get-ServiceInfoByCIMFragment "C:\Orb\Radium").Name
|
|
Indicator = "stopped overflow"
|
|
}
|
|
}
|
|
$serviceInfo
|
|
$stoppedInstancesNotOverflowServices += $serviceInfo
|
|
} catch {
|
|
Write-Host $_.Exception.Message
|
|
Write-Host "Instance failed on data collection, will proceed to shutdown process"
|
|
}
|
|
|
|
try {
|
|
# Turn the server back off again
|
|
Write-Host "Stopping instance [$($instance.InstanceID)]"
|
|
Stop-Computer -ComputerName $instance.Hostname
|
|
} catch {
|
|
try {
|
|
Write-Host "Stopping instance [$($instance.InstanceID)]"
|
|
Stop-Computer -ComputerName $instance.Hostname
|
|
} catch {
|
|
Write-Host "Stop-Computer -ComputerName $($instance.Hostname)"
|
|
}
|
|
}
|
|
Write-Host "----------"
|
|
Write-Host ""
|
|
}
|
|
ConvertTo-Json $stoppedInstancesNotOverflowServices -Depth 10 | Set-Content -Path Z:\stuff\stoppedNotOverflow.txt
|
|
|
|
Write-Host "`n`tGathering overflow app servers`n"
|
|
|
|
# $runningInstancesOverflowServices = Invoke-Command -ComputerName $runningInstancesOverflowApp.Hostname -ScriptBlock {
|
|
# return @{
|
|
# Hostname = (Get-FullyQualifiedServerName)
|
|
# NagServiceName = (Get-ServiceInfoByCIMFragment "C:\Orb\Nag").Name
|
|
# RadiumServiceName = (Get-ServiceInfoByCIMFragment "C:\Orb\Radium").Name
|
|
# Indicator = "running overflow"
|
|
# }
|
|
# }
|
|
# ConvertTo-Json $runningInstancesOverflowServices -Depth 10 | Set-Content -Path Z:\stuff\runningOverflow.txt
|
|
|
|
# Write-Host "`n`tGathering not-overflow app servers`n"
|
|
|
|
# $runningInstancesNonOverflowServices = Invoke-Command -ComputerName $runningInstancesNotOverflowApp.Hostname -ScriptBlock {
|
|
# return @{
|
|
# Hostname = (Get-FullyQualifiedServerName)
|
|
# NagServiceName = (Get-ServiceInfoByCIMFragment "C:\Orb\Nag").Name
|
|
# RadiumServiceName = (Get-ServiceInfoByCIMFragment "C:\Orb\Radium").Name
|
|
# Indicator = "running -not overflow"
|
|
# }
|
|
# }
|
|
# ConvertTo-Json $runningInstancesNonOverflowServices -Depth 10 | Set-Content -Path Z:\stuff\runningNotOverflow.txt
|
|
|
|
Stop-Transcript
|
|
|
|
$audit = @()
|
|
$audit += $stoppedInstancesNotOverflowServices
|
|
$audit += $stoppedInstancesOverflowServices
|
|
$audit += $runningInstancesNonOverflowServices
|
|
$audit += $runningInstancesOverflowServices
|