166 lines
6.0 KiB
PowerShell
166 lines
6.0 KiB
PowerShell
function Get-ServerStatusReport {
|
|
<#
|
|
.SYNOPSIS
|
|
Start the Server Status check, typically run after a Scale-Up event.
|
|
|
|
.DESCRIPTION
|
|
Performs several sanity tests to ensure the servers are fully operational after a Scale-Up event.
|
|
Main entrypoint that calls this is in teamcity.sre.code/ScaleEnvironments/Invoke-ReportServerStatus.ps1
|
|
|
|
.PARAMETER Servers
|
|
[string[]]Array of servers to run tests against. Usually this is a list of servers by Designation (LC3, Morph, etc.)
|
|
|
|
.PARAMETER ProfileName
|
|
[string] Specific AWS CLI Profile to use in AWS API calls.
|
|
|
|
.PARAMETER Region
|
|
[string] Specific AWS CLI Region to use in AWS API calls.
|
|
|
|
.NOTES
|
|
Outputs a table with the results of the test by host.
|
|
Returns custom object with the following properties:
|
|
"Hostname"
|
|
"Designation"
|
|
"InLoadBalancer"
|
|
"IsNagRunning"
|
|
"IsSubServiceRunning"
|
|
"AllAlkServicesRunning"
|
|
"TagsCorrect"
|
|
"IsAppServer"
|
|
"IsWebServer"
|
|
"IsMicServer"
|
|
#>
|
|
[CmdletBinding()]
|
|
[OutputType([Object])]
|
|
Param (
|
|
[Parameter(Mandatory = $true)]
|
|
[string[]] $Servers,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string] $ProfileName,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string] $Region
|
|
)
|
|
|
|
$logLead = (Get-LogLeadName)
|
|
|
|
Write-Host "$logLead : Running for servers:`n$Servers"
|
|
|
|
# Script Block to do the tests.
|
|
$testsScriptBlock = {
|
|
param ($hostname, $arguments)
|
|
|
|
Write-Host "##teamcity[blockOpened name='Running tests for hostname $hostname']"
|
|
|
|
$testResults = Invoke-Command -ComputerName $hostname -ScriptBlock {
|
|
$computername = (Get-FullyQualifiedServerName)
|
|
$isAppServer = Test-IsAppServer -ComputerName $computername
|
|
$isWebServer = Test-IsWebServer -ComputerName $computername
|
|
$isMicServer = Test-IsMicroServer -ComputerName $computername
|
|
|
|
# Make an object to hold the results of the tests
|
|
$testResult = New-Object PSObject -Property @{
|
|
"Hostname" = $computername
|
|
"Designation" = "Unknown"
|
|
"InLoadBalancer" = $null
|
|
"IsNagRunning" = $null
|
|
"IsSubServiceRunning" = $null
|
|
"AllAlkServicesRunning" = $null
|
|
"TagsCorrect" = $null
|
|
"IsAppServer" = $isAppServer
|
|
"IsWebServer" = $isWebServer
|
|
"IsMicServer" = $isMicServer
|
|
}
|
|
|
|
# NAG test
|
|
try {
|
|
if($IsAppServer) {
|
|
Write-Host "Running NAG test"
|
|
$nagStatus = Test-IsNagRunning -Server $computername 2>$null
|
|
$testResult.IsNagRunning = $nagStatus
|
|
} else {
|
|
$testResult.IsNagRunning = "N/A"
|
|
}
|
|
} catch {
|
|
Resolve-Error -ErrorRecord $_
|
|
$testResult.IsNagRunning = "Unknown"
|
|
}
|
|
|
|
|
|
# Subscription Service test
|
|
try {
|
|
if($isMicServer -or $isWebServer -or $isAppServer) {
|
|
Write-Host "Running Subscription Service test"
|
|
$chocoServices = Get-ChocolateyServices 2>$null
|
|
$subscriptionService = ($chocoServices | Where-Object {$_.Name -eq "Alkami.Services.Subscriptions.Host"})
|
|
if($null -ne $subscriptionService) {
|
|
$testResult.IsSubServiceRunning = ($subscriptionService.State -eq "Running")
|
|
} else {
|
|
$testResult.IsSubServiceRunning = $false
|
|
}
|
|
} else {
|
|
$testResult.IsSubServiceRunning = "N/A"
|
|
}
|
|
} catch {
|
|
Resolve-Error -ErrorRecord $_
|
|
$testResult.IsSubServiceRunning = "Unknown"
|
|
}
|
|
|
|
|
|
# All ALK Services running test?
|
|
Write-Host "Running All Alk Services test"
|
|
$alkServices = ($chocoServices | Where-Object {$_.Name -like "Alkami.*"})
|
|
$stoppedAlkServiceCount = ($alkServices | Where-Object {$_.State -eq "Stopped"}).count
|
|
$testResult.AllAlkServicesRunning = !($stoppedAlkServiceCount -gt 0)
|
|
|
|
|
|
# Tags test
|
|
try {
|
|
Write-Host "Running Tags test"
|
|
$tags = Get-CurrentInstanceTags
|
|
$autoShutdownTagKey = "alk:autoshutdown"
|
|
$autoShutdownTag = ($tags | Where-Object { ($_.Key -eq $autoShutdownTagKey -and $_.Value -eq "true") })
|
|
$isAutoShutdown = ($null -ne $autoShutdownTag)
|
|
$testResult.TagsCorrect = (!($isAutoShutdown))
|
|
} catch {
|
|
Resolve-Error -ErrorRecord $_
|
|
$testResult.TagsCorrect = "Unknown"
|
|
}
|
|
|
|
return $testResult
|
|
}
|
|
|
|
# LoadBalancer test
|
|
# This test is done outside of the above script block as it needs to be run off-host.
|
|
# Running this on host will fail due to the IAM creds not having permissions for some
|
|
# resources (Only for NGinx lookups).
|
|
try {
|
|
if($testResults.IsAppServer -or $testResults.IsWebServer) {
|
|
Write-Host "Running LoadBalancer test"
|
|
$lbState = Get-LoadBalancerState -Server $hostname -AwsProfileName $arguments.ProfileName -AwsRegion $arguments.Region
|
|
$testResults.InLoadBalancer = ($lbState -eq "Active")
|
|
} else {
|
|
$testResults.InLoadBalancer = "N/A"
|
|
}
|
|
} catch {
|
|
Resolve-Error -ErrorRecord $_
|
|
$testResults.InLoadBalancer = "Unknown"
|
|
}
|
|
|
|
Write-Host "##teamcity[blockClosed name='Running tests for hostname $hostname']"
|
|
|
|
# Return the test
|
|
return $testResults
|
|
}
|
|
|
|
# Parallel over the servers to test.
|
|
try {
|
|
$results = (Invoke-Parallel2 -Objects $Servers -Arguments @{ProfileName = $ProfileName; Region = $Region;} -Script $testsScriptBlock).Result
|
|
} catch {
|
|
Resolve-Error -ErrorRecord $_
|
|
}
|
|
|
|
# Send results to caller.
|
|
return $results
|
|
} |