function Get-ServerStatusReport { <# .SYNOPSIS Start the Server Status check, typically run after a Scale-Up event. .DESCRIPTION Performs several sanity tests to ensure the servers are fully operational after a Scale-Up event. Main entrypoint that calls this is in teamcity.sre.code/ScaleEnvironments/Invoke-ReportServerStatus.ps1 .PARAMETER Servers [string[]]Array of servers to run tests against. Usually this is a list of servers by Designation (LC3, Morph, etc.) .PARAMETER ProfileName [string] Specific AWS CLI Profile to use in AWS API calls. .PARAMETER Region [string] Specific AWS CLI Region to use in AWS API calls. .NOTES Outputs a table with the results of the test by host. Returns custom object with the following properties: "Hostname" "Designation" "InLoadBalancer" "IsNagRunning" "IsSubServiceRunning" "AllAlkServicesRunning" "TagsCorrect" "IsAppServer" "IsWebServer" "IsMicServer" #> [CmdletBinding()] [OutputType([Object])] Param ( [Parameter(Mandatory = $true)] [string[]] $Servers, [Parameter(Mandatory = $true)] [string] $ProfileName, [Parameter(Mandatory = $true)] [string] $Region ) $logLead = (Get-LogLeadName) Write-Host "$logLead : Running for servers:`n$Servers" # Script Block to do the tests. $testsScriptBlock = { param ($hostname, $arguments) Write-Host "##teamcity[blockOpened name='Running tests for hostname $hostname']" $testResults = Invoke-Command -ComputerName $hostname -ScriptBlock { $computername = (Get-FullyQualifiedServerName) $isAppServer = Test-IsAppServer -ComputerName $computername $isWebServer = Test-IsWebServer -ComputerName $computername $isMicServer = Test-IsMicroServer -ComputerName $computername # Make an object to hold the results of the tests $testResult = New-Object PSObject -Property @{ "Hostname" = $computername "Designation" = "Unknown" "InLoadBalancer" = $null "IsNagRunning" = $null "IsSubServiceRunning" = $null "AllAlkServicesRunning" = $null "TagsCorrect" = $null "IsAppServer" = $isAppServer "IsWebServer" = $isWebServer "IsMicServer" = $isMicServer } # NAG test try { if($IsAppServer) { Write-Host "Running NAG test" $nagStatus = Test-IsNagRunning -Server $computername 2>$null $testResult.IsNagRunning = $nagStatus } else { $testResult.IsNagRunning = "N/A" } } catch { Resolve-Error -ErrorRecord $_ $testResult.IsNagRunning = "Unknown" } # Subscription Service test try { if($isMicServer -or $isWebServer -or $isAppServer) { Write-Host "Running Subscription Service test" $chocoServices = Get-ChocolateyServices 2>$null $subscriptionService = ($chocoServices | Where-Object {$_.Name -eq "Alkami.Services.Subscriptions.Host"}) if($null -ne $subscriptionService) { $testResult.IsSubServiceRunning = ($subscriptionService.State -eq "Running") } else { $testResult.IsSubServiceRunning = $false } } else { $testResult.IsSubServiceRunning = "N/A" } } catch { Resolve-Error -ErrorRecord $_ $testResult.IsSubServiceRunning = "Unknown" } # All ALK Services running test? Write-Host "Running All Alk Services test" $alkServices = ($chocoServices | Where-Object {$_.Name -like "Alkami.*"}) $stoppedAlkServiceCount = ($alkServices | Where-Object {$_.State -eq "Stopped"}).count $testResult.AllAlkServicesRunning = !($stoppedAlkServiceCount -gt 0) # Tags test try { Write-Host "Running Tags test" $tags = Get-CurrentInstanceTags $autoShutdownTagKey = "alk:autoshutdown" $autoShutdownTag = ($tags | Where-Object { ($_.Key -eq $autoShutdownTagKey -and $_.Value -eq "true") }) $isAutoShutdown = ($null -ne $autoShutdownTag) $testResult.TagsCorrect = (!($isAutoShutdown)) } catch { Resolve-Error -ErrorRecord $_ $testResult.TagsCorrect = "Unknown" } return $testResult } # LoadBalancer test # This test is done outside of the above script block as it needs to be run off-host. # Running this on host will fail due to the IAM creds not having permissions for some # resources (Only for NGinx lookups). try { if($testResults.IsAppServer -or $testResults.IsWebServer) { Write-Host "Running LoadBalancer test" $lbState = Get-LoadBalancerState -Server $hostname -AwsProfileName $arguments.ProfileName -AwsRegion $arguments.Region $testResults.InLoadBalancer = ($lbState -eq "Active") } else { $testResults.InLoadBalancer = "N/A" } } catch { Resolve-Error -ErrorRecord $_ $testResults.InLoadBalancer = "Unknown" } Write-Host "##teamcity[blockClosed name='Running tests for hostname $hostname']" # Return the test return $testResults } # Parallel over the servers to test. try { $results = (Invoke-Parallel2 -Objects $Servers -Arguments @{ProfileName = $ProfileName; Region = $Region;} -Script $testsScriptBlock).Result } catch { Resolve-Error -ErrorRecord $_ } # Send results to caller. return $results }