ps/Modules/Alkami.DevOps.Common/Public/Get-ServerStatusReport.ps1

function Get-ServerStatusReport {
<#
.SYNOPSIS
Start the Server Status check, typically run after a Scale-Up event.

.DESCRIPTION
Performs several sanity tests to ensure the servers are fully operational after a Scale-Up event.
Main entrypoint that calls this is in teamcity.sre.code/ScaleEnvironments/Invoke-ReportServerStatus.ps1

.PARAMETER Servers
[string[]]Array of servers to run tests against.  Usually this is a list of servers by Designation (LC3, Morph, etc.)

.PARAMETER ProfileName
[string] Specific AWS CLI Profile to use in AWS API calls.

.PARAMETER Region
[string] Specific AWS CLI Region to use in AWS API calls.

.NOTES
Outputs a table with the results of the test by host.
Returns custom object with the following properties:
    "Hostname"
    "Designation"
    "InLoadBalancer"
    "IsNagRunning"
    "IsSubServiceRunning"
    "AllAlkServicesRunning"
    "TagsCorrect"
    "IsAppServer"
    "IsWebServer"
    "IsMicServer"
#>
    [CmdletBinding()]
    [OutputType([Object])]
    Param (
        [Parameter(Mandatory = $true)]
        [string[]] $Servers,

        [Parameter(Mandatory = $true)]
        [string] $ProfileName,

        [Parameter(Mandatory = $true)]
        [string] $Region
    )

    $logLead = (Get-LogLeadName)

    Write-Host "$logLead : Running for servers:`n$Servers"

    # Script Block to do the tests.
    $testsScriptBlock = {
        param ($hostname, $arguments)

        Write-Host "##teamcity[blockOpened name='Running tests for hostname $hostname']"

        $testResults = Invoke-Command -ComputerName $hostname -ScriptBlock {
            $computername = (Get-FullyQualifiedServerName)
            $isAppServer = Test-IsAppServer -ComputerName $computername
            $isWebServer = Test-IsWebServer -ComputerName $computername
            $isMicServer = Test-IsMicroServer -ComputerName $computername

            # Make an object to hold the results of the tests
            $testResult = New-Object PSObject -Property @{
                "Hostname"              = $computername
                "Designation"           = "Unknown"
                "InLoadBalancer"        = $null
                "IsNagRunning"          = $null
                "IsSubServiceRunning"   = $null
                "AllAlkServicesRunning" = $null
                "TagsCorrect"           = $null
                "IsAppServer"           = $isAppServer
                "IsWebServer"           = $isWebServer
                "IsMicServer"           = $isMicServer
            }

            # NAG test
            try {
                if($IsAppServer) {
                    Write-Host "Running NAG test"
                    $nagStatus = Test-IsNagRunning -Server $computername 2>$null
                    $testResult.IsNagRunning = $nagStatus
                } else {
                    $testResult.IsNagRunning = "N/A"
                }
            } catch {
                Resolve-Error -ErrorRecord $_
                $testResult.IsNagRunning = "Unknown"
            }


            # Subscription Service test
            try {
                if($isMicServer -or $isWebServer -or $isAppServer) {
                    Write-Host "Running Subscription Service test"
                    $chocoServices = Get-ChocolateyServices 2>$null
                    $subscriptionService = ($chocoServices | Where-Object {$_.Name -eq "Alkami.Services.Subscriptions.Host"})
                    if($null -ne $subscriptionService) {
                        $testResult.IsSubServiceRunning = ($subscriptionService.State -eq "Running")
                    } else {
                        $testResult.IsSubServiceRunning = $false
                    }
                } else {
                    $testResult.IsSubServiceRunning = "N/A"
                }
            } catch {
                Resolve-Error -ErrorRecord $_
                $testResult.IsSubServiceRunning = "Unknown"
            }


            # All ALK Services running test?
            Write-Host "Running All Alk Services test"
            $alkServices = ($chocoServices | Where-Object {$_.Name -like "Alkami.*"})
            $stoppedAlkServiceCount = ($alkServices | Where-Object {$_.State -eq "Stopped"}).count
            $testResult.AllAlkServicesRunning = !($stoppedAlkServiceCount -gt 0)


            # Tags test
            try {
                Write-Host "Running Tags test"
                $tags = Get-CurrentInstanceTags
                $autoShutdownTagKey = "alk:autoshutdown"
                $autoShutdownTag = ($tags | Where-Object { ($_.Key -eq $autoShutdownTagKey -and $_.Value -eq "true") })
                $isAutoShutdown = ($null -ne $autoShutdownTag)
                $testResult.TagsCorrect = (!($isAutoShutdown))
            } catch {
                Resolve-Error -ErrorRecord $_
                $testResult.TagsCorrect = "Unknown"
            }

            return $testResult
        }

        # LoadBalancer test
        # This test is done outside of the above script block as it needs to be run off-host.
        # Running this on host will fail due to the IAM creds not having permissions for some
        # resources (Only for NGinx lookups).
        try {
            if($testResults.IsAppServer -or $testResults.IsWebServer) {
                Write-Host "Running LoadBalancer test"
                $lbState = Get-LoadBalancerState -Server $hostname -AwsProfileName $arguments.ProfileName -AwsRegion $arguments.Region
                $testResults.InLoadBalancer = ($lbState -eq "Active")
            } else {
                $testResults.InLoadBalancer = "N/A"
            }
        } catch {
            Resolve-Error -ErrorRecord $_
            $testResults.InLoadBalancer = "Unknown"
        }

        Write-Host "##teamcity[blockClosed name='Running tests for hostname $hostname']"

        # Return the test
        return $testResults
    }

    # Parallel over the servers to test.
    try {
        $results = (Invoke-Parallel2 -Objects $Servers -Arguments @{ProfileName = $ProfileName; Region = $Region;} -Script $testsScriptBlock).Result
    } catch {
        Resolve-Error -ErrorRecord $_
    }

    # Send results to caller.
    return $results
}