157 lines
8.5 KiB
PowerShell
157 lines
8.5 KiB
PowerShell
function Invoke-AWSConfigureAWSPackage {
|
|
<#
|
|
.SYNOPSIS
|
|
Used to help facilitate updating various driver packages on TeamCity agents
|
|
|
|
.PARAMETER JobName
|
|
Used to specify which driver set to update
|
|
|
|
.PARAMETER InstanceId
|
|
One or more instance ids to operate on. Defaults to the TC agent machine list
|
|
|
|
.PARAMETER Comment
|
|
Please provide a Jira ticket number associated with the work you are doing
|
|
|
|
.PARAMETER ProfileName
|
|
The AWS Profile name to use (think temp-prod, unless this is implemented as a job on TC itself, then Prod, etc)
|
|
|
|
.PARAMETER Region
|
|
The AWS Region where the command should be run
|
|
|
|
.LINK
|
|
https://confluence.alkami.com/display/SRE/How+To+update+AWS+EC2+Drivers+on+TeamCity+Agents
|
|
#>
|
|
[CmdletBinding()]
|
|
[OutputType([System.Void])]
|
|
param (
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateSet('AwsEnaNetworkDriver', 'AWSPVDriver', 'AWSNVMe')]
|
|
[ArgumentCompleter( { $possibleValues = @('AwsEnaNetworkDriver', 'AWSPVDriver', 'AWSNVMe'); return $possibleValues | ForEach-Object { $_ } })]
|
|
[string]$JobName,
|
|
[Parameter(Mandatory = $false)]
|
|
[ValidateNotNullOrEmpty()]
|
|
[string[]]$InstanceId = @("i-05993777a46817dfe", "i-0117159b672f24a1a", "i-01c107a3e148d503a", "i-0cc56f29481ad409b", "i-0dfbf5cf56d898966", "i-0b17585457e090315"),
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateNotNullOrEmpty()]
|
|
[Alias('JiraTicketNumber')]
|
|
[string]$Comment,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateNotNullOrEmpty()]
|
|
[string]$ProfileName,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateNotNullOrEmpty()]
|
|
[string]$Region
|
|
)
|
|
|
|
$logLead = Get-LogLeadName
|
|
|
|
$targets = @(
|
|
@{
|
|
Key = "InstanceIds"
|
|
Values = @($instanceId) #this value has to be an array even if only one value is given, I think
|
|
}
|
|
)
|
|
# The requirements on the AWS CLI are very odd. I must be doing something wrong
|
|
$targetsString = '"' + (ConvertTo-Json $targets -Compress).Replace('"', '""') + '"'
|
|
|
|
$parameters = @{
|
|
action = @("Install")
|
|
installationType = @("Uninstall and reinstall")
|
|
version = @("")
|
|
additionalArguments = @("{}")
|
|
name = @($JobName)
|
|
}
|
|
$parametersString = '"' + (ConvertTo-Json $parameters -Compress).Replace('"', '""') + '"'
|
|
|
|
Write-Host $targetsString
|
|
Write-Host $parametersString
|
|
|
|
Write-Host "$logLead : Sending this command to AWS CLI:`n`n aws ssm send-command --document-name `"AWS-ConfigureAWSPackage`" --document-version `"1`" --targets=$targetsString --parameters=$parametersString --comment `"$Comment`" --timeout-seconds `"600`" --max-concurrency `"50`" --max-errors `"0`" --output-s3-bucket-name `"teamcity-alkami-bucket`" --region `"$Region`" --profile `"$ProfileName`"`n`n"
|
|
|
|
# https://docs.aws.amazon.com/cli/latest/reference/ssm/send-command.html
|
|
$resultString = (aws ssm send-command --document-name "AWS-ConfigureAWSPackage" --document-version "1" --targets=$targetsString --parameters=$parametersString --comment $Comment --timeout-seconds "600" --max-concurrency "50" --max-errors "0" --output-s3-bucket-name "teamcity-alkami-bucket" --region $Region --profile $ProfileName)
|
|
|
|
$result = ConvertFrom-Json $resultString -Depth 100
|
|
|
|
$command = $result.Command
|
|
|
|
if ($command.ErroCount -gt 0) {
|
|
Write-Host "$logLead : response was the following:`n$resultString"
|
|
throw "$logLead : Result had non-zero error count. Please review the logs above."
|
|
}
|
|
|
|
$commandId = $command.CommandId
|
|
$instanceIds = $command.Targets[0].Values
|
|
|
|
<#
|
|
Potential status fields https://docs.aws.amazon.com/cli/latest/reference/ssm/get-command-invocation.html
|
|
Pending: The command hasn't been sent to the managed node.
|
|
In Progress: The command has been sent to the managed node but hasn't reached a terminal state.
|
|
Delayed: The system attempted to send the command to the target, but the target wasn't available. The managed node might not be available because of network issues, because the node was stopped, or for similar reasons. The system will try to send the command again.
|
|
Success: The command or plugin ran successfully. This is a terminal state.
|
|
Delivery Timed Out: The command wasn't delivered to the managed node before the delivery timeout expired. Delivery timeouts don't count against the parent command's MaxErrors limit, but they do contribute to whether the parent command status is Success or Incomplete. This is a terminal state.
|
|
Execution Timed Out: The command started to run on the managed node, but the execution wasn't complete before the timeout expired. Execution timeouts count against the MaxErrors limit of the parent command. This is a terminal state.
|
|
Failed: The command wasn't run successfully on the managed node. For a plugin, this indicates that the result code wasn't zero. For a command invocation, this indicates that the result code for one or more plugins wasn't zero. Invocation failures count against the MaxErrors limit of the parent command. This is a terminal state.
|
|
Cancelled: The command was terminated before it was completed. This is a terminal state.
|
|
Undeliverable: The command can't be delivered to the managed node. The node might not exist or might not be responding. Undeliverable invocations don't count against the parent command's MaxErrors limit and don't contribute to whether the parent command status is Success or Incomplete. This is a terminal state.
|
|
Terminated: The parent command exceeded its MaxErrors limit and subsequent command invocations were canceled by the system. This is a terminal state.
|
|
#>
|
|
|
|
# keep trying again
|
|
$keepLoopingStatuses = @('Pending', 'In Progress', 'Delayed')
|
|
# stop trying again
|
|
$stopLoopingStatuses = @('Success', 'Delivery Timed Out', 'Execution Timed Out', 'Failed', 'Cancelled', 'Undeliverable', 'Terminated')
|
|
# show an error
|
|
$failureStatuses = @('Delivery Timed Out', 'Execution Timed Out', 'Failed', 'Cancelled', 'Undeliverable', 'Terminated')
|
|
# show a success
|
|
$successStatuses = @('Success')
|
|
|
|
while ($instanceIds.Count -gt 0) {
|
|
$finishedInstanceIds = @()
|
|
foreach ($instanceId in $instanceIds) {
|
|
try {
|
|
$instanceResultString = (aws ssm get-command-invocation --command-id $commandId --instance-id $instanceId)
|
|
|
|
$instanceResult = ConvertFrom-Json $instanceResultString -Depth 100
|
|
|
|
if ($keepLoopingStatuses -contains $instanceResult.StatusDetails) {
|
|
Write-Host "$logLead : Will recheck status on [$instanceId] after sleep because it is still state [$($instanceResult.StatusDetails)]"
|
|
}
|
|
if ($stopLoopingStatuses -contains $instanceResult.StatusDetails) {
|
|
if ($successStatuses -contains $instanceResult.StatusDetails) {
|
|
Write-Host "$logLead : Instance [$instanceId] successfully finished the process"
|
|
}
|
|
if ($failureStatuses -contains $instanceResult.StatusDetails) {
|
|
Write-Warning "$logLead : Instance [$instanceId] finished the process with state [$(instanceResult.StatusDetails)]"
|
|
Write-Host "$logLead : For the following results, please see https://docs.aws.amazon.com/cli/latest/reference/ssm/get-command-invocation.html"
|
|
}
|
|
# log "normal" properties
|
|
foreach ($property in @('Comment', 'StandardOutputContent', 'StandardOutputUrl')) {
|
|
if (![string]::IsNullOrWhiteSpace($instanceResult.$property)) {
|
|
Write-Host "$logLead : $instanceId : $property : $($instanceResult.$property)"
|
|
}
|
|
}
|
|
# log "warning" properties
|
|
foreach ($property in @('StandardErrorUrl', 'StandardErrorContent')) {
|
|
if (![string]::IsNullOrWhiteSpace($instanceResult.$property)) {
|
|
Write-Warning "$logLead : $instanceId : $property : $($instanceResult.$property)"
|
|
}
|
|
}
|
|
|
|
$finishedInstanceIds += $instanceId
|
|
}
|
|
} catch {
|
|
$finishedInstanceIds += $instanceId
|
|
}
|
|
}
|
|
|
|
$instanceIds = $instanceIds | Where-Object { $finishedInstanceIds -notcontains $PSItem }
|
|
|
|
if ($instanceIds.Count -gt 0) {
|
|
Start-Sleep -Seconds 5
|
|
}
|
|
|
|
Write-Host "$logLead : Still waiting to check the results on [$($instanceIds -join ',')]"
|
|
}
|
|
}
|