114 lines
4.4 KiB
PowerShell
114 lines
4.4 KiB
PowerShell
|
function Invoke-CommandWithRetry {
|
|||
|
|
<#
|
|||
|
|
.SYNOPSIS
|
|||
|
|
Executes a script block and retries the operation a given number of times if it fails.
|
|||
|
|
Executions are applied by either linear or exponential format and always contain an amount of random jitter in milliseconds.
|
|||
|
|
The jitter range can be managed by appropriate flags. Defaults are 50ms to 500ms.
|
|||
|
|
|
|||
|
|
.EXAMPLE
|
|||
|
|
$result = Invoke-CommandWithRetry -ScriptBlock $script -MaxRetries 3 -SecondsDelay 1 -Exponential
|
|||
|
|
|
|||
|
|
.PARAMETER ScriptBlock
|
|||
|
|
The script block to execute
|
|||
|
|
|
|||
|
|
.PARAMETER Arguments
|
|||
|
|
An array of objects to pass into the scriptblock
|
|||
|
|
|
|||
|
|
.PARAMETER MaxRetries
|
|||
|
|
The maxumim number of retries to attempt. Defaults to 5.
|
|||
|
|
|
|||
|
|
.PARAMETER Seconds
|
|||
|
|
The number of seconds to delay for each retry. Defaults to 1 when used.
|
|||
|
|
|
|||
|
|
.PARAMETER Milliseconds
|
|||
|
|
The number of milliseconds to delay for each retry. Defaults to 1000 ms.
|
|||
|
|
|
|||
|
|
.PARAMETER Exponential
|
|||
|
|
[switch] Exponentially increment the time delay by a power of 2 for each attempt
|
|||
|
|
The alternative to exponential is linear.
|
|||
|
|
|
|||
|
|
.PARAMETER JitterMin
|
|||
|
|
[int] The min number of milliseconds to jitter by
|
|||
|
|
|
|||
|
|
.PARAMETER JitterMax
|
|||
|
|
[int] The max number of milliseconds to jitter by
|
|||
|
|
#>
|
|||
|
|
[CmdletBinding(DefaultParameterSetName='Milliseconds')]
|
|||
|
|
[OutputType([System.Object])]
|
|||
|
|
Param(
|
|||
|
|
[Parameter(Mandatory=$true)]
|
|||
|
|
[scriptblock]$ScriptBlock,
|
|||
|
|
|
|||
|
|
[Parameter(Mandatory=$false)]
|
|||
|
|
[object[]]$Arguments = $null,
|
|||
|
|
|
|||
|
|
[Parameter(Mandatory=$false)]
|
|||
|
|
[int]$MaxRetries = 5,
|
|||
|
|
|
|||
|
|
[Parameter(ParameterSetName='Seconds', Mandatory=$false)]
|
|||
|
|
[Alias('SecondsDelay')]
|
|||
|
|
[int]$Seconds = 1,
|
|||
|
|
|
|||
|
|
[Parameter(ParameterSetName='Milliseconds', Mandatory=$false)]
|
|||
|
|
[int]$Milliseconds = 1000,
|
|||
|
|
|
|||
|
|
[Parameter(Mandatory=$false)]
|
|||
|
|
[switch]$Exponential,
|
|||
|
|
|
|||
|
|
[Parameter(Mandatory=$false)]
|
|||
|
|
[int]$JitterMin = -500,
|
|||
|
|
|
|||
|
|
[Parameter(Mandatory=$false)]
|
|||
|
|
[int]$JitterMax = 500
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
$logLead = (Get-LogLeadName)
|
|||
|
|
$retryCount = 0
|
|||
|
|
|
|||
|
|
if ($PSCmdlet.ParameterSetName -eq 'Seconds') {
|
|||
|
|
$Milliseconds = $Seconds * 1000
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
do {
|
|||
|
|
# Not being picked up by above call
|
|||
|
|
$logLead = "[Invoke-CommandWithRetry]"
|
|||
|
|
$retryCount++
|
|||
|
|
try {
|
|||
|
|
$ScriptBlock.Invoke($Arguments)
|
|||
|
|
return
|
|||
|
|
} catch {
|
|||
|
|
Write-Warning $_.Exception.InnerException.Message
|
|||
|
|
|
|||
|
|
# Don't stall at the end of the failed run otherwise we delay error reporting by one more sleep cycle
|
|||
|
|
if ($retryCount -lt $MaxRetries) {
|
|||
|
|
# As a deconstructed equation because the below block covers a lot of ground
|
|||
|
|
# $sleepTimeout = ($milliseconds * (1 OR a power of 2) ) + random jitter amount
|
|||
|
|
# 1 -shl 0 = 1 1 -shl 3 = 8 3rd retry = 8 multiplier
|
|||
|
|
# $sleepTimeout = ($milliseconds * (1 -shl (0,($retryCount - 1))[$Exponential])) + (Get-Random -Maximum $JitterMax -Minimum $JitterMin)
|
|||
|
|
# $sleepTimeoutModifier
|
|||
|
|
# -shiftleft
|
|||
|
|
# $retryCount - 1 when exponential, 1 when linear
|
|||
|
|
# $sleepTimeout = ($milliseconds * $sleepTimeoutModifier ) + $jitterAmount
|
|||
|
|
|
|||
|
|
$jitterAmount = Get-Random -Maximum $JitterMax -Minimum $JitterMin
|
|||
|
|
$sleepTimeoutModifier = 1
|
|||
|
|
if ($Exponential) {
|
|||
|
|
# When you want 2 to the power, you just shift the binary leader left
|
|||
|
|
# 1 = 0001
|
|||
|
|
# 2 = 0010
|
|||
|
|
# 4 = 0100
|
|||
|
|
# 8 = 1000
|
|||
|
|
$sleepTimeoutModifier = $sleepTimeoutModifier -shl ($retryCount - 1) #decremented because we start at 1 retry count
|
|||
|
|
}
|
|||
|
|
# 3rd retry would be base timeout * 8 + jitter
|
|||
|
|
$sleepTimeout = ($milliseconds * $sleepTimeoutModifier) + $jitterAmount
|
|||
|
|
Write-Warning "$logLead : Attempt #$RetryCount failed. Start-Sleep -Milliseconds [$sleepTimeout] for retry"
|
|||
|
|
Start-Sleep -Milliseconds $sleepTimeout
|
|||
|
|
} else {
|
|||
|
|
Write-Warning "$logLead : Attempt #$RetryCount failed."
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} while ($retryCount -lt $MaxRetries)
|
|||
|
|
|
|||
|
|
Write-Error "$logLead : Execution failed. Maximum retries attempted."
|
|||
|
|
}
|