function Start-ServicesInParallel { <# .SYNOPSIS Starts Multiple Windows Services in Parallel. .DESCRIPTION Starts Multiple Windows Services in Parallel. Max service start parallelism defaults to 0, which means to start services as quickly as possible. Limited by CPU capacity. If max parallelism is greater than 0, the number of services that can start at a time will be limited to that many services at a time, in addition to limitation by CPU capacity. .PARAMETER ServiceNamestoStart A string array of service names to start .PARAMETER MaxParallel The maximum number of services to start in parallel. Defaults to 0, which starts every service as fast as possible. .PARAMETER MicroserviceCpuGuess The estimate of how much CPU a microservice will utilize as it starts. Defaults to 16. .PARAMETER CpuTarget The target maximum CPU percentage, as an Integer to use while starting services, in order to leave some overhead for the rest of the system. Defaults to 85. Overridable via environment variable named 'ALKAMI_STARTSERVICESINPARALLEL_WITHTIMEOUTANDRETRY' .PARAMETER ReturnResults Whether to return an array of result objects. Currently only exceptions. .PARAMETER WithoutTimeoutAndRetry Causes the parallel jobs to use Start-Service instead of Start-AlkamiService. Start-AlkamiService has a default Timeout of 60 seconds and retries 3 times. .EXAMPLE Start-ServicesInParallel -serviceNamesToStart @("Alkami Radium Scheduler Service", "Alkami Nag Service") -maxParallel 2 Starting Service Alkami Radium Scheduler Service Starting Service Alkami Nag Service .. Done Starting Services #> [CmdletBinding()] [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseUsingScopeModifierInNewRunspaces', '', Scope = 'Function', Justification = 'Per https://github.com/PowerShell/PSScriptAnalyzer/issues/1504 this is a known regression, skip this validation step')] Param( [Parameter(Mandatory = $true)] [ValidateNotNull()] [string[]]$ServiceNamestoStart, [Parameter(Mandatory = $false)] [ValidateRange(0, [int]::MaxValue)] [int]$MaxParallel = 0, [Parameter(Mandatory = $false)] [ValidateRange(1, [int]::MaxValue)] [int]$MicroserviceCpuGuess = 16, [Parameter(Mandatory = $false)] [ValidateRange(1, 100)] [int]$CpuTarget = 85, [Parameter(Mandatory = $false)] [switch]$ReturnResults, [Parameter(Mandatory = $false)] [switch]$WithoutTimeoutAndRetry ) $loglead = Get-LogLeadName # Return if there are no services to start. if (Test-IsCollectionNullOrEmpty -Collection $ServiceNamestoStart) { return } # For each input object. $jobs = @() $useStartAlkamiService = $true $envVarWithTimeoutAndRetry = Get-EnvironmentVariable -Name "ALKAMI_STARTSERVICESINPARALLEL_WITHTIMEOUTANDRETRY" if ($WithoutTimeoutAndRetry -or $envVarWithTimeoutAndRetry -eq "false") { $useStartAlkamiService = $false } $envVarCpuTarget = Get-EnvironmentVariable -Name "ALKAMI_STARTSERVICESINPARALLEL_CPUTARGET" if(!(Test-StringIsNullOrEmpty -Value $envVarCpuTarget)) { # Get-EnvironmentVariable will return $null if the $env:ALKAMI_STARTSERVICESINPARALLEL_CPUTARGET has no value. # Reusing the param $CpuTarget will still enforce the datatype and ValidateRange rules. Write-Host "$loglead : Found environment variable 'ALKAMI_STARTSERVICESINPARALLEL_CPUTARGET' with a value of '$envVarCpuTarget'" Write-Host "$loglead : Setting CpuTarget to $envVarCpuTarget" try { $CpuTarget = $envVarCpuTarget } catch { Write-Warning "$loglead : Caught exception trying to set CpuTarget param from environment variable. Using default value '$CpuTarget'." Write-Warning $_ } } Write-Host "$loglead : Using CpuTarget '$CpuTarget'" $serviceCounter = 0 # How many services we have started jobs for. $numServicesToStart = 0 # The number of services in the 'queue' to start. $errors = @() # Errors thrown from the service-starts. do { # If the max parallelism param is set and we are running too many jobs, wait for any job to complete. if ( ($MaxParallel -gt 0) -and ($jobs.Count -ge $MaxParallel) ) { (Wait-Job -Job $jobs -Any) | Out-Null } # Scrub the jobs array of jobs that have finished, and receive their outputs. $runningJobs = $jobs | Where-Object { ($_.State -eq "Running") -or ($_.State -eq "NotStarted") } $completedJobs = $jobs | Where-Object { $_.State -eq "Completed" } if ( !(Test-IsCollectionNullOrEmpty -Collection $completedJobs) ) { foreach ($completedJob in $completedJobs) { try { Receive-Job -Job $completedJob -ErrorAction Stop } catch { $errors += $_ } } } [array]$jobs = $runningJobs # Figure out how many services we can start, if any. if ($numServicesToStart -eq 0) { # Keep looping until we find the bandwidth to start another microservice. while ($numServicesToStart -eq 0) { $cpuUsage = Get-CPUUsage $remainingCPU = $CpuTarget - $cpuUsage if ($remainingCPU -lt 0) { $remainingCPU = 0 } $extraMicroservicesToStart = $remainingCPU / $MicroserviceCpuGuess $extraMicroservicesToStart = [Math]::Floor($extraMicroservicesToStart) if ($extraMicroservicesToStart -gt 0) { $numServicesToStart += $extraMicroservicesToStart } else { Start-Sleep -Milliseconds 30 } } # Limit the number of services to start by the max parallelism param, if applicable. if ($maxParallel -gt 0) { $numStartableJobs = $maxParallel - $jobs.Count if ($numServicesToStart -gt $numStartableJobs) { $numServicesToStart = $numStartableJobs } } } # Get the service that we are starting, and decrement the services to start count. $serviceName = $ServiceNamestoStart[$serviceCounter++] $numServicesToStart-- # Start a new job. $jobs += Start-Job -ArgumentList ($serviceName, $useStartAlkamiService, $logLead) -ScriptBlock { param($sbServiceName, $sbUseStartAlkamiService, $sbLoglead) Write-Host "$sbLogLead : Starting Service $sbServiceName" Write-Host "$sbLogLead : UseStartAlkamiService is $sbUseStartAlkamiService" if ($sbUseStartAlkamiService) { Write-Host "$sbLogLead : Calling Start-AlkamiService..." Start-AlkamiService -ServiceName $sbServiceName Write-Host "$sbLogLead : Finished Start-AlkamiService" } else { Write-Host "$sbLogLead : Calling Start-Service..." Start-Service -Name $sbServiceName -WarningAction SilentlyContinue Write-Host "$sbLogLead : Finished Start-Service" } } } while ($serviceCounter -lt $ServiceNamestoStart.Count) # If there are outstanding jobs... if ( !(Test-IsCollectionNullOrEmpty $jobs) ) { # Wait for all outstanding jobs to complete. (Wait-Job -Job $jobs) | Out-Null # Receive all the jobs. foreach($job in $jobs) { try { Receive-Job -Job $job -ErrorAction Stop } catch { $errors += $_ } } } # Report if there were errors. if ( !(Test-IsCollectionNullOrEmpty $errors) ) { $errorString = $errors -join "`n" # TODO: Evaluate risk of making this function fail. # throw "$loglead There were issues starting microservices. Errors:`n$errorString" Write-Warning "$loglead : There were issues starting microservices. Errors:`n$errorString" } else { Write-Host "`n$loglead : Done Starting Services" } if ($ReturnResults) { return $errors } }