function Checkpoint-AllAvailableRepos { <# .SYNOPSIS Backup all the available repositories you have access to from bitbucket to a known location .PARAMETER StartingFolder [string] Where do you want to start storing the repos. .PARAMETER ApiKey [string] A bitbucket API key .PARAMETER BranchName [string] A target branch name. If not present will default to, in order, 'main', 'master', 'develop' .PARAMETER ReplacePath [string] If you want to replace part of your checkout path from the clone url. This is mostly useful with ssh config files. .PARAMETER ReplacePrefix [string] Used with ReplacePath, only really useful if you use ssh config files with heavy modification #> [CmdletBinding()] Param( [Parameter(Mandatory = $false, Position = 0)] [string]$Path = (Get-RepoCheckpointPath), [Parameter(Mandatory = $false, Position = 1)] [string]$ApiKey = "", [Parameter(Mandatory = $false, Position = 2)] [string]$BranchName, [Parameter(Mandatory = $false, Position = 3)] [string]$ReplacePath = '', [Parameter(Mandatory = $false, Position = 4)] [string]$ReplacePrefix = '', # TODO: Project-only list instead of project-skip list [Parameter(Mandatory = $false, Position = 5)] [string[]]$ProjectSkipList = @(), [Parameter(Mandatory = $false, Position = 6)] [string[]]$RepositorySkipList = @('iosdev','dba','themes'), [Parameter(Mandatory = $false, Position = 7)] [object[]]$SpecificProjectRepoSkipList = @(@{Key='dsn';Slug='old-design-repo-old';},@{Key='dsn';Slug='internal';}) ) $logLead = (Get-LogLeadName) # Fix some git nonsense Set-EnvironmentVariable -Name "GIT_REDIRECT_STDERR" -Value '2>&1' -StoreName Machine if ([string]::IsNullOrWhiteSpace($Path)) { throw "$logLead : No path found for checkpoint location!" } $branchNameEmpty = [string]::IsNullOrWhiteSpace($BranchName) $currentLocation = (Get-Location) if ([string]::IsNullOrWhiteSpace($ApiKey)) { $ApiKey = (Get-EnvironmentVariable -Name "Checkpoint_AllAvailableRepos_ApiKey" 6>$null 5>$null 4>$null 3>$null) } if (!(Test-Path $Path)) { (New-Item -ItemType Directory -Path $Path -Force) | Out-Null } Set-Location $Path $removalFilePath = (Join-Path -Path $Path -ChildPath "RemoveFiles.txt") $pathReferencePath = (Join-Path -Path $Path -ChildPath "PathReference.txt") (New-Item -Path $removalFilePath -ItemType File -Force -ErrorAction Ignore) | Out-Null (New-Item -Path $pathReferencePath -ItemType File -Force -ErrorAction Ignore) | Out-Null $replaceUrlParts = ![string]::IsNullOrWhiteSpace($ReplacePath) -and ![string]::IsNullOrWhiteSpace($ReplacePrefix) $sshConfigEntries = @() $hasSshConfigEntries = $false if (!$replaceUrlParts) { try { $sshConfigEntries = @(Get-SSHConfigEntries) $hasSshConfigEntries = $null -ne $sshConfigEntries } catch { Write-Verbose "$logLead : Was not able to get a config entry, this might be a problem, but probably ok" } } $currentUserCredential = $null try { $currentUserCredential = Get-CredentialFromEnvironmentVariables } catch {} if ([string]::IsNullOrWhiteSpace($ApiKey)) { Write-Warning "$logLead : Can not continue without an API Key for Bitbucket." Write-Warning "$logLead : Please go here https://bitbucket.corp.alkami.net/plugins/servlet/access-tokens/manage and create a key (Personal Access Token) with project and repository read permissions." Write-Warning "$logLead : Once you have a key, call Set-EnvironmentVariable -Name `"Checkpoint_AllAvailableRepos_ApiKey`" -StoreName User -Value ''" Write-Warning "$logLead : This value can also be passed in on a command line parameter -ApiKey" throw "$logLead : Must provide API key to continue" } # TODO: Refactor this to a cacheable object that can be ps1xml'd as well for friendly output $projectResponse = (Invoke-WebRequest -Uri "https://bitbucket.corp.alkami.net/rest/api/1.0/projects?limit=1000" -UseBasicParsing -Headers @{ "Authorization" = "Bearer $ApiKey"}).Content | ConvertFrom-Json $projectList = $projectResponse.values $projectList | ConvertTo-Json -Depth 10 | Set-Content -Path (Join-Path $Path "projects.json") $allRepos = Invoke-JobRunner -JobInputs $projectList -ReturnObjects -Credential $currentUserCredential -AdditionalArguments $logLead,$Path,$ProjectSkipList,$RepositorySkipList,$SpecificProjectRepoSkipList,$ReplacePrefix,$ReplacePath,$hasSshConfigEntries,$sshConfigEntries,$ApiKey -ScriptBlock { param( $project, $logLead,$Path,$ProjectSkipList,$RepositorySkipList,$SpecificProjectRepoSkipList,$ReplacePrefix,$ReplacePath,$hasSshConfigEntries,$sshConfigEntries,$ApiKey ) # $logLead = $otherParams[0] # $Path = $otherParams[1] # $ProjectSkipList = $otherParams[2] # $SpecificProjectRepoSkipList = $otherParams[3] # $ReplacePrefix = $otherParams[4] # $ReplacePath = $otherParams[5] # $hasSshConfigEntries = $otherParams[6] # $ApiKey = $otherParams[7] if (Any $ProjectSkipList.Where({$_ -eq $project.Key})) { Write-Host "$logLead : Skipping project [$($project.Key)] per parameter list instruction" continue } # TODO: Change this to pull a common bb url $projectKey = $project.Key $repoResponse = (Invoke-WebRequest -Uri "https://bitbucket.corp.alkami.net/rest/api/1.0/projects/$($projectKey)/repos?limit=1000" -UseBasicParsing -Headers @{ "Authorization" = "Bearer $ApiKey"}).Content | ConvertFrom-Json $repoResponse | ConvertTo-Json -Depth 10 | Set-Content -Path (Join-Path $Path "projects.$projectKey.json") $repoList = $repoResponse.values $repos = @() foreach($repo in $repoList) { if (Any $RepositorySkipList.Where({$_ -eq $repo.slug})) { Write-Host "$logLead : Skipping repository [$($project.Key)/$($repo.slug)] per parameter list instruction" continue } if ($null -ne $repo.links.clone) { if (Any $SpecificProjectRepoSkipList.Where({$_.Key -eq $project.Key -and $_.Slug -eq $repo.slug})) { # Helpful for skipping specific project/repo combos, and/or for skipping the design repos Write-Host "$logLead : Skipping repository [$($project.Key)/$($repo.slug)] per parameter list instruction" continue } $cloneUrl = $repo.links.clone.Where({$_.name -eq 'ssh'}).href if ([string]::IsNullOrWhiteSpace($cloneUrl)) { $cloneUrl = "unknown" } $clonePath = (Join-Path (Join-Path $Path $project.Key) $repo.slug) $bbUrl = $cloneUrl if ($replaceUrlParts) { $bbUrl = $cloneUrl -replace $ReplacePrefix, $ReplacePath } elseif ($hasSshConfigEntries) { $uriBuilder = [System.UriBuilder]::new($cloneUrl) $sshConfigEntry = @($sshConfigEntries.Where({$_.Hostname -eq $uriBuilder.Host}))[0] if ($null -ne $sshConfigEntry) { $uriBuilder.Host = $sshConfigEntry.Host if (![string]::IsNullOrWhiteSpace($sshConfigEntry.User)) { $uriBuilder.UserName = $null } if (![string]::IsNullOrWhiteSpace($sshConfigEntry.Port)) { $uriBuilder.Port = -1 } $bbUrl = "$uriBuilder" # fast convert to a url } } $repos += @{ key = $project.Key; slug = $repo.slug; cloneUrl = $cloneUrl; clonePath = $clonePath; bbUrl = $bbUrl } } } return $repos } # Write-Output $allRepos $allRepos | ConvertTo-Json -Depth 10 | Set-Content -Path (Join-Path $Path "repos.json") $allProjectFolders = (Get-ChildItem $Path -Directory) foreach($project in $allProjectFolders) { $projectName = $project.Name $projectRepos = @($allRepos.Where({ $_.Key -eq $projectName })) if ($projectRepos.Length -eq 0) { "Remove-Item $($project.FullName) -Recurse -Force" | Add-Content -Path $removalFilePath continue } $allRepoFolders = (Get-ChildItem $project.FullName -Directory) foreach($repo in $allRepoFolders) { $repoName = $repo.Name $foundRepo = @($projectRepos.Where({ $_.Slug -eq $repoName })) if ($foundRepo.Length -eq 0) { $movedMaybe = @($allRepos.Where({$_.Slug -eq $repoName})) if ($movedMaybe.Length -gt 0) { foreach($maybe in $movedMaybe) { $maybePath = (Join-Path $Path (Join-Path $maybe.Key $maybe.Slug)) if (Test-Path $maybePath) { "# Did $repoName move to here? $maybePath" | Add-Content -Path $removalFilePath } } } "Remove-Item $($repo.FullName) -Recurse -Force" | Add-Content -Path $removalFilePath } } } Invoke-JobRunner -JobInputs $allRepos -ReturnObjects -Credential $currentUserCredential -AdditionalArguments $Path,$BranchName,$branchNameEmpty,$removalFilePath -ScriptBlock { param($repo,$Path,$BranchName,$branchNameEmpty,$removalFilePath) $directoryName = $repo.key $slug = $repo.slug $projectDirectory = $repo.clonePath $bbUrl = $repo.bbUrl $cloneUrl = $repo.cloneUrl $currentTime = [DateTime]::Now.ToString("yyyy-MM-dd HH:mm:ss") if(!(Test-IsGitFolderRoot $projectDirectory)) { (New-Item -ItemType Directory -Path $projectDirectory -Force) | Out-Null Write-Host "git clone $bbUrl $projectDirectory" (git clone $bbUrl $projectDirectory) | Out-Null } else { (Set-Location $projectDirectory) | Out-Null if (@(git diff --stat).length) { (git add -A . --quiet) | Out-Null (git stash --quiet) | Out-Null (git reset --hard --quiet) | Out-Null } (git remote set-url origin $bbUrl --quiet) | Out-Null (git fetch origin --quiet) | Out-Null $branches = (Get-GitBranchNames -Path $projectDirectory) $checkoutBranchName = "develop" if ((!$branchNameEmpty) -and ($branches -contains $BranchName)) { $checkoutBranchName = $BranchName } else { foreach($branch in $branches) { if ($branches -contains $branch) { $checkoutBranchName = $branch continue } } } (git checkout $checkoutBranchName --quiet) | Out-Null (git pull origin $checkoutBranchName --quiet) | Out-Null } if ((Get-ChildItem -Path $projectDirectory).Count -eq 0) { # Folder is empty "# Could not find any files in $projectDirectory" | Add-Content -Path $removalFilePath } return "$currentTime | $projectDirectory | $directoryName | $slug | $bbUrl | $cloneUrl" } | Add-Content -Path $pathReferencePath foreach($folder in $allRepos.clonePath) { if ((Get-ChildItem -Path $folder).Count -eq 0) { # Folder is empty Write-Host "Could not find any files in $folder" } } Set-Location $currentLocation }