self-hosted/teleport
self-hosted/teleport
1
0
Fork 0
mirror of https://github.com/gravitational/teleport synced 2024-10-19 16:53:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

[v10] Fix OS package repo promotion parallelism issue (#15531)

Browse source 
* Checkpoint

* Fixed OS package repo promotion parallelism

* Added migrations for v10.1.4, v10.1.2, and v9.3.14

* Marked migrations as completed

* Added migration docs

* Update build.assets/README.md

Co-authored-by: Trent Clarke <trent@goteleport.com>

Co-authored-by: Trent Clarke <trent@goteleport.com>
This commit is contained in:
fheinecke 2022-08-16 18:29:36 -05:00 committed by GitHub
parent 3dc49fdb25
commit b209b98f0d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 121 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
.drone.yml
View file

@ -5395,7 +5395,7 @@ volumes:
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/os_repos.go:270
# Generated at dronegen/os_repos.go:254
################################################
kind: pipeline
@ -5423,7 +5423,7 @@ steps:
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/os_repos.go:294
# Generated at dronegen/os_repos.go:278
################################################
kind: pipeline
@ -5452,16 +5452,21 @@ steps:
- name: Check out code
image: alpine/git:latest
commands:
- mkdir -p "/go/src/github.com/gravitational/teleport"
- mkdir -pv "/go/src/github.com/gravitational/teleport"
- cd "/go/src/github.com/gravitational/teleport"
- git clone https://github.com/gravitational/${DRONE_REPO_NAME}.git .
- git checkout "${DRONE_TAG}"
- git init && git remote add origin ${DRONE_REMOTE_URL}
- git fetch origin
- git checkout -qf "${DRONE_TAG}"
depends_on:
- Verify build is tagged
- name: Check if tag is prerelease
image: golang:1.17-alpine
image: golang:1.18-alpine
commands:
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is
a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78)
a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)
depends_on:
- Check out code
- name: Download artifacts for "${DRONE_TAG}"
image: amazon/aws-cli
commands:
@ -5532,7 +5537,7 @@ volumes:
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/os_repos.go:270
# Generated at dronegen/os_repos.go:254
################################################
kind: pipeline
@ -5560,7 +5565,7 @@ steps:
################################################
# Generated using dronegen, do not edit by hand!
# Use 'make dronegen' to update.
# Generated at dronegen/os_repos.go:294
# Generated at dronegen/os_repos.go:278
################################################
kind: pipeline
@ -5589,16 +5594,21 @@ steps:
- name: Check out code
image: alpine/git:latest
commands:
- mkdir -p "/go/src/github.com/gravitational/teleport"
- mkdir -pv "/go/src/github.com/gravitational/teleport"
- cd "/go/src/github.com/gravitational/teleport"
- git clone https://github.com/gravitational/${DRONE_REPO_NAME}.git .
- git checkout "${DRONE_TAG}"
- git init && git remote add origin ${DRONE_REMOTE_URL}
- git fetch origin
- git checkout -qf "${DRONE_TAG}"
depends_on:
- Verify build is tagged
- name: Check if tag is prerelease
image: golang:1.17-alpine
image: golang:1.18-alpine
commands:
- cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
- go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is
a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78)
a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)
depends_on:
- Check out code
- name: Download artifacts for "${DRONE_TAG}"
image: amazon/aws-cli
commands:
@ -6280,6 +6290,6 @@ volumes:
medium: memory
---
kind: signature
hmac: bc780a15ccc0771e6ec9208f853a9a065b78a16c4d6f6599aad3147856408165
hmac: 882be7c351f77a0ab527027fbf878c319a5337a664ba2347595a4a5068f0a552
...

29
build.assets/README.md
View file

@ -26,3 +26,32 @@ Or simply copy the binary out of the image using a volume (it will be copied to
```
docker run -v $(pwd)/build:/builds -it teleportbuilder cp /gopath/src/github.com/gravitational/teleport/teleport.tgz /builds
```
# OS package repo migrations
An OS package repo migration is semi-manually publishing specific releases to the new APT and YUM repos. This is required in several situations:
* A customer requests that we add an older version to the repos
* We add another OS package repo (for example APK)
* A OS package promotion fails (for example https://drone.platform.teleport.sh/gravitational/teleport/14666/1/3), requires a PR to fix, and we don't want to cut another minor version
Multiple migrations can be performed at once. To run a migration do the following:
1. Clone https://github.com/gravitational/teleport.git.
2. Change to the directory the repo was cloned to.
3. Create a new branch from master.
4. Add the Teleport versions you wish to migration as demonstrated here: https://github.com/gravitational/teleport/commit/151a2f489e3116fc7ce8f55e056529361d3233a6#diff-2e3a64c97d186491e06fb2c7ead081b7ace2b67c4a4d974a563daf7c117a2c50.
5. Set the `migrationBranch` variable to the name of the branch you created in (3) as demonstrated here: https://github.com/gravitational/teleport/commit/151a2f489e3116fc7ce8f55e056529361d3233a6#diff-2e3a64c97d186491e06fb2c7ead081b7ace2b67c4a4d974a563daf7c117a2c50.
6. Get your Drone credentials from here: https://drone.platform.teleport.sh/account.
7. Export your drone credentials as shown under "Example CLI Usage" on the Drone account page
8. Open a new terminal.
9. Run `tsh app login drone` and follow any prompts.
10. Run `tsh proxy app drone` and copy the printed socket. This should look something like `127.0.0.1:60982`
11. Switch back to your previous terminal.
12. Run `export DRONE_SERVER=http://{host:port}`, replacing `{host:port}` with the data you copied in (10)
13. Run `make dronegen`
14. Commit the two changed files and push/publish the branch
15. Open a PR merging your changes into master via https://github.com/gravitational/teleport/compare
16. Under the "checks" section, click "details" on the check labeled "continuous-integration/drone/push"
17. Once the pipelines complete, comment out the versions you added and blank out the `migrationBranch` string set in (4, 5) as demonstrated here: https://github.com/gravitational/teleport/pull/15531/commits/9095880560cfe6c93e491e39a7604b1faf72c600#diff-2e3a64c97d186491e06fb2c7ead081b7ace2b67c4a4d974a563daf7c117a2c50
18. Run `make dronegen`
19. Commit and push the changes.
20. Merge the PR and backport if required.

61
dronegen/common.go
View file

@ -19,6 +19,7 @@ import (
"fmt"
"log"
"os/exec"
"path"
"strings"
)
@ -96,6 +97,16 @@ func pushTriggerForBranch(branches ...string) trigger {
return t
}
func cloneRepoCommands(cloneDirectory, commit string) []string {
return []string{
fmt.Sprintf("mkdir -pv %q", cloneDirectory),
fmt.Sprintf("cd %q", cloneDirectory),
`git init && git remote add origin ${DRONE_REMOTE_URL}`,
`git fetch origin`,
fmt.Sprintf("git checkout -qf %q", commit),
}
}
type buildType struct {
os string
arch string
@ -105,9 +116,10 @@ type buildType struct {
}
// Description provides a human-facing description of the artifact, e.g.:
// Windows 64-bit (tsh client only)
// Linux ARMv7 (32-bit)
// MacOS Intel .pkg installer
//
// Windows 64-bit (tsh client only)
// Linux ARMv7 (32-bit)
// MacOS Intel .pkg installer
func (b *buildType) Description(packageType string, extraQualifications ...string) string {
var result string
@ -241,3 +253,46 @@ func waitForDockerStep() step {
Volumes: dockerVolumeRefs(),
}
}
func verifyValidPromoteRunSteps(checkoutPath, commit string, isParallelismEnabled bool) []step {
tagStep := verifyTaggedStep()
cloneStep := cloneRepoStep(checkoutPath, commit)
verifyStep := verifyNotPrereleaseStep(checkoutPath)
if isParallelismEnabled {
cloneStep.DependsOn = []string{tagStep.Name}
verifyStep.DependsOn = []string{cloneStep.Name}
}
return []step{tagStep, cloneStep, verifyStep}
}
func verifyTaggedStep() step {
return step{
Name: "Verify build is tagged",
Image: "alpine:latest",
Commands: []string{
"[ -n ${DRONE_TAG} ] || (echo 'DRONE_TAG is not set. Is the commit tagged?' && exit 1)",
},
}
}
// Note that tags are also valid here as a tag refers to a specific commit
func cloneRepoStep(clonePath, commit string) step {
return step{
Name: "Check out code",
Image: "alpine/git:latest",
Commands: cloneRepoCommands(clonePath, commit),
}
}
func verifyNotPrereleaseStep(checkoutPath string) step {
return step{
Name: "Check if tag is prerelease",
Image: "golang:1.18-alpine",
Commands: []string{
fmt.Sprintf("cd %q", path.Join(checkoutPath, "build.assets", "tooling")),
"go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)",
},
}
}

34
dronegen/os_repos.go
View file

@ -31,11 +31,11 @@ func promoteBuildOsRepoPipelines() []pipeline {
// Used for one-off migrations of older versions.
// Use cases include:
// * We want to support another OS while providing backwards compatibility
// * We want to support another OS version while providing backwards compatibility
// * A customer wants to be able to install an older version via APT/YUM even if we
// no longer support it
// * RPM migrations after new YUM pipeline is done
// - We want to support another OS while providing backwards compatibility
// - We want to support another OS version while providing backwards compatibility
// - A customer wants to be able to install an older version via APT/YUM even if we
// no longer support it
// - RPM migrations after new YUM pipeline is done
func artifactMigrationPipeline() []pipeline {
migrationVersions := []string{
// These versions were migrated as a part of the new `promoteAptPipeline`
@ -83,9 +83,12 @@ func artifactMigrationPipeline() []pipeline {
// "v9.3.10",
// "v9.3.12",
// "v9.3.13",
// "v9.3.14",
// "v10.0.0",
// "v10.0.1",
// "v10.0.2",
// "v10.1.2",
// "v10.1.4",
}
// Pushing to this branch will trigger the listed versions to be migrated. Typically this should be
// the branch that these changes are being committed to.
@ -192,26 +195,7 @@ func (optpb *OsPackageToolPipelineBuilder) buildPromoteOsPackagePipeline() pipel
p.Trigger = triggerPromote
p.Trigger.Repo.Include = []string{"gravitational/teleport"}
setupSteps := []step{
{
Name: "Verify build is tagged",
Image: "alpine:latest",
Commands: []string{
"[ -n ${DRONE_TAG} ] || (echo 'DRONE_TAG is not set. Is the commit tagged?' && exit 1)",
},
},
}
setupSteps = append(setupSteps, p.Steps...)
setupSteps = append(setupSteps,
step{
Name: "Check if tag is prerelease",
Image: "golang:1.17-alpine",
Commands: []string{
fmt.Sprintf("cd %q", path.Join(checkoutPath, "build.assets", "tooling")),
"go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78)",
},
},
)
setupSteps := verifyValidPromoteRunSteps(checkoutPath, commitName, true)
setupStepNames := make([]string, 0, len(setupSteps))
for _, setupStep := range setupSteps {