diff --git a/.drone.yml b/.drone.yml index 23c10006f1c..59bc9fa530a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -5395,7 +5395,7 @@ volumes: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/os_repos.go:270 +# Generated at dronegen/os_repos.go:254 ################################################ kind: pipeline @@ -5423,7 +5423,7 @@ steps: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/os_repos.go:294 +# Generated at dronegen/os_repos.go:278 ################################################ kind: pipeline @@ -5452,16 +5452,21 @@ steps: - name: Check out code image: alpine/git:latest commands: - - mkdir -p "/go/src/github.com/gravitational/teleport" + - mkdir -pv "/go/src/github.com/gravitational/teleport" - cd "/go/src/github.com/gravitational/teleport" - - git clone https://github.com/gravitational/${DRONE_REPO_NAME}.git . - - git checkout "${DRONE_TAG}" + - git init && git remote add origin ${DRONE_REMOTE_URL} + - git fetch origin + - git checkout -qf "${DRONE_TAG}" + depends_on: + - Verify build is tagged - name: Check if tag is prerelease - image: golang:1.17-alpine + image: golang:1.18-alpine commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is - a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78) + a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78) + depends_on: + - Check out code - name: Download artifacts for "${DRONE_TAG}" image: amazon/aws-cli commands: @@ -5532,7 +5537,7 @@ volumes: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/os_repos.go:270 +# Generated at dronegen/os_repos.go:254 ################################################ kind: pipeline @@ -5560,7 +5565,7 @@ steps: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/os_repos.go:294 +# Generated at dronegen/os_repos.go:278 ################################################ kind: pipeline @@ -5589,16 +5594,21 @@ steps: - name: Check out code image: alpine/git:latest commands: - - mkdir -p "/go/src/github.com/gravitational/teleport" + - mkdir -pv "/go/src/github.com/gravitational/teleport" - cd "/go/src/github.com/gravitational/teleport" - - git clone https://github.com/gravitational/${DRONE_REPO_NAME}.git . - - git checkout "${DRONE_TAG}" + - git init && git remote add origin ${DRONE_REMOTE_URL} + - git fetch origin + - git checkout -qf "${DRONE_TAG}" + depends_on: + - Verify build is tagged - name: Check if tag is prerelease - image: golang:1.17-alpine + image: golang:1.18-alpine commands: - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is - a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78) + a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78) + depends_on: + - Check out code - name: Download artifacts for "${DRONE_TAG}" image: amazon/aws-cli commands: @@ -6280,6 +6290,6 @@ volumes: medium: memory --- kind: signature -hmac: bc780a15ccc0771e6ec9208f853a9a065b78a16c4d6f6599aad3147856408165 +hmac: 882be7c351f77a0ab527027fbf878c319a5337a664ba2347595a4a5068f0a552 ... diff --git a/build.assets/README.md b/build.assets/README.md index cc7b9a372d3..f79703dde70 100644 --- a/build.assets/README.md +++ b/build.assets/README.md @@ -26,3 +26,32 @@ Or simply copy the binary out of the image using a volume (it will be copied to ``` docker run -v $(pwd)/build:/builds -it teleportbuilder cp /gopath/src/github.com/gravitational/teleport/teleport.tgz /builds ``` + +# OS package repo migrations + +An OS package repo migration is semi-manually publishing specific releases to the new APT and YUM repos. This is required in several situations: +* A customer requests that we add an older version to the repos +* We add another OS package repo (for example APK) +* A OS package promotion fails (for example https://drone.platform.teleport.sh/gravitational/teleport/14666/1/3), requires a PR to fix, and we don't want to cut another minor version + +Multiple migrations can be performed at once. To run a migration do the following: +1. Clone https://github.com/gravitational/teleport.git. +2. Change to the directory the repo was cloned to. +3. Create a new branch from master. +4. Add the Teleport versions you wish to migration as demonstrated here: https://github.com/gravitational/teleport/commit/151a2f489e3116fc7ce8f55e056529361d3233a6#diff-2e3a64c97d186491e06fb2c7ead081b7ace2b67c4a4d974a563daf7c117a2c50. +5. Set the `migrationBranch` variable to the name of the branch you created in (3) as demonstrated here: https://github.com/gravitational/teleport/commit/151a2f489e3116fc7ce8f55e056529361d3233a6#diff-2e3a64c97d186491e06fb2c7ead081b7ace2b67c4a4d974a563daf7c117a2c50. +6. Get your Drone credentials from here: https://drone.platform.teleport.sh/account. +7. Export your drone credentials as shown under "Example CLI Usage" on the Drone account page +8. Open a new terminal. +9. Run `tsh app login drone` and follow any prompts. +10. Run `tsh proxy app drone` and copy the printed socket. This should look something like `127.0.0.1:60982` +11. Switch back to your previous terminal. +12. Run `export DRONE_SERVER=http://{host:port}`, replacing `{host:port}` with the data you copied in (10) +13. Run `make dronegen` +14. Commit the two changed files and push/publish the branch +15. Open a PR merging your changes into master via https://github.com/gravitational/teleport/compare +16. Under the "checks" section, click "details" on the check labeled "continuous-integration/drone/push" +17. Once the pipelines complete, comment out the versions you added and blank out the `migrationBranch` string set in (4, 5) as demonstrated here: https://github.com/gravitational/teleport/pull/15531/commits/9095880560cfe6c93e491e39a7604b1faf72c600#diff-2e3a64c97d186491e06fb2c7ead081b7ace2b67c4a4d974a563daf7c117a2c50 +18. Run `make dronegen` +19. Commit and push the changes. +20. Merge the PR and backport if required. \ No newline at end of file diff --git a/dronegen/common.go b/dronegen/common.go index 586f8214c28..e1ad715f5a0 100644 --- a/dronegen/common.go +++ b/dronegen/common.go @@ -19,6 +19,7 @@ import ( "fmt" "log" "os/exec" + "path" "strings" ) @@ -96,6 +97,16 @@ func pushTriggerForBranch(branches ...string) trigger { return t } +func cloneRepoCommands(cloneDirectory, commit string) []string { + return []string{ + fmt.Sprintf("mkdir -pv %q", cloneDirectory), + fmt.Sprintf("cd %q", cloneDirectory), + `git init && git remote add origin ${DRONE_REMOTE_URL}`, + `git fetch origin`, + fmt.Sprintf("git checkout -qf %q", commit), + } +} + type buildType struct { os string arch string @@ -105,9 +116,10 @@ type buildType struct { } // Description provides a human-facing description of the artifact, e.g.: -// Windows 64-bit (tsh client only) -// Linux ARMv7 (32-bit) -// MacOS Intel .pkg installer +// +// Windows 64-bit (tsh client only) +// Linux ARMv7 (32-bit) +// MacOS Intel .pkg installer func (b *buildType) Description(packageType string, extraQualifications ...string) string { var result string @@ -241,3 +253,46 @@ func waitForDockerStep() step { Volumes: dockerVolumeRefs(), } } + +func verifyValidPromoteRunSteps(checkoutPath, commit string, isParallelismEnabled bool) []step { + tagStep := verifyTaggedStep() + cloneStep := cloneRepoStep(checkoutPath, commit) + verifyStep := verifyNotPrereleaseStep(checkoutPath) + + if isParallelismEnabled { + cloneStep.DependsOn = []string{tagStep.Name} + verifyStep.DependsOn = []string{cloneStep.Name} + } + + return []step{tagStep, cloneStep, verifyStep} +} + +func verifyTaggedStep() step { + return step{ + Name: "Verify build is tagged", + Image: "alpine:latest", + Commands: []string{ + "[ -n ${DRONE_TAG} ] || (echo 'DRONE_TAG is not set. Is the commit tagged?' && exit 1)", + }, + } +} + +// Note that tags are also valid here as a tag refers to a specific commit +func cloneRepoStep(clonePath, commit string) step { + return step{ + Name: "Check out code", + Image: "alpine/git:latest", + Commands: cloneRepoCommands(clonePath, commit), + } +} + +func verifyNotPrereleaseStep(checkoutPath string) step { + return step{ + Name: "Check if tag is prerelease", + Image: "golang:1.18-alpine", + Commands: []string{ + fmt.Sprintf("cd %q", path.Join(checkoutPath, "build.assets", "tooling")), + "go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)", + }, + } +} diff --git a/dronegen/os_repos.go b/dronegen/os_repos.go index 8df081ba37b..3ad05acf2aa 100644 --- a/dronegen/os_repos.go +++ b/dronegen/os_repos.go @@ -31,11 +31,11 @@ func promoteBuildOsRepoPipelines() []pipeline { // Used for one-off migrations of older versions. // Use cases include: -// * We want to support another OS while providing backwards compatibility -// * We want to support another OS version while providing backwards compatibility -// * A customer wants to be able to install an older version via APT/YUM even if we -// no longer support it -// * RPM migrations after new YUM pipeline is done +// - We want to support another OS while providing backwards compatibility +// - We want to support another OS version while providing backwards compatibility +// - A customer wants to be able to install an older version via APT/YUM even if we +// no longer support it +// - RPM migrations after new YUM pipeline is done func artifactMigrationPipeline() []pipeline { migrationVersions := []string{ // These versions were migrated as a part of the new `promoteAptPipeline` @@ -83,9 +83,12 @@ func artifactMigrationPipeline() []pipeline { // "v9.3.10", // "v9.3.12", // "v9.3.13", + // "v9.3.14", // "v10.0.0", // "v10.0.1", // "v10.0.2", + // "v10.1.2", + // "v10.1.4", } // Pushing to this branch will trigger the listed versions to be migrated. Typically this should be // the branch that these changes are being committed to. @@ -192,26 +195,7 @@ func (optpb *OsPackageToolPipelineBuilder) buildPromoteOsPackagePipeline() pipel p.Trigger = triggerPromote p.Trigger.Repo.Include = []string{"gravitational/teleport"} - setupSteps := []step{ - { - Name: "Verify build is tagged", - Image: "alpine:latest", - Commands: []string{ - "[ -n ${DRONE_TAG} ] || (echo 'DRONE_TAG is not set. Is the commit tagged?' && exit 1)", - }, - }, - } - setupSteps = append(setupSteps, p.Steps...) - setupSteps = append(setupSteps, - step{ - Name: "Check if tag is prerelease", - Image: "golang:1.17-alpine", - Commands: []string{ - fmt.Sprintf("cd %q", path.Join(checkoutPath, "build.assets", "tooling")), - "go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78)", - }, - }, - ) + setupSteps := verifyValidPromoteRunSteps(checkoutPath, commitName, true) setupStepNames := make([]string, 0, len(setupSteps)) for _, setupStep := range setupSteps {