From 61376de9b8b510ef998f6ce1155d4593f90b2604 Mon Sep 17 00:00:00 2001 From: godofredoc Date: Mon, 28 Nov 2022 20:14:24 -0800 Subject: [PATCH] Generate local metadata even when not publishing. (#116087) * Generate local metadata even when not publishing. For SLSA compliance we need to separate the fetch, compile and upload steps of release artifacts. Translating this to the packaging workflows the fetch step will checkout the prepare_package script at main ToT, the compile step generate the bundle archives and the recipes will upload the artifact bundles as part of the upload stage. This change adds functionality to generate both the release bundle and the updated metadata file in a way that both files can be uploaded as part of the upload stage. Bug: https://github.com/flutter/flutter/issues/115487 * Address comments. * Update tests. --- dev/bots/prepare_package.dart | 30 ++++++++++++++++++------- dev/bots/test/prepare_package_test.dart | 15 ++++++++----- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/dev/bots/prepare_package.dart b/dev/bots/prepare_package.dart index e7b4e4b74a6..1a73b54bfb6 100644 --- a/dev/bots/prepare_package.dart +++ b/dev/bots/prepare_package.dart @@ -637,6 +637,12 @@ class ArchivePublisher { dest: destGsPath, ); assert(tempDir.existsSync()); + final String gcsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}'; + await _publishMetadata(gcsPath); + } + + /// Downloads and updates the metadata file without publishing it. + Future generateLocalMetadata() async { await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}'); } @@ -705,6 +711,13 @@ class ArchivePublisher { const JsonEncoder encoder = JsonEncoder.withIndent(' '); metadataFile.writeAsStringSync(encoder.convert(jsonData)); + } + + /// Publishes the metadata file to GCS. + Future _publishMetadata(String gsPath) async { + final File metadataFile = File( + path.join(tempDir.absolute.path, getMetadataFilename(platform)), + ); await _cloudCopy( src: metadataFile.absolute.path, dest: gsPath, @@ -899,15 +912,16 @@ Future main(List rawArguments) async { try { final Map version = await creator.initializeRepo(); final File outputFile = await creator.createArchive(); + final ArchivePublisher publisher = ArchivePublisher( + tempDir, + revision, + branch, + version, + outputFile, + dryRun, + ); + await publisher.generateLocalMetadata(); if (parsedArguments['publish'] as bool) { - final ArchivePublisher publisher = ArchivePublisher( - tempDir, - revision, - branch, - version, - outputFile, - dryRun, - ); await publisher.publishArchive(parsedArguments['force'] as bool); } } on PreparePackageException catch (e) { diff --git a/dev/bots/test/prepare_package_test.dart b/dev/bots/test/prepare_package_test.dart index 93baa9c8b91..611764241a0 100644 --- a/dev/bots/test/prepare_package_test.dart +++ b/dev/bots/test/prepare_package_test.dart @@ -434,10 +434,10 @@ void main() { File(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ // This process fails because the file does NOT already exist + '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- stat $gsArchivePath': [ProcessResult(0, 1, '', '')], '$gsutilCall -- rm $gsArchivePath': null, '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $gsArchivePath': null, - '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; @@ -461,6 +461,7 @@ void main() { platform: platform, ); assert(tempDir.existsSync()); + await publisher.generateLocalMetadata(); await publisher.publishArchive(); final File releaseFile = File(jsonPath); @@ -534,10 +535,10 @@ void main() { File(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ // This process fails because the file does NOT already exist + '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- stat $gsArchivePath': [ProcessResult(0, 1, '', '')], '$gsutilCall -- rm $gsArchivePath': null, '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $gsArchivePath': null, - '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; @@ -561,6 +562,7 @@ void main() { platform: platform, ); assert(tempDir.existsSync()); + await publisher.generateLocalMetadata(); await publisher.publishArchive(); final File releaseFile = File(jsonPath); @@ -598,10 +600,10 @@ void main() { File(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ // This process fails because the file does NOT already exist + '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- stat $gsArchivePath': [ProcessResult(0, 1, '', '')], '$gsutilCall -- rm $gsArchivePath': null, '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $gsArchivePath': null, - '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; @@ -625,6 +627,7 @@ void main() { platform: platform, ); assert(tempDir.existsSync()); + await publisher.generateLocalMetadata(); await publisher.publishArchive(); final File releaseFile = File(jsonPath); @@ -678,10 +681,10 @@ void main() { File(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ // This process fails because the file does NOT already exist + '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- stat $gsArchivePath': [ProcessResult(0, 1, '', '')], '$gsutilCall -- rm $gsArchivePath': null, '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $gsArchivePath': null, - '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; @@ -705,6 +708,7 @@ void main() { platform: platform, ); assert(tempDir.existsSync()); + await publisher.generateLocalMetadata(); await publisher.publishArchive(); final File releaseFile = File(jsonPath); @@ -799,14 +803,15 @@ void main() { File(jsonPath).writeAsStringSync(releasesJson); File(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ + '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsArchivePath': null, '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $gsArchivePath': null, - '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; processManager.addCommands(convertResults(calls)); assert(tempDir.existsSync()); + await publisher.generateLocalMetadata(); await publisher.publishArchive(true); }); });