Make PRSS CDN optional (#198631)

* empty commit

* mooncake creds expired

* more logs

* prevent exit

* log

* async is hard

* 💄

* temporarily ignore PRSS errors

* whitespace

build/azure-pipelines/common/publish.ts

@@ -353,7 +353,6 @@ async function releaseAndProvision(
 	const credential = new ClientSecretCredential(provisionTenantId, provisionAADUsername, provisionAADPassword);
 	const accessToken = await credential.getToken(['https://microsoft.onmicrosoft.com/DS.Provisioning.WebApi/.default']);
 	const service = new ProvisionService(log, accessToken.token);
-
 	await service.provision(release.releaseId, release.fileId, fileName);
 
 	return result;
@@ -628,7 +627,7 @@ function getRealType(type: string) {
 	}
 }
 
-async function uploadAssetLegacy(log: (...args: any[]) => void, quality: string, commit: string, filePath: string): Promise<{ assetUrl: string; mooncakeUrl: string }> {
+async function uploadAssetLegacy(log: (...args: any[]) => void, quality: string, commit: string, filePath: string): Promise<string> {
 	const fileName = path.basename(filePath);
 	const blobName = commit + '/' + fileName;
 
@@ -645,60 +644,17 @@ async function uploadAssetLegacy(log: (...args: any[]) => void, quality: string,
 		}
 	};
 
-	const uploadPromises: Promise<void>[] = [];
+	log(`Checking for blob in Azure...`);
 
-	uploadPromises.push((async (): Promise<void> => {
-		log(`Checking for blob in Azure...`);
-
-		if (await blobClient.exists()) {
-			log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
-			throw new Error(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
-		} else {
-			log(`Uploading blobs to Azure storage...`);
-			await blobClient.uploadFile(filePath, blobOptions);
-			log('Blob successfully uploaded to Azure storage.');
-		}
-	})());
-
-	const shouldUploadToMooncake = /true/i.test(e('VSCODE_PUBLISH_TO_MOONCAKE'));
-
-	if (shouldUploadToMooncake) {
-		const mooncakeCredential = new ClientSecretCredential(e('AZURE_MOONCAKE_TENANT_ID'), e('AZURE_MOONCAKE_CLIENT_ID'), e('AZURE_MOONCAKE_CLIENT_SECRET'));
-		const mooncakeBlobServiceClient = new BlobServiceClient(`https://vscode.blob.core.chinacloudapi.cn`, mooncakeCredential, { retryOptions: { retryPolicyType: StorageRetryPolicyType.FIXED, tryTimeoutInMs: 5 * 60 * 1000 } });
-		const mooncakeContainerClient = mooncakeBlobServiceClient.getContainerClient(quality);
-		const mooncakeBlobClient = mooncakeContainerClient.getBlockBlobClient(blobName);
-
-		uploadPromises.push((async (): Promise<void> => {
-			log(`Checking for blob in Mooncake Azure...`);
-
-			if (await mooncakeBlobClient.exists()) {
-				log(`Mooncake Blob ${quality}, ${blobName} already exists, not publishing again.`);
-				throw new Error(`Mooncake Blob ${quality}, ${blobName} already exists, not publishing again.`);
-			} else {
-				log(`Uploading blobs to Mooncake Azure storage...`);
-				await mooncakeBlobClient.uploadFile(filePath, blobOptions);
-				log('Blob successfully uploaded to Mooncake Azure storage.');
-			}
-		})());
-	}
-
-	const promiseResults = await Promise.allSettled(uploadPromises);
-	const rejectedPromiseResults = promiseResults.filter(result => result.status === 'rejected') as PromiseRejectedResult[];
-
-	if (rejectedPromiseResults.length === 0) {
-		log('All blobs successfully uploaded.');
-	} else if (rejectedPromiseResults[0]?.reason?.message?.includes('already exists')) {
-		log('Some blobs successfully uploaded.');
+	if (await blobClient.exists()) {
+		log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
 	} else {
-		// eslint-disable-next-line no-throw-literal
-		throw rejectedPromiseResults[0]?.reason;
+		log(`Uploading blobs to Azure storage...`);
+		await blobClient.uploadFile(filePath, blobOptions);
+		log('Blob successfully uploaded to Azure storage.');
 	}
 
-	const assetUrl = `${e('AZURE_CDN_URL')}/${quality}/${blobName}`;
-	const blobPath = new URL(assetUrl).pathname;
-	const mooncakeUrl = `${e('MOONCAKE_CDN_URL')}${blobPath}`;
-
-	return { assetUrl, mooncakeUrl };
+	return `${e('AZURE_CDN_URL')}/${quality}/${blobName}`;
 }
 
 async function processArtifact(artifact: Artifact, artifactFilePath: string): Promise<void> {
@@ -719,7 +675,7 @@ async function processArtifact(artifact: Artifact, artifactFilePath: string): Pr
 	const stream = fs.createReadStream(artifactFilePath);
 	const [sha1hash, sha256hash] = await Promise.all([hashStream('sha1', stream), hashStream('sha256', stream)]);
 
-	const [{ assetUrl, mooncakeUrl }, prssUrl] = await Promise.all([
+	const [cdnSettledResult, prssSettledResult] = await Promise.allSettled([
 		uploadAssetLegacy(log, quality, commit, artifactFilePath),
 		releaseAndProvision(
 			log,
@@ -736,11 +692,20 @@ async function processArtifact(artifact: Artifact, artifactFilePath: string): Pr
 		)
 	]);
 
-	const asset: Asset = { platform, type, url: assetUrl, hash: sha1hash, mooncakeUrl, prssUrl, sha256hash, size, supportsFastUpdate: true };
+	if (cdnSettledResult.status === 'rejected') {
+		throw cdnSettledResult.reason;
+	} else if (prssSettledResult.status === 'rejected') { // TODO@joaomoreno, let's temporarily ignore these errors
+		console.error(prssSettledResult.reason);
+	}
+
+	const assetUrl = cdnSettledResult.value;
+	const prssUrl = prssSettledResult.status === 'fulfilled' ? prssSettledResult.value : undefined;
+
+	const asset: Asset = { platform, type, url: assetUrl, hash: sha1hash, mooncakeUrl: prssUrl, prssUrl, sha256hash, size, supportsFastUpdate: true };
 	log('Creating asset...', JSON.stringify(asset));
 
 	await retry(async (attempt) => {
-		log(`Creating asset in Cosmos DB(attempt ${attempt})...`);
+		log(`Creating asset in Cosmos DB (attempt ${attempt})...`);
 		const aadCredentials = new ClientSecretCredential(e('AZURE_TENANT_ID'), e('AZURE_CLIENT_ID'), e('AZURE_CLIENT_SECRET'));
 		const client = new CosmosClient({ endpoint: e('AZURE_DOCUMENTDB_ENDPOINT'), aadCredentials });
 		const scripts = client.database('builds').container(quality).scripts;
@@ -777,6 +742,7 @@ async function main() {
 	if (e('VSCODE_BUILD_STAGE_MACOS') === 'True') { stages.add('macOS'); }
 	if (e('VSCODE_BUILD_STAGE_WEB') === 'True') { stages.add('Web'); }
 
+	let resultPromise = Promise.resolve<PromiseSettledResult<void>[]>([]);
 	const operations: { name: string; operation: Promise<void> }[] = [];
 
 	while (true) {
@@ -806,7 +772,7 @@ async function main() {
 
 			await retry(async (attempt) => {
 				const start = Date.now();
-				console.log(`[${artifact.name}]Downloading(attempt ${attempt})...`);
+				console.log(`[${artifact.name}] Downloading (attempt ${attempt})...`);
 				await downloadArtifact(artifact, artifactZipPath);
 				const archiveSize = fs.statSync(artifactZipPath).size;
 				const downloadDurationS = (Date.now() - start) / 1000;
@@ -830,7 +796,7 @@ async function main() {
 					if (code === 0) {
 						resolve();
 					} else {
-						reject(new Error('Worker stopped with exit code ${code}'));
+						reject(new Error(`[${artifact.name}] Worker stopped with exit code ${code}`));
 					}
 				});
 			});
@@ -842,6 +808,7 @@ async function main() {
 			});
 
 			operations.push({ name: artifact.name, operation });
+			resultPromise = Promise.allSettled(operations.map(o => o.operation));
 		}
 
 		await new Promise(c => setTimeout(c, 10_000));
@@ -855,7 +822,7 @@ async function main() {
 		console.log('Artifacts in progress:', artifactsInProgress.map(a => a.name).join(', '));
 	}
 
-	const results = await Promise.allSettled(operations.map(o => o.operation));
+	const results = await resultPromise;
 
 	for (let i = 0; i < operations.length; i++) {
 		const result = results[i];

build/azure-pipelines/product-build.yml

@@ -76,10 +76,6 @@ parameters:
     displayName: "Publish to builds.code.visualstudio.com"
     type: boolean
     default: true
-  - name: VSCODE_PUBLISH_TO_MOONCAKE
-    displayName: "Publish to Azure China"
-    type: boolean
-    default: true
   - name: VSCODE_RELEASE
     displayName: "Release build if successful"
     type: boolean
@@ -116,8 +112,6 @@ variables:
     value: ${{ in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI') }}
   - name: VSCODE_PUBLISH
     value: ${{ and(eq(parameters.VSCODE_PUBLISH, true), eq(variables.VSCODE_CIBUILD, false), eq(parameters.VSCODE_COMPILE_ONLY, false)) }}
-  - name: VSCODE_PUBLISH_TO_MOONCAKE
-    value: ${{ eq(parameters.VSCODE_PUBLISH_TO_MOONCAKE, true) }}
   - name: VSCODE_SCHEDULEDBUILD
     value: ${{ eq(variables['Build.Reason'], 'Schedule') }}
   - name: VSCODE_7PM_BUILD
@@ -138,8 +132,6 @@ variables:
     value: https://az764295.vo.msecnd.net
   - name: AZURE_DOCUMENTDB_ENDPOINT
     value: https://vscode.documents.azure.com:443/
-  - name: MOONCAKE_CDN_URL
-    value: https://vscode.cdn.azure.cn
   - name: VSCODE_MIXIN_REPO
     value: microsoft/vscode-distro
   - name: skipComponentGovernanceDetection

build/azure-pipelines/product-publish.yml

@@ -49,18 +49,6 @@ steps:
         Write-Host "##vso[task.setvariable variable=AZURE_CLIENT_ID]$env:servicePrincipalId"
         Write-Host "##vso[task.setvariable variable=AZURE_CLIENT_SECRET;issecret=true]$env:servicePrincipalKey"
 
-  - task: AzureCLI@2
-    displayName: Fetch Mooncake secrets
-    inputs:
-      azureSubscription: "vscode-builds-mooncake-subscription"
-      scriptType: pscore
-      scriptLocation: inlineScript
-      addSpnToEnvironment: true
-      inlineScript: |
-        Write-Host "##vso[task.setvariable variable=AZURE_MOONCAKE_TENANT_ID]$env:tenantId"
-        Write-Host "##vso[task.setvariable variable=AZURE_MOONCAKE_CLIENT_ID]$env:servicePrincipalId"
-        Write-Host "##vso[task.setvariable variable=AZURE_MOONCAKE_CLIENT_SECRET;issecret=true]$env:servicePrincipalKey"
-
   - pwsh: |
       . build/azure-pipelines/win32/exec.ps1
 
@@ -102,9 +90,6 @@ steps:
       AZURE_TENANT_ID: "$(AZURE_TENANT_ID)"
       AZURE_CLIENT_ID: "$(AZURE_CLIENT_ID)"
       AZURE_CLIENT_SECRET: "$(AZURE_CLIENT_SECRET)"
-      AZURE_MOONCAKE_TENANT_ID: "$(AZURE_MOONCAKE_TENANT_ID)"
-      AZURE_MOONCAKE_CLIENT_ID: "$(AZURE_MOONCAKE_CLIENT_ID)"
-      AZURE_MOONCAKE_CLIENT_SECRET: "$(AZURE_MOONCAKE_CLIENT_SECRET)"
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
       RELEASE_TENANT_ID: "$(PRSS_RELEASE_TENANT_ID)"
       RELEASE_CLIENT_ID: "$(PRSS_RELEASE_CLIENT_ID)"