mirror of
https://github.com/Microsoft/vscode
synced 2024-09-13 13:46:13 +00:00
Make PRSS CDN optional (#198631)
* empty commit
* mooncake creds expired
* more logs
* prevent exit
* log
* async is hard
* 💄
* temporarily ignore PRSS errors
* whitespace
This commit is contained in:
parent
55181913b9
commit
244c48768d
File diff suppressed because one or more lines are too long
|
@ -353,7 +353,6 @@ async function releaseAndProvision(
|
|||
const credential = new ClientSecretCredential(provisionTenantId, provisionAADUsername, provisionAADPassword);
|
||||
const accessToken = await credential.getToken(['https://microsoft.onmicrosoft.com/DS.Provisioning.WebApi/.default']);
|
||||
const service = new ProvisionService(log, accessToken.token);
|
||||
|
||||
await service.provision(release.releaseId, release.fileId, fileName);
|
||||
|
||||
return result;
|
||||
|
@ -628,7 +627,7 @@ function getRealType(type: string) {
|
|||
}
|
||||
}
|
||||
|
||||
async function uploadAssetLegacy(log: (...args: any[]) => void, quality: string, commit: string, filePath: string): Promise<{ assetUrl: string; mooncakeUrl: string }> {
|
||||
async function uploadAssetLegacy(log: (...args: any[]) => void, quality: string, commit: string, filePath: string): Promise<string> {
|
||||
const fileName = path.basename(filePath);
|
||||
const blobName = commit + '/' + fileName;
|
||||
|
||||
|
@ -645,60 +644,17 @@ async function uploadAssetLegacy(log: (...args: any[]) => void, quality: string,
|
|||
}
|
||||
};
|
||||
|
||||
const uploadPromises: Promise<void>[] = [];
|
||||
log(`Checking for blob in Azure...`);
|
||||
|
||||
uploadPromises.push((async (): Promise<void> => {
|
||||
log(`Checking for blob in Azure...`);
|
||||
|
||||
if (await blobClient.exists()) {
|
||||
log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
throw new Error(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
} else {
|
||||
log(`Uploading blobs to Azure storage...`);
|
||||
await blobClient.uploadFile(filePath, blobOptions);
|
||||
log('Blob successfully uploaded to Azure storage.');
|
||||
}
|
||||
})());
|
||||
|
||||
const shouldUploadToMooncake = /true/i.test(e('VSCODE_PUBLISH_TO_MOONCAKE'));
|
||||
|
||||
if (shouldUploadToMooncake) {
|
||||
const mooncakeCredential = new ClientSecretCredential(e('AZURE_MOONCAKE_TENANT_ID'), e('AZURE_MOONCAKE_CLIENT_ID'), e('AZURE_MOONCAKE_CLIENT_SECRET'));
|
||||
const mooncakeBlobServiceClient = new BlobServiceClient(`https://vscode.blob.core.chinacloudapi.cn`, mooncakeCredential, { retryOptions: { retryPolicyType: StorageRetryPolicyType.FIXED, tryTimeoutInMs: 5 * 60 * 1000 } });
|
||||
const mooncakeContainerClient = mooncakeBlobServiceClient.getContainerClient(quality);
|
||||
const mooncakeBlobClient = mooncakeContainerClient.getBlockBlobClient(blobName);
|
||||
|
||||
uploadPromises.push((async (): Promise<void> => {
|
||||
log(`Checking for blob in Mooncake Azure...`);
|
||||
|
||||
if (await mooncakeBlobClient.exists()) {
|
||||
log(`Mooncake Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
throw new Error(`Mooncake Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
} else {
|
||||
log(`Uploading blobs to Mooncake Azure storage...`);
|
||||
await mooncakeBlobClient.uploadFile(filePath, blobOptions);
|
||||
log('Blob successfully uploaded to Mooncake Azure storage.');
|
||||
}
|
||||
})());
|
||||
}
|
||||
|
||||
const promiseResults = await Promise.allSettled(uploadPromises);
|
||||
const rejectedPromiseResults = promiseResults.filter(result => result.status === 'rejected') as PromiseRejectedResult[];
|
||||
|
||||
if (rejectedPromiseResults.length === 0) {
|
||||
log('All blobs successfully uploaded.');
|
||||
} else if (rejectedPromiseResults[0]?.reason?.message?.includes('already exists')) {
|
||||
log('Some blobs successfully uploaded.');
|
||||
if (await blobClient.exists()) {
|
||||
log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
} else {
|
||||
// eslint-disable-next-line no-throw-literal
|
||||
throw rejectedPromiseResults[0]?.reason;
|
||||
log(`Uploading blobs to Azure storage...`);
|
||||
await blobClient.uploadFile(filePath, blobOptions);
|
||||
log('Blob successfully uploaded to Azure storage.');
|
||||
}
|
||||
|
||||
const assetUrl = `${e('AZURE_CDN_URL')}/${quality}/${blobName}`;
|
||||
const blobPath = new URL(assetUrl).pathname;
|
||||
const mooncakeUrl = `${e('MOONCAKE_CDN_URL')}${blobPath}`;
|
||||
|
||||
return { assetUrl, mooncakeUrl };
|
||||
return `${e('AZURE_CDN_URL')}/${quality}/${blobName}`;
|
||||
}
|
||||
|
||||
async function processArtifact(artifact: Artifact, artifactFilePath: string): Promise<void> {
|
||||
|
@ -719,7 +675,7 @@ async function processArtifact(artifact: Artifact, artifactFilePath: string): Pr
|
|||
const stream = fs.createReadStream(artifactFilePath);
|
||||
const [sha1hash, sha256hash] = await Promise.all([hashStream('sha1', stream), hashStream('sha256', stream)]);
|
||||
|
||||
const [{ assetUrl, mooncakeUrl }, prssUrl] = await Promise.all([
|
||||
const [cdnSettledResult, prssSettledResult] = await Promise.allSettled([
|
||||
uploadAssetLegacy(log, quality, commit, artifactFilePath),
|
||||
releaseAndProvision(
|
||||
log,
|
||||
|
@ -736,11 +692,20 @@ async function processArtifact(artifact: Artifact, artifactFilePath: string): Pr
|
|||
)
|
||||
]);
|
||||
|
||||
const asset: Asset = { platform, type, url: assetUrl, hash: sha1hash, mooncakeUrl, prssUrl, sha256hash, size, supportsFastUpdate: true };
|
||||
if (cdnSettledResult.status === 'rejected') {
|
||||
throw cdnSettledResult.reason;
|
||||
} else if (prssSettledResult.status === 'rejected') { // TODO@joaomoreno, let's temporarily ignore these errors
|
||||
console.error(prssSettledResult.reason);
|
||||
}
|
||||
|
||||
const assetUrl = cdnSettledResult.value;
|
||||
const prssUrl = prssSettledResult.status === 'fulfilled' ? prssSettledResult.value : undefined;
|
||||
|
||||
const asset: Asset = { platform, type, url: assetUrl, hash: sha1hash, mooncakeUrl: prssUrl, prssUrl, sha256hash, size, supportsFastUpdate: true };
|
||||
log('Creating asset...', JSON.stringify(asset));
|
||||
|
||||
await retry(async (attempt) => {
|
||||
log(`Creating asset in Cosmos DB(attempt ${attempt})...`);
|
||||
log(`Creating asset in Cosmos DB (attempt ${attempt})...`);
|
||||
const aadCredentials = new ClientSecretCredential(e('AZURE_TENANT_ID'), e('AZURE_CLIENT_ID'), e('AZURE_CLIENT_SECRET'));
|
||||
const client = new CosmosClient({ endpoint: e('AZURE_DOCUMENTDB_ENDPOINT'), aadCredentials });
|
||||
const scripts = client.database('builds').container(quality).scripts;
|
||||
|
@ -777,6 +742,7 @@ async function main() {
|
|||
if (e('VSCODE_BUILD_STAGE_MACOS') === 'True') { stages.add('macOS'); }
|
||||
if (e('VSCODE_BUILD_STAGE_WEB') === 'True') { stages.add('Web'); }
|
||||
|
||||
let resultPromise = Promise.resolve<PromiseSettledResult<void>[]>([]);
|
||||
const operations: { name: string; operation: Promise<void> }[] = [];
|
||||
|
||||
while (true) {
|
||||
|
@ -806,7 +772,7 @@ async function main() {
|
|||
|
||||
await retry(async (attempt) => {
|
||||
const start = Date.now();
|
||||
console.log(`[${artifact.name}]Downloading(attempt ${attempt})...`);
|
||||
console.log(`[${artifact.name}] Downloading (attempt ${attempt})...`);
|
||||
await downloadArtifact(artifact, artifactZipPath);
|
||||
const archiveSize = fs.statSync(artifactZipPath).size;
|
||||
const downloadDurationS = (Date.now() - start) / 1000;
|
||||
|
@ -830,7 +796,7 @@ async function main() {
|
|||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error('Worker stopped with exit code ${code}'));
|
||||
reject(new Error(`[${artifact.name}] Worker stopped with exit code ${code}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
@ -842,6 +808,7 @@ async function main() {
|
|||
});
|
||||
|
||||
operations.push({ name: artifact.name, operation });
|
||||
resultPromise = Promise.allSettled(operations.map(o => o.operation));
|
||||
}
|
||||
|
||||
await new Promise(c => setTimeout(c, 10_000));
|
||||
|
@ -855,7 +822,7 @@ async function main() {
|
|||
console.log('Artifacts in progress:', artifactsInProgress.map(a => a.name).join(', '));
|
||||
}
|
||||
|
||||
const results = await Promise.allSettled(operations.map(o => o.operation));
|
||||
const results = await resultPromise;
|
||||
|
||||
for (let i = 0; i < operations.length; i++) {
|
||||
const result = results[i];
|
||||
|
|
|
@ -76,10 +76,6 @@ parameters:
|
|||
displayName: "Publish to builds.code.visualstudio.com"
|
||||
type: boolean
|
||||
default: true
|
||||
- name: VSCODE_PUBLISH_TO_MOONCAKE
|
||||
displayName: "Publish to Azure China"
|
||||
type: boolean
|
||||
default: true
|
||||
- name: VSCODE_RELEASE
|
||||
displayName: "Release build if successful"
|
||||
type: boolean
|
||||
|
@ -116,8 +112,6 @@ variables:
|
|||
value: ${{ in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI') }}
|
||||
- name: VSCODE_PUBLISH
|
||||
value: ${{ and(eq(parameters.VSCODE_PUBLISH, true), eq(variables.VSCODE_CIBUILD, false), eq(parameters.VSCODE_COMPILE_ONLY, false)) }}
|
||||
- name: VSCODE_PUBLISH_TO_MOONCAKE
|
||||
value: ${{ eq(parameters.VSCODE_PUBLISH_TO_MOONCAKE, true) }}
|
||||
- name: VSCODE_SCHEDULEDBUILD
|
||||
value: ${{ eq(variables['Build.Reason'], 'Schedule') }}
|
||||
- name: VSCODE_7PM_BUILD
|
||||
|
@ -138,8 +132,6 @@ variables:
|
|||
value: https://az764295.vo.msecnd.net
|
||||
- name: AZURE_DOCUMENTDB_ENDPOINT
|
||||
value: https://vscode.documents.azure.com:443/
|
||||
- name: MOONCAKE_CDN_URL
|
||||
value: https://vscode.cdn.azure.cn
|
||||
- name: VSCODE_MIXIN_REPO
|
||||
value: microsoft/vscode-distro
|
||||
- name: skipComponentGovernanceDetection
|
||||
|
|
|
@ -49,18 +49,6 @@ steps:
|
|||
Write-Host "##vso[task.setvariable variable=AZURE_CLIENT_ID]$env:servicePrincipalId"
|
||||
Write-Host "##vso[task.setvariable variable=AZURE_CLIENT_SECRET;issecret=true]$env:servicePrincipalKey"
|
||||
|
||||
- task: AzureCLI@2
|
||||
displayName: Fetch Mooncake secrets
|
||||
inputs:
|
||||
azureSubscription: "vscode-builds-mooncake-subscription"
|
||||
scriptType: pscore
|
||||
scriptLocation: inlineScript
|
||||
addSpnToEnvironment: true
|
||||
inlineScript: |
|
||||
Write-Host "##vso[task.setvariable variable=AZURE_MOONCAKE_TENANT_ID]$env:tenantId"
|
||||
Write-Host "##vso[task.setvariable variable=AZURE_MOONCAKE_CLIENT_ID]$env:servicePrincipalId"
|
||||
Write-Host "##vso[task.setvariable variable=AZURE_MOONCAKE_CLIENT_SECRET;issecret=true]$env:servicePrincipalKey"
|
||||
|
||||
- pwsh: |
|
||||
. build/azure-pipelines/win32/exec.ps1
|
||||
|
||||
|
@ -102,9 +90,6 @@ steps:
|
|||
AZURE_TENANT_ID: "$(AZURE_TENANT_ID)"
|
||||
AZURE_CLIENT_ID: "$(AZURE_CLIENT_ID)"
|
||||
AZURE_CLIENT_SECRET: "$(AZURE_CLIENT_SECRET)"
|
||||
AZURE_MOONCAKE_TENANT_ID: "$(AZURE_MOONCAKE_TENANT_ID)"
|
||||
AZURE_MOONCAKE_CLIENT_ID: "$(AZURE_MOONCAKE_CLIENT_ID)"
|
||||
AZURE_MOONCAKE_CLIENT_SECRET: "$(AZURE_MOONCAKE_CLIENT_SECRET)"
|
||||
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
|
||||
RELEASE_TENANT_ID: "$(PRSS_RELEASE_TENANT_ID)"
|
||||
RELEASE_CLIENT_ID: "$(PRSS_RELEASE_CLIENT_ID)"
|
||||
|
|
Loading…
Reference in a new issue