development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-08-27 21:09:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

cache builtn extensions locations from gallery (#183467)

Browse source 
- revert hosted extensions
- cache builtn extensions locations from gallery
This commit is contained in:
Sandeep Somavarapu 2023-05-26 00:41:29 +02:00 committed by GitHub
parent 710d6ea32c
commit 79b9f746dd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 125 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/vs/platform/extensionResourceLoader/common/extensionResourceLoader.ts
View file

@ -40,6 +40,11 @@ export interface IExtensionResourceLoaderService {
*/
readonly supportsExtensionGalleryResources: boolean;
/**
* Return true if the given URI is a extension gallery resource.
*/
isExtensionGalleryResource(uri: URI): boolean;
/**
* Computes the URL of a extension gallery resource. Returns `undefined` if gallery does not provide extension resources.
*/
@ -104,8 +109,8 @@ export abstract class AbstractExtensionResourceLoaderService implements IExtensi
public abstract readExtensionResource(uri: URI): Promise<string>;
protected isExtensionGalleryResource(uri: URI) {
return this._extensionGalleryAuthority && this._extensionGalleryAuthority === this._getExtensionGalleryAuthority(uri);
isExtensionGalleryResource(uri: URI): boolean {
return !!this._extensionGalleryAuthority && this._extensionGalleryAuthority === this._getExtensionGalleryAuthority(uri);
}
protected async getExtensionGalleryRequestHeaders(): Promise<IHeaders> {

13
src/vs/workbench/browser/web.api.ts
View file

@ -18,8 +18,6 @@ import type { IProgress, IProgressCompositeOptions, IProgressDialogOptions, IPro
import type { ITextEditorOptions } from 'vs/platform/editor/common/editor';
import type { EditorGroupLayout } from 'vs/workbench/services/editor/common/editorGroupsService';
import type { IEmbedderTerminalOptions } from 'vs/workbench/services/terminal/common/embedderTerminalService';
import { IExtensionManifest } from 'vs/platform/extensions/common/extensions';
import { ITranslations } from 'vs/platform/extensionManagement/common/extensionNls';
/**
* The `IWorkbench` interface is the API facade for web embedders
@ -222,7 +220,7 @@ export interface IWorkbenchConstructionOptions {
* - an extension in the Marketplace
* - location of the extension where it is hosted.
*/
readonly additionalBuiltinExtensions?: readonly (MarketplaceExtension | UriComponents | HostedExtension)[];
readonly additionalBuiltinExtensions?: readonly (MarketplaceExtension | UriComponents)[];
/**
* List of extensions to be enabled if they are installed.
@ -375,15 +373,6 @@ export interface IResourceUriProvider {
export type ExtensionId = string;
export type MarketplaceExtension = ExtensionId | { readonly id: ExtensionId; preRelease?: boolean; migrateStorageFrom?: ExtensionId };
export interface HostedExtension {
readonly location: UriComponents;
readonly preRelease?: boolean;
readonly packageJSON?: IExtensionManifest;
readonly defaultPackageTranslations?: ITranslations | null;
readonly packageNLSUris?: Map<string, UriComponents>;
readonly readmeUri?: UriComponents;
readonly changelogUri?: UriComponents;
}
export interface ICommonTelemetryPropertiesResolver {
(): { [key: string]: any };

177
src/vs/workbench/services/extensionManagement/browser/webExtensionsScannerService.ts
View file

@ -45,15 +45,6 @@ import { IUserDataProfilesService } from 'vs/platform/userDataProfile/common/use
import { IUriIdentityService } from 'vs/platform/uriIdentity/common/uriIdentity';
type GalleryExtensionInfo = { readonly id: string; preRelease?: boolean; migrateStorageFrom?: string };
interface HostedExtensionInfo {
readonly location: UriComponents;
readonly preRelease?: boolean;
readonly packageJSON?: IExtensionManifest;
readonly defaultPackageTranslations?: ITranslations | null;
readonly packageNLSUris?: Map<string, UriComponents>;
readonly readmeUri?: UriComponents;
readonly changelogUri?: UriComponents;
}
type ExtensionInfo = { readonly id: string; preRelease: boolean };
function isGalleryExtensionInfo(obj: unknown): obj is GalleryExtensionInfo {
@ -63,16 +54,6 @@ function isGalleryExtensionInfo(obj: unknown): obj is GalleryExtensionInfo {
&& (galleryExtensionInfo.migrateStorageFrom === undefined || typeof galleryExtensionInfo.migrateStorageFrom === 'string');
}
function isHostedExtensionInfo(obj: unknown): obj is HostedExtensionInfo {
const hostedExtensionInfo = obj as HostedExtensionInfo | undefined;
return isUriComponents(hostedExtensionInfo?.location)
&& (hostedExtensionInfo?.preRelease === undefined || typeof hostedExtensionInfo.preRelease === 'boolean')
&& (hostedExtensionInfo?.packageJSON === undefined || typeof hostedExtensionInfo.packageJSON === 'object')
&& (hostedExtensionInfo?.defaultPackageTranslations === undefined || hostedExtensionInfo?.defaultPackageTranslations === null || typeof hostedExtensionInfo.defaultPackageTranslations === 'object')
&& (hostedExtensionInfo?.changelogUri === undefined || isUriComponents(hostedExtensionInfo?.changelogUri))
&& (hostedExtensionInfo?.readmeUri === undefined || isUriComponents(hostedExtensionInfo?.readmeUri));
}
function isUriComponents(thing: unknown): thing is UriComponents {
if (!thing) {
return false;
@ -144,12 +125,13 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
}
}
private _customBuiltinExtensionsInfoPromise: Promise<{ extensions: ExtensionInfo[]; extensionsToMigrate: [string, string][]; extensionLocations: HostedExtensionInfo[] }> | undefined;
private readCustomBuiltinExtensionsInfoFromEnv(): Promise<{ extensions: ExtensionInfo[]; extensionsToMigrate: [string, string][]; extensionLocations: HostedExtensionInfo[] }> {
private _customBuiltinExtensionsInfoPromise: Promise<{ extensions: ExtensionInfo[]; extensionsToMigrate: [string, string][]; extensionLocations: URI[]; extensionGalleryResources: URI[] }> | undefined;
private readCustomBuiltinExtensionsInfoFromEnv(): Promise<{ extensions: ExtensionInfo[]; extensionsToMigrate: [string, string][]; extensionLocations: URI[]; extensionGalleryResources: URI[] }> {
if (!this._customBuiltinExtensionsInfoPromise) {
this._customBuiltinExtensionsInfoPromise = (async () => {
let extensions: ExtensionInfo[] = [];
const extensionLocations: HostedExtensionInfo[] = [];
const extensionLocations: URI[] = [];
const extensionGalleryResources: URI[] = [];
const extensionsToMigrate: [string, string][] = [];
const customBuiltinExtensionsInfo = this.environmentService.options && Array.isArray(this.environmentService.options.additionalBuiltinExtensions)
? this.environmentService.options.additionalBuiltinExtensions.map(additionalBuiltinExtension => isString(additionalBuiltinExtension) ? { id: additionalBuiltinExtension } : additionalBuiltinExtension)
@ -160,11 +142,12 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
if (e.migrateStorageFrom) {
extensionsToMigrate.push([e.migrateStorageFrom, e.id]);
}
} else {
if (isHostedExtensionInfo(e)) {
extensionLocations.push(e);
} else if (isUriComponents(e)) {
const extensionLocation = URI.revive(e);
if (this.extensionResourceLoaderService.isExtensionGalleryResource(extensionLocation)) {
extensionGalleryResources.push(extensionLocation);
} else {
extensionLocations.push({ location: e });
extensionLocations.push(extensionLocation);
}
}
}
@ -177,7 +160,10 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
if (extensionLocations.length) {
this.logService.info('Found additional builtin location extensions in env', extensionLocations.map(e => e.toString()));
}
return { extensions, extensionsToMigrate, extensionLocations };
if (extensionGalleryResources.length) {
this.logService.info('Found additional builtin extension gallery resources in env', extensionGalleryResources.map(e => e.toString()));
}
return { extensions, extensionsToMigrate, extensionLocations, extensionGalleryResources };
})();
}
return this._customBuiltinExtensionsInfoPromise;
@ -243,15 +229,9 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
return [];
}
const result: IScannedExtension[] = [];
await Promise.allSettled(extensionLocations.map(async ({ location, preRelease, packageNLSUris, packageJSON, defaultPackageTranslations, readmeUri, changelogUri }) => {
await Promise.allSettled(extensionLocations.map(async extensionLocation => {
try {
const webExtension = await this.toWebExtension(URI.revive(location), undefined,
packageJSON,
packageNLSUris ? [...packageNLSUris.entries()].reduce((result, [key, value]) => { result.set(key, URI.revive(value)); return result; }, new Map<string, URI>()) : undefined,
defaultPackageTranslations,
URI.revive(readmeUri),
URI.revive(changelogUri),
{ isPreReleaseVersion: preRelease });
const webExtension = await this.toWebExtension(extensionLocation);
const extension = await this.toScannedExtension(webExtension, true);
if (extension.isValid || !scanOptions?.skipInvalidExtensions) {
result.push(extension);
@ -271,9 +251,13 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
return [];
}
const result: IScannedExtension[] = [];
const { extensions } = await this.readCustomBuiltinExtensionsInfoFromEnv();
const { extensions, extensionGalleryResources } = await this.readCustomBuiltinExtensionsInfoFromEnv();
try {
const useCache = this.storageService.get('additionalBuiltinExtensions', StorageScope.APPLICATION, '[]') === JSON.stringify(extensions);
const cacheValue = JSON.stringify({
extensions: extensions.sort((a, b) => a.id.localeCompare(b.id)),
extensionGalleryResources: extensionGalleryResources.map(e => e.toString()).sort()
});
const useCache = this.storageService.get('additionalBuiltinExtensions', StorageScope.APPLICATION, '{}') === cacheValue;
const webExtensions = await (useCache ? this.getCustomBuiltinExtensionsFromCache() : this.updateCustomBuiltinExtensionsCache());
if (webExtensions.length) {
await Promise.all(webExtensions.map(async webExtension => {
@ -289,7 +273,7 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
}
}));
}
this.storageService.store('additionalBuiltinExtensions', JSON.stringify(extensions), StorageScope.APPLICATION, StorageTarget.MACHINE);
this.storageService.store('additionalBuiltinExtensions', cacheValue, StorageScope.APPLICATION, StorageTarget.MACHINE);
} catch (error) {
this.logService.info('Ignoring following additional builtin extensions as there is an error while fetching them from gallery', extensions.map(({ id }) => id), getErrorMessage(error));
}
@ -366,23 +350,17 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
if (!this._updateCustomBuiltinExtensionsCachePromise) {
this._updateCustomBuiltinExtensionsCachePromise = (async () => {
this.logService.info('Updating additional builtin extensions cache');
const webExtensions: IWebExtension[] = [];
const { extensions } = await this.readCustomBuiltinExtensionsInfoFromEnv();
if (extensions.length) {
const galleryExtensionsMap = await this.getExtensionsWithDependenciesAndPackedExtensions(extensions);
const missingExtensions = extensions.filter(({ id }) => !galleryExtensionsMap.has(id.toLowerCase()));
if (missingExtensions.length) {
this.logService.info('Skipping the additional builtin extensions because their compatible versions are not found.', missingExtensions);
}
await Promise.all([...galleryExtensionsMap.values()].map(async gallery => {
try {
const webExtension = await this.toWebExtensionFromGallery(gallery, { isPreReleaseVersion: gallery.properties.isPreReleaseVersion, preRelease: gallery.properties.isPreReleaseVersion, isBuiltin: true });
webExtensions.push(webExtension);
} catch (error) {
this.logService.info(`Ignoring additional builtin extension ${gallery.identifier.id} because there is an error while converting it into web extension`, getErrorMessage(error));
}
}));
const { extensions, extensionGalleryResources } = await this.readCustomBuiltinExtensionsInfoFromEnv();
const [galleryWebExtensions, extensionGalleryResourceWebExtensions] = await Promise.all([
this.resolveBuiltinGalleryExtensions(extensions),
this.resolveBuiltinExtensionGalleryResources(extensionGalleryResources)
]);
const webExtensionsMap = new Map<string, IWebExtension>();
for (const webExtension of [...galleryWebExtensions, ...extensionGalleryResourceWebExtensions]) {
webExtensionsMap.set(webExtension.identifier.id.toLowerCase(), webExtension);
}
await this.resolveDependenciesAndPackedExtensions(extensionGalleryResourceWebExtensions, webExtensionsMap);
const webExtensions = [...webExtensionsMap.values()];
await this.writeCustomBuiltinExtensionsCache(() => webExtensions);
return webExtensions;
})();
@ -390,7 +368,77 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
return this._updateCustomBuiltinExtensionsCachePromise;
}
private async getExtensionsWithDependenciesAndPackedExtensions(toGet: IExtensionInfo[], result: Map<string, IGalleryExtension> = new Map<string, IGalleryExtension>()): Promise<Map<string, IGalleryExtension>> {
private async resolveBuiltinExtensionGalleryResources(extensionGalleryResources: URI[]): Promise<IWebExtension[]> {
if (extensionGalleryResources.length === 0) {
return [];
}
const result = new Map<string, IWebExtension>();
const extensionInfos: IExtensionInfo[] = [];
await Promise.all(extensionGalleryResources.map(async extensionGalleryResource => {
const webExtension = await this.toWebExtensionFromExtensionGalleryResource(extensionGalleryResource);
result.set(webExtension.identifier.id.toLowerCase(), webExtension);
extensionInfos.push({ id: webExtension.identifier.id, version: webExtension.version });
}));
const galleryExtensions = await this.galleryService.getExtensions(extensionInfos, CancellationToken.None);
for (const galleryExtension of galleryExtensions) {
const webExtension = result.get(galleryExtension.identifier.id.toLowerCase());
if (webExtension) {
result.set(galleryExtension.identifier.id.toLowerCase(), {
...webExtension,
readmeUri: galleryExtension.assets.readme ? URI.parse(galleryExtension.assets.readme.uri) : undefined,
changelogUri: galleryExtension.assets.changelog ? URI.parse(galleryExtension.assets.changelog.uri) : undefined,
metadata: { isPreReleaseVersion: galleryExtension.properties.isPreReleaseVersion, preRelease: galleryExtension.properties.isPreReleaseVersion, isBuiltin: true }
});
}
}
return [...result.values()];
}
private async resolveBuiltinGalleryExtensions(extensions: IExtensionInfo[]): Promise<IWebExtension[]> {
if (extensions.length === 0) {
return [];
}
const webExtensions: IWebExtension[] = [];
const galleryExtensionsMap = await this.getExtensionsWithDependenciesAndPackedExtensions(extensions);
const missingExtensions = extensions.filter(({ id }) => !galleryExtensionsMap.has(id.toLowerCase()));
if (missingExtensions.length) {
this.logService.info('Skipping the additional builtin extensions because their compatible versions are not found.', missingExtensions);
}
await Promise.all([...galleryExtensionsMap.values()].map(async gallery => {
try {
const webExtension = await this.toWebExtensionFromGallery(gallery, { isPreReleaseVersion: gallery.properties.isPreReleaseVersion, preRelease: gallery.properties.isPreReleaseVersion, isBuiltin: true });
webExtensions.push(webExtension);
} catch (error) {
this.logService.info(`Ignoring additional builtin extension ${gallery.identifier.id} because there is an error while converting it into web extension`, getErrorMessage(error));
}
}));
return webExtensions;
}
private async resolveDependenciesAndPackedExtensions(webExtensions: IWebExtension[], result: Map<string, IWebExtension>): Promise<void> {
const extensionInfos: IExtensionInfo[] = [];
for (const webExtension of webExtensions) {
for (const e of [...(webExtension.manifest?.extensionDependencies ?? []), ...(webExtension.manifest?.extensionPack ?? [])]) {
if (!result.has(e.toLowerCase())) {
extensionInfos.push({ id: e, version: webExtension.version });
}
}
}
if (extensionInfos.length === 0) {
return;
}
const galleryExtensions = await this.getExtensionsWithDependenciesAndPackedExtensions(extensionInfos, new Set<string>([...result.keys()]));
await Promise.all([...galleryExtensions.values()].map(async gallery => {
try {
const webExtension = await this.toWebExtensionFromGallery(gallery, { isPreReleaseVersion: gallery.properties.isPreReleaseVersion, preRelease: gallery.properties.isPreReleaseVersion, isBuiltin: true });
result.set(webExtension.identifier.id.toLowerCase(), webExtension);
} catch (error) {
this.logService.info(`Ignoring additional builtin extension ${gallery.identifier.id} because there is an error while converting it into web extension`, getErrorMessage(error));
}
}));
}
private async getExtensionsWithDependenciesAndPackedExtensions(toGet: IExtensionInfo[], seen: Set<string> = new Set<string>(), result: Map<string, IGalleryExtension> = new Map<string, IGalleryExtension>()): Promise<Map<string, IGalleryExtension>> {
if (toGet.length === 0) {
return result;
}
@ -399,13 +447,13 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
for (const extension of extensions) {
result.set(extension.identifier.id.toLowerCase(), extension);
for (const id of [...(isNonEmptyArray(extension.properties.dependencies) ? extension.properties.dependencies : []), ...(isNonEmptyArray(extension.properties.extensionPack) ? extension.properties.extensionPack : [])]) {
if (!result.has(id.toLowerCase()) && !packsAndDependencies.has(id.toLowerCase())) {
if (!result.has(id.toLowerCase()) && !packsAndDependencies.has(id.toLowerCase()) && !seen.has(id.toLowerCase())) {
const extensionInfo = toGet.find(e => areSameExtensions(e, extension.identifier));
packsAndDependencies.set(id.toLowerCase(), { id, preRelease: extensionInfo?.preRelease });
}
}
}
return this.getExtensionsWithDependenciesAndPackedExtensions([...packsAndDependencies.values()].filter(({ id }) => !result.has(id.toLowerCase())), result);
return this.getExtensionsWithDependenciesAndPackedExtensions([...packsAndDependencies.values()].filter(({ id }) => !result.has(id.toLowerCase())), seen, result);
}
async scanSystemExtensions(): Promise<IExtension[]> {
@ -606,6 +654,15 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
if (!extensionLocation) {
throw new Error('No extension gallery service configured.');
}
return this.toWebExtensionFromExtensionGalleryResource(extensionLocation,
galleryExtension.identifier,
galleryExtension.assets.readme ? URI.parse(galleryExtension.assets.readme.uri) : undefined,
galleryExtension.assets.changelog ? URI.parse(galleryExtension.assets.changelog.uri) : undefined,
metadata);
}
private async toWebExtensionFromExtensionGalleryResource(extensionLocation: URI, identifier?: IExtensionIdentifier, readmeUri?: URI, changelogUri?: URI, metadata?: Metadata): Promise<IWebExtension> {
const extensionResources = await this.listExtensionResources(extensionLocation);
const packageNLSResources = this.getPackageNLSResourceMapFromResources(extensionResources);
@ -613,12 +670,12 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
const fallbackPackageNLSResource = extensionResources.find(e => basename(e) === 'package.nls.json');
return this.toWebExtension(
extensionLocation,
galleryExtension.identifier,
identifier,
undefined,
packageNLSResources,
fallbackPackageNLSResource ? URI.parse(fallbackPackageNLSResource) : null,
galleryExtension.assets.readme ? URI.parse(galleryExtension.assets.readme.uri) : undefined,
galleryExtension.assets.changelog ? URI.parse(galleryExtension.assets.changelog.uri) : undefined,
readmeUri,
changelogUri,
metadata);
}