diff --git a/homeassistant/bootstrap.py b/homeassistant/bootstrap.py index 2c7b40ab24c3..77714f8f10d5 100644 --- a/homeassistant/bootstrap.py +++ b/homeassistant/bootstrap.py @@ -26,8 +26,8 @@ ERROR_LOG_FILENAME = 'home-assistant.log' # hass.data key for logging information. DATA_LOGGING = 'logging' +CORE_INTEGRATIONS = ('homeassistant', 'persistent_notification') LOGGING_INTEGRATIONS = {'logger', 'system_log'} - STAGE_1_INTEGRATIONS = { # To record data 'recorder', @@ -91,60 +91,7 @@ async def async_from_config_dict(config: Dict[str, Any], hass.config_entries = config_entries.ConfigEntries(hass, config) await hass.config_entries.async_initialize() - domains = _get_domains(hass, config) - - # Resolve all dependencies of all components so we can find the logging - # and integrations that need faster initialization. - resolved_domains_task = asyncio.gather(*[ - loader.async_component_dependencies(hass, domain) - for domain in domains - ], return_exceptions=True) - - # Set up core. - if not all(await asyncio.gather( - async_setup_component(hass, 'homeassistant', config), - async_setup_component(hass, 'persistent_notification', config), - )): - _LOGGER.error("Home Assistant core failed to initialize. " - "Further initialization aborted") - return hass - - _LOGGER.debug("Home Assistant core initialized") - - # Finish resolving domains - for dep_domains in await resolved_domains_task: - # Result is either a set or an exception. We ignore exceptions - # It will be properly handled during setup of the domain. - if isinstance(dep_domains, set): - domains.update(dep_domains) - - # setup components - logging_domains = domains & LOGGING_INTEGRATIONS - stage_1_domains = domains & STAGE_1_INTEGRATIONS - stage_2_domains = domains - logging_domains - stage_1_domains - - await asyncio.gather(*[ - async_setup_component(hass, domain, config) - for domain in logging_domains - ]) - - # Kick off loading the registries. They don't need to be awaited. - asyncio.gather( - hass.helpers.device_registry.async_get_registry(), - hass.helpers.entity_registry.async_get_registry(), - hass.helpers.area_registry.async_get_registry()) - - # Continue setting up the components - for to_load in (stage_1_domains, stage_2_domains): - if not to_load: - continue - - await asyncio.gather(*[ - async_setup_component(hass, domain, config) - for domain in to_load - ]) - - await hass.async_block_till_done() + await _async_set_up_integrations(hass, config) stop = time() _LOGGER.info("Home Assistant initialized in %.2fs", stop-start) @@ -352,3 +299,113 @@ def _get_domains(hass: core.HomeAssistant, config: Dict[str, Any]) -> Set[str]: domains.add('hassio') return domains + + +async def _async_set_up_integrations( + hass: core.HomeAssistant, config: Dict[str, Any]) -> None: + """Set up all the integrations.""" + domains = _get_domains(hass, config) + + # Resolve all dependencies of all components so we can find the logging + # and integrations that need faster initialization. + resolved_domains_task = asyncio.gather(*[ + loader.async_component_dependencies(hass, domain) + for domain in domains + ], return_exceptions=True) + + # Set up core. + _LOGGER.debug("Setting up %s", CORE_INTEGRATIONS) + + if not all(await asyncio.gather(*[ + async_setup_component(hass, domain, config) + for domain in CORE_INTEGRATIONS + ])): + _LOGGER.error("Home Assistant core failed to initialize. " + "Further initialization aborted") + return + + _LOGGER.debug("Home Assistant core initialized") + + # Finish resolving domains + for dep_domains in await resolved_domains_task: + # Result is either a set or an exception. We ignore exceptions + # It will be properly handled during setup of the domain. + if isinstance(dep_domains, set): + domains.update(dep_domains) + + # setup components + logging_domains = domains & LOGGING_INTEGRATIONS + stage_1_domains = domains & STAGE_1_INTEGRATIONS + stage_2_domains = domains - logging_domains - stage_1_domains + + if logging_domains: + _LOGGER.debug("Setting up %s", logging_domains) + + await asyncio.gather(*[ + async_setup_component(hass, domain, config) + for domain in logging_domains + ]) + + # Kick off loading the registries. They don't need to be awaited. + asyncio.gather( + hass.helpers.device_registry.async_get_registry(), + hass.helpers.entity_registry.async_get_registry(), + hass.helpers.area_registry.async_get_registry()) + + if stage_1_domains: + await asyncio.gather(*[ + async_setup_component(hass, domain, config) + for domain in logging_domains + ]) + + # Load all integrations + after_dependencies = {} # type: Dict[str, Set[str]] + + for int_or_exc in await asyncio.gather(*[ + loader.async_get_integration(hass, domain) + for domain in stage_2_domains + ], return_exceptions=True): + # Exceptions are handled in async_setup_component. + if (isinstance(int_or_exc, loader.Integration) and + int_or_exc.after_dependencies): + after_dependencies[int_or_exc.domain] = set( + int_or_exc.after_dependencies + ) + + last_load = None + while stage_2_domains: + domains_to_load = set() + + for domain in stage_2_domains: + after_deps = after_dependencies.get(domain) + # Load if integration has no after_dependencies or they are + # all loaded + if (not after_deps or + not after_deps-hass.config.components): + domains_to_load.add(domain) + + if not domains_to_load or domains_to_load == last_load: + break + + _LOGGER.debug("Setting up %s", domains_to_load) + + await asyncio.gather(*[ + async_setup_component(hass, domain, config) + for domain in domains_to_load + ]) + + last_load = domains_to_load + stage_2_domains -= domains_to_load + + # These are stage 2 domains that never have their after_dependencies + # satisfied. + if stage_2_domains: + _LOGGER.debug("Final set up: %s", stage_2_domains) + + await asyncio.gather(*[ + async_setup_component(hass, domain, config) + for domain in stage_2_domains + ]) + + # Wrap up startup + await hass.async_block_till_done() diff --git a/homeassistant/components/camera/manifest.json b/homeassistant/components/camera/manifest.json index afa6f0d9bb7e..3af6a15ca524 100644 --- a/homeassistant/components/camera/manifest.json +++ b/homeassistant/components/camera/manifest.json @@ -6,5 +6,8 @@ "dependencies": [ "http" ], + "after_dependencies": [ + "stream" + ], "codeowners": [] } diff --git a/homeassistant/components/mysensors/manifest.json b/homeassistant/components/mysensors/manifest.json index 2b94c2678aa2..f18f5d4f8dda 100644 --- a/homeassistant/components/mysensors/manifest.json +++ b/homeassistant/components/mysensors/manifest.json @@ -6,5 +6,8 @@ "pymysensors==0.18.0" ], "dependencies": [], + "after_dependencies": [ + "mqtt" + ], "codeowners": [] } diff --git a/homeassistant/components/owntracks/manifest.json b/homeassistant/components/owntracks/manifest.json index 3646f32093a6..60bce1bca3dc 100644 --- a/homeassistant/components/owntracks/manifest.json +++ b/homeassistant/components/owntracks/manifest.json @@ -8,5 +8,8 @@ "dependencies": [ "webhook" ], + "after_dependencies": [ + "mqtt" + ], "codeowners": [] } diff --git a/homeassistant/loader.py b/homeassistant/loader.py index ba37c5114d80..ecc39f8db8f6 100644 --- a/homeassistant/loader.py +++ b/homeassistant/loader.py @@ -21,6 +21,8 @@ from typing import ( TypeVar, List, Dict, + Union, + cast, ) # Typing imports that create a circular dependency @@ -116,6 +118,8 @@ class Integration: self.name = manifest['name'] # type: str self.domain = manifest['domain'] # type: str self.dependencies = manifest['dependencies'] # type: List[str] + self.after_dependencies = manifest.get( + 'after_dependencies') # type: Optional[List[str]] self.requirements = manifest['requirements'] # type: List[str] _LOGGER.info("Loaded %s from %s", self.domain, pkg_path) @@ -150,7 +154,8 @@ async def async_get_integration(hass: 'HomeAssistant', domain: str)\ raise IntegrationNotFound(domain) cache = hass.data[DATA_INTEGRATIONS] = {} - int_or_evt = cache.get(domain, _UNDEF) # type: Optional[Integration] + int_or_evt = cache.get( + domain, _UNDEF) # type: Optional[Union[Integration, asyncio.Event]] if isinstance(int_or_evt, asyncio.Event): await int_or_evt.wait() @@ -161,7 +166,7 @@ async def async_get_integration(hass: 'HomeAssistant', domain: str)\ elif int_or_evt is None: raise IntegrationNotFound(domain) else: - return int_or_evt + return cast(Integration, int_or_evt) event = cache[domain] = asyncio.Event() diff --git a/script/hassfest/dependencies.py b/script/hassfest/dependencies.py index af8a782906bb..dfdd3434045e 100644 --- a/script/hassfest/dependencies.py +++ b/script/hassfest/dependencies.py @@ -35,6 +35,7 @@ def validate_dependencies(integration: Integration): r"hass\.components\.(\w+)") referenced -= ALLOWED_USED_COMPONENTS referenced -= set(integration.manifest['dependencies']) + referenced -= set(integration.manifest.get('after_dependencies', [])) if referenced: for domain in sorted(referenced): diff --git a/script/hassfest/manifest.py b/script/hassfest/manifest.py index b644ec7d055f..30f892312993 100644 --- a/script/hassfest/manifest.py +++ b/script/hassfest/manifest.py @@ -13,6 +13,7 @@ MANIFEST_SCHEMA = vol.Schema({ vol.Required('documentation'): str, vol.Required('requirements'): [str], vol.Required('dependencies'): [str], + vol.Optional('after_dependencies'): [str], vol.Required('codeowners'): [str], }) diff --git a/tests/common.py b/tests/common.py index 6e3b95725b00..99afd4fdb95b 100644 --- a/tests/common.py +++ b/tests/common.py @@ -442,13 +442,16 @@ class MockModule: requirements=None, config_schema=None, platform_schema=None, platform_schema_base=None, async_setup=None, async_setup_entry=None, async_unload_entry=None, - async_migrate_entry=None, async_remove_entry=None): + async_migrate_entry=None, async_remove_entry=None, + partial_manifest=None): """Initialize the mock module.""" self.__name__ = 'homeassistant.components.{}'.format(domain) self.__file__ = 'homeassistant/components/{}'.format(domain) self.DOMAIN = domain self.DEPENDENCIES = dependencies or [] self.REQUIREMENTS = requirements or [] + # Overlay to be used when generating manifest from this module + self._partial_manifest = partial_manifest if config_schema is not None: self.CONFIG_SCHEMA = config_schema @@ -481,6 +484,13 @@ class MockModule: if async_remove_entry is not None: self.async_remove_entry = async_remove_entry + def mock_manifest(self): + """Generate a mock manifest to represent this module.""" + return { + **loader.manifest_from_legacy_module(self), + **(self._partial_manifest or {}) + } + class MockPlatform: """Provide a fake platform.""" @@ -906,7 +916,7 @@ def mock_integration(hass, module): """Mock an integration.""" integration = loader.Integration( hass, 'homeassisant.components.{}'.format(module.DOMAIN), None, - loader.manifest_from_legacy_module(module)) + module.mock_manifest()) _LOGGER.info("Adding mock integration: %s", module.DOMAIN) hass.data.setdefault( diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 4dfef3212072..7196c83b67eb 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -9,7 +9,9 @@ import homeassistant.config as config_util from homeassistant import bootstrap import homeassistant.util.dt as dt_util -from tests.common import patch_yaml_files, get_test_config_dir, mock_coro +from tests.common import ( + patch_yaml_files, get_test_config_dir, mock_coro, mock_integration, + MockModule) ORIG_TIMEZONE = dt_util.DEFAULT_TIME_ZONE VERSION_PATH = os.path.join(get_test_config_dir(), config_util.VERSION_FILE) @@ -87,3 +89,154 @@ async def test_load_hassio(hass): with patch.dict(os.environ, {'HASSIO': '1'}): assert bootstrap._get_domains(hass, {}) == {'hassio'} + + +async def test_empty_setup(hass): + """Test an empty set up loads the core.""" + await bootstrap._async_set_up_integrations(hass, {}) + for domain in bootstrap.CORE_INTEGRATIONS: + assert domain in hass.config.components, domain + + +async def test_core_failure_aborts(hass, caplog): + """Test failing core setup aborts further setup.""" + with patch('homeassistant.components.homeassistant.async_setup', + return_value=mock_coro(False)): + await bootstrap._async_set_up_integrations(hass, { + 'group': {} + }) + + assert 'core failed to initialize' in caplog.text + # We aborted early, group not set up + assert 'group' not in hass.config.components + + +async def test_setting_up_config(hass, caplog): + """Test we set up domains in config.""" + await bootstrap._async_set_up_integrations(hass, { + 'group hello': {}, + 'homeassistant': {} + }) + + assert 'group' in hass.config.components + + +async def test_setup_after_deps_all_present(hass, caplog): + """Test after_dependencies when all present.""" + caplog.set_level(logging.DEBUG) + order = [] + + def gen_domain_setup(domain): + async def async_setup(hass, config): + order.append(domain) + return True + + return async_setup + + mock_integration(hass, MockModule( + domain='root', + async_setup=gen_domain_setup('root') + )) + mock_integration(hass, MockModule( + domain='first_dep', + async_setup=gen_domain_setup('first_dep'), + partial_manifest={ + 'after_dependencies': ['root'] + } + )) + mock_integration(hass, MockModule( + domain='second_dep', + async_setup=gen_domain_setup('second_dep'), + partial_manifest={ + 'after_dependencies': ['first_dep'] + } + )) + + await bootstrap._async_set_up_integrations(hass, { + 'root': {}, + 'first_dep': {}, + 'second_dep': {}, + }) + + assert 'root' in hass.config.components + assert 'first_dep' in hass.config.components + assert 'second_dep' in hass.config.components + assert order == ['root', 'first_dep', 'second_dep'] + + +async def test_setup_after_deps_not_trigger_load(hass, caplog): + """Test after_dependencies does not trigger loading it.""" + caplog.set_level(logging.DEBUG) + order = [] + + def gen_domain_setup(domain): + async def async_setup(hass, config): + order.append(domain) + return True + + return async_setup + + mock_integration(hass, MockModule( + domain='root', + async_setup=gen_domain_setup('root') + )) + mock_integration(hass, MockModule( + domain='first_dep', + async_setup=gen_domain_setup('first_dep'), + partial_manifest={ + 'after_dependencies': ['root'] + } + )) + mock_integration(hass, MockModule( + domain='second_dep', + async_setup=gen_domain_setup('second_dep'), + partial_manifest={ + 'after_dependencies': ['first_dep'] + } + )) + + await bootstrap._async_set_up_integrations(hass, { + 'root': {}, + 'second_dep': {}, + }) + + assert 'root' in hass.config.components + assert 'first_dep' not in hass.config.components + assert 'second_dep' in hass.config.components + assert order == ['root', 'second_dep'] + + +async def test_setup_after_deps_not_present(hass, caplog): + """Test after_dependencies when referenced integration doesn't exist.""" + caplog.set_level(logging.DEBUG) + order = [] + + def gen_domain_setup(domain): + async def async_setup(hass, config): + order.append(domain) + return True + + return async_setup + + mock_integration(hass, MockModule( + domain='root', + async_setup=gen_domain_setup('root') + )) + mock_integration(hass, MockModule( + domain='second_dep', + async_setup=gen_domain_setup('second_dep'), + partial_manifest={ + 'after_dependencies': ['first_dep'] + } + )) + + await bootstrap._async_set_up_integrations(hass, { + 'root': {}, + 'first_dep': {}, + 'second_dep': {}, + }) + + assert 'root' in hass.config.components + assert 'first_dep' not in hass.config.components + assert 'second_dep' in hass.config.components + assert order == ['root', 'second_dep']