Make the frontend available sooner (Part 1 of 2) (#36263)

* Part 1 of 2 (no breaking changes in part 1).

When integrations configured via the UI block startup or fail to start,
the webserver can remain offline which make it is impossible
to recover without manually changing files in
.storage since the UI is not available.

This change is the foundation that part 2 will build on
and enable a listener to start the webserver when the frontend
is finished loading.

Frontend Changes (home-assistant/frontend#6068)

* Address review comments

* bump timeout to 1800s, adjust comment

* bump timeout to 4h

* remove timeout failsafe

* and the test

homeassistant/bootstrap.py

@@ -44,6 +44,13 @@ STAGE_1_INTEGRATIONS = {
     "mqtt_eventstream",
     # To provide account link implementations
     "cloud",
+    # Ensure supervisor is available
+    "hassio",
+    # Get the frontend up and running as soon
+    # as possible so problem integrations can
+    # be removed
+    "frontend",
+    "config",
 }
 
 
@@ -399,6 +406,8 @@ async def _async_set_up_integrations(
     )
 
     if stage_1_domains:
+        _LOGGER.info("Setting up %s", stage_1_domains)
+
         await async_setup_multi_components(stage_1_domains)
 
     # Load all integrations

homeassistant/components/http/__init__.py

@@ -4,7 +4,7 @@ import logging
 import os
 import ssl
 from traceback import extract_stack
-from typing import Optional, cast
+from typing import Dict, Optional, cast
 
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPMovedPermanently
@@ -15,7 +15,7 @@ from homeassistant.const import (
     EVENT_HOMEASSISTANT_STOP,
     SERVER_PORT,
 )
-from homeassistant.core import HomeAssistant
+from homeassistant.core import Event, HomeAssistant
 from homeassistant.helpers import storage
 import homeassistant.helpers.config_validation as cv
 from homeassistant.loader import bind_hass
@@ -216,29 +216,25 @@ async def async_setup(hass, config):
         ssl_profile=ssl_profile,
     )
 
-    async def stop_server(event):
+    startup_listeners = []
+
+    async def stop_server(event: Event) -> None:
         """Stop the server."""
         await server.stop()
 
-    async def start_server(event):
+    async def start_server(event: Event) -> None:
         """Start the server."""
+
+        for listener in startup_listeners:
+            listener()
+
         hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, stop_server)
-        await server.start()
 
-        # If we are set up successful, we store the HTTP settings for safe mode.
-        store = storage.Store(hass, STORAGE_VERSION, STORAGE_KEY)
+        await start_http_server_and_save_config(hass, dict(conf), server)
 
-        if CONF_TRUSTED_PROXIES in conf:
-            conf_to_save = dict(conf)
-            conf_to_save[CONF_TRUSTED_PROXIES] = [
-                str(ip.network_address) for ip in conf_to_save[CONF_TRUSTED_PROXIES]
-            ]
-        else:
-            conf_to_save = conf
-
-        await store.async_save(conf_to_save)
-
-    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_START, start_server)
+    startup_listeners.append(
+        hass.bus.async_listen(EVENT_HOMEASSISTANT_START, start_server)
+    )
 
     hass.http = server
 
@@ -418,3 +414,20 @@ class HomeAssistantHTTP:
         """Stop the aiohttp server."""
         await self.site.stop()
         await self.runner.cleanup()
+
+
+async def start_http_server_and_save_config(
+    hass: HomeAssistant, conf: Dict, server: HomeAssistantHTTP
+) -> None:
+    """Startup the http server and save the config."""
+    await server.start()  # type: ignore
+
+    # If we are set up successful, we store the HTTP settings for safe mode.
+    store = storage.Store(hass, STORAGE_VERSION, STORAGE_KEY)
+
+    if CONF_TRUSTED_PROXIES in conf:
+        conf[CONF_TRUSTED_PROXIES] = [
+            str(ip.network_address) for ip in conf[CONF_TRUSTED_PROXIES]
+        ]
+
+    await store.async_save(conf)

homeassistant/components/http/view.py

@@ -14,7 +14,7 @@ from aiohttp.web_exceptions import (
 import voluptuous as vol
 
 from homeassistant import exceptions
-from homeassistant.const import CONTENT_TYPE_JSON, HTTP_OK
+from homeassistant.const import CONTENT_TYPE_JSON, HTTP_OK, HTTP_SERVICE_UNAVAILABLE
 from homeassistant.core import Context, is_callback
 from homeassistant.helpers.json import JSONEncoder
 
@@ -107,8 +107,8 @@ def request_handler_factory(view: HomeAssistantView, handler: Callable) -> Calla
 
     async def handle(request: web.Request) -> web.StreamResponse:
         """Handle incoming request."""
-        if not request.app[KEY_HASS].is_running:
-            return web.Response(status=503)
+        if request.app[KEY_HASS].is_stopping:
+            return web.Response(status=HTTP_SERVICE_UNAVAILABLE)
 
         authenticated = request.get(KEY_AUTHENTICATED, False)
 

homeassistant/components/websocket_api/decorators.py

@@ -1,4 +1,5 @@
 """Decorators for the Websocket API."""
+import asyncio
 from functools import wraps
 import logging
 from typing import Awaitable, Callable
@@ -31,7 +32,9 @@ def async_response(
     @wraps(func)
     def schedule_handler(hass, connection, msg):
         """Schedule the handler."""
-        hass.async_create_task(_handle_async_response(func, hass, connection, msg))
+        # As the webserver is now started before the start
+        # event we do not want to block for websocket responders
+        asyncio.create_task(_handle_async_response(func, hass, connection, msg))
 
     return schedule_handler
 

homeassistant/components/websocket_api/http.py

@@ -165,7 +165,9 @@ class WebSocketHandler:
             EVENT_HOMEASSISTANT_STOP, handle_hass_stop
         )
 
-        self._writer_task = self.hass.async_create_task(self._writer())
+        # As the webserver is now started before the start
+        # event we do not want to block for websocket responses
+        self._writer_task = asyncio.create_task(self._writer())
 
         auth = AuthPhase(self._logger, self.hass, self._send_message, request)
         connection = None

homeassistant/core.py

@@ -209,6 +209,11 @@ class HomeAssistant:
         """Return if Home Assistant is running."""
         return self.state in (CoreState.starting, CoreState.running)
 
+    @property
+    def is_stopping(self) -> bool:
+        """Return if Home Assistant is stopping."""
+        return self.state in (CoreState.stopping, CoreState.final_write)
+
     def start(self) -> int:
         """Start Home Assistant.
 
@@ -260,6 +265,7 @@ class HomeAssistant:
 
         setattr(self.loop, "_thread_ident", threading.get_ident())
         self.bus.async_fire(EVENT_HOMEASSISTANT_START)
+        self.bus.async_fire(EVENT_CORE_CONFIG_UPDATE)
 
         try:
             # Only block for EVENT_HOMEASSISTANT_START listener
@@ -1391,6 +1397,7 @@ class Config:
             "version": __version__,
             "config_source": self.config_source,
             "safe_mode": self.safe_mode,
+            "state": self.hass.state.value,
             "external_url": self.external_url,
             "internal_url": self.internal_url,
         }

tests/components/hassio/test_discovery.py

@@ -60,6 +60,9 @@ async def test_hassio_discovery_startup(hass, aioclient_mock, hassio_client):
 
 async def test_hassio_discovery_startup_done(hass, aioclient_mock, hassio_client):
     """Test startup and discovery with hass discovery."""
+    aioclient_mock.post(
+        "http://127.0.0.1/supervisor/options", json={"result": "ok", "data": {}},
+    )
     aioclient_mock.get(
         "http://127.0.0.1/discovery",
         json={
@@ -101,7 +104,7 @@ async def test_hassio_discovery_startup_done(hass, aioclient_mock, hassio_client
         await async_setup_component(hass, "hassio", {})
         await hass.async_block_till_done()
 
-        assert aioclient_mock.call_count == 2
+        assert aioclient_mock.call_count == 3
         assert mock_mqtt.called
         mock_mqtt.assert_called_with(
             {

tests/components/http/test_data_validator.py

@@ -11,7 +11,7 @@ from tests.async_mock import Mock
 async def get_client(aiohttp_client, validator):
     """Generate a client that hits a view decorated with validator."""
     app = web.Application()
-    app["hass"] = Mock(is_running=True)
+    app["hass"] = Mock(is_stopping=False)
 
     class TestView(HomeAssistantView):
         url = "/"

tests/components/http/test_view.py

@@ -19,7 +19,13 @@ from tests.async_mock import AsyncMock, Mock
 @pytest.fixture
 def mock_request():
     """Mock a request."""
-    return Mock(app={"hass": Mock(is_running=True)}, match_info={})
+    return Mock(app={"hass": Mock(is_stopping=False)}, match_info={})
+
+
+@pytest.fixture
+def mock_request_with_stopping():
+    """Mock a request."""
+    return Mock(app={"hass": Mock(is_stopping=True)}, match_info={})
 
 
 async def test_invalid_json(caplog):
@@ -55,3 +61,11 @@ async def test_handling_service_not_found(mock_request):
             Mock(requires_auth=False),
             AsyncMock(side_effect=ServiceNotFound("test", "test")),
         )(mock_request)
+
+
+async def test_not_running(mock_request_with_stopping):
+    """Test we get a 503 when not running."""
+    response = await request_handler_factory(
+        Mock(requires_auth=False), AsyncMock(side_effect=Unauthorized)
+    )(mock_request_with_stopping)
+    assert response.status == 503

tests/components/logbook/test_init.py

@@ -42,7 +42,8 @@ class TestComponentLogbook(unittest.TestCase):
         """Set up things to be run when tests are started."""
         self.hass = get_test_home_assistant()
         init_recorder_component(self.hass)  # Force an in memory DB
-        assert setup_component(self.hass, logbook.DOMAIN, self.EMPTY_CONFIG)
+        with patch("homeassistant.components.http.start_http_server_and_save_config"):
+            assert setup_component(self.hass, logbook.DOMAIN, self.EMPTY_CONFIG)
 
     def tearDown(self):
         """Stop everything that was started."""

tests/components/panel_iframe/test_init.py

@@ -4,6 +4,7 @@ import unittest
 from homeassistant import setup
 from homeassistant.components import frontend
 
+from tests.async_mock import patch
 from tests.common import get_test_home_assistant
 
 
@@ -26,38 +27,42 @@ class TestPanelIframe(unittest.TestCase):
         ]
 
         for conf in to_try:
-            assert not setup.setup_component(
-                self.hass, "panel_iframe", {"panel_iframe": conf}
-            )
+            with patch(
+                "homeassistant.components.http.start_http_server_and_save_config"
+            ):
+                assert not setup.setup_component(
+                    self.hass, "panel_iframe", {"panel_iframe": conf}
+                )
 
     def test_correct_config(self):
         """Test correct config."""
-        assert setup.setup_component(
-            self.hass,
-            "panel_iframe",
-            {
-                "panel_iframe": {
-                    "router": {
-                        "icon": "mdi:network-wireless",
-                        "title": "Router",
-                        "url": "http://192.168.1.1",
-                        "require_admin": True,
-                    },
-                    "weather": {
-                        "icon": "mdi:weather",
-                        "title": "Weather",
-                        "url": "https://www.wunderground.com/us/ca/san-diego",
-                        "require_admin": True,
-                    },
-                    "api": {"icon": "mdi:weather", "title": "Api", "url": "/api"},
-                    "ftp": {
-                        "icon": "mdi:weather",
-                        "title": "FTP",
-                        "url": "ftp://some/ftp",
-                    },
-                }
-            },
-        )
+        with patch("homeassistant.components.http.start_http_server_and_save_config"):
+            assert setup.setup_component(
+                self.hass,
+                "panel_iframe",
+                {
+                    "panel_iframe": {
+                        "router": {
+                            "icon": "mdi:network-wireless",
+                            "title": "Router",
+                            "url": "http://192.168.1.1",
+                            "require_admin": True,
+                        },
+                        "weather": {
+                            "icon": "mdi:weather",
+                            "title": "Weather",
+                            "url": "https://www.wunderground.com/us/ca/san-diego",
+                            "require_admin": True,
+                        },
+                        "api": {"icon": "mdi:weather", "title": "Api", "url": "/api"},
+                        "ftp": {
+                            "icon": "mdi:weather",
+                            "title": "FTP",
+                            "url": "ftp://some/ftp",
+                        },
+                    }
+                },
+            )
 
         panels = self.hass.data[frontend.DATA_PANELS]
 

tests/test_bootstrap.py

@@ -261,7 +261,9 @@ async def test_setup_hass(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"browser": {}, "frontend": {}},
-    ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000):
+    ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000), patch(
+        "homeassistant.components.http.start_http_server_and_save_config"
+    ):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=verbose,
@@ -338,7 +340,7 @@ async def test_setup_hass_invalid_yaml(
     """Test it works."""
     with patch(
         "homeassistant.config.async_hass_config_yaml", side_effect=HomeAssistantError
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -391,7 +393,9 @@ async def test_setup_hass_safe_mode(
     hass.config_entries._async_schedule_save()
     await flush_store(hass.config_entries._store)
 
-    with patch("homeassistant.components.browser.setup") as browser_setup:
+    with patch("homeassistant.components.browser.setup") as browser_setup, patch(
+        "homeassistant.components.http.start_http_server_and_save_config"
+    ):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -421,7 +425,7 @@ async def test_setup_hass_invalid_core_config(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"homeassistant": {"non-existing": 1}},
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -451,7 +455,7 @@ async def test_setup_safe_mode_if_no_frontend(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"map": {}, "person": {"invalid": True}},
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=verbose,

tests/test_core.py

@@ -35,7 +35,7 @@ from homeassistant.exceptions import InvalidEntityFormatError, InvalidStateError
 import homeassistant.util.dt as dt_util
 from homeassistant.util.unit_system import METRIC_SYSTEM
 
-from tests.async_mock import MagicMock, Mock, patch
+from tests.async_mock import MagicMock, Mock, PropertyMock, patch
 from tests.common import async_mock_service, get_test_home_assistant
 
 PST = pytz.timezone("America/Los_Angeles")
@@ -901,6 +901,8 @@ class TestConfig(unittest.TestCase):
     def test_as_dict(self):
         """Test as dict."""
         self.config.config_dir = "/test/ha-config"
+        self.config.hass = MagicMock()
+        type(self.config.hass.state).value = PropertyMock(return_value="RUNNING")
         expected = {
             "latitude": 0,
             "longitude": 0,
@@ -914,6 +916,7 @@ class TestConfig(unittest.TestCase):
             "version": __version__,
             "config_source": "default",
             "safe_mode": False,
+            "state": "RUNNING",
             "external_url": None,
             "internal_url": None,
         }