From d0021a61711d163f0cf70e0a0d179c6fa384d32f Mon Sep 17 00:00:00 2001
From: Paulus Schoutsen <paulus@paulusschoutsen.nl>
Date: Thu, 1 Jun 2017 23:23:39 -0700
Subject: [PATCH] Make monkey patch work in Python 3.6 (#7848)

* Make monkey patch work in Python 3.6

* Update dockerfiles back to 3.6

* Lint

* Do not set env variable for dockerfile

* Lint
---
 Dockerfile                           |  2 +-
 homeassistant/__main__.py            | 12 ++++-
 homeassistant/monkey_patch.py        | 75 ++++++++++++++++++++++++++++
 script/bootstrap_server              |  2 +-
 virtualization/Docker/Dockerfile.dev |  2 +-
 5 files changed, 89 insertions(+), 4 deletions(-)
 create mode 100644 homeassistant/monkey_patch.py

diff --git a/Dockerfile b/Dockerfile
index 3d899f2404bd..8c4cd0f54402 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.5
+FROM python:3.6
 MAINTAINER Paulus Schoutsen <Paulus@PaulusSchoutsen.nl>
 
 # Uncomment any of the following lines to disable the installation.
diff --git a/homeassistant/__main__.py b/homeassistant/__main__.py
index 0b07e5aa6f66..219d413db122 100644
--- a/homeassistant/__main__.py
+++ b/homeassistant/__main__.py
@@ -10,6 +10,7 @@ import threading
 
 from typing import Optional, List
 
+from homeassistant import monkey_patch
 from homeassistant.const import (
     __version__,
     EVENT_HOMEASSISTANT_START,
@@ -17,7 +18,6 @@ from homeassistant.const import (
     REQUIRED_PYTHON_VER_WIN,
     RESTART_EXIT_CODE,
 )
-from homeassistant.util.async import run_callback_threadsafe
 
 
 def attempt_use_uvloop():
@@ -310,6 +310,9 @@ def setup_and_run_hass(config_dir: str,
         return None
 
     if args.open_ui:
+        # Imported here to avoid importing asyncio before monkey patch
+        from homeassistant.util.async import run_callback_threadsafe
+
         def open_browser(event):
             """Open the webinterface in a browser."""
             if hass.config.api is not None:
@@ -371,6 +374,13 @@ def main() -> int:
     """Start Home Assistant."""
     validate_python()
 
+    if os.environ.get('HASS_MONKEYPATCH_ASYNCIO') == '1':
+        if sys.version_info[:3] >= (3, 6):
+            monkey_patch.disable_c_asyncio()
+        monkey_patch.patch_weakref_tasks()
+    elif sys.version_info[:3] < (3, 5, 3):
+        monkey_patch.patch_weakref_tasks()
+
     attempt_use_uvloop()
 
     if sys.version_info[:3] < (3, 5, 3):
diff --git a/homeassistant/monkey_patch.py b/homeassistant/monkey_patch.py
new file mode 100644
index 000000000000..819d8de48e02
--- /dev/null
+++ b/homeassistant/monkey_patch.py
@@ -0,0 +1,75 @@
+"""Monkey patch Python to work around issues causing segfaults.
+
+Under heavy threading operations that schedule calls into
+the asyncio event loop, Task objects are created. Due to
+a bug in Python, GC may have an issue when switching between
+the threads and objects with __del__ (which various components
+in HASS have).
+
+This monkey-patch removes the weakref.Weakset, and replaces it
+with an object that ignores the only call utilizing it (the
+Task.__init__ which calls _all_tasks.add(self)). It also removes
+the __del__ which could trigger the future objects __del__ at
+unpredictable times.
+
+The side-effect of this manipulation of the Task is that
+Task.all_tasks() is no longer accurate, and there will be no
+warning emitted if a Task is GC'd while in use.
+
+Related Python bugs:
+ - https://bugs.python.org/issue26617
+"""
+import sys
+
+
+def patch_weakref_tasks():
+    """Replace weakref.WeakSet to address Python 3 bug."""
+    # pylint: disable=no-self-use, protected-access, bare-except
+    import asyncio.tasks
+
+    class IgnoreCalls:
+        """Ignore add calls."""
+
+        def add(self, other):
+            """No-op add."""
+            return
+
+    asyncio.tasks.Task._all_tasks = IgnoreCalls()
+    try:
+        del asyncio.tasks.Task.__del__
+    except:
+        pass
+
+
+def disable_c_asyncio():
+    """Disable using C implementation of asyncio.
+
+    Required to be able to apply the weakref monkey patch.
+
+    Requires Python 3.6+.
+    """
+    class AsyncioImportFinder:
+        """Finder that blocks C version of asyncio being loaded."""
+
+        PATH_TRIGGER = '_asyncio'
+
+        def __init__(self, path_entry):
+            if path_entry != self.PATH_TRIGGER:
+                raise ImportError()
+            return
+
+        def find_module(self, fullname, path=None):
+            """Find a module."""
+            if fullname == self.PATH_TRIGGER:
+                # We lint in Py34, exception is introduced in Py36
+                # pylint: disable=undefined-variable
+                raise ModuleNotFoundError()  # noqa
+            return None
+
+    sys.path_hooks.append(AsyncioImportFinder)
+    sys.path.insert(0, AsyncioImportFinder.PATH_TRIGGER)
+
+    try:
+        import _asyncio  # noqa
+    except ImportError:
+        pass
diff --git a/script/bootstrap_server b/script/bootstrap_server
index 7929a00fe551..791adc3a0c18 100755
--- a/script/bootstrap_server
+++ b/script/bootstrap_server
@@ -7,4 +7,4 @@ set -e
 cd "$(dirname "$0")/.."
 
 echo "Installing test dependencies..."
-python3 -m pip install tox
+python3 -m pip install tox colorlog
diff --git a/virtualization/Docker/Dockerfile.dev b/virtualization/Docker/Dockerfile.dev
index 58d99285bc91..2f40ea5f409a 100644
--- a/virtualization/Docker/Dockerfile.dev
+++ b/virtualization/Docker/Dockerfile.dev
@@ -2,7 +2,7 @@
 # Based on the production Dockerfile, but with development additions.
 # Keep this file as close as possible to the production Dockerfile, so the environments match.
 
-FROM python:3.5
+FROM python:3.6
 MAINTAINER Paulus Schoutsen <Paulus@PaulusSchoutsen.nl>
 
 # Uncomment any of the following lines to disable the installation.