diff --git a/homeassistant/config_entries.py b/homeassistant/config_entries.py
index a2a9ce78989..83bf9d22de3 100644
--- a/homeassistant/config_entries.py
+++ b/homeassistant/config_entries.py
@@ -340,7 +340,10 @@ class ConfigEntries:
         entry = self._entries.pop(found)
         self._async_schedule_save()
 
-        unloaded = await entry.async_unload(self.hass)
+        if entry.state == ENTRY_STATE_LOADED:
+            unloaded = await entry.async_unload(self.hass)
+        else:
+            unloaded = True
 
         device_registry = await \
             self.hass.helpers.device_registry.async_get_registry()
diff --git a/tests/components/config/test_config_entries.py b/tests/components/config/test_config_entries.py
index 66d29aac757..1e3b507727c 100644
--- a/tests/components/config/test_config_entries.py
+++ b/tests/components/config/test_config_entries.py
@@ -72,7 +72,7 @@ def test_get_entries(hass, client):
 @asyncio.coroutine
 def test_remove_entry(hass, client):
     """Test removing an entry via the API."""
-    entry = MockConfigEntry(domain='demo')
+    entry = MockConfigEntry(domain='demo', state=core_ce.ENTRY_STATE_LOADED)
     entry.add_to_hass(hass)
     resp = yield from client.delete(
         '/api/config/config_entries/entry/{}'.format(entry.entry_id))
diff --git a/tests/test_config_entries.py b/tests/test_config_entries.py
index afb13d71c2e..8f12407c6b7 100644
--- a/tests/test_config_entries.py
+++ b/tests/test_config_entries.py
@@ -49,7 +49,11 @@ def test_remove_entry(hass, manager):
         MockModule('comp', async_unload_entry=mock_unload_entry))
 
     MockConfigEntry(domain='test', entry_id='test1').add_to_manager(manager)
-    MockConfigEntry(domain='test', entry_id='test2').add_to_manager(manager)
+    MockConfigEntry(
+        domain='test',
+        entry_id='test2',
+        state=config_entries.ENTRY_STATE_LOADED
+    ).add_to_manager(manager)
     MockConfigEntry(domain='test', entry_id='test3').add_to_manager(manager)
 
     assert [item.entry_id for item in manager.async_entries()] == \
@@ -79,7 +83,11 @@ def test_remove_entry_raises(hass, manager):
         MockModule('comp', async_unload_entry=mock_unload_entry))
 
     MockConfigEntry(domain='test', entry_id='test1').add_to_manager(manager)
-    MockConfigEntry(domain='test', entry_id='test2').add_to_manager(manager)
+    MockConfigEntry(
+        domain='test',
+        entry_id='test2',
+        state=config_entries.ENTRY_STATE_LOADED
+    ).add_to_manager(manager)
     MockConfigEntry(domain='test', entry_id='test3').add_to_manager(manager)
 
     assert [item.entry_id for item in manager.async_entries()] == \
@@ -94,6 +102,33 @@ def test_remove_entry_raises(hass, manager):
         ['test1', 'test3']
 
 
+@asyncio.coroutine
+def test_remove_entry_if_not_loaded(hass, manager):
+    """Test that we can remove an entry."""
+    mock_unload_entry = MagicMock(return_value=mock_coro(True))
+
+    loader.set_component(
+        hass, 'test',
+        MockModule('comp', async_unload_entry=mock_unload_entry))
+
+    MockConfigEntry(domain='test', entry_id='test1').add_to_manager(manager)
+    MockConfigEntry(domain='test', entry_id='test2').add_to_manager(manager)
+    MockConfigEntry(domain='test', entry_id='test3').add_to_manager(manager)
+
+    assert [item.entry_id for item in manager.async_entries()] == \
+        ['test1', 'test2', 'test3']
+
+    result = yield from manager.async_remove('test2')
+
+    assert result == {
+        'require_restart': False
+    }
+    assert [item.entry_id for item in manager.async_entries()] == \
+        ['test1', 'test3']
+
+    assert len(mock_unload_entry.mock_calls) == 0
+
+
 @asyncio.coroutine
 def test_add_entry_calls_setup_entry(hass, manager):
     """Test we call setup_config_entry."""