Add file selector and file upload integration (#76672)

.strict-typing

@@ -98,6 +98,7 @@ homeassistant.components.energy.*
 homeassistant.components.evil_genius_labs.*
 homeassistant.components.fan.*
 homeassistant.components.fastdotcom.*
+homeassistant.components.file_upload.*
 homeassistant.components.filesize.*
 homeassistant.components.fitbit.*
 homeassistant.components.flunearyou.*

CODEOWNERS

@@ -329,6 +329,8 @@ build.json @home-assistant/supervisor
 /tests/components/fibaro/ @rappenze
 /homeassistant/components/file/ @fabaff
 /tests/components/file/ @fabaff
+/homeassistant/components/file_upload/ @home-assistant/core
+/tests/components/file_upload/ @home-assistant/core
 /homeassistant/components/filesize/ @gjohansson-ST
 /tests/components/filesize/ @gjohansson-ST
 /homeassistant/components/filter/ @dgomes

homeassistant/components/file_upload/__init__.py

@@ -0,0 +1,182 @@
+"""The File Upload integration."""
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+import shutil
+import tempfile
+
+from aiohttp import web
+import voluptuous as vol
+
+from homeassistant.components.http import HomeAssistantView
+from homeassistant.components.http.data_validator import RequestDataValidator
+from homeassistant.const import EVENT_HOMEASSISTANT_STOP
+from homeassistant.core import Event, HomeAssistant, callback
+from homeassistant.helpers.typing import ConfigType
+from homeassistant.util import raise_if_invalid_filename
+from homeassistant.util.ulid import ulid_hex
+
+DOMAIN = "file_upload"
+
+# If increased, change upload view to streaming
+# https://docs.aiohttp.org/en/stable/web_quickstart.html#file-uploads
+MAX_SIZE = 1024 * 1024 * 10
+TEMP_DIR_NAME = f"home-assistant-{DOMAIN}"
+
+
+@contextmanager
+def process_uploaded_file(hass: HomeAssistant, file_id: str) -> Iterator[Path]:
+    """Get an uploaded file.
+
+    File is removed at the end of the context.
+    """
+    if DOMAIN not in hass.data:
+        raise ValueError("File does not exist")
+
+    file_upload_data: FileUploadData = hass.data[DOMAIN]
+
+    if not file_upload_data.has_file(file_id):
+        raise ValueError("File does not exist")
+
+    try:
+        yield file_upload_data.file_path(file_id)
+    finally:
+        file_upload_data.files.pop(file_id)
+        shutil.rmtree(file_upload_data.file_dir(file_id))
+
+
+async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
+    """Set up File Upload."""
+    hass.http.register_view(FileUploadView)
+    return True
+
+
+@dataclass(frozen=True)
+class FileUploadData:
+    """File upload data."""
+
+    temp_dir: Path
+    files: dict[str, str]
+
+    @classmethod
+    async def create(cls, hass: HomeAssistant) -> FileUploadData:
+        """Initialize the file upload data."""
+
+        def _create_temp_dir() -> Path:
+            """Create temporary directory."""
+            temp_dir = Path(tempfile.gettempdir()) / TEMP_DIR_NAME
+
+            # If it exists, it's an old one and Home Assistant didn't shut down correctly.
+            if temp_dir.exists():
+                shutil.rmtree(temp_dir)
+
+            temp_dir.mkdir(0o700)
+            return temp_dir
+
+        temp_dir = await hass.async_add_executor_job(_create_temp_dir)
+
+        def cleanup_unused_files(ev: Event) -> None:
+            """Clean up unused files."""
+            shutil.rmtree(temp_dir)
+
+        hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, cleanup_unused_files)
+
+        return cls(temp_dir, {})
+
+    def has_file(self, file_id: str) -> bool:
+        """Return if file exists."""
+        return file_id in self.files
+
+    def file_dir(self, file_id: str) -> Path:
+        """Return the file directory."""
+        return self.temp_dir / file_id
+
+    def file_path(self, file_id: str) -> Path:
+        """Return the file path."""
+        return self.file_dir(file_id) / self.files[file_id]
+
+
+class FileUploadView(HomeAssistantView):
+    """HTTP View to upload files."""
+
+    url = "/api/file_upload"
+    name = "api:file_upload"
+
+    _upload_lock: asyncio.Lock | None = None
+
+    @callback
+    def _get_upload_lock(self) -> asyncio.Lock:
+        """Get upload lock."""
+        if self._upload_lock is None:
+            self._upload_lock = asyncio.Lock()
+
+        return self._upload_lock
+
+    async def post(self, request: web.Request) -> web.Response:
+        """Upload a file."""
+        async with self._get_upload_lock():
+            return await self._upload_file(request)
+
+    async def _upload_file(self, request: web.Request) -> web.Response:
+        """Handle uploaded file."""
+        # Increase max payload
+        request._client_max_size = MAX_SIZE  # pylint: disable=protected-access
+
+        data = await request.post()
+        file_field = data.get("file")
+
+        if not isinstance(file_field, web.FileField):
+            raise vol.Invalid("Expected a file")
+
+        try:
+            raise_if_invalid_filename(file_field.filename)
+        except ValueError as err:
+            raise web.HTTPBadRequest from err
+
+        hass: HomeAssistant = request.app["hass"]
+        file_id = ulid_hex()
+
+        if DOMAIN not in hass.data:
+            hass.data[DOMAIN] = await FileUploadData.create(hass)
+
+        file_upload_data: FileUploadData = hass.data[DOMAIN]
+        file_dir = file_upload_data.file_dir(file_id)
+
+        def _sync_work() -> None:
+            file_dir.mkdir()
+
+            # MyPy forgets about the isinstance check because we're in a function scope
+            assert isinstance(file_field, web.FileField)
+
+            with (file_dir / file_field.filename).open("wb") as target_fileobj:
+                shutil.copyfileobj(file_field.file, target_fileobj)
+
+        await hass.async_add_executor_job(_sync_work)
+
+        file_upload_data.files[file_id] = file_field.filename
+
+        return self.json({"file_id": file_id})
+
+    @RequestDataValidator({vol.Required("file_id"): str})
+    async def delete(self, request: web.Request, data: dict[str, str]) -> web.Response:
+        """Delete a file."""
+        hass: HomeAssistant = request.app["hass"]
+
+        if DOMAIN not in hass.data:
+            raise web.HTTPNotFound()
+
+        file_id = data["file_id"]
+        file_upload_data: FileUploadData = hass.data[DOMAIN]
+
+        if file_upload_data.files.pop(file_id, None) is None:
+            raise web.HTTPNotFound()
+
+        await hass.async_add_executor_job(
+            lambda: shutil.rmtree(file_upload_data.file_dir(file_id))
+        )
+
+        return self.json_message("File deleted")

homeassistant/components/file_upload/manifest.json

@@ -0,0 +1,8 @@
+{
+  "domain": "file_upload",
+  "name": "File Upload",
+  "documentation": "https://www.home-assistant.io/integrations/file_upload",
+  "dependencies": ["http"],
+  "codeowners": ["@home-assistant/core"],
+  "quality_scale": "internal"
+}

homeassistant/components/frontend/manifest.json

@@ -9,6 +9,7 @@
     "config",
     "device_automation",
     "diagnostics",
+    "file_upload",
     "http",
     "lovelace",
     "onboarding",

homeassistant/helpers/selector.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 
 from collections.abc import Callable, Sequence
 from typing import Any, TypedDict, cast
+from uuid import UUID
 
 import voluptuous as vol
 
@@ -10,7 +11,7 @@ from homeassistant.backports.enum import StrEnum
 from homeassistant.const import CONF_MODE, CONF_UNIT_OF_MEASUREMENT
 from homeassistant.core import split_entity_id, valid_entity_id
 from homeassistant.util import decorator
-from homeassistant.util.yaml.dumper import add_representer, represent_odict
+from homeassistant.util.yaml import dumper
 
 from . import config_validation as cv
 
@@ -888,9 +889,42 @@ class TimeSelector(Selector):
         return cast(str, data)
 
 
-add_representer(
+class FileSelectorConfig(TypedDict):
+    """Class to represent a file selector config."""
+
+    accept: str  # required
+
+
+@SELECTORS.register("file")
+class FileSelector(Selector):
+    """Selector of a file."""
+
+    selector_type = "file"
+
+    CONFIG_SCHEMA = vol.Schema(
+        {
+            # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/file#accept
+            vol.Required("accept"): str,
+        }
+    )
+
+    def __init__(self, config: FileSelectorConfig | None = None) -> None:
+        """Instantiate a selector."""
+        super().__init__(config)
+
+    def __call__(self, data: Any) -> str:
+        """Validate the passed selection."""
+        if not isinstance(data, str):
+            raise vol.Invalid("Value should be a string")
+
+        UUID(data)
+
+        return data
+
+
+dumper.add_representer(
     Selector,
-    lambda dumper, value: represent_odict(
+    lambda dumper, value: dumper.represent_odict(
         dumper, "tag:yaml.org,2002:map", value.serialize()
     ),
 )

mypy.ini

@@ -739,6 +739,16 @@ disallow_untyped_defs = true
 warn_return_any = true
 warn_unreachable = true
 
+[mypy-homeassistant.components.file_upload.*]
+check_untyped_defs = true
+disallow_incomplete_defs = true
+disallow_subclassing_any = true
+disallow_untyped_calls = true
+disallow_untyped_decorators = true
+disallow_untyped_defs = true
+warn_return_any = true
+warn_unreachable = true
+
 [mypy-homeassistant.components.filesize.*]
 check_untyped_defs = true
 disallow_incomplete_defs = true

script/hassfest/manifest.py

@@ -51,6 +51,7 @@ NO_IOT_CLASS = [
     "discovery",
     "downloader",
     "ffmpeg",
+    "file_upload",
     "frontend",
     "hardkernel",
     "hardware",

tests/components/file_upload/__init__.py

@@ -0,0 +1 @@
+"""Tests for the File Upload integration."""

tests/components/file_upload/test_init.py

@@ -0,0 +1,66 @@
+"""Test the File Upload integration."""
+from pathlib import Path
+from random import getrandbits
+from unittest.mock import patch
+
+import pytest
+
+from homeassistant.components import file_upload
+from homeassistant.core import HomeAssistant
+from homeassistant.setup import async_setup_component
+
+from tests.components.image import TEST_IMAGE
+
+
+@pytest.fixture
+async def uploaded_file_dir(hass: HomeAssistant, hass_client) -> Path:
+    """Test uploading and using a file."""
+    assert await async_setup_component(hass, "file_upload", {})
+    client = await hass_client()
+
+    with patch(
+        # Patch temp dir name to avoid tests fail running in parallel
+        "homeassistant.components.file_upload.TEMP_DIR_NAME",
+        file_upload.TEMP_DIR_NAME + f"-{getrandbits(10):03x}",
+    ), TEST_IMAGE.open("rb") as fp:
+        res = await client.post("/api/file_upload", data={"file": fp})
+
+    assert res.status == 200
+    response = await res.json()
+
+    file_dir = hass.data[file_upload.DOMAIN].file_dir(response["file_id"])
+    assert file_dir.is_dir()
+    return file_dir
+
+
+async def test_using_file(hass: HomeAssistant, uploaded_file_dir):
+    """Test uploading and using a file."""
+    # Test we can use it
+    with file_upload.process_uploaded_file(hass, uploaded_file_dir.name) as file_path:
+        assert file_path.is_file()
+        assert file_path.parent == uploaded_file_dir
+        assert file_path.read_bytes() == TEST_IMAGE.read_bytes()
+
+    # Test it's removed
+    assert not uploaded_file_dir.exists()
+
+
+async def test_removing_file(hass: HomeAssistant, hass_client, uploaded_file_dir):
+    """Test uploading and using a file."""
+    client = await hass_client()
+
+    response = await client.delete(
+        "/api/file_upload", json={"file_id": uploaded_file_dir.name}
+    )
+    assert response.status == 200
+
+    # Test it's removed
+    assert not uploaded_file_dir.exists()
+
+
+async def test_removed_on_stop(hass: HomeAssistant, hass_client, uploaded_file_dir):
+    """Test uploading and using a file."""
+    await hass.async_stop()
+
+    # Test it's removed
+    assert not uploaded_file_dir.exists()

tests/components/image/__init__.py

@@ -1 +1,4 @@
 """Tests for the Image integration."""
+import pathlib
+
+TEST_IMAGE = pathlib.Path(__file__).parent / "logo.png"

tests/components/image/test_init.py

@@ -9,11 +9,12 @@ from homeassistant.components.websocket_api import const as ws_const
 from homeassistant.setup import async_setup_component
 from homeassistant.util import dt as util_dt
 
+from . import TEST_IMAGE
+
 
 async def test_upload_image(hass, hass_client, hass_ws_client):
     """Test we can upload an image."""
     now = util_dt.utcnow()
-    test_image = pathlib.Path(__file__).parent / "logo.png"
 
     with tempfile.TemporaryDirectory() as tempdir, patch.object(
         hass.config, "path", return_value=tempdir
@@ -22,7 +23,7 @@ async def test_upload_image(hass, hass_client, hass_ws_client):
         ws_client: ClientWebSocketResponse = await hass_ws_client()
         client: ClientSession = await hass_client()
 
-        with test_image.open("rb") as fp:
+        with TEST_IMAGE.open("rb") as fp:
             res = await client.post("/api/image/upload", data={"file": fp})
 
         assert res.status == 200
@@ -36,7 +37,7 @@ async def test_upload_image(hass, hass_client, hass_ws_client):
 
         tempdir = pathlib.Path(tempdir)
         item_folder: pathlib.Path = tempdir / item["id"]
-        assert (item_folder / "original").read_bytes() == test_image.read_bytes()
+        assert (item_folder / "original").read_bytes() == TEST_IMAGE.read_bytes()
 
         # fetch non-existing image
         res = await client.get("/api/image/serve/non-existing/256x256")

tests/helpers/test_selector.py

@@ -646,3 +646,19 @@ def test_datetime_selector_schema(schema, valid_selections, invalid_selections):
 def test_template_selector_schema(schema, valid_selections, invalid_selections):
     """Test template selector."""
     _test_selector("template", schema, valid_selections, invalid_selections)
+
+
+@pytest.mark.parametrize(
+    "schema,valid_selections,invalid_selections",
+    (
+        (
+            {"accept": "image/*"},
+            ("0182a1b99dbc5ae24aecd90c346605fa",),
+            (None, "not-a-uuid", "abcd", 1),
+        ),
+    ),
+)
+def test_file_selector_schema(schema, valid_selections, invalid_selections):
+    """Test file selector."""
+
+    _test_selector("file", schema, valid_selections, invalid_selections)