From 5bbb5f43cbf8abadddd08ce8d11eb2be9a1bf60e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Feb 2020 23:06:52 +0000 Subject: [PATCH] Support XML conversion for RESTful sensors Many devices continue to use XML for RESTful APIs. Interfacing with these APIs requires custom integrations or command line fork()/exec() overhead which many of these devices can work with as if they were JSON using xmltojson via this spec: https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html This change implements converting XML output to JSON via xmltojson so it can work with the existing rest sensor component. As the attributes that usually need to be scraped are deeper in the document support for passing in a template to find the JSON attributes that have been added. JSON APIs that do not have their attributes at the top level can also benefit from this change. --- homeassistant/components/rest/manifest.json | 2 +- homeassistant/components/rest/sensor.py | 38 ++++ requirements_all.txt | 1 + requirements_test_all.txt | 1 + tests/components/rest/test_sensor.py | 182 ++++++++++++++++++++ 5 files changed, 223 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/rest/manifest.json b/homeassistant/components/rest/manifest.json index 8c8b7f39609..d03dce3017b 100644 --- a/homeassistant/components/rest/manifest.json +++ b/homeassistant/components/rest/manifest.json @@ -2,7 +2,7 @@ "domain": "rest", "name": "RESTful", "documentation": "https://www.home-assistant.io/integrations/rest", - "requirements": [], + "requirements": ["xmltodict==0.12.0"], "dependencies": [], "codeowners": [] } diff --git a/homeassistant/components/rest/sensor.py b/homeassistant/components/rest/sensor.py index 51120cb350c..bdff22d5a3b 100644 --- a/homeassistant/components/rest/sensor.py +++ b/homeassistant/components/rest/sensor.py @@ -1,10 +1,13 @@ """Support for RESTful API sensors.""" +import ast import json import logging +from xml.parsers.expat import ExpatError import requests from requests.auth import HTTPBasicAuth, HTTPDigestAuth import voluptuous as vol +import xmltodict from homeassistant.components.sensor import DEVICE_CLASSES_SCHEMA, PLATFORM_SCHEMA from homeassistant.const import ( @@ -38,7 +41,11 @@ DEFAULT_VERIFY_SSL = True DEFAULT_FORCE_UPDATE = False DEFAULT_TIMEOUT = 10 +CONF_CONVERT_XML = "convert_xml" +DEFAULT_CONVERT_XML = False + CONF_JSON_ATTRS = "json_attributes" +CONF_JSON_ATTRS_TEMPLATE = "json_attributes_template" METHODS = ["POST", "GET"] PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( @@ -50,6 +57,7 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( ), vol.Optional(CONF_HEADERS): vol.Schema({cv.string: cv.string}), vol.Optional(CONF_JSON_ATTRS, default=[]): cv.ensure_list_csv, + vol.Optional(CONF_CONVERT_XML, default=DEFAULT_CONVERT_XML): cv.boolean, vol.Optional(CONF_METHOD, default=DEFAULT_METHOD): vol.In(METHODS), vol.Optional(CONF_NAME, default=DEFAULT_NAME): cv.string, vol.Optional(CONF_PASSWORD): cv.string, @@ -57,6 +65,7 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( vol.Optional(CONF_UNIT_OF_MEASUREMENT): cv.string, vol.Optional(CONF_DEVICE_CLASS): DEVICE_CLASSES_SCHEMA, vol.Optional(CONF_USERNAME): cv.string, + vol.Optional(CONF_JSON_ATTRS_TEMPLATE): cv.template, vol.Optional(CONF_VALUE_TEMPLATE): cv.template, vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean, vol.Optional(CONF_FORCE_UPDATE, default=DEFAULT_FORCE_UPDATE): cv.boolean, @@ -84,9 +93,14 @@ def setup_platform(hass, config, add_entities, discovery_info=None): device_class = config.get(CONF_DEVICE_CLASS) value_template = config.get(CONF_VALUE_TEMPLATE) json_attrs = config.get(CONF_JSON_ATTRS) + convert_xml = config.get(CONF_CONVERT_XML) + json_attrs_template = config.get(CONF_JSON_ATTRS_TEMPLATE) force_update = config.get(CONF_FORCE_UPDATE) timeout = config.get(CONF_TIMEOUT) + if json_attrs_template is not None: + json_attrs_template.hass = hass + if value_template is not None: value_template.hass = hass @@ -120,6 +134,8 @@ def setup_platform(hass, config, add_entities, discovery_info=None): json_attrs, force_update, resource_template, + convert_xml, + json_attrs_template, ) ], True, @@ -140,6 +156,8 @@ class RestSensor(Entity): json_attrs, force_update, resource_template, + convert_xml, + json_attrs_template, ): """Initialize the REST sensor.""" self._hass = hass @@ -153,6 +171,8 @@ class RestSensor(Entity): self._attributes = None self._force_update = force_update self._resource_template = resource_template + self._convert_xml = convert_xml + self._json_attrs_template = json_attrs_template @property def name(self): @@ -191,12 +211,30 @@ class RestSensor(Entity): self.rest.update() value = self.rest.data + json_dict = None + + if self._convert_xml: + try: + value = json.dumps(xmltodict.parse(value)) + except ExpatError: + _LOGGER.warning( + "REST xml result could not be parsed and converted to JSON." + ) + _LOGGER.debug("Erroneous XML: %s", value) if self._json_attrs: self._attributes = {} if value: try: json_dict = json.loads(value) + if self._json_attrs_template is not None: + # render_with_possible_json_value returns single quoted + # strings so we cannot use json.loads to read it back here + json_dict = ast.literal_eval( + self._json_attrs_template.render_with_possible_json_value( + value, None + ) + ) if isinstance(json_dict, list): json_dict = json_dict[0] if isinstance(json_dict, dict): diff --git a/requirements_all.txt b/requirements_all.txt index ac811f0889f..f30de81f153 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -2094,6 +2094,7 @@ xfinity-gateway==0.0.4 xknx==0.11.2 # homeassistant.components.bluesound +# homeassistant.components.rest # homeassistant.components.startca # homeassistant.components.ted5000 # homeassistant.components.yr diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 130f9c4530d..5c6282cf4dc 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -711,6 +711,7 @@ withings-api==2.1.3 wled==0.2.1 # homeassistant.components.bluesound +# homeassistant.components.rest # homeassistant.components.startca # homeassistant.components.ted5000 # homeassistant.components.yr diff --git a/tests/components/rest/test_sensor.py b/tests/components/rest/test_sensor.py index 7e03eb0fd41..d2ce631b4db 100644 --- a/tests/components/rest/test_sensor.py +++ b/tests/components/rest/test_sensor.py @@ -166,6 +166,34 @@ class TestRestSensorSetup(unittest.TestCase): ) assert 2 == mock_req.call_count + @requests_mock.Mocker() + def test_setup_get_xml(self, mock_req): + """Test setup with valid configuration.""" + mock_req.get("http://localhost", status_code=200) + with assert_setup_component(1, "sensor"): + assert setup_component( + self.hass, + "sensor", + { + "sensor": { + "platform": "rest", + "resource": "http://localhost", + "method": "GET", + "value_template": "{{ value_json.key }}", + "name": "foo", + "unit_of_measurement": DATA_MEGABYTES, + "verify_ssl": "true", + "timeout": 30, + "convert_xml": True, + "authentication": "basic", + "username": "my username", + "password": "my password", + "headers": {"Accept": "text/xml"}, + } + }, + ) + assert 2 == mock_req.call_count + class TestRestSensor(unittest.TestCase): """Tests for REST sensor platform.""" @@ -185,9 +213,12 @@ class TestRestSensor(unittest.TestCase): self.unit_of_measurement = DATA_MEGABYTES self.device_class = None self.value_template = template("{{ value_json.key }}") + self.json_attrs_template = template("{{ value_json }}") self.value_template.hass = self.hass + self.json_attrs_template.hass = self.hass self.force_update = False self.resource_template = None + self.convert_xml = False self.sensor = rest.RestSensor( self.hass, @@ -199,6 +230,8 @@ class TestRestSensor(unittest.TestCase): [], self.force_update, self.resource_template, + self.convert_xml, + self.json_attrs_template, ) def tearDown(self): @@ -260,6 +293,8 @@ class TestRestSensor(unittest.TestCase): [], self.force_update, self.resource_template, + self.convert_xml, + self.json_attrs_template, ) self.sensor.update() assert "plain_state" == self.sensor.state @@ -281,6 +316,8 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.convert_xml, + self.json_attrs_template, ) self.sensor.update() assert "some_json_value" == self.sensor.device_state_attributes["key"] @@ -301,6 +338,8 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.convert_xml, + self.json_attrs_template, ) self.sensor.update() assert "another_value" == self.sensor.device_state_attributes["key"] @@ -321,6 +360,8 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.convert_xml, + self.json_attrs_template, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -343,6 +384,8 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.convert_xml, + self.json_attrs_template, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -365,6 +408,8 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.convert_xml, + self.json_attrs_template, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -389,6 +434,8 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.convert_xml, + self.json_attrs_template, ) self.sensor.update() @@ -397,6 +444,141 @@ class TestRestSensor(unittest.TestCase): "json_state_updated_value" == self.sensor.device_state_attributes["key"] ), self.force_update + def test_update_with_json_attrs_with_json_attrs_template(self): + """Test attributes get extracted from a JSON result with a template for the attributes.""" + json_attrs_template = template("{{ value_json.toplevel.second_level }}") + json_attrs_template.hass = self.hass + value_template = template("{{ value_json.toplevel.master_value }}") + value_template.hass = self.hass + + self.rest.update = Mock( + "rest.RestData.update", + side_effect=self.update_side_effect( + '{ "toplevel": {"master_value": "master", "second_level": {"some_json_key": "some_json_value", "some_json_key2": "some_json_value2" } } }' + ), + ) + self.sensor = rest.RestSensor( + self.hass, + self.rest, + self.name, + self.unit_of_measurement, + self.device_class, + value_template, + ["some_json_key", "some_json_key2"], + self.force_update, + self.resource_template, + self.convert_xml, + json_attrs_template, + ) + + self.sensor.update() + assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"] + assert ( + "some_json_value2" == self.sensor.device_state_attributes["some_json_key2"] + ) + assert "master" == self.sensor.state + + def test_update_with_xml_convert_json_attrs_with_json_attrs_template(self): + """Test attributes get extracted from a JSON result that was converted from XML with a template for the attributes.""" + json_attrs_template = template("{{ value_json.toplevel.second_level }}") + json_attrs_template.hass = self.hass + value_template = template("{{ value_json.toplevel.master_value }}") + value_template.hass = self.hass + convert_xml = True + + self.rest.update = Mock( + "rest.RestData.update", + side_effect=self.update_side_effect( + "mastersome_json_valuesome_json_value2" + ), + ) + self.sensor = rest.RestSensor( + self.hass, + self.rest, + self.name, + self.unit_of_measurement, + self.device_class, + value_template, + ["some_json_key", "some_json_key2"], + self.force_update, + self.resource_template, + convert_xml, + json_attrs_template, + ) + + self.sensor.update() + assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"] + assert ( + "some_json_value2" == self.sensor.device_state_attributes["some_json_key2"] + ) + assert "master" == self.sensor.state + + def test_update_with_xml_convert_json_attrs_with_jsonattr_template(self): + """Test attributes get extracted from a JSON result that was converted from XML.""" + json_attrs_template = template("{{ value_json.response }}") + json_attrs_template.hass = self.hass + value_template = template("{{ value_json.response.bss.wlan }}") + value_template.hass = self.hass + convert_xml = True + + self.rest.update = Mock( + "rest.RestData.update", + side_effect=self.update_side_effect( + '01255648alexander000bogus000000000upupupup000x0XF0x0XF 0' + ), + ) + self.sensor = rest.RestSensor( + self.hass, + self.rest, + self.name, + self.unit_of_measurement, + self.device_class, + value_template, + ["led0", "led1", "temp0", "time0", "ver"], + self.force_update, + self.resource_template, + convert_xml, + json_attrs_template, + ) + + self.sensor.update() + assert "0" == self.sensor.device_state_attributes["led0"] + assert "0" == self.sensor.device_state_attributes["led1"] + assert "0x0XF0x0XF" == self.sensor.device_state_attributes["temp0"] + assert "0" == self.sensor.device_state_attributes["time0"] + assert "12556" == self.sensor.device_state_attributes["ver"] + assert "bogus" == self.sensor.state + + @patch("homeassistant.components.rest.sensor._LOGGER") + def test_update_with_xml_convert_bad_xml(self, mock_logger): + """Test attributes get extracted from a XML result with bad xml.""" + value_template = template("{{ value_json.toplevel.master_value }}") + value_template.hass = self.hass + convert_xml = True + + self.rest.update = Mock( + "rest.RestData.update", + side_effect=self.update_side_effect("this is not xml"), + ) + self.sensor = rest.RestSensor( + self.hass, + self.rest, + self.name, + self.unit_of_measurement, + self.device_class, + value_template, + ["key"], + self.force_update, + self.resource_template, + convert_xml, + self.json_attrs_template, + ) + + self.sensor.update() + assert {} == self.sensor.device_state_attributes + assert mock_logger.warning.called + assert mock_logger.debug.called + class TestRestData(unittest.TestCase): """Tests for RestData."""