diff --git a/homeassistant/components/rest/manifest.json b/homeassistant/components/rest/manifest.json index d03dce3017b..fd7eea12f7e 100644 --- a/homeassistant/components/rest/manifest.json +++ b/homeassistant/components/rest/manifest.json @@ -2,7 +2,7 @@ "domain": "rest", "name": "RESTful", "documentation": "https://www.home-assistant.io/integrations/rest", - "requirements": ["xmltodict==0.12.0"], + "requirements": ["jsonpath==0.82", "xmltodict==0.12.0"], "dependencies": [], "codeowners": [] } diff --git a/homeassistant/components/rest/sensor.py b/homeassistant/components/rest/sensor.py index bdff22d5a3b..9372d76cd8a 100644 --- a/homeassistant/components/rest/sensor.py +++ b/homeassistant/components/rest/sensor.py @@ -1,9 +1,9 @@ """Support for RESTful API sensors.""" -import ast import json import logging from xml.parsers.expat import ExpatError +from jsonpath import jsonpath import requests from requests.auth import HTTPBasicAuth, HTTPDigestAuth import voluptuous as vol @@ -41,11 +41,9 @@ DEFAULT_VERIFY_SSL = True DEFAULT_FORCE_UPDATE = False DEFAULT_TIMEOUT = 10 -CONF_CONVERT_XML = "convert_xml" -DEFAULT_CONVERT_XML = False CONF_JSON_ATTRS = "json_attributes" -CONF_JSON_ATTRS_TEMPLATE = "json_attributes_template" +CONF_JSON_ATTRS_PATH = "json_attributes_path" METHODS = ["POST", "GET"] PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( @@ -57,7 +55,6 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( ), vol.Optional(CONF_HEADERS): vol.Schema({cv.string: cv.string}), vol.Optional(CONF_JSON_ATTRS, default=[]): cv.ensure_list_csv, - vol.Optional(CONF_CONVERT_XML, default=DEFAULT_CONVERT_XML): cv.boolean, vol.Optional(CONF_METHOD, default=DEFAULT_METHOD): vol.In(METHODS), vol.Optional(CONF_NAME, default=DEFAULT_NAME): cv.string, vol.Optional(CONF_PASSWORD): cv.string, @@ -65,7 +62,7 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( vol.Optional(CONF_UNIT_OF_MEASUREMENT): cv.string, vol.Optional(CONF_DEVICE_CLASS): DEVICE_CLASSES_SCHEMA, vol.Optional(CONF_USERNAME): cv.string, - vol.Optional(CONF_JSON_ATTRS_TEMPLATE): cv.template, + vol.Optional(CONF_JSON_ATTRS_PATH): cv.string, vol.Optional(CONF_VALUE_TEMPLATE): cv.template, vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean, vol.Optional(CONF_FORCE_UPDATE, default=DEFAULT_FORCE_UPDATE): cv.boolean, @@ -93,14 +90,10 @@ def setup_platform(hass, config, add_entities, discovery_info=None): device_class = config.get(CONF_DEVICE_CLASS) value_template = config.get(CONF_VALUE_TEMPLATE) json_attrs = config.get(CONF_JSON_ATTRS) - convert_xml = config.get(CONF_CONVERT_XML) - json_attrs_template = config.get(CONF_JSON_ATTRS_TEMPLATE) + json_attrs_path = config.get(CONF_JSON_ATTRS_PATH) force_update = config.get(CONF_FORCE_UPDATE) timeout = config.get(CONF_TIMEOUT) - if json_attrs_template is not None: - json_attrs_template.hass = hass - if value_template is not None: value_template.hass = hass @@ -134,8 +127,7 @@ def setup_platform(hass, config, add_entities, discovery_info=None): json_attrs, force_update, resource_template, - convert_xml, - json_attrs_template, + json_attrs_path, ) ], True, @@ -156,8 +148,7 @@ class RestSensor(Entity): json_attrs, force_update, resource_template, - convert_xml, - json_attrs_template, + json_attrs_path, ): """Initialize the REST sensor.""" self._hass = hass @@ -171,8 +162,7 @@ class RestSensor(Entity): self._attributes = None self._force_update = force_update self._resource_template = resource_template - self._convert_xml = convert_xml - self._json_attrs_template = json_attrs_template + self._json_attrs_path = json_attrs_path @property def name(self): @@ -211,9 +201,9 @@ class RestSensor(Entity): self.rest.update() value = self.rest.data - json_dict = None + content_type = self.rest.headers.get("content-type") - if self._convert_xml: + if content_type and content_type.startswith("text/xml"): try: value = json.dumps(xmltodict.parse(value)) except ExpatError: @@ -227,14 +217,8 @@ class RestSensor(Entity): if value: try: json_dict = json.loads(value) - if self._json_attrs_template is not None: - # render_with_possible_json_value returns single quoted - # strings so we cannot use json.loads to read it back here - json_dict = ast.literal_eval( - self._json_attrs_template.render_with_possible_json_value( - value, None - ) - ) + if self._json_attrs_path is not None: + json_dict = jsonpath(json_dict, self._json_attrs_path) if isinstance(json_dict, list): json_dict = json_dict[0] if isinstance(json_dict, dict): @@ -278,6 +262,7 @@ class RestData: self._verify_ssl = verify_ssl self._timeout = timeout self.data = None + self.headers = None def set_url(self, url): """Set url.""" @@ -297,6 +282,8 @@ class RestData: verify=self._verify_ssl, ) self.data = response.text + self.headers = response.headers except requests.exceptions.RequestException as ex: _LOGGER.error("Error fetching data: %s failed with %s", self._resource, ex) self.data = None + self.headers = None diff --git a/requirements_all.txt b/requirements_all.txt index f30de81f153..3cc7163d4f3 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -742,6 +742,7 @@ iperf3==0.1.11 # homeassistant.components.route53 ipify==1.0.0 +# homeassistant.components.rest # homeassistant.components.verisure jsonpath==0.82 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 5c6282cf4dc..fafea1c49ba 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -278,6 +278,7 @@ iaqualink==0.3.1 # homeassistant.components.influxdb influxdb==5.2.3 +# homeassistant.components.rest # homeassistant.components.verisure jsonpath==0.82 diff --git a/tests/components/rest/test_sensor.py b/tests/components/rest/test_sensor.py index d2ce631b4db..30eeae9a8e3 100644 --- a/tests/components/rest/test_sensor.py +++ b/tests/components/rest/test_sensor.py @@ -6,6 +6,7 @@ import pytest from pytest import raises import requests from requests.exceptions import RequestException, Timeout +from requests.structures import CaseInsensitiveDict import requests_mock import homeassistant.components.rest.sensor as rest @@ -184,7 +185,6 @@ class TestRestSensorSetup(unittest.TestCase): "unit_of_measurement": DATA_MEGABYTES, "verify_ssl": "true", "timeout": 30, - "convert_xml": True, "authentication": "basic", "username": "my username", "password": "my password", @@ -206,19 +206,18 @@ class TestRestSensor(unittest.TestCase): self.rest.update = Mock( "rest.RestData.update", side_effect=self.update_side_effect( - '{ "key": "' + self.initial_state + '" }' + '{ "key": "' + self.initial_state + '" }', + CaseInsensitiveDict({"Content-Type": "application/json"}), ), ) self.name = "foo" self.unit_of_measurement = DATA_MEGABYTES self.device_class = None self.value_template = template("{{ value_json.key }}") - self.json_attrs_template = template("{{ value_json }}") + self.json_attrs_path = None self.value_template.hass = self.hass - self.json_attrs_template.hass = self.hass self.force_update = False self.resource_template = None - self.convert_xml = False self.sensor = rest.RestSensor( self.hass, @@ -230,17 +229,17 @@ class TestRestSensor(unittest.TestCase): [], self.force_update, self.resource_template, - self.convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) def tearDown(self): """Stop everything that was started.""" self.hass.stop() - def update_side_effect(self, data): + def update_side_effect(self, data, headers): """Side effect function for mocking RestData.update().""" self.rest.data = data + self.rest.headers = headers def test_name(self): """Test the name.""" @@ -262,7 +261,8 @@ class TestRestSensor(unittest.TestCase): def test_update_when_value_is_none(self): """Test state gets updated to unknown when sensor returns no data.""" self.rest.update = Mock( - "rest.RestData.update", side_effect=self.update_side_effect(None) + "rest.RestData.update", + side_effect=self.update_side_effect(None, CaseInsensitiveDict()), ) self.sensor.update() assert self.sensor.state is None @@ -272,7 +272,10 @@ class TestRestSensor(unittest.TestCase): """Test state gets updated when sensor returns a new status.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect('{ "key": "updated_state" }'), + side_effect=self.update_side_effect( + '{ "key": "updated_state" }', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), ) self.sensor.update() assert "updated_state" == self.sensor.state @@ -281,7 +284,10 @@ class TestRestSensor(unittest.TestCase): def test_update_with_no_template(self): """Test update when there is no value template.""" self.rest.update = Mock( - "rest.RestData.update", side_effect=self.update_side_effect("plain_state") + "rest.RestData.update", + side_effect=self.update_side_effect( + "plain_state", CaseInsensitiveDict({"Content-Type": "application/json"}) + ), ) self.sensor = rest.RestSensor( self.hass, @@ -293,8 +299,7 @@ class TestRestSensor(unittest.TestCase): [], self.force_update, self.resource_template, - self.convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) self.sensor.update() assert "plain_state" == self.sensor.state @@ -304,7 +309,10 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a JSON result.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect('{ "key": "some_json_value" }'), + side_effect=self.update_side_effect( + '{ "key": "some_json_value" }', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), ) self.sensor = rest.RestSensor( self.hass, @@ -316,8 +324,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, - self.convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) self.sensor.update() assert "some_json_value" == self.sensor.device_state_attributes["key"] @@ -326,7 +333,10 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a JSON list[0] result.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect('[{ "key": "another_value" }]'), + side_effect=self.update_side_effect( + '[{ "key": "another_value" }]', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), ) self.sensor = rest.RestSensor( self.hass, @@ -338,8 +348,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, - self.convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) self.sensor.update() assert "another_value" == self.sensor.device_state_attributes["key"] @@ -348,7 +357,10 @@ class TestRestSensor(unittest.TestCase): def test_update_with_json_attrs_no_data(self, mock_logger): """Test attributes when no JSON result fetched.""" self.rest.update = Mock( - "rest.RestData.update", side_effect=self.update_side_effect(None) + "rest.RestData.update", + side_effect=self.update_side_effect( + None, CaseInsensitiveDict({"Content-Type": "application/json"}) + ), ) self.sensor = rest.RestSensor( self.hass, @@ -360,8 +372,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, - self.convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -372,7 +383,10 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a JSON result.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect('["list", "of", "things"]'), + side_effect=self.update_side_effect( + '["list", "of", "things"]', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), ) self.sensor = rest.RestSensor( self.hass, @@ -384,8 +398,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, - self.convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -396,7 +409,10 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a JSON result.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect("This is text rather than JSON data."), + side_effect=self.update_side_effect( + "This is text rather than JSON data.", + CaseInsensitiveDict({"Content-Type": "text/plain"}), + ), ) self.sensor = rest.RestSensor( self.hass, @@ -408,8 +424,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, - self.convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -421,7 +436,8 @@ class TestRestSensor(unittest.TestCase): self.rest.update = Mock( "rest.RestData.update", side_effect=self.update_side_effect( - '{ "key": "json_state_updated_value" }' + '{ "key": "json_state_updated_value" }', + CaseInsensitiveDict({"Content-Type": "application/json"}), ), ) self.sensor = rest.RestSensor( @@ -434,8 +450,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, - self.convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) self.sensor.update() @@ -444,17 +459,17 @@ class TestRestSensor(unittest.TestCase): "json_state_updated_value" == self.sensor.device_state_attributes["key"] ), self.force_update - def test_update_with_json_attrs_with_json_attrs_template(self): + def test_update_with_json_attrs_with_json_attrs_path(self): """Test attributes get extracted from a JSON result with a template for the attributes.""" - json_attrs_template = template("{{ value_json.toplevel.second_level }}") - json_attrs_template.hass = self.hass + json_attrs_path = "$.toplevel.second_level" value_template = template("{{ value_json.toplevel.master_value }}") value_template.hass = self.hass self.rest.update = Mock( "rest.RestData.update", side_effect=self.update_side_effect( - '{ "toplevel": {"master_value": "master", "second_level": {"some_json_key": "some_json_value", "some_json_key2": "some_json_value2" } } }' + '{ "toplevel": {"master_value": "master", "second_level": {"some_json_key": "some_json_value", "some_json_key2": "some_json_value2" } } }', + CaseInsensitiveDict({"Content-Type": "application/json"}), ), ) self.sensor = rest.RestSensor( @@ -467,8 +482,7 @@ class TestRestSensor(unittest.TestCase): ["some_json_key", "some_json_key2"], self.force_update, self.resource_template, - self.convert_xml, - json_attrs_template, + json_attrs_path, ) self.sensor.update() @@ -478,18 +492,17 @@ class TestRestSensor(unittest.TestCase): ) assert "master" == self.sensor.state - def test_update_with_xml_convert_json_attrs_with_json_attrs_template(self): + def test_update_with_xml_convert_json_attrs_with_json_attrs_path(self): """Test attributes get extracted from a JSON result that was converted from XML with a template for the attributes.""" - json_attrs_template = template("{{ value_json.toplevel.second_level }}") - json_attrs_template.hass = self.hass + json_attrs_path = "$.toplevel.second_level" value_template = template("{{ value_json.toplevel.master_value }}") value_template.hass = self.hass - convert_xml = True self.rest.update = Mock( "rest.RestData.update", side_effect=self.update_side_effect( - "mastersome_json_valuesome_json_value2" + "mastersome_json_valuesome_json_value2", + CaseInsensitiveDict({"Content-Type": "text/xml+svg"}), ), ) self.sensor = rest.RestSensor( @@ -502,8 +515,7 @@ class TestRestSensor(unittest.TestCase): ["some_json_key", "some_json_key2"], self.force_update, self.resource_template, - convert_xml, - json_attrs_template, + json_attrs_path, ) self.sensor.update() @@ -515,16 +527,15 @@ class TestRestSensor(unittest.TestCase): def test_update_with_xml_convert_json_attrs_with_jsonattr_template(self): """Test attributes get extracted from a JSON result that was converted from XML.""" - json_attrs_template = template("{{ value_json.response }}") - json_attrs_template.hass = self.hass + json_attrs_path = "$.response" value_template = template("{{ value_json.response.bss.wlan }}") value_template.hass = self.hass - convert_xml = True self.rest.update = Mock( "rest.RestData.update", side_effect=self.update_side_effect( - '01255648alexander000bogus000000000upupupup000x0XF0x0XF 0' + '01255648alexander000bogus000000000upupupup000x0XF0x0XF 0', + CaseInsensitiveDict({"Content-Type": "text/xml"}), ), ) self.sensor = rest.RestSensor( @@ -537,8 +548,7 @@ class TestRestSensor(unittest.TestCase): ["led0", "led1", "temp0", "time0", "ver"], self.force_update, self.resource_template, - convert_xml, - json_attrs_template, + json_attrs_path, ) self.sensor.update() @@ -554,11 +564,12 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a XML result with bad xml.""" value_template = template("{{ value_json.toplevel.master_value }}") value_template.hass = self.hass - convert_xml = True self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect("this is not xml"), + side_effect=self.update_side_effect( + "this is not xml", CaseInsensitiveDict({"Content-Type": "text/xml"}) + ), ) self.sensor = rest.RestSensor( self.hass, @@ -570,8 +581,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, - convert_xml, - self.json_attrs_template, + self.json_attrs_path, ) self.sensor.update()