home-assistant/core
1
0
Fork 1
mirror of https://github.com/home-assistant/core.git synced 2025-08-01 03:35:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Support XML conversion for RESTful sensors

Browse Source 
Many devices continue to use XML for RESTful
APIs.  Interfacing with these APIs requires custom
integrations or command line fork()/exec() overhead
which many of these devices can work with as if
they were JSON using xmltojson via this spec:
https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html

This change implements converting XML output to
JSON via xmltojson so it can work with the existing
rest sensor component.  As the attributes that
usually need to be scraped are deeper in the document
support for passing in a template to find the
JSON attributes that have been added.  JSON APIs that
do not have their attributes at the top level
can also benefit from this change.
This commit is contained in:
J. Nick Koston
2020-02-13 23:06:52 +00:00
parent 3e23a3a860
commit 5bbb5f43cb
5 changed files with 223 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
homeassistant/components/rest/manifest.json
View File

@@ -2,7 +2,7 @@
"domain": "rest",
"name": "RESTful",
"documentation": "https://www.home-assistant.io/integrations/rest",
"requirements": [],
"requirements": ["xmltodict==0.12.0"],
"dependencies": [],
"codeowners": []
}

38
homeassistant/components/rest/sensor.py
View File

@@ -1,10 +1,13 @@
"""Support for RESTful API sensors."""
import ast
import json
import logging
from xml.parsers.expat import ExpatError
import requests
from requests.auth import HTTPBasicAuth, HTTPDigestAuth
import voluptuous as vol
import xmltodict
from homeassistant.components.sensor import DEVICE_CLASSES_SCHEMA, PLATFORM_SCHEMA
from homeassistant.const import (
@@ -38,7 +41,11 @@ DEFAULT_VERIFY_SSL = True
DEFAULT_FORCE_UPDATE = False
DEFAULT_TIMEOUT = 10
CONF_CONVERT_XML = "convert_xml"
DEFAULT_CONVERT_XML = False
CONF_JSON_ATTRS = "json_attributes"
CONF_JSON_ATTRS_TEMPLATE = "json_attributes_template"
METHODS = ["POST", "GET"]
PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
@@ -50,6 +57,7 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
),
vol.Optional(CONF_HEADERS): vol.Schema({cv.string: cv.string}),
vol.Optional(CONF_JSON_ATTRS, default=[]): cv.ensure_list_csv,
vol.Optional(CONF_CONVERT_XML, default=DEFAULT_CONVERT_XML): cv.boolean,
vol.Optional(CONF_METHOD, default=DEFAULT_METHOD): vol.In(METHODS),
vol.Optional(CONF_NAME, default=DEFAULT_NAME): cv.string,
vol.Optional(CONF_PASSWORD): cv.string,
@@ -57,6 +65,7 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
vol.Optional(CONF_UNIT_OF_MEASUREMENT): cv.string,
vol.Optional(CONF_DEVICE_CLASS): DEVICE_CLASSES_SCHEMA,
vol.Optional(CONF_USERNAME): cv.string,
vol.Optional(CONF_JSON_ATTRS_TEMPLATE): cv.template,
vol.Optional(CONF_VALUE_TEMPLATE): cv.template,
vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean,
vol.Optional(CONF_FORCE_UPDATE, default=DEFAULT_FORCE_UPDATE): cv.boolean,
@@ -84,9 +93,14 @@ def setup_platform(hass, config, add_entities, discovery_info=None):
device_class = config.get(CONF_DEVICE_CLASS)
value_template = config.get(CONF_VALUE_TEMPLATE)
json_attrs = config.get(CONF_JSON_ATTRS)
convert_xml = config.get(CONF_CONVERT_XML)
json_attrs_template = config.get(CONF_JSON_ATTRS_TEMPLATE)
force_update = config.get(CONF_FORCE_UPDATE)
timeout = config.get(CONF_TIMEOUT)
if json_attrs_template is not None:
json_attrs_template.hass = hass
if value_template is not None:
value_template.hass = hass
@@ -120,6 +134,8 @@ def setup_platform(hass, config, add_entities, discovery_info=None):
json_attrs,
force_update,
resource_template,
convert_xml,
json_attrs_template,
)
],
True,
@@ -140,6 +156,8 @@ class RestSensor(Entity):
json_attrs,
force_update,
resource_template,
convert_xml,
json_attrs_template,
):
"""Initialize the REST sensor."""
self._hass = hass
@@ -153,6 +171,8 @@ class RestSensor(Entity):
self._attributes = None
self._force_update = force_update
self._resource_template = resource_template
self._convert_xml = convert_xml
self._json_attrs_template = json_attrs_template
@property
def name(self):
@@ -191,12 +211,30 @@ class RestSensor(Entity):
self.rest.update()
value = self.rest.data
json_dict = None
if self._convert_xml:
try:
value = json.dumps(xmltodict.parse(value))
except ExpatError:
_LOGGER.warning(
"REST xml result could not be parsed and converted to JSON."
)
_LOGGER.debug("Erroneous XML: %s", value)
if self._json_attrs:
self._attributes = {}
if value:
try:
json_dict = json.loads(value)
if self._json_attrs_template is not None:
# render_with_possible_json_value returns single quoted
# strings so we cannot use json.loads to read it back here
json_dict = ast.literal_eval(
self._json_attrs_template.render_with_possible_json_value(
value, None
)
)
if isinstance(json_dict, list):
json_dict = json_dict[0]
if isinstance(json_dict, dict):

1
requirements_all.txt
View File

@@ -2094,6 +2094,7 @@ xfinity-gateway==0.0.4
xknx==0.11.2
# homeassistant.components.bluesound
# homeassistant.components.rest
# homeassistant.components.startca
# homeassistant.components.ted5000
# homeassistant.components.yr

1
requirements_test_all.txt
View File

@@ -711,6 +711,7 @@ withings-api==2.1.3
wled==0.2.1
# homeassistant.components.bluesound
# homeassistant.components.rest
# homeassistant.components.startca
# homeassistant.components.ted5000
# homeassistant.components.yr

182
tests/components/rest/test_sensor.py
View File

@@ -166,6 +166,34 @@ class TestRestSensorSetup(unittest.TestCase):
)
assert 2 == mock_req.call_count
@requests_mock.Mocker()
def test_setup_get_xml(self, mock_req):
"""Test setup with valid configuration."""
mock_req.get("http://localhost", status_code=200)
with assert_setup_component(1, "sensor"):
assert setup_component(
self.hass,
"sensor",
{
"sensor": {
"platform": "rest",
"resource": "http://localhost",
"method": "GET",
"value_template": "{{ value_json.key }}",
"name": "foo",
"unit_of_measurement": DATA_MEGABYTES,
"verify_ssl": "true",
"timeout": 30,
"convert_xml": True,
"authentication": "basic",
"username": "my username",
"password": "my password",
"headers": {"Accept": "text/xml"},
}
},
)
assert 2 == mock_req.call_count
class TestRestSensor(unittest.TestCase):
"""Tests for REST sensor platform."""
@@ -185,9 +213,12 @@ class TestRestSensor(unittest.TestCase):
self.unit_of_measurement = DATA_MEGABYTES
self.device_class = None
self.value_template = template("{{ value_json.key }}")
self.json_attrs_template = template("{{ value_json }}")
self.value_template.hass = self.hass
self.json_attrs_template.hass = self.hass
self.force_update = False
self.resource_template = None
self.convert_xml = False
self.sensor = rest.RestSensor(
self.hass,
@@ -199,6 +230,8 @@ class TestRestSensor(unittest.TestCase):
[],
self.force_update,
self.resource_template,
self.convert_xml,
self.json_attrs_template,
)
def tearDown(self):
@@ -260,6 +293,8 @@ class TestRestSensor(unittest.TestCase):
[],
self.force_update,
self.resource_template,
self.convert_xml,
self.json_attrs_template,
)
self.sensor.update()
assert "plain_state" == self.sensor.state
@@ -281,6 +316,8 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
self.convert_xml,
self.json_attrs_template,
)
self.sensor.update()
assert "some_json_value" == self.sensor.device_state_attributes["key"]
@@ -301,6 +338,8 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
self.convert_xml,
self.json_attrs_template,
)
self.sensor.update()
assert "another_value" == self.sensor.device_state_attributes["key"]
@@ -321,6 +360,8 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
self.convert_xml,
self.json_attrs_template,
)
self.sensor.update()
assert {} == self.sensor.device_state_attributes
@@ -343,6 +384,8 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
self.convert_xml,
self.json_attrs_template,
)
self.sensor.update()
assert {} == self.sensor.device_state_attributes
@@ -365,6 +408,8 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
self.convert_xml,
self.json_attrs_template,
)
self.sensor.update()
assert {} == self.sensor.device_state_attributes
@@ -389,6 +434,8 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
self.convert_xml,
self.json_attrs_template,
)
self.sensor.update()
@@ -397,6 +444,141 @@ class TestRestSensor(unittest.TestCase):
"json_state_updated_value" == self.sensor.device_state_attributes["key"]
), self.force_update
def test_update_with_json_attrs_with_json_attrs_template(self):
"""Test attributes get extracted from a JSON result with a template for the attributes."""
json_attrs_template = template("{{ value_json.toplevel.second_level }}")
json_attrs_template.hass = self.hass
value_template = template("{{ value_json.toplevel.master_value }}")
value_template.hass = self.hass
self.rest.update = Mock(
"rest.RestData.update",
side_effect=self.update_side_effect(
'{ "toplevel": {"master_value": "master", "second_level": {"some_json_key": "some_json_value", "some_json_key2": "some_json_value2" } } }'
),
)
self.sensor = rest.RestSensor(
self.hass,
self.rest,
self.name,
self.unit_of_measurement,
self.device_class,
value_template,
["some_json_key", "some_json_key2"],
self.force_update,
self.resource_template,
self.convert_xml,
json_attrs_template,
)
self.sensor.update()
assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"]
assert (
"some_json_value2" == self.sensor.device_state_attributes["some_json_key2"]
)
assert "master" == self.sensor.state
def test_update_with_xml_convert_json_attrs_with_json_attrs_template(self):
"""Test attributes get extracted from a JSON result that was converted from XML with a template for the attributes."""
json_attrs_template = template("{{ value_json.toplevel.second_level }}")
json_attrs_template.hass = self.hass
value_template = template("{{ value_json.toplevel.master_value }}")
value_template.hass = self.hass
convert_xml = True
self.rest.update = Mock(
"rest.RestData.update",
side_effect=self.update_side_effect(
"<toplevel><master_value>master</master_value><second_level><some_json_key>some_json_value</some_json_key><some_json_key2>some_json_value2</some_json_key2></second_level></toplevel>"
),
)
self.sensor = rest.RestSensor(
self.hass,
self.rest,
self.name,
self.unit_of_measurement,
self.device_class,
value_template,
["some_json_key", "some_json_key2"],
self.force_update,
self.resource_template,
convert_xml,
json_attrs_template,
)
self.sensor.update()
assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"]
assert (
"some_json_value2" == self.sensor.device_state_attributes["some_json_key2"]
)
assert "master" == self.sensor.state
def test_update_with_xml_convert_json_attrs_with_jsonattr_template(self):
"""Test attributes get extracted from a JSON result that was converted from XML."""
json_attrs_template = template("{{ value_json.response }}")
json_attrs_template.hass = self.hass
value_template = template("{{ value_json.response.bss.wlan }}")
value_template.hass = self.hass
convert_xml = True
self.rest.update = Mock(
"rest.RestData.update",
side_effect=self.update_side_effect(
'<?xml version="1.0" encoding="utf-8"?><response><scan>0</scan><ver>12556</ver><count>48</count><ssid>alexander</ssid><bss><valid>0</valid><name>0</name><privacy>0</privacy><wlan>bogus</wlan><strength>0</strength></bss><led0>0</led0><led1>0</led1><led2>0</led2><led3>0</led3><led4>0</led4><led5>0</led5><led6>0</led6><led7>0</led7><btn0>up</btn0><btn1>up</btn1><btn2>up</btn2><btn3>up</btn3><pot0>0</pot0><usr0>0</usr0><temp0>0x0XF0x0XF</temp0><time0> 0</time0></response>'
),
)
self.sensor = rest.RestSensor(
self.hass,
self.rest,
self.name,
self.unit_of_measurement,
self.device_class,
value_template,
["led0", "led1", "temp0", "time0", "ver"],
self.force_update,
self.resource_template,
convert_xml,
json_attrs_template,
)
self.sensor.update()
assert "0" == self.sensor.device_state_attributes["led0"]
assert "0" == self.sensor.device_state_attributes["led1"]
assert "0x0XF0x0XF" == self.sensor.device_state_attributes["temp0"]
assert "0" == self.sensor.device_state_attributes["time0"]
assert "12556" == self.sensor.device_state_attributes["ver"]
assert "bogus" == self.sensor.state
@patch("homeassistant.components.rest.sensor._LOGGER")
def test_update_with_xml_convert_bad_xml(self, mock_logger):
"""Test attributes get extracted from a XML result with bad xml."""
value_template = template("{{ value_json.toplevel.master_value }}")
value_template.hass = self.hass
convert_xml = True
self.rest.update = Mock(
"rest.RestData.update",
side_effect=self.update_side_effect("this is not xml"),
)
self.sensor = rest.RestSensor(
self.hass,
self.rest,
self.name,
self.unit_of_measurement,
self.device_class,
value_template,
["key"],
self.force_update,
self.resource_template,
convert_xml,
self.json_attrs_template,
)
self.sensor.update()
assert {} == self.sensor.device_state_attributes
assert mock_logger.warning.called
assert mock_logger.debug.called
class TestRestData(unittest.TestCase):
"""Tests for RestData."""