feedc0de/platformio-core
1
0 Fork 0
forked from platformio/platformio-core
Code Pull Requests Activity

Significantly improve Package Manager // Resolve #913

Browse Source 
* Handle dependencies when installing non-registry package/library (VCS, archive, local folder)
This commit is contained in:
Ivan Kravets committed 2017-03-08 17:24:58 +02:00
1 parent 41cea76603
commit 58942c3f38
13 files changed
+657 -433
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/commands/test_lib.py
+121 -42
View File
@@ -16,7 +16,7 @@ import json
import re
from os.path import basename
from platformio import util
from platformio import exception, util
from platformio.commands.init import cli as cmd_init
from platformio.commands.lib import cli as cmd_lib
@@ -37,15 +37,35 @@ def test_search(clirunner, validate_cliresult):
def test_global_install_registry(clirunner, validate_cliresult,
isolated_pio_home):
result = clirunner.invoke(cmd_lib, [
"-g", "install", "58", "OneWire",
"-g", "install", "58", "547@2.2.4", "DallasTemperature",
"http://dl.platformio.org/libraries/archives/3/5174.tar.gz",
"ArduinoJson@5.6.7", "ArduinoJson@~5.7.0"
"ArduinoJson@5.6.7", "ArduinoJson@~5.7.0", "1089@fee16e880b"
])
validate_cliresult(result)
# check lib with duplicate URL
result = clirunner.invoke(cmd_lib, [
"-g", "install",
"http://dl.platformio.org/libraries/archives/3/5174.tar.gz"
])
validate_cliresult(result)
assert "is already installed" in result.output
# check lib with duplicate ID
result = clirunner.invoke(cmd_lib, ["-g", "install", "305"])
validate_cliresult(result)
assert "is already installed" in result.output
# install unknown library
result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, exception.LibNotFound)
items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
items2 = [
"DHT22_ID58", "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7",
"OneWire_ID1", "ESPAsyncTCP_ID305"
"ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54",
"DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "OneWire_ID1",
"IRremoteESP8266_ID1089"
]
assert set(items1) == set(items2)
@@ -55,11 +75,29 @@ def test_global_install_archive(clirunner, validate_cliresult,
result = clirunner.invoke(cmd_lib, [
"-g", "install", "https://github.com/adafruit/Adafruit-ST7735-Library/"
"archive/master.zip",
"http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2"
])
validate_cliresult(result)
# incorrect requirements
result = clirunner.invoke(cmd_lib, [
"-g", "install",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3"
])
assert result.exit_code != 0
# check lib with duplicate URL
result = clirunner.invoke(cmd_lib, [
"-g", "install",
"http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip"
])
validate_cliresult(result)
assert "is already installed" in result.output
items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
items2 = ["Adafruit ST7735 Library", "RadioHead"]
items2 = ["Adafruit ST7735 Library", "RadioHead-1.62"]
assert set(items1) >= set(items2)
@@ -71,14 +109,20 @@ def test_global_install_repository(clirunner, validate_cliresult,
"-g",
"install",
"https://github.com/gioblu/PJON.git#3.0",
"https://github.com/gioblu/PJON.git#6.2",
"https://github.com/bblanchon/ArduinoJson.git",
"https://gitlab.com/ivankravets/rs485-nodeproto.git",
# "https://developer.mbed.org/users/simon/code/TextLCD/",
"knolleary/pubsubclient"
])
validate_cliresult(result)
items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
items2 = ["PJON", "ESPAsyncTCP", "PubSubClient"]
assert set(items2) & set(items1)
items2 = [
"PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d",
"ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "rs485-nodeproto",
"PubSubClient"
]
assert set(items1) >= set(items2)
def test_global_lib_list(clirunner, validate_cliresult, isolated_pio_home):
@@ -89,13 +133,15 @@ def test_global_lib_list(clirunner, validate_cliresult, isolated_pio_home):
result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
assert all([
n in result.output
for n in ("PJON", "git+https://github.com/knolleary/pubsubclient")
for n in ("PJON", "git+https://github.com/knolleary/pubsubclient",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip"
)
])
items1 = [i['name'] for i in json.loads(result.output)]
items2 = [
"OneWire", "DHT22", "PJON", "ESPAsyncTCP", "ArduinoJson",
"pubsubclient", "rs485-nodeproto", "Adafruit ST7735 Library",
"RadioHead"
"PubSubClient", "rs485-nodeproto", "Adafruit ST7735 Library",
"RadioHead-1.62", "DallasTemperature", "NeoPixelBus", "IRremoteESP8266"
]
assert set(items1) == set(items2)
@@ -106,31 +152,71 @@ def test_global_lib_update_check(clirunner, validate_cliresult,
cmd_lib, ["-g", "update", "--only-check", "--json-output"])
validate_cliresult(result)
output = json.loads(result.output)
assert set(["ArduinoJson", "ESPAsyncTCP", "RadioHead"]) == set(
assert set(["ArduinoJson", "IRremoteESP8266", "NeoPixelBus"]) == set(
[l['name'] for l in output])
def test_global_lib_update(clirunner, validate_cliresult, isolated_pio_home):
# update library using package directory
result = clirunner.invoke(
cmd_lib,
["-g", "update", "NeoPixelBus", "--only-check", "--json-output"])
validate_cliresult(result)
oudated = json.loads(result.output)
assert len(oudated) == 1
assert "__pkg_dir" in oudated[0]
result = clirunner.invoke(cmd_lib,
["-g", "update", oudated[0]['__pkg_dir']])
validate_cliresult(result)
assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output
# update rest libraries
result = clirunner.invoke(cmd_lib, ["-g", "update"])
validate_cliresult(result)
assert "[Up-to-date]" in result.output
validate_cliresult(result)
assert result.output.count("[Skip]") == 5
assert result.output.count("[Up-to-date]") == 9
assert "Uninstalling ArduinoJson @ 5.7.3" in result.output
assert "Uninstalling IRremoteESP8266 @ fee16e880b" in result.output
# update unknown library
result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, exception.UnknownPackage)
def test_global_lib_uninstall(clirunner, validate_cliresult,
isolated_pio_home):
# uninstall using package directory
result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
validate_cliresult(result)
items = json.loads(result.output)
result = clirunner.invoke(cmd_lib,
["-g", "uninstall", items[0]['__pkg_dir']])
validate_cliresult(result)
assert "Uninstalling Adafruit ST7735 Library" in result.output
# uninstall the rest libraries
result = clirunner.invoke(cmd_lib, [
"-g", "uninstall", "1", "ArduinoJson@!=5.6.7", "TextLCD",
"Adafruit ST7735 Library"
"-g", "uninstall", "1", "ArduinoJson@!=5.6.7",
"https://github.com/bblanchon/ArduinoJson.git", "IRremoteESP8266@>=0.2"
])
validate_cliresult(result)
items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
items2 = [
"DHT22_ID58", "ArduinoJson_ID64", "ESPAsyncTCP_ID305",
"pubsubclient", "PJON", "rs485-nodeproto", "RadioHead_ID124"
"ArduinoJson", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54",
"DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "PJON",
"PJON@src-79de467ebe19de18287becff0a1fb42d", "PubSubClient",
"RadioHead-1.62", "rs485-nodeproto"
]
assert set(items1) == set(items2)
# uninstall unknown library
result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, exception.UnknownPackage)
def test_lib_show(clirunner, validate_cliresult, isolated_pio_home):
result = clirunner.invoke(cmd_lib, ["show", "64"])
@@ -142,31 +228,24 @@ def test_lib_show(clirunner, validate_cliresult, isolated_pio_home):
assert "OneWire" in result.output
def test_project_lib_complex(clirunner, validate_cliresult, tmpdir):
with tmpdir.as_cwd():
# init
result = clirunner.invoke(cmd_init)
validate_cliresult(result)
def test_lib_builtin(clirunner, validate_cliresult, isolated_pio_home):
result = clirunner.invoke(cmd_lib, ["builtin"])
validate_cliresult(result)
result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"])
validate_cliresult(result)
# isntall
result = clirunner.invoke(cmd_lib, ["install", "54", "ArduinoJson"])
validate_cliresult(result)
items1 = [
d.basename
for d in tmpdir.join(basename(util.get_projectlibdeps_dir()))
.listdir()
]
items2 = ["DallasTemperature_ID54", "OneWire_ID1", "ArduinoJson_ID64"]
assert set(items1) == set(items2)
# list
result = clirunner.invoke(cmd_lib, ["list", "--json-output"])
validate_cliresult(result)
items1 = [i['name'] for i in json.loads(result.output)]
items2 = ["DallasTemperature", "OneWire", "ArduinoJson"]
assert set(items1) == set(items2)
def test_lib_stats(clirunner, validate_cliresult, isolated_pio_home):
result = clirunner.invoke(cmd_lib, ["stats"])
validate_cliresult(result)
assert all([
s in result.output
for s in ("UPDATED", "ago", "http://platformio.org/lib/show")
])
# update
result = clirunner.invoke(cmd_lib, ["update"])
validate_cliresult(result)
assert "[Up-to-date]" in result.output
result = clirunner.invoke(cmd_lib, ["stats", "--json-output"])
validate_cliresult(result)
assert set([
"dlweek", "added", "updated", "topkeywords", "dlmonth", "dlday",
"lastkeywords"
]) == set(json.loads(result.output).keys())
tests/test_maintenance.py
+1 -1
View File
@@ -156,7 +156,7 @@ def test_check_platform_updates(clirunner, validate_cliresult,
manifest['version'] = "0.0.0"
manifest_path.write(json.dumps(manifest))
# reset cached manifests
PlatformManager().reset_cache()
PlatformManager().cache_reset()
# reset check time
interval = int(app.get_setting("check_platforms_interval")) * 3600 * 24
tests/test_managers.py
+138 -24
View File
@@ -12,70 +12,118 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from os.path import join
from platformio import util
from platformio.managers.package import BasePkgManager
from platformio.managers.package import PackageManager
def test_pkg_name_parser():
def test_pkg_input_parser():
items = [
["PkgName", ("PkgName", None, None)],
[("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)],
["PkgName@1.2.3", ("PkgName", "1.2.3", None)],
[("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)],
["id:13", ("id:13", None, None)],
["id:13@~1.2.3", ("id:13", "~1.2.3", None)], [
["id:13@~1.2.3", ("id:13", "~1.2.3", None)],
[
util.get_home_dir(),
(".platformio", None, "file://" + util.get_home_dir())
], [
],
[
"LocalName=" + util.get_home_dir(),
("LocalName", None, "file://" + util.get_home_dir())
], [
],
[
"LocalName=%s@>2.3.0" % util.get_home_dir(),
("LocalName", ">2.3.0", "file://" + util.get_home_dir())
],
[
"https://github.com/user/package.git",
("package", None, "git+https://github.com/user/package.git")
], [
"https://gitlab.com/user/package.git",
("package", None, "git+https://gitlab.com/user/package.git")
], [
],
[
"MyPackage=https://gitlab.com/user/package.git",
("MyPackage", None, "git+https://gitlab.com/user/package.git")
],
[
"MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2",
("MyPackage", "3.2.1,!=2",
"git+https://gitlab.com/user/package.git")
],
[
"https://somedomain.com/path/LibraryName-1.2.3.zip",
("LibraryName-1.2.3", None,
"https://somedomain.com/path/LibraryName-1.2.3.zip")
],
[
"https://github.com/user/package/archive/branch.zip",
("branch", None,
"https://github.com/user/package/archive/branch.zip")
], [
],
[
"https://github.com/user/package/archive/branch.zip@~1.2.3",
("branch", "~1.2.3",
"https://github.com/user/package/archive/branch.zip")
],
[
"https://github.com/user/package/archive/branch.tar.gz",
("branch", None,
("branch.tar", None,
"https://github.com/user/package/archive/branch.tar.gz")
], [
],
[
"https://github.com/user/package/archive/branch.tar.gz@!=5",
("branch.tar", "!=5",
"https://github.com/user/package/archive/branch.tar.gz")
],
[
"https://developer.mbed.org/users/user/code/package/",
("package", None,
"hg+https://developer.mbed.org/users/user/code/package/")
], [
],
[
"https://github.com/user/package#v1.2.3",
("package", None, "git+https://github.com/user/package#v1.2.3")
], [
],
[
"https://github.com/user/package.git#branch",
("package", None, "git+https://github.com/user/package.git#branch")
], [
],
[
"PkgName=https://github.com/user/package.git#a13d344fg56",
("PkgName", None,
"git+https://github.com/user/package.git#a13d344fg56")
], [
],
[
"user/package",
("package", None, "git+https://github.com/user/package")
],
[
"PkgName=user/package",
("PkgName", None, "git+https://github.com/user/package")
], [
],
[
"PkgName=user/package#master",
("PkgName", None, "git+https://github.com/user/package#master")
], [
],
[
"git+https://github.com/user/package",
("package", None, "git+https://github.com/user/package")
], [
],
[
"hg+https://example.com/user/package",
("package", None, "hg+https://example.com/user/package")
], [
],
[
"git@github.com:user/package.git",
("package", None, "git@github.com:user/package.git")
], [
],
[
"git@github.com:user/package.git#v1.2.0",
("package", None, "git@github.com:user/package.git#v1.2.0")
], [
],
[
"git+ssh://git@gitlab.private-server.com/user/package#1.2.0",
("package", None,
"git+ssh://git@gitlab.private-server.com/user/package#1.2.0")
@@ -83,6 +131,72 @@ def test_pkg_name_parser():
]
for params, result in items:
if isinstance(params, tuple):
assert BasePkgManager.parse_pkg_name(*params) == result
assert PackageManager.parse_pkg_input(*params) == result
else:
assert BasePkgManager.parse_pkg_name(params) == result
assert PackageManager.parse_pkg_input(params) == result
def test_install_packages(isolated_pio_home, tmpdir):
packages = [
dict(id=1, name="name_1", version="shasum"),
dict(id=1, name="name_1", version="2.0.0"),
dict(id=1, name="name_1", version="2.1.0"),
dict(id=1, name="name_1", version="1.2.0"),
dict(id=1, name="name_1", version="1.0.0"),
dict(name="name_2", version="1.0.0"),
dict(name="name_2", version="2.0.0",
__src_url="git+https://github.com"),
dict(name="name_2", version="3.0.0",
__src_url="git+https://github2.com"),
dict(name="name_2", version="4.0.0",
__src_url="git+https://github2.com")
]
pm = PackageManager(join(util.get_home_dir(), "packages"))
for package in packages:
tmp_dir = tmpdir.mkdir("tmp-package")
tmp_dir.join("package.json").write(json.dumps(package))
pm._install_from_url(package['name'], "file://%s" % str(tmp_dir))
tmp_dir.remove(rec=1)
assert len(pm.get_installed()) == len(packages) - 1
pkg_dirnames = [
'name_1_ID1', 'name_1_ID1@1.0.0', 'name_1_ID1@1.2.0',
'name_1_ID1@2.0.0', 'name_1_ID1@shasum', 'name_2',
'name_2@src-177cbce1f0705580d17790fda1cc2ef5',
'name_2@src-f863b537ab00f4c7b5011fc44b120e1f'
]
assert set([p.basename for p in isolated_pio_home.join(
"packages").listdir()]) == set(pkg_dirnames)
def test_get_package(isolated_pio_home):
tests = [
[("unknown", ), None],
[("1", ), None],
[("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")],
[("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")],
[("id=1", "^1"), dict(id=1, name="name_1", version="1.2.0")],
[("id=1", "^1"), dict(id=1, name="name_1", version="1.2.0")],
[("name_1", "<2"), dict(id=1, name="name_1", version="1.2.0")],
[("name_1", ">2"), None],
[("name_1", "2-0-0"), dict(id=1, name="name_1", version="2.1.0")],
[("name_1", "2-0-0"), dict(id=1, name="name_1", version="2.1.0")],
[("name_2", ), dict(name="name_2", version="4.0.0")],
[("url_has_higher_priority", None, "git+https://github.com"),
dict(name="name_2", version="2.0.0",
__src_url="git+https://github.com")],
[("name_2", None, "git+https://github.com"),
dict(name="name_2", version="2.0.0",
__src_url="git+https://github.com")],
]
pm = PackageManager(join(util.get_home_dir(), "packages"))
for test in tests:
manifest = pm.get_package(*test[0])
if test[1] is None:
assert manifest is None, test
continue
for key, value in test[1].items():
assert manifest[key] == value, test
tests/test_pkgmanifest.py
+16 -20
View File
@@ -16,19 +16,6 @@ import pytest
import requests
def pytest_generate_tests(metafunc):
if "package_data" not in metafunc.fixturenames:
return
pkgs_manifest = requests.get(
"https://dl.bintray.com/platformio/dl-packages/manifest.json").json()
assert isinstance(pkgs_manifest, dict)
packages = []
for _, variants in pkgs_manifest.iteritems():
for item in variants:
packages.append(item)
metafunc.parametrize("package_data", packages)
def validate_response(req):
assert req.status_code == 200
assert int(req.headers['Content-Length']) > 0
@@ -36,13 +23,22 @@ def validate_response(req):
"application/octet-stream")
def test_package(package_data):
assert package_data['url'].endswith(".tar.gz")
def test_packages():
pkgs_manifest = requests.get(
"https://dl.bintray.com/platformio/dl-packages/manifest.json").json()
assert isinstance(pkgs_manifest, dict)
items = []
for _, variants in pkgs_manifest.iteritems():
for item in variants:
items.append(item)
r = requests.head(package_data['url'], allow_redirects=True)
validate_response(r)
for item in items:
assert item['url'].endswith(".tar.gz"), item
if "X-Checksum-Sha1" not in r.headers:
return pytest.skip("X-Checksum-Sha1 is not provided")
r = requests.head(item['url'], allow_redirects=True)
validate_response(r)
assert package_data['sha1'] == r.headers.get("X-Checksum-Sha1")
if "X-Checksum-Sha1" not in r.headers:
return pytest.skip("X-Checksum-Sha1 is not provided")
assert item['sha1'] == r.headers.get("X-Checksum-Sha1"), item