platformio-core/tests/package/test_manifest.py

# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re

import jsondiff
import pytest

from platformio.compat import WINDOWS
from platformio.package.manifest import parser
from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError


def test_library_json_parser():
    contents = """
{
    "name": "TestPackage",
    "keywords": "kw1, KW2, kw3",
    "platforms": ["atmelavr", "espressif"],
    "repository": {
        "type": "git",
        "url": "http://github.com/username/repo/"
    },
    "url": "http://old.url.format",
    "exclude": [".gitignore", "tests"],
    "include": "mylib",
    "build": {
        "flags": ["-DHELLO"]
    },
    "examples": ["examples/*/*.pde"],
    "dependencies": {
        "deps1": "1.2.0",
        "deps2": "https://github.com/username/package.git",
        "@owner/deps3": "^2.1.3"
    },
    "customField": "Custom Value"
}
"""
    raw_data = parser.LibraryJsonManifestParser(contents).as_dict()
    raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"])
    assert not jsondiff.diff(
        raw_data,
        {
            "name": "TestPackage",
            "platforms": ["atmelavr", "espressif8266"],
            "repository": {
                "type": "git",
                "url": "https://github.com/username/repo.git",
            },
            "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]},
            "keywords": ["kw1", "kw2", "kw3"],
            "homepage": "http://old.url.format",
            "build": {"flags": ["-DHELLO"]},
            "dependencies": [
                {"name": "@owner/deps3", "version": "^2.1.3"},
                {"name": "deps1", "version": "1.2.0"},
                {"name": "deps2", "version": "https://github.com/username/package.git"},
            ],
            "customField": "Custom Value",
        }
    )

    contents = """
{
    "keywords": ["sound", "audio", "music", "SD", "card", "playback"],
    "frameworks": "arduino",
    "platforms": "atmelavr",
    "export": {
        "exclude": "audio_samples"
    },
    "dependencies": [
        {"name": "deps1", "version": "1.0.0"},
        {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino, espidf"},
        {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]}
    ]
}
"""
    mp = parser.LibraryJsonManifestParser(contents)
    assert not jsondiff.diff(
        mp.as_dict(),
        {
            "keywords": ["sound", "audio", "music", "sd", "card", "playback"],
            "frameworks": ["arduino"],
            "export": {"exclude": ["audio_samples"]},
            "platforms": ["atmelavr"],
            "dependencies": [
                {"name": "deps1", "version": "1.0.0"},
                {
                    "name": "@owner/deps2",
                    "version": "1.0.0",
                    "frameworks": ["arduino", "espidf"],
                },
                {
                    "name": "deps3",
                    "version": "1.0.0",
                    "platforms": ["ststm32", "sifive"],
                },
            ],
        },
    )

    # broken dependencies
    with pytest.raises(parser.ManifestParserError):
        mp = parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]})


def test_module_json_parser():
    contents = """
{
  "author": "Name Surname <name@surname.com>",
  "description": "This is Yotta library",
  "homepage": "https://yottabuild.org",
  "keywords": [
    "mbed",
    "Yotta"
  ],
  "licenses": [
    {
      "type": "Apache-2.0",
      "url": "https://spdx.org/licenses/Apache-2.0"
    }
  ],
  "name": "YottaLibrary",
  "repository": {
    "type": "git",
    "url": "git@github.com:username/repo.git"
  },
  "version": "1.2.3",
  "customField": "Custom Value"
}
"""

    mp = parser.ModuleJsonManifestParser(contents)
    assert not jsondiff.diff(
        mp.as_dict(),
        {
            "name": "YottaLibrary",
            "description": "This is Yotta library",
            "homepage": "https://yottabuild.org",
            "keywords": ["mbed", "Yotta"],
            "license": "Apache-2.0",
            "platforms": ["*"],
            "frameworks": ["mbed"],
            "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]},
            "authors": [{"email": "name@surname.com", "name": "Name Surname"}],
            "version": "1.2.3",
            "repository": {"type": "git", "url": "git@github.com:username/repo.git"},
            "customField": "Custom Value",
        },
    )


def test_library_properties_parser():
    # Base
    contents = """
name=TestPackage
version=1.2.3
author=SomeAuthor <info AT author.com>
sentence=This is Arduino library
customField=Custom Value
depends=First Library (=2.0.0), Second Library (>=1.2.0), Third
"""
    mp = parser.LibraryPropertiesManifestParser(contents)
    assert not jsondiff.diff(
        mp.as_dict(),
        {
            "name": "TestPackage",
            "version": "1.2.3",
            "description": "This is Arduino library",
            "sentence": "This is Arduino library",
            "platforms": ["*"],
            "frameworks": ["arduino"],
            "export": {
                "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]
            },
            "authors": [{"email": "info@author.com", "name": "SomeAuthor"}],
            "keywords": ["uncategorized"],
            "customField": "Custom Value",
            "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third",
            "dependencies": [
                {
                    "name": "First Library",
                    "version": "=2.0.0",
                    "frameworks": ["arduino"],
                },
                {
                    "name": "Second Library",
                    "version": ">=1.2.0",
                    "frameworks": ["arduino"],
                },
                {"name": "Third", "frameworks": ["arduino"]},
            ],
        },
    )

    # Platforms ALL
    data = parser.LibraryPropertiesManifestParser(
        "architectures=*\n" + contents
    ).as_dict()
    assert data["platforms"] == ["*"]

    # Platforms specific
    data = parser.LibraryPropertiesManifestParser(
        "architectures=avr, esp32\n" + contents
    ).as_dict()
    assert data["platforms"] == ["atmelavr", "espressif32"]

    # Remote URL
    data = parser.LibraryPropertiesManifestParser(
        contents,
        remote_url=(
            "https://raw.githubusercontent.com/username/reponame/master/"
            "libraries/TestPackage/library.properties"
        ),
    ).as_dict()
    assert data["export"] == {
        "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"],
        "include": ["libraries/TestPackage"],
    }
    assert data["repository"] == {
        "url": "https://github.com/username/reponame.git",
        "type": "git",
    }

    # Home page
    data = parser.LibraryPropertiesManifestParser(
        "url=https://github.com/username/reponame.git\n" + contents
    ).as_dict()
    assert data["repository"] == {
        "type": "git",
        "url": "https://github.com/username/reponame.git",
    }

    # Author + Maintainer
    data = parser.LibraryPropertiesManifestParser(
        """
author=Rocket Scream Electronics
maintainer=Rocket Scream Electronics
"""
    ).as_dict()
    assert data["authors"] == [
        {"name": "Rocket Scream Electronics", "maintainer": True}
    ]


def test_library_json_schema():
    contents = """
{
  "name": "ArduinoJson",
  "keywords": "JSON, rest, http, web",
  "description": "An elegant and efficient JSON library for embedded systems",
  "homepage": "https://arduinojson.org",
  "repository": {
    "type": "git",
    "url": "https://github.com/bblanchon/ArduinoJson.git"
  },
  "version": "6.12.0",
  "authors": {
    "name": "Benoit Blanchon",
    "url": "https://blog.benoitblanchon.fr"
  },
  "downloadUrl": "https://example.com/package.tar.gz",
  "exclude": [
    "fuzzing",
    "scripts",
    "test",
    "third-party"
  ],
  "frameworks": "arduino",
  "platforms": "*",
  "license": "MIT",
  "examples": [
    {
        "name": "JsonConfigFile",
        "base": "examples/JsonConfigFile",
        "files": ["JsonConfigFile.ino"]
    },
    {
        "name": "JsonHttpClient",
        "base": "examples/JsonHttpClient",
        "files": ["JsonHttpClient.ino"]
    }
  ],
  "dependencies": [
    {"name": "deps1", "version": "1.0.0"},
    {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino"},
    {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]}
  ]
}
"""
    raw_data = parser.ManifestParserFactory.new(
        contents, parser.ManifestFileType.LIBRARY_JSON
    ).as_dict()

    data = ManifestSchema().load_manifest(raw_data)

    assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git"
    assert data["examples"][1]["base"] == "examples/JsonHttpClient"
    assert data["examples"][1]["files"] == ["JsonHttpClient.ino"]

    assert not jsondiff.diff(
        data,
        {
            "name": "ArduinoJson",
            "keywords": ["json", "rest", "http", "web"],
            "description": "An elegant and efficient JSON library for embedded systems",
            "homepage": "https://arduinojson.org",
            "repository": {
                "url": "https://github.com/bblanchon/ArduinoJson.git",
                "type": "git",
            },
            "version": "6.12.0",
            "authors": [
                {"name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr"}
            ],
            "downloadUrl": "https://example.com/package.tar.gz",
            "export": {"exclude": ["fuzzing", "scripts", "test", "third-party"]},
            "frameworks": ["arduino"],
            "platforms": ["*"],
            "license": "MIT",
            "examples": [
                {
                    "name": "JsonConfigFile",
                    "base": "examples/JsonConfigFile",
                    "files": ["JsonConfigFile.ino"],
                },
                {
                    "name": "JsonHttpClient",
                    "base": "examples/JsonHttpClient",
                    "files": ["JsonHttpClient.ino"],
                },
            ],
            "dependencies": [
                {"name": "deps1", "version": "1.0.0"},
                {"name": "@owner/deps2", "version": "1.0.0", "frameworks": ["arduino"]},
                {
                    "name": "deps3",
                    "version": "1.0.0",
                    "platforms": ["ststm32", "sifive"],
                },
            ],
        },
    )


def test_library_properties_schema():
    contents = """
name=U8glib
version=1.19.1
author=oliver <olikraus@gmail.com>
maintainer=oliver <olikraus@gmail.com>
sentence=A library for monochrome TFTs and OLEDs
paragraph=Supported display controller: SSD1306, SSD1309, SSD1322, SSD1325
category=Display
url=https://github.com/olikraus/u8glib
architectures=avr,sam
depends=First Library (=2.0.0), Second Library (>=1.2.0), Third
"""
    raw_data = parser.ManifestParserFactory.new(
        contents, parser.ManifestFileType.LIBRARY_PROPERTIES
    ).as_dict()

    data = ManifestSchema().load_manifest(raw_data)

    assert not jsondiff.diff(
        data,
        {
            "description": (
                "A library for monochrome TFTs and OLEDs. Supported display "
                "controller: SSD1306, SSD1309, SSD1322, SSD1325"
            ),
            "repository": {
                "url": "https://github.com/olikraus/u8glib.git",
                "type": "git",
            },
            "frameworks": ["arduino"],
            "platforms": ["atmelavr", "atmelsam"],
            "version": "1.19.1",
            "export": {
                "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]
            },
            "authors": [
                {"maintainer": True, "email": "olikraus@gmail.com", "name": "oliver"}
            ],
            "keywords": ["display"],
            "name": "U8glib",
            "dependencies": [
                {
                    "name": "First Library",
                    "version": "=2.0.0",
                    "frameworks": ["arduino"],
                },
                {
                    "name": "Second Library",
                    "version": ">=1.2.0",
                    "frameworks": ["arduino"],
                },
                {"name": "Third", "frameworks": ["arduino"]},
            ],
        },
    )

    # Broken fields
    contents = """
name=Mozzi
version=1.0.3
author=Tim Barrass and contributors as documented in source, and at https://github.com/sensorium/Mozzi/graphs/contributors
maintainer=Tim Barrass <faveflave@gmail.com>
sentence=Sound synthesis library for Arduino
paragraph=With Mozzi, you can construct sounds using familiar synthesis units like oscillators, delays, filters and envelopes.
category=Signal Input/Output
url=https://sensorium.github.io/Mozzi/
architectures=*
dot_a_linkage=false
includes=MozziGuts.h
"""
    raw_data = parser.ManifestParserFactory.new(
        contents,
        parser.ManifestFileType.LIBRARY_PROPERTIES,
        remote_url=(
            "https://raw.githubusercontent.com/sensorium/Mozzi/"
            "master/library.properties"
        ),
    ).as_dict()

    try:
        ManifestSchema().load_manifest(raw_data)
    except ManifestValidationError as e:
        data = e.valid_data
        errors = e.messages

    assert errors["authors"]

    assert not jsondiff.diff(
        data,
        {
            "name": "Mozzi",
            "version": "1.0.3",
            "description": (
                "Sound synthesis library for Arduino. With Mozzi, you can construct "
                "sounds using familiar synthesis units like oscillators, delays, "
                "filters and envelopes."
            ),
            "repository": {
                "url": "https://github.com/sensorium/Mozzi.git",
                "type": "git",
            },
            "platforms": ["*"],
            "frameworks": ["arduino"],
            "export": {
                "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]
            },
            "authors": [
                {
                    "maintainer": True,
                    "email": "faveflave@gmail.com",
                    "name": "Tim Barrass",
                }
            ],
            "keywords": ["signal", "input", "output"],
            "homepage": "https://sensorium.github.io/Mozzi/",
        },
    )


def test_platform_json_schema():
    contents = """
{
  "name": "atmelavr",
  "title": "Atmel AVR",
  "description": "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of performance, power efficiency and design flexibility. Optimized to speed time to market-and easily adapt to new ones-they are based on the industrys most code-efficient architecture for C and assembly programming.",
  "url": "http://www.atmel.com/products/microcontrollers/avr/default.aspx",
  "homepage": "http://platformio.org/platforms/atmelavr",
  "license": "Apache-2.0",
  "engines": {
    "platformio": "<5"
  },
  "repository": {
    "type": "git",
    "url": "https://github.com/platformio/platform-atmelavr.git"
  },
  "version": "1.15.0",
  "frameworks": {
    "arduino": {
      "package": "framework-arduinoavr",
      "script": "builder/frameworks/arduino.py"
    },
    "simba": {
      "package": "framework-simba",
      "script": "builder/frameworks/simba.py"
    }
  },
  "packages": {
    "toolchain-atmelavr": {
      "type": "toolchain",
      "version": "~1.50400.0"
    },
    "framework-arduinoavr": {
      "type": "framework",
      "optional": true,
      "version": "~4.2.0"
    },
    "tool-avrdude": {
      "type": "uploader",
      "optional": true,
      "version": "~1.60300.0"
    }
  }
}
"""
    raw_data = parser.ManifestParserFactory.new(
        contents, parser.ManifestFileType.PLATFORM_JSON
    ).as_dict()
    raw_data["frameworks"] = sorted(raw_data["frameworks"])
    data = ManifestSchema().load_manifest(raw_data)

    assert not jsondiff.diff(
        data,
        {
            "name": "atmelavr",
            "title": "Atmel AVR",
            "description": (
                "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of "
                "performance, power efficiency and design flexibility. Optimized to "
                "speed time to market-and easily adapt to new ones-they are based "
                "on the industrys most code-efficient architecture for C and "
                "assembly programming."
            ),
            "homepage": "http://platformio.org/platforms/atmelavr",
            "license": "Apache-2.0",
            "repository": {
                "url": "https://github.com/platformio/platform-atmelavr.git",
                "type": "git",
            },
            "frameworks": sorted(["arduino", "simba"]),
            "version": "1.15.0",
            "dependencies": [
                {"name": "toolchain-atmelavr", "version": "~1.50400.0"},
                {"name": "framework-arduinoavr", "version": "~4.2.0"},
                {"name": "tool-avrdude", "version": "~1.60300.0"},
            ],
        },
    )


def test_package_json_schema():
    contents = """
{
    "name": "tool-scons",
    "description": "SCons software construction tool",
    "url": "http://www.scons.org",
    "version": "3.30101.0"
}
"""
    raw_data = parser.ManifestParserFactory.new(
        contents, parser.ManifestFileType.PACKAGE_JSON
    ).as_dict()

    data = ManifestSchema().load_manifest(raw_data)

    assert not jsondiff.diff(
        data,
        {
            "name": "tool-scons",
            "description": "SCons software construction tool",
            "homepage": "http://www.scons.org",
            "version": "3.30101.0",
        },
    )

    mp = parser.ManifestParserFactory.new(
        '{"system": "*"}', parser.ManifestFileType.PACKAGE_JSON
    )
    assert "system" not in mp.as_dict()

    mp = parser.ManifestParserFactory.new(
        '{"system": "all"}', parser.ManifestFileType.PACKAGE_JSON
    )
    assert "system" not in mp.as_dict()

    mp = parser.ManifestParserFactory.new(
        '{"system": "darwin_x86_64"}', parser.ManifestFileType.PACKAGE_JSON
    )
    assert mp.as_dict()["system"] == ["darwin_x86_64"]


def test_parser_from_dir(tmpdir_factory):
    pkg_dir = tmpdir_factory.mktemp("package")
    pkg_dir.join("package.json").write('{"name": "package.json"}')
    pkg_dir.join("library.json").write('{"name": "library.json"}')
    pkg_dir.join("library.properties").write("name=library.properties")

    data = parser.ManifestParserFactory.new_from_dir(str(pkg_dir)).as_dict()
    assert data["name"] == "library.json"

    data = parser.ManifestParserFactory.new_from_dir(
        str(pkg_dir), remote_url="http://localhost/library.properties"
    ).as_dict()
    assert data["name"] == "library.properties"


def test_examples_from_dir(tmpdir_factory):
    package_dir = tmpdir_factory.mktemp("project")
    package_dir.join("library.json").write(
        '{"name": "pkg", "version": "1.0.0", "examples": ["examples/*/*.pde"]}'
    )
    examples_dir = package_dir.mkdir("examples")

    # PlatformIO project #1
    pio_dir = examples_dir.mkdir("PlatformIO").mkdir("hello")
    pio_dir.join(".vimrc").write("")
    pio_ini = pio_dir.join("platformio.ini")
    pio_ini.write("")
    if not WINDOWS:
        pio_dir.join("platformio.ini.copy").mksymlinkto(pio_ini)
    pio_dir.mkdir("include").join("main.h").write("")
    pio_dir.mkdir("src").join("main.cpp").write("")

    # wiring examples
    arduino_dir = examples_dir.mkdir("1. General")
    arduino_dir.mkdir("SomeSketchIno").join("SomeSketchIno.ino").write("")
    arduino_dir.mkdir("SomeSketchPde").join("SomeSketchPde.pde").write("")

    # custom examples
    demo_dir = examples_dir.mkdir("demo")
    demo_dir.join("demo.cpp").write("")
    demo_dir.join("demo.h").write("")
    demo_dir.join("util.h").write("")

    # PlatformIO project #2
    pio_dir = examples_dir.mkdir("world")
    pio_dir.join("platformio.ini").write("")
    pio_dir.join("README").write("")
    pio_dir.join("extra.py").write("")
    pio_dir.mkdir("include").join("world.h").write("")
    pio_dir.mkdir("src").join("world.c").write("")

    # example files in root
    examples_dir.join("root.c").write("")
    examples_dir.join("root.h").write("")

    # invalid example
    examples_dir.mkdir("invalid-example").join("hello.json")

    # Do testing

    raw_data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict()
    assert isinstance(raw_data["examples"], list)
    assert len(raw_data["examples"]) == 6

    def _to_unix_path(path):
        return re.sub(r"[\\/]+", "/", path)

    def _sort_examples(items):
        for i, item in enumerate(items):
            items[i]["base"] = _to_unix_path(items[i]["base"])
            items[i]["files"] = [_to_unix_path(f) for f in sorted(items[i]["files"])]
        return sorted(items, key=lambda item: item["name"])

    raw_data["examples"] = _sort_examples(raw_data["examples"])

    data = ManifestSchema().load_manifest(raw_data)

    assert not jsondiff.diff(
        data,
        {
            "version": "1.0.0",
            "name": "pkg",
            "examples": _sort_examples(
                [
                    {
                        "name": "PlatformIO/hello",
                        "base": os.path.join("examples", "PlatformIO", "hello"),
                        "files": [
                            "platformio.ini",
                            os.path.join("include", "main.h"),
                            os.path.join("src", "main.cpp"),
                        ],
                    },
                    {
                        "name": "1_General/SomeSketchIno",
                        "base": os.path.join("examples", "1. General", "SomeSketchIno"),
                        "files": ["SomeSketchIno.ino"],
                    },
                    {
                        "name": "1_General/SomeSketchPde",
                        "base": os.path.join("examples", "1. General", "SomeSketchPde"),
                        "files": ["SomeSketchPde.pde"],
                    },
                    {
                        "name": "demo",
                        "base": os.path.join("examples", "demo"),
                        "files": ["demo.h", "util.h", "demo.cpp"],
                    },
                    {
                        "name": "world",
                        "base": "examples/world",
                        "files": [
                            "platformio.ini",
                            os.path.join("include", "world.h"),
                            os.path.join("src", "world.c"),
                            "README",
                            "extra.py",
                        ],
                    },
                    {
                        "name": "Examples",
                        "base": "examples",
                        "files": ["root.c", "root.h"],
                    },
                ]
            ),
        },
    )


def test_broken_schemas():
    # missing required field
    with pytest.raises(
        ManifestValidationError, match=("Invalid semantic versioning format")
    ) as exc_info:
        ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version"))
    assert exc_info.value.valid_data == {"name": "MyPackage"}

    # invalid StrictList
    with pytest.raises(
        ManifestValidationError, match=("Invalid manifest fields.+keywords")
    ) as exc_info:
        ManifestSchema().load_manifest(
            dict(name="MyPackage", version="1.0.0", keywords=["kw1", "*^[]"])
        )
    assert list(exc_info.value.messages.keys()) == ["keywords"]
    assert exc_info.value.valid_data["keywords"] == ["kw1"]

    # broken SemVer
    with pytest.raises(
        ManifestValidationError, match=("Invalid semantic versioning format")
    ):
        ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version"))

    # broken value for Nested
    with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"):
        ManifestSchema().load_manifest(
            dict(
                name="MyPackage",
                description="MyDescription",
                keywords=["a", "b"],
                authors=["should be dict here"],
                version="1.2.3",
            )
        )