Use package parser for package manager and LDF

platformio/builder/tools/piolib.py

@@ -17,7 +17,6 @@
 
 from __future__ import absolute_import
 
-import codecs
 import hashlib
 import os
 import re
@@ -34,6 +33,7 @@ from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
 from platformio.compat import WINDOWS, hashlib_encode_data, string_types
 from platformio.managers.lib import LibraryManager
+from platformio.package.manifest.parser import ManifestParserFactory
 
 
 class LibBuilderFactory(object):
@@ -456,17 +456,10 @@ class UnknownLibBuilder(LibBuilderBase):
 
 class ArduinoLibBuilder(LibBuilderBase):
     def load_manifest(self):
-        manifest = {}
-        if not isfile(join(self.path, "library.properties")):
-            return manifest
         manifest_path = join(self.path, "library.properties")
-        with codecs.open(manifest_path, encoding="utf-8") as fp:
+        if not isfile(manifest_path):
-            for line in fp.readlines():
+            return {}
-                if "=" not in line:
+        return ManifestParserFactory.new_from_file(manifest_path).as_dict()
-                    continue
-                key, value = line.split("=", 1)
-                manifest[key.strip()] = value.strip()
-        return manifest
 
     def get_include_dirs(self):
         include_dirs = LibBuilderBase.get_include_dirs(self)
@@ -510,24 +503,7 @@ class ArduinoLibBuilder(LibBuilderBase):
         return util.items_in_list(frameworks, ["arduino", "energia"])
 
     def is_platforms_compatible(self, platforms):
-        platforms_map = {
+        items = self._manifest.get("platforms", [])
-            "avr": ["atmelavr"],
-            "sam": ["atmelsam"],
-            "samd": ["atmelsam"],
-            "esp8266": ["espressif8266"],
-            "esp32": ["espressif32"],
-            "arc32": ["intel_arc32"],
-            "stm32": ["ststm32"],
-            "nrf5": ["nordicnrf51", "nordicnrf52"],
-        }
-        items = []
-        for arch in self._manifest.get("architectures", "").split(","):
-            arch = arch.strip().lower()
-            if arch == "*":
-                items = "*"
-                break
-            if arch in platforms_map:
-                items.extend(platforms_map[arch])
         if not items:
             return LibBuilderBase.is_platforms_compatible(self, platforms)
         return util.items_in_list(platforms, items)
@@ -535,9 +511,10 @@ class ArduinoLibBuilder(LibBuilderBase):
 
 class MbedLibBuilder(LibBuilderBase):
     def load_manifest(self):
-        if not isfile(join(self.path, "module.json")):
+        manifest_path = join(self.path, "module.json")
+        if not isfile(manifest_path):
             return {}
-        return fs.load_json(join(self.path, "module.json"))
+        return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
     @property
     def include_dir(self):
@@ -682,20 +659,12 @@ class MbedLibBuilder(LibBuilderBase):
 
 class PlatformIOLibBuilder(LibBuilderBase):
     def load_manifest(self):
-        assert isfile(join(self.path, "library.json"))
+        manifest_path = join(self.path, "library.json")
-        manifest = fs.load_json(join(self.path, "library.json"))
+        if not isfile(manifest_path):
-        assert "name" in manifest
+            return {}
+        return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
-        # replace "espressif" old name dev/platform with ESP8266
+    def _has_arduino_manifest(self):
-        if "platforms" in manifest:
-            manifest["platforms"] = [
-                "espressif8266" if p == "espressif" else p
-                for p in util.items_to_list(manifest["platforms"])
-            ]
-
-        return manifest
-
-    def _is_arduino_manifest(self):
         return isfile(join(self.path, "library.properties"))
 
     @property
@@ -718,7 +687,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
             return self._manifest.get("build").get("srcFilter")
         if self.env["SRC_FILTER"]:
             return self.env["SRC_FILTER"]
-        if self._is_arduino_manifest():
+        if self._has_arduino_manifest():
             return ArduinoLibBuilder.src_filter.fget(self)
         return LibBuilderBase.src_filter.fget(self)
 
@@ -789,7 +758,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
         # backwards compatibility with PlatformIO 2.0
         if (
             "build" not in self._manifest
-            and self._is_arduino_manifest()
+            and self._has_arduino_manifest()
             and not isdir(join(self.path, "src"))
             and isdir(join(self.path, "utility"))
         ):
@@ -954,9 +923,8 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)))
         if verbose:
             sys.stderr.write("Platform incompatible library %s\n" % lb.path)
         return False
-    if (
+    if compat_mode in ("soft", "strict") and not lb.is_frameworks_compatible(
-        compat_mode in ("soft", "strict")
+        env.get("PIOFRAMEWORK", [])
-        and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", []))
     ):
         if verbose:
             sys.stderr.write("Framework incompatible library %s\n" % lb.path)

platformio/managers/lib.py

@@ -16,7 +16,6 @@
 # pylint: disable=too-many-return-statements
 
 import json
-import re
 from glob import glob
 from os.path import isdir, join
 
@@ -62,73 +61,6 @@ class LibraryManager(BasePkgManager):
 
         return None
 
-    def load_manifest(self, pkg_dir):
-        manifest = BasePkgManager.load_manifest(self, pkg_dir)
-        if not manifest:
-            return manifest
-
-        # if Arduino library.properties
-        if "sentence" in manifest:
-            manifest["frameworks"] = ["arduino"]
-            manifest["description"] = manifest["sentence"]
-            del manifest["sentence"]
-
-        if "author" in manifest:
-            if isinstance(manifest["author"], dict):
-                manifest["authors"] = [manifest["author"]]
-            else:
-                manifest["authors"] = [{"name": manifest["author"]}]
-            del manifest["author"]
-
-        if "authors" in manifest and not isinstance(manifest["authors"], list):
-            manifest["authors"] = [manifest["authors"]]
-
-        if "keywords" not in manifest:
-            keywords = []
-            for keyword in re.split(
-                r"[\s/]+", manifest.get("category", "Uncategorized")
-            ):
-                keyword = keyword.strip()
-                if not keyword:
-                    continue
-                keywords.append(keyword.lower())
-            manifest["keywords"] = keywords
-            if "category" in manifest:
-                del manifest["category"]
-
-        # don't replace VCS URL
-        if "url" in manifest and "description" in manifest:
-            manifest["homepage"] = manifest["url"]
-            del manifest["url"]
-
-        if "architectures" in manifest:
-            platforms = []
-            platforms_map = {
-                "avr": "atmelavr",
-                "sam": "atmelsam",
-                "samd": "atmelsam",
-                "esp8266": "espressif8266",
-                "esp32": "espressif32",
-                "arc32": "intel_arc32",
-            }
-            for arch in manifest["architectures"].split(","):
-                arch = arch.strip()
-                if arch == "*":
-                    platforms = "*"
-                    break
-                if arch in platforms_map:
-                    platforms.append(platforms_map[arch])
-            manifest["platforms"] = platforms
-            del manifest["architectures"]
-
-        # convert listed items via comma to array
-        for key in ("keywords", "frameworks", "platforms"):
-            if key not in manifest or not isinstance(manifest[key], string_types):
-                continue
-            manifest[key] = [i.strip() for i in manifest[key].split(",") if i.strip()]
-
-        return manifest
-
     @staticmethod
     def normalize_dependencies(dependencies):
         if not dependencies:

platformio/managers/package.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import codecs
 import hashlib
 import json
 import os
@@ -29,6 +28,10 @@ from platformio import __version__, app, exception, fs, telemetry, util
 from platformio.compat import hashlib_encode_data
 from platformio.downloader import FileDownloader
 from platformio.lockfile import LockFile
+from platformio.package.manifest.parser import (
+    ManifestParserError,
+    ManifestParserFactory,
+)
 from platformio.unpacker import FileUnpacker
 from platformio.vcsclient import VCSClientFactory
 
@@ -326,7 +329,7 @@ class PkgInstallerMixin(object):
     def manifest_exists(self, pkg_dir):
         return self.get_manifest_path(pkg_dir) or self.get_src_manifest_path(pkg_dir)
 
-    def load_manifest(self, pkg_dir):
+    def load_manifest(self, pkg_dir):  # pylint: disable=too-many-branches
         cache_key = "load_manifest-%s" % pkg_dir
         result = self.cache_get(cache_key)
         if result:
@@ -342,15 +345,10 @@ class PkgInstallerMixin(object):
         if not manifest_path and not src_manifest_path:
             return None
 
-        if manifest_path and manifest_path.endswith(".json"):
+        try:
-            manifest = fs.load_json(manifest_path)
+            manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict()
-        elif manifest_path and manifest_path.endswith(".properties"):
+        except ManifestParserError:
-            with codecs.open(manifest_path, encoding="utf-8") as fp:
+            pass
-                for line in fp.readlines():
-                    if "=" not in line:
-                        continue
-                    key, value = line.split("=", 1)
-                    manifest[key.strip()] = value.strip()
 
         if src_manifest:
             if "version" in src_manifest:

platformio/package/manifest/parser.py

@@ -46,6 +46,8 @@ class ManifestFileType(object):
             return ManifestFileType.MODULE_JSON
         if uri.endswith("package.json"):
             return ManifestFileType.PACKAGE_JSON
+        if uri.endswith("library.json"):
+            return ManifestFileType.LIBRARY_JSON
         return None
 
 
@@ -331,18 +333,16 @@ class LibraryJsonManifestParser(BaseManifestParser):
 class ModuleJsonManifestParser(BaseManifestParser):
     def parse(self, contents):
         data = json.loads(contents)
-        return dict(
+        data["frameworks"] = ["mbed"]
-            name=data["name"],
+        data["platforms"] = ["*"]
-            version=data["version"],
+        data["export"] = {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}
-            keywords=data.get("keywords"),
+        if "author" in data:
-            description=data["description"],
+            data["authors"] = self._parse_authors(data.get("author"))
-            frameworks=["mbed"],
+            del data["author"]
-            platforms=["*"],
+        if "licenses" in data:
-            homepage=data.get("homepage"),
+            data["license"] = self._parse_license(data.get("licenses"))
-            export={"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]},
+            del data["licenses"]
-            authors=self._parse_authors(data.get("author")),
+        return data
-            license=self._parse_license(data.get("licenses")),
-        )
 
     def _parse_authors(self, raw):
         if not raw:
@@ -364,23 +364,26 @@ class ModuleJsonManifestParser(BaseManifestParser):
 
 class LibraryPropertiesManifestParser(BaseManifestParser):
     def parse(self, contents):
-        properties = self._parse_properties(contents)
+        data = self._parse_properties(contents)
-        repository = self._parse_repository(properties)
+        repository = self._parse_repository(data)
-        homepage = properties.get("url")
+        homepage = data.get("url")
         if repository and repository["url"] == homepage:
             homepage = None
-        return dict(
+        data.update(
+            dict(
                 frameworks=["arduino"],
                 homepage=homepage,
                 repository=repository or None,
-            name=properties.get("name"),
+                description=self._parse_description(data),
-            version=properties.get("version"),
+                platforms=self._parse_platforms(data) or ["*"],
-            description=self._parse_description(properties),
+                keywords=self._parse_keywords(data),
-            platforms=self._parse_platforms(properties) or ["*"],
-            keywords=self._parse_keywords(properties),
-            authors=self._parse_authors(properties) or None,
                 export=self._parse_export(),
             )
+        )
+        if "author" in data:
+            data["authors"] = self._parse_authors(data)
+            del data["author"]
+        return data
 
     @staticmethod
     def _parse_properties(contents):

tests/test_pkgmanifest.py

@@ -31,7 +31,11 @@ def test_library_json_parser():
     "platforms": ["atmelavr", "espressif"],
     "url": "http://old.url.format",
     "exclude": [".gitignore", "tests"],
-    "include": "mylib"
+    "include": "mylib",
+    "build": {
+        "flags": ["-DHELLO"]
+    },
+    "customField": "Custom Value"
 }
 """
     mp = parser.LibraryJsonManifestParser(contents)
@@ -43,6 +47,8 @@ def test_library_json_parser():
             "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]},
             "keywords": ["kw1", "kw2", "kw3"],
             "homepage": "http://old.url.format",
+            "build": {"flags": ["-DHELLO"]},
+            "customField": "Custom Value",
         },
     )
 
@@ -89,9 +95,11 @@ def test_module_json_parser():
     "type": "git",
     "url": "git@github.com:username/repo.git"
   },
-  "version": "1.2.3"
+  "version": "1.2.3",
+  "customField": "Custom Value"
 }
 """
+
     mp = parser.ModuleJsonManifestParser(contents)
     assert not jsondiff.diff(
         mp.as_dict(),
@@ -106,6 +114,8 @@ def test_module_json_parser():
             "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]},
             "authors": [{"email": "name@surname.com", "name": "Name Surname"}],
             "version": "1.2.3",
+            "repository": {"type": "git", "url": "git@github.com:username/repo.git"},
+            "customField": "Custom Value",
         },
     )
 
@@ -117,6 +127,7 @@ name=TestPackage
 version=1.2.3
 author=SomeAuthor <info AT author.com>
 sentence=This is Arduino library
+customField=Custom Value
 """
     mp = parser.LibraryPropertiesManifestParser(contents)
     assert not jsondiff.diff(
@@ -125,6 +136,7 @@ sentence=This is Arduino library
             "name": "TestPackage",
             "version": "1.2.3",
             "description": "This is Arduino library",
+            "sentence": "This is Arduino library",
             "platforms": ["*"],
             "frameworks": ["arduino"],
             "export": {
@@ -132,6 +144,7 @@ sentence=This is Arduino library
             },
             "authors": [{"email": "info@author.com", "name": "SomeAuthor"}],
             "keywords": ["uncategorized"],
+            "customField": "Custom Value",
         },
     )