From 7e0fb43dbe440e7b49b69f2e43548aa63b961fb0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 22 Dec 2019 23:45:12 +0200 Subject: [PATCH] Normalise repository url when parsing package manifest --- Makefile | 4 +-- platformio/package/manifest/parser.py | 48 ++++++++++++++++++--------- tests/test_pkgmanifest.py | 20 +++++++++-- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 30cfe178..548f96f3 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,14 @@ isort: isort -rc ./platformio isort -rc ./tests -black: +format: black --target-version py27 ./platformio black --target-version py27 ./tests test: py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py -before-commit: isort black lint test +before-commit: isort format lint test clean-docs: rm -rf docs/_build diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 22c21b7b..3e2ff5ac 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -125,6 +125,8 @@ class BaseManifestParser(object): self._data = self.parse(contents) except Exception as e: raise ManifestParserError("Could not parse manifest -> %s" % e) + + self._data = self.normalize_repository(self._data) self._data = self.parse_examples(self._data) # remove None fields @@ -139,7 +141,7 @@ class BaseManifestParser(object): return self._data @staticmethod - def cleanup_author(author): + def normalize_author(author): assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) @@ -160,6 +162,22 @@ class BaseManifestParser(object): email = raw[raw.index(ldel) + 1 : raw.index(rdel)] return (name.strip(), email.strip() if email else None) + @staticmethod + def normalize_repository(data): + url = (data.get("repository") or {}).get("url") + if not url or "://" not in url: + return data + url_attrs = urlparse(url) + if url_attrs.netloc not in ("github.com", "bitbucket.org", "gitlab.com"): + return data + url = "https://%s%s" % (url_attrs.netloc, url_attrs.path) + if url.endswith("/"): + url = url[:-1] + if not url.endswith(".git"): + url += ".git" + data["repository"]["url"] = url + return data + def parse_examples(self, data): examples = data.get("examples") if ( @@ -305,7 +323,7 @@ class LibraryJsonManifestParser(BaseManifestParser): # normalize Union[dict, list] fields if not isinstance(raw, list): raw = [raw] - return [self.cleanup_author(author) for author in raw] + return [self.normalize_author(author) for author in raw] @staticmethod def _parse_platforms(raw): @@ -352,7 +370,7 @@ class ModuleJsonManifestParser(BaseManifestParser): name, email = self.parse_author_name_and_email(author) if not name: continue - result.append(self.cleanup_author(dict(name=name, email=email))) + result.append(self.normalize_author(dict(name=name, email=email))) return result @staticmethod @@ -451,7 +469,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): name, email = self.parse_author_name_and_email(author) if not name: continue - authors.append(self.cleanup_author(dict(name=name, email=email))) + authors.append(self.normalize_author(dict(name=name, email=email))) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) if not name: @@ -466,27 +484,25 @@ class LibraryPropertiesManifestParser(BaseManifestParser): item["email"] = email if not found: authors.append( - self.cleanup_author(dict(name=name, email=email, maintainer=True)) + self.normalize_author(dict(name=name, email=email, maintainer=True)) ) return authors def _parse_repository(self, properties): if self.remote_url: - repo_parse = urlparse(self.remote_url) - repo_path_tokens = repo_parse.path[1:].split("/")[:-1] - if "github" in repo_parse.netloc: + url_attrs = urlparse(self.remote_url) + repo_path_tokens = url_attrs.path[1:].split("/")[:-1] + if "github" in url_attrs.netloc: return dict( type="git", - url="%s://github.com/%s" - % (repo_parse.scheme, "/".join(repo_path_tokens[:2])), + url="https://github.com/" + "/".join(repo_path_tokens[:2]), ) if "raw" in repo_path_tokens: return dict( type="git", - url="%s://%s/%s" + url="https://%s/%s" % ( - repo_parse.scheme, - repo_parse.netloc, + url_attrs.netloc, "/".join(repo_path_tokens[: repo_path_tokens.index("raw")]), ), ) @@ -498,9 +514,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser): result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]} include = None if self.remote_url: - repo_parse = urlparse(self.remote_url) - repo_path_tokens = repo_parse.path[1:].split("/")[:-1] - if "github" in repo_parse.netloc: + url_attrs = urlparse(self.remote_url) + repo_path_tokens = url_attrs.path[1:].split("/")[:-1] + if "github" in url_attrs.netloc: include = "/".join(repo_path_tokens[3:]) or None elif "raw" in repo_path_tokens: include = ( diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 72b347c5..204ee07f 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -29,6 +29,10 @@ def test_library_json_parser(): "name": "TestPackage", "keywords": "kw1, KW2, kw3", "platforms": ["atmelavr", "espressif"], + "repository": { + "type": "git", + "url": "http://github.com/username/repo/" + }, "url": "http://old.url.format", "exclude": [".gitignore", "tests"], "include": "mylib", @@ -44,6 +48,10 @@ def test_library_json_parser(): { "name": "TestPackage", "platforms": ["atmelavr", "espressif8266"], + "repository": { + "type": "git", + "url": "https://github.com/username/repo.git", + }, "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, "keywords": ["kw1", "kw2", "kw3"], "homepage": "http://old.url.format", @@ -172,7 +180,7 @@ customField=Custom Value "include": ["libraries/TestPackage"], } assert data["repository"] == { - "url": "https://github.com/username/reponame", + "url": "https://github.com/username/reponame.git", "type": "git", } @@ -299,7 +307,10 @@ architectures=avr,sam "A library for monochrome TFTs and OLEDs. Supported display " "controller: SSD1306, SSD1309, SSD1322, SSD1325" ), - "repository": {"url": "https://github.com/olikraus/u8glib", "type": "git"}, + "repository": { + "url": "https://github.com/olikraus/u8glib.git", + "type": "git", + }, "frameworks": ["arduino"], "platforms": ["atmelavr", "atmelsam"], "version": "1.19.1", @@ -350,7 +361,10 @@ includes=MozziGuts.h "sounds using familiar synthesis units like oscillators, delays, " "filters and envelopes." ), - "repository": {"url": "https://github.com/sensorium/Mozzi", "type": "git"}, + "repository": { + "url": "https://github.com/sensorium/Mozzi.git", + "type": "git", + }, "platforms": ["*"], "frameworks": ["arduino"], "export": {