diff --git a/.appveyor.yml b/.appveyor.yml index a4cc2f40..72fc32ac 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,6 +7,8 @@ platform: environment: matrix: - TOXENV: "py27" + - TOXENV: "py35" + - TOXENV: "py36" install: - cmd: git submodule update --init --recursive diff --git a/.pylintrc b/.pylintrc index aad06b28..09569a4f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -20,4 +20,4 @@ confidence= # --disable=W" # disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating -disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens +disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias diff --git a/.travis.yml b/.travis.yml index 589289f8..fe90e097 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,18 @@ matrix: sudo: false python: 2.7 env: TOX_ENV=docs - - os: linux - sudo: false - python: 2.7 - env: TOX_ENV=lint - os: linux sudo: required python: 2.7 env: TOX_ENV=py27 + - os: linux + sudo: required + python: 3.5 + env: TOX_ENV=py35 + - os: linux + sudo: required + python: 3.6 + env: TOX_ENV=py36 - os: osx language: generic env: TOX_ENV=skipexamples @@ -24,7 +28,7 @@ install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi # ChipKIT issue: install 32-bit support for GCC PIC32 - - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi script: - tox -e $TOX_ENV diff --git a/HISTORY.rst b/HISTORY.rst index 651451d4..80144ad7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,10 +1,19 @@ Release Notes ============= +PlatformIO 4.0 +-------------- + +4.0.0 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + +* Added Python 3.5+ support + (`issue #895 `_) + PlatformIO 3.0 -------------- -3.6.4 (2018-??-??) +3.6.4 (2019-??-??) ~~~~~~~~~~~~~~~~~~ * Improved Project Generator for IDEs: diff --git a/platformio/__init__.py b/platformio/__init__.py index 07fa8a5d..8a5c9124 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -VERSION = (3, 6, "4b1") +VERSION = (4, 0, "0a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -33,10 +31,3 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" - -if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0): - msg = ("PlatformIO Core v%s does not run under Python version %s.\n" - "Minimum supported version is 2.7, please upgrade Python.\n" - "Python 3 is not yet supported.\n") - sys.stderr.write(msg % (__version__, sys.version)) - sys.exit(1) diff --git a/platformio/__main__.py b/platformio/__main__.py index 161acac3..61db78de 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -53,7 +53,7 @@ class PlatformioCLI(click.MultiCommand): # pylint: disable=R0904 if name == "platforms": from platformio.commands import platform return platform.cli - elif name == "serialports": + if name == "serialports": from platformio.commands import device return device.cli raise AttributeError() diff --git a/platformio/app.py b/platformio/app.py index 14c4de22..c7dfdc43 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -169,8 +169,11 @@ class ContentCache(object): @staticmethod def key_from_args(*args): h = hashlib.md5() - for data in args: - h.update(str(data)) + for arg in args: + if not arg: + continue + arg = str(arg) + h.update(arg if util.PY2 else arg.encode()) return h.hexdigest() def get(self, key): @@ -191,7 +194,7 @@ class ContentCache(object): if not isdir(self.cache_dir): os.makedirs(self.cache_dir) tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} - assert valid.endswith(tuple(tdmap.keys())) + assert valid.endswith(tuple(tdmap)) expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) if not self._lock_dbindex(): @@ -339,21 +342,22 @@ def is_disabled_progressbar(): def get_cid(): cid = get_state_item("cid") - if not cid: - _uid = None - if getenv("C9_UID"): - _uid = getenv("C9_UID") - elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): - try: - _uid = requests.get("{api}/user?token={token}".format( - api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), - token=getenv("USER_TOKEN"))).json().get("id") - except: # pylint: disable=bare-except - pass - cid = str( - uuid.UUID( - bytes=hashlib.md5(str(_uid if _uid else uuid.getnode())). - digest())) - if "windows" in util.get_systype() or os.getuid() > 0: - set_state_item("cid", cid) + if cid: + return cid + uid = None + if getenv("C9_UID"): + uid = getenv("C9_UID") + elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): + try: + uid = requests.get("{api}/user?token={token}".format( + api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), + token=getenv("USER_TOKEN"))).json().get("id") + except: # pylint: disable=bare-except + pass + if not uid: + uid = uuid.getnode() + cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest()) + cid = str(cid) + if "windows" in util.get_systype() or os.getuid() > 0: + set_state_item("cid", cid) return cid diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 5f2856ac..2afb2289 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -92,7 +92,7 @@ DEFAULT_ENV_OPTIONS = dict( variables=commonvars, # Propagating External Environment - PIOVARIABLES=commonvars.keys(), + PIOVARIABLES=list(commonvars.keys()), ENV=environ, UNIX_TIME=int(time()), PIOHOME_DIR=util.get_home_dir(), @@ -125,9 +125,11 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) # decode common variables -for k in commonvars.keys(): +for k in list(commonvars.keys()): if k in env: env[k] = base64.b64decode(env[k]) + if isinstance(env[k], bytes): + env[k] = env[k].decode() if k in MULTILINE_VARS: env[k] = util.parse_conf_multi_values(env[k]) @@ -161,7 +163,9 @@ env['LIBSOURCE_DIRS'] = [ env.LoadPioPlatform(commonvars) env.SConscriptChdir(0) -env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite")) +env.SConsignFile( + join("$PROJECTBUILD_DIR", + ".sconsign.dblite" if util.PY2 else ".sconsign3.dblite")) for item in env.GetExtraScripts("pre"): env.SConscript(item, exports="env") diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 17cefb0c..02fee30c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -74,12 +74,19 @@ class LibBuilderFactory(object): if not env.IsFileWithExt( fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): continue - with open(join(root, fname)) as f: - content = f.read() - if "Arduino.h" in content and include_re.search(content): - return ["arduino"] - elif "mbed.h" in content and include_re.search(content): - return ["mbed"] + content = "" + try: + with open(join(root, fname)) as f: + content = f.read() + except UnicodeDecodeError: + with open(join(root, fname), encoding="latin-1") as f: + content = f.read() + if not content: + continue + if "Arduino.h" in content and include_re.search(content): + return ["arduino"] + if "mbed.h" in content and include_re.search(content): + return ["mbed"] return [] @@ -183,9 +190,9 @@ class LibBuilderBase(object): @property def build_dir(self): - return join("$BUILD_DIR", - "lib%s" % hashlib.sha1(self.path).hexdigest()[:3], - basename(self.path)) + lib_hash = hashlib.sha1(self.path if util.PY2 else self.path. + encode()).hexdigest()[:3] + return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property def build_flags(self): @@ -227,7 +234,7 @@ class LibBuilderBase(object): @staticmethod def validate_ldf_mode(mode): - if isinstance(mode, basestring): + if isinstance(mode, util.string_types): mode = mode.strip().lower() if mode in LibBuilderBase.LDF_MODES: return mode @@ -239,7 +246,7 @@ class LibBuilderBase(object): @staticmethod def validate_compat_mode(mode): - if isinstance(mode, basestring): + if isinstance(mode, util.string_types): mode = mode.strip().lower() if mode in LibBuilderBase.COMPAT_MODES: return mode @@ -612,9 +619,9 @@ class PlatformIOLibBuilder(LibBuilderBase): def src_filter(self): if "srcFilter" in self._manifest.get("build", {}): return self._manifest.get("build").get("srcFilter") - elif self.env['SRC_FILTER']: + if self.env['SRC_FILTER']: return self.env['SRC_FILTER'] - elif self._is_arduino_manifest(): + if self._is_arduino_manifest(): return ArduinoLibBuilder.src_filter.fget(self) return LibBuilderBase.src_filter.fget(self) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index be030f39..878cb486 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -162,7 +162,7 @@ class InoToCPPConverter(object): if not prototypes: return contents - prototype_names = set([m.group(3).strip() for m in prototypes]) + prototype_names = set(m.group(3).strip() for m in prototypes) split_pos = prototypes[0].start() match_ptrs = re.search( self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), @@ -212,7 +212,7 @@ def _get_compiler_type(env): output = "".join([result['out'], result['err']]).lower() if "clang" in output and "LLVM" in output: return "clang" - elif "gcc" in output: + if "gcc" in output: return "gcc" return None diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 9d37ae77..358aee7d 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -95,8 +95,10 @@ def LoadPioPlatform(env, variables): for key, value in variables.UnknownVariables().items(): if not key.startswith("BOARD_"): continue - env.Replace( - **{key.upper().replace("BUILD.", ""): base64.b64decode(value)}) + value = base64.b64decode(value) + if isinstance(value, bytes): + value = value.decode() + env.Replace(**{key.upper().replace("BUILD.", ""): value}) return # update board manifest with a custom data @@ -104,10 +106,13 @@ def LoadPioPlatform(env, variables): for key, value in variables.UnknownVariables().items(): if not key.startswith("BOARD_"): continue - board_config.update(key.lower()[6:], base64.b64decode(value)) + value = base64.b64decode(value) + if isinstance(value, bytes): + value = value.decode() + board_config.update(key.lower()[6:], value) # update default environment variables - for key in variables.keys(): + for key in list(variables.keys()): if key in env or \ not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]): continue diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index 7db4f943..c893ffd5 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platform import system + +from platformio import util # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options @@ -58,7 +61,9 @@ def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) - tmp_file = join(build_dir, "longcmd-%s" % md5(data).hexdigest()) + tmp_file = join( + build_dir, + "longcmd-%s" % md5(data if util.PY2 else data.encode()).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: @@ -71,7 +76,7 @@ def exists(_): def generate(env): - if system() != "Windows": + if "windows" not in util.get_systype(): return None env.Replace(_long_sources_hook=long_sources_hook) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index b93499ae..2a539e10 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -24,7 +24,7 @@ from SCons import Builder, Util from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, DefaultEnvironment, Export, SConscript) -from platformio.util import glob_escape, pioversion_to_intstr +from platformio.util import glob_escape, pioversion_to_intstr, string_types SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] @@ -189,7 +189,7 @@ def ProcessFlags(env, flags): # pylint: disable=too-many-branches # provided with a -U option // Issue #191 undefines = [ u for u in env.get("CCFLAGS", []) - if isinstance(u, basestring) and u.startswith("-U") + if isinstance(u, string_types) and u.startswith("-U") ] if undefines: for undef in undefines: diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 450b52bd..0a8ea38f 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -44,7 +44,7 @@ def device_list( # pylint: disable=too-many-branches if mdns: data['mdns'] = util.get_mdns_services() - single_key = data.keys()[0] if len(data.keys()) == 1 else None + single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: return click.echo(json.dumps(data[single_key] if single_key else data)) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index c305dad6..2f09cd9c 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -17,7 +17,6 @@ import json import time from os.path import isdir, join -from urllib import quote import click @@ -25,6 +24,11 @@ from platformio import exception, util from platformio.managers.lib import LibraryManager, get_builtin_libs from platformio.util import get_api_result +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + @click.group(short_help="Library Manager") @click.option( @@ -142,9 +146,9 @@ def lib_update(lm, libraries, only_check, json_output): manifest['versionLatest'] = latest result.append(manifest) return click.echo(json.dumps(result)) - else: - for library in libraries: - lm.update(library, only_check=only_check) + + for library in libraries: + lm.update(library, only_check=only_check) return True diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 4ccd65ff..cb69c453 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -62,7 +62,7 @@ def _original_version(version): return None if len(y) % 2 != 0: y = "0" + y - parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(len(y) / 2)] + parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(int(len(y) / 2))] return ".".join(parts) @@ -88,8 +88,8 @@ def _get_installed_platform_data(platform, docs=p.docs_url, license=p.license, forDesktop=not p.is_embedded(), - frameworks=sorted(p.frameworks.keys() if p.frameworks else []), - packages=p.packages.keys() if p.packages else []) + frameworks=sorted(list(p.frameworks) if p.frameworks else []), + packages=list(p.packages) if p.packages else []) # if dump to API # del data['version'] @@ -374,15 +374,14 @@ def platform_update(platforms, only_packages, only_check, json_output): data['versionLatest'] = latest result.append(data) return click.echo(json.dumps(result)) - else: - # cleanup cached board and platform lists - app.clean_cache() - for platform in platforms: - click.echo("Platform %s" % click.style( - pkg_dir_to_name.get(platform, platform), fg="cyan")) - click.echo("--------") - pm.update( - platform, only_packages=only_packages, only_check=only_check) - click.echo() + + # cleanup cached board and platform lists + app.clean_cache() + for platform in platforms: + click.echo("Platform %s" % click.style( + pkg_dir_to_name.get(platform, platform), fg="cyan")) + click.echo("--------") + pm.update(platform, only_packages=only_packages, only_check=only_check) + click.echo() return True diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 47f14f81..ad4e61e0 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -357,7 +357,7 @@ def _clean_build_dir(build_dir): def print_header(label, is_error=False): terminal_width, _ = click.get_terminal_size() width = len(click.unstyle(label)) - half_line = "=" * ((terminal_width - width - 2) / 2) + half_line = "=" * int((terminal_width - width - 2) / 2) click.echo("%s %s %s" % (half_line, label, half_line), err=is_error) @@ -394,7 +394,7 @@ def print_summary(results, start_time): def check_project_defopts(config): if not config.has_section("platformio"): return True - unknown = set([k for k, _ in config.items("platformio")]) - set( + unknown = set(k for k, _ in config.items("platformio")) - set( EnvironmentProcessor.KNOWN_PLATFORMIO_OPTIONS) if not unknown: return True @@ -409,7 +409,7 @@ def check_project_envs(config, environments=None): if not config.sections(): raise exception.ProjectEnvsNotAvailable() - known = set([s[4:] for s in config.sections() if s.startswith("env:")]) + known = set(s[4:] for s in config.sections() if s.startswith("env:")) unknown = set(environments or []) - known if unknown: raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) @@ -432,4 +432,5 @@ def calculate_project_hash(): # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() - return sha1(chunks_to_str).hexdigest() + return sha1(chunks_to_str if util.PY2 else chunks_to_str. + encode()).hexdigest() diff --git a/platformio/downloader.py b/platformio/downloader.py index 62c19385..d8b2fd24 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -45,14 +45,16 @@ class FileDownloader(object): if disposition and "filename=" in disposition: self._fname = disposition[disposition.index("filename=") + 9:].replace('"', "").replace("'", "") - self._fname = self._fname.encode("utf8") + if util.PY2: + self._fname = self._fname.encode("utf8") else: self._fname = [p for p in url.split("/") if p][-1] self._destination = self._fname if dest_dir: - self.set_destination( - join(dest_dir.decode(getfilesystemencoding()), self._fname)) + if util.PY2: + dest_dir = dest_dir.decode(getfilesystemencoding()) + self.set_destination(join(dest_dir, self._fname)) def set_destination(self, destination): self._destination = destination diff --git a/platformio/exception.py b/platformio/exception.py index 48f831c9..7829b648 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=not-an-iterable + class PlatformioException(Exception): @@ -20,7 +22,7 @@ class PlatformioException(Exception): def __str__(self): # pragma: no cover if self.MESSAGE: return self.MESSAGE.format(*self.args) - return Exception.__str__(self) + return super(PlatformioException, self).__str__() class ReturnErrorCode(PlatformioException): diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 8ec43aed..e5c392fa 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -116,9 +116,10 @@ class ProjectGenerator(object): os.makedirs(dst_dir) file_name = basename(tpl_path)[:-4] + contents = self._render_tpl(tpl_path) self._merge_contents( join(dst_dir, file_name), - self._render_tpl(tpl_path).encode("utf8")) + contents.encode("utf8") if util.PY2 else contents) def _render_tpl(self, tpl_path): content = "" diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 7894dbda..47105a5b 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -127,7 +127,7 @@ class Upgrader(object): if not item.endswith(".json"): continue data = util.load_json(join(boards_dir, item)) - if set(["name", "url", "vendor"]) <= set(data.keys()): + if set(["name", "url", "vendor"]) <= set(data): continue os.remove(join(boards_dir, item)) for key, value in data.items(): diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 2a516f04..8962cbb7 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -26,9 +26,9 @@ from platformio.managers.package import PackageManager CORE_PACKAGES = { "contrib-piohome": "^2.0.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.0.0", + "tool-pioplus": "^2.0.2", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" + "tool-scons": "~2.20501.7" if util.PY2 else "~3.30100.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 9042e624..eaea4412 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -120,7 +120,7 @@ class LibraryManager(BasePkgManager): # convert listed items via comma to array for key in ("keywords", "frameworks", "platforms"): if key not in manifest or \ - not isinstance(manifest[key], basestring): + not isinstance(manifest[key], util.string_types): continue manifest[key] = [ i.strip() for i in manifest[key].split(",") if i.strip() @@ -147,7 +147,7 @@ class LibraryManager(BasePkgManager): continue if item[k] == "*": del item[k] - elif isinstance(item[k], basestring): + elif isinstance(item[k], util.string_types): item[k] = [ i.strip() for i in item[k].split(",") if i.strip() ] @@ -275,7 +275,7 @@ class LibraryManager(BasePkgManager): break if not lib_info: - if filters.keys() == ["name"]: + if list(filters) == ["name"]: raise exception.LibNotFound(filters['name']) else: raise exception.LibNotFound(str(filters)) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 2f73452d..8242a649 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -71,7 +71,7 @@ class PackageRepoIterator(object): if self.package in manifest: return manifest[self.package] - return self.next() + return next(self) class PkgRepoMixin(object): @@ -544,7 +544,7 @@ class PkgInstallerMixin(object): def _install_from_tmp_dir( # pylint: disable=too-many-branches self, tmp_dir, requirements=None): tmp_manifest = self.load_manifest(tmp_dir) - assert set(["name", "version"]) <= set(tmp_manifest.keys()) + assert set(["name", "version"]) <= set(tmp_manifest) pkg_dirname = self.get_install_dirname(tmp_manifest) pkg_dir = join(self.package_dir, pkg_dirname) @@ -587,8 +587,10 @@ class PkgInstallerMixin(object): cur_manifest['version']) if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( - pkg_dirname, hashlib.md5( - cur_manifest['__src_url']).hexdigest()) + pkg_dirname, + hashlib.md5(cur_manifest['__src_url'] if util. + PY2 else cur_manifest['__src_url']. + encode()).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -596,8 +598,10 @@ class PkgInstallerMixin(object): tmp_manifest['version']) if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( - pkg_dirname, hashlib.md5( - tmp_manifest['__src_url']).hexdigest()) + pkg_dirname, + hashlib.md5(tmp_manifest['__src_url'] if util. + PY2 else tmp_manifest['__src_url']. + encode()).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 5f7e0adc..76a0550c 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -18,7 +18,6 @@ import re from imp import load_source from multiprocessing import cpu_count from os.path import basename, dirname, isdir, isfile, join -from urllib import quote import click import semantic_version @@ -27,6 +26,11 @@ from platformio import __version__, app, exception, util from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + class PlatformManager(BasePkgManager): @@ -81,7 +85,7 @@ class PlatformManager(BasePkgManager): skip_default_package, silent=silent, force=force) - return self.cleanup_packages(p.packages.keys()) + return self.cleanup_packages(list(p.packages)) def uninstall(self, package, requirements=None, after_update=False): if isdir(package): @@ -101,7 +105,7 @@ class PlatformManager(BasePkgManager): if after_update: return True - return self.cleanup_packages(p.packages.keys()) + return self.cleanup_packages(list(p.packages)) def update( # pylint: disable=arguments-differ self, @@ -119,17 +123,17 @@ class PlatformManager(BasePkgManager): raise exception.UnknownPlatform(package) p = PlatformFactory.newPlatform(pkg_dir) - pkgs_before = p.get_installed_packages().keys() + pkgs_before = list(p.get_installed_packages()) missed_pkgs = set() if not only_packages: BasePkgManager.update(self, pkg_dir, requirements, only_check) p = PlatformFactory.newPlatform(pkg_dir) - missed_pkgs = set(pkgs_before) & set(p.packages.keys()) - missed_pkgs -= set(p.get_installed_packages().keys()) + missed_pkgs = set(pkgs_before) & set(p.packages) + missed_pkgs -= set(p.get_installed_packages()) p.update_packages(only_check) - self.cleanup_packages(p.packages.keys()) + self.cleanup_packages(list(p.packages)) if missed_pkgs: p.install_packages( @@ -265,7 +269,7 @@ class PlatformPackagesMixin(object): without_packages = set(self.find_pkg_names(without_packages or [])) upkgs = with_packages | without_packages - ppkgs = set(self.packages.keys()) + ppkgs = set(self.packages) if not upkgs.issubset(ppkgs): raise exception.UnknownPackage(", ".join(upkgs - ppkgs)) @@ -384,7 +388,12 @@ class PlatformRunMixin(object): # encode and append variables for key, value in variables.items(): - cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) + if util.PY2: + cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) + else: + cmd.append( + "%s=%s" % (key.upper(), base64.b64encode( + value.encode()).decode())) util.copy_pythonpath_to_osenv() result = util.exec_command( @@ -550,7 +559,7 @@ class PlatformBase( # pylint: disable=too-many-public-methods config = PlatformBoardConfig(manifest_path) if "platform" in config and config.get("platform") != self.name: return - elif "platforms" in config \ + if "platforms" in config \ and self.name not in config.get("platforms"): return config.manifest['platform'] = self.name @@ -652,7 +661,7 @@ class PlatformBoardConfig(object): self._manifest = util.load_json(manifest_path) except ValueError: raise exception.InvalidBoardManifest(manifest_path) - if not set(["name", "url", "vendor"]) <= set(self._manifest.keys()): + if not set(["name", "url", "vendor"]) <= set(self._manifest): raise exception.PlatformioException( "Please specify name, url and vendor fields for " + manifest_path) @@ -666,8 +675,7 @@ class PlatformBoardConfig(object): except KeyError: if default is not None: return default - else: - raise KeyError("Invalid board option '%s'" % path) + raise KeyError("Invalid board option '%s'" % path) def update(self, path, value): newdict = None @@ -752,7 +760,7 @@ class PlatformBoardConfig(object): return tool_name raise exception.DebugInvalidOptions( "Unknown debug tool `%s`. Please use one of `%s` or `custom`" % - (tool_name, ", ".join(sorted(debug_tools.keys())))) + (tool_name, ", ".join(sorted(list(debug_tools))))) # automatically select best tool data = {"default": [], "onboard": [], "external": []} diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 7394654c..9d379c23 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -14,7 +14,6 @@ import atexit import platform -import Queue import re import sys import threading @@ -29,6 +28,11 @@ import requests from platformio import __version__, app, exception, util +try: + import queue +except ImportError: + import Queue as queue + class TelemetryBase(object): @@ -74,12 +78,12 @@ class MeasurementProtocol(TelemetryBase): def __getitem__(self, name): if name in self.PARAMS_MAP: name = self.PARAMS_MAP[name] - return TelemetryBase.__getitem__(self, name) + return super(MeasurementProtocol, self).__getitem__(name) def __setitem__(self, name, value): if name in self.PARAMS_MAP: name = self.PARAMS_MAP[name] - TelemetryBase.__setitem__(self, name, value) + super(MeasurementProtocol, self).__setitem__(name, value) def _prefill_appinfo(self): self['av'] = __version__ @@ -178,7 +182,7 @@ class MPDataPusher(object): MAX_WORKERS = 5 def __init__(self): - self._queue = Queue.LifoQueue() + self._queue = queue.LifoQueue() self._failedque = deque() self._http_session = requests.Session() self._http_offline = False @@ -203,7 +207,7 @@ class MPDataPusher(object): try: while True: items.append(self._queue.get_nowait()) - except Queue.Empty: + except queue.Empty: pass return items @@ -384,7 +388,7 @@ def backup_reports(items): for params in items: # skip static options - for key in params.keys(): + for key in params: if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"): del params[key] diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 8f55b42d..be0fa552 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -64,7 +64,7 @@ class ZIPArchive(ArchiveBase): @staticmethod def preserve_permissions(item, dest_dir): - attrs = item.external_attr >> 16L + attrs = item.external_attr >> 16 if attrs: chmod(join(dest_dir, item.filename), attrs) @@ -72,7 +72,7 @@ class ZIPArchive(ArchiveBase): def preserve_mtime(item, dest_dir): util.change_filemtime( join(dest_dir, item.filename), - mktime(list(item.date_time) + [0] * 3)) + mktime(tuple(item.date_time) + tuple([0, 0, 0]))) def get_items(self): return self._afo.infolist() diff --git a/platformio/util.py b/platformio/util.py index c5433ee3..a8415ab0 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -34,12 +34,15 @@ import requests from platformio import __apiurl__, __version__, exception -# pylint: disable=wrong-import-order, too-many-ancestors +# pylint: disable=too-many-ancestors -try: - import configparser as ConfigParser -except ImportError: +PY2 = sys.version_info[0] == 2 +if PY2: import ConfigParser as ConfigParser + string_types = basestring # pylint: disable=undefined-variable +else: + import configparser as ConfigParser + string_types = str class ProjectConfig(ConfigParser.ConfigParser): @@ -52,7 +55,8 @@ class ProjectConfig(ConfigParser.ConfigParser): items.append((option, self.get(section, option))) return items - def get(self, section, option, **kwargs): + def get( # pylint: disable=arguments-differ + self, section, option, **kwargs): try: value = ConfigParser.ConfigParser.get(self, section, option, **kwargs) @@ -177,6 +181,8 @@ def singleton(cls): def path_to_unicode(path): + if not PY2: + return path return path.decode(sys.getfilesystemencoding()).encode("utf-8") @@ -315,8 +321,11 @@ def get_projectbuild_dir(force=False): path = get_project_optional_dir("build_dir", join(get_project_dir(), ".pioenvs")) if "$PROJECT_HASH" in path: - path = path.replace("$PROJECT_HASH", - sha1(get_project_dir()).hexdigest()[:10]) + project_dir = get_project_dir() + path = path.replace( + "$PROJECT_HASH", + sha1(project_dir if PY2 else project_dir.encode()).hexdigest() + [:10]) try: if not isdir(path): os.makedirs(path) @@ -414,8 +423,10 @@ def exec_command(*args, **kwargs): result[s[3:]] = "\n".join(kwargs[s].get_buffer()) for k, v in result.items(): - if v and isinstance(v, basestring): - result[k].strip() + if not PY2 and isinstance(result[k], bytes): + result[k] = result[k].decode() + if v and isinstance(v, string_types): + result[k] = result[k].strip() return result @@ -445,10 +456,13 @@ def get_serial_ports(filter_hwid=False): if not p: continue if "windows" in get_systype(): - try: - d = unicode(d, errors="ignore") - except TypeError: - pass + if PY2: + try: + d = unicode( # pylint: disable=undefined-variable + d, + errors="ignore") + except TypeError: + pass if not filter_hwid or "VID:PID" in h: result.append({"port": p, "description": d, "hwid": h}) @@ -490,17 +504,17 @@ def get_logical_devices(): for device in re.findall(r"[A-Z]:\\", result): items.append({"path": device, "name": None}) return items - else: - result = exec_command(["df"]).get("out") - devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) - for line in result.split("\n"): - match = devicenamere.match(line.strip()) - if not match: - continue - items.append({ - "path": match.group(1), - "name": basename(match.group(1)) - }) + + result = exec_command(["df"]).get("out") + devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) + for line in result.split("\n"): + match = devicenamere.match(line.strip()) + if not match: + continue + items.append({ + "path": match.group(1), + "name": basename(match.group(1)) + }) return items @@ -557,12 +571,16 @@ def get_mdns_services(): time.sleep(3) for service in mdns.get_services(): properties = None - try: - if service.properties: - json.dumps(service.properties) - properties = service.properties - except UnicodeDecodeError: - pass + if service.properties: + try: + properties = { + k.decode("utf8"): + v.decode("utf8") if isinstance(v, bytes) else v + for k, v in service.properties.items() + } + json.dumps(properties) + except UnicodeDecodeError: + properties = None items.append({ "type": @@ -570,7 +588,10 @@ def get_mdns_services(): "name": service.name, "ip": - ".".join([str(ord(c)) for c in service.address]), + ".".join([ + str(c if isinstance(c, int) else ord(c)) + for c in service.address + ]), "port": service.port, "properties": @@ -735,7 +756,7 @@ def where_is_program(program, envpath=None): for bin_dir in env.get("PATH", "").split(os.pathsep): if isfile(join(bin_dir, program)): return join(bin_dir, program) - elif isfile(join(bin_dir, "%s.exe" % program)): + if isfile(join(bin_dir, "%s.exe" % program)): return join(bin_dir, "%s.exe" % program) return program diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 6a924370..67ef29d2 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -16,11 +16,15 @@ import re from os.path import join from subprocess import CalledProcessError, check_call from sys import modules -from urlparse import urlparse from platformio import util from platformio.exception import PlatformioException, UserSideException +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + class VCSClientFactory(object): diff --git a/scripts/install_devplatforms.py b/scripts/install_devplatforms.py index 434ab7cd..4d8daaa6 100644 --- a/scripts/install_devplatforms.py +++ b/scripts/install_devplatforms.py @@ -21,7 +21,7 @@ from platformio import util def main(): platforms = json.loads( subprocess.check_output( - ["platformio", "platform", "search", "--json-output"])) + ["platformio", "platform", "search", "--json-output"]).decode()) for platform in platforms: if platform['forDesktop']: continue diff --git a/setup.py b/setup.py index dc9b7bf4..747d6cc2 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,8 @@ setup( author_email=__email__, url=__url__, license=__license__, - python_requires='>=2.7, <3', + python_requires=", ".join([ + ">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"]), install_requires=install_requires, packages=find_packages() + ["scripts"], package_data={ @@ -65,6 +66,7 @@ setup( "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python", + "Programming Language :: Python :: 3", "Topic :: Software Development", "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Compilers" diff --git a/tests/test_examples.py b/tests/test_examples.py index d9588956..d1ab17d8 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -74,7 +74,7 @@ def test_run(pioproject_dir): ["platformio", "run", "-e", random.choice(env_names)]) if result['returncode'] != 0: - pytest.fail(result) + pytest.fail(str(result)) assert isdir(build_dir) diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index ec9d749a..4a69227e 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -44,7 +44,7 @@ def test_after_upgrade_2_to_3(clirunner, validate_cliresult, result = clirunner.invoke(cli_pio, ["settings", "get"]) validate_cliresult(result) - assert "upgraded to 3" in result.output + assert "upgraded to " in result.output # check PlatformIO 3.0 boards assert board_ids == set([p.basename[:-5] for p in boards.listdir()]) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 92da706f..e0875a16 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -28,7 +28,7 @@ def test_packages(): "https://dl.bintray.com/platformio/dl-packages/manifest.json").json() assert isinstance(pkgs_manifest, dict) items = [] - for _, variants in pkgs_manifest.iteritems(): + for _, variants in pkgs_manifest.items(): for item in variants: items.append(item) diff --git a/tox.ini b/tox.ini index 26327fd7..597c7b8f 100644 --- a/tox.ini +++ b/tox.ini @@ -13,10 +13,10 @@ # limitations under the License. [tox] -envlist = py27, docs, lint +envlist = py27, py35, py36, py37, docs -[testenv:develop] -basepython = python2.7 +[testenv] +passenv = * usedevelop = True deps = isort @@ -24,10 +24,15 @@ deps = pylint pytest pytest-xdist -commands = python --version +commands = + {envpython} --version + pylint --rcfile=./.pylintrc ./platformio + {envpython} -c "print('travis_fold:start:install_devplatforms')" + {envpython} scripts/install_devplatforms.py + {envpython} -c "print('travis_fold:end:install_devplatforms')" + py.test -v --basetemp="{envtmpdir}" tests [testenv:docs] -basepython = python2.7 deps = sphinx sphinx_rtd_theme @@ -37,44 +42,23 @@ commands = sphinx-build -W -b latex -d {envtmpdir}/doctrees docs docs/_build/latex [testenv:docslinkcheck] -basepython = python2.7 deps = sphinx sphinx_rtd_theme commands = sphinx-build -W -b linkcheck docs docs/_build/html -[testenv:lint] -basepython = python2.7 -deps = - pylint -commands = - pylint --rcfile=./.pylintrc ./platformio - -[testenv] -basepython = python2.7 -passenv = * -deps = - pytest -commands = - {envpython} --version - {envpython} -c "print 'travis_fold:start:install_devplatforms'" - {envpython} scripts/install_devplatforms.py - {envpython} -c "print 'travis_fold:end:install_devplatforms'" - py.test -v --basetemp="{envtmpdir}" tests - [testenv:skipexamples] -basepython = python2.7 deps = pytest commands = py.test -v --basetemp="{envtmpdir}" tests --ignore tests/test_examples.py -[testenv:coverage] -basepython = python2.7 -passenv = * -deps = - pytest - pytest-cov -commands = - py.test --cov=platformio --cov-report term --cov-report xml --ignore=tests/test_examples.py --ignore=tests/test_pkgmanifest.py -v tests +; [testenv:coverage] +; basepython = python2 +; passenv = * +; deps = +; pytest +; pytest-cov +; commands = +; py.test --cov=platformio --cov-report term --cov-report xml --ignore=tests/test_examples.py --ignore=tests/test_pkgmanifest.py -v tests