diff --git a/.appveyor.yml b/.appveyor.yml index a4cc2f40..c9703e56 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,19 +1,20 @@ build: off platform: - - x86 - x64 environment: matrix: - TOXENV: "py27" + PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build} + + - TOXENV: "py36" + PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build} install: - cmd: git submodule update --init --recursive - cmd: SET PATH=C:\MinGW\bin;%PATH% - - if %PLATFORM% == x64 SET PATH=C:\Python27-x64;C:\Python27-x64\Scripts;%PATH% - - if %PLATFORM% == x86 SET PATH=C:\Python27;C:\Python27\Scripts;%PATH% - - cmd: pip install tox + - cmd: pip install --force-reinstall tox test_script: - cmd: tox diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..6f70f7e9 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://platformio.org/donate diff --git a/.isort.cfg b/.isort.cfg index 9b58f629..c147908f 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc diff --git a/.pylintrc b/.pylintrc index aad06b28..180a05b8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,23 +1,12 @@ [MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -# disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating - -disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens +disable= + missing-docstring, + ungrouped-imports, + invalid-name, + cyclic-import, + duplicate-code, + superfluous-parens, + too-few-public-methods, + useless-object-inheritance, + useless-import-alias, + fixme diff --git a/.travis.yml b/.travis.yml index 589289f8..e4b65227 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,14 @@ matrix: sudo: false python: 2.7 env: TOX_ENV=docs - - os: linux - sudo: false - python: 2.7 - env: TOX_ENV=lint - os: linux sudo: required python: 2.7 - env: TOX_ENV=py27 + env: TOX_ENV=py27 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d) + - os: linux + sudo: required + python: 3.6 + env: TOX_ENV=py36 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d) - os: osx language: generic env: TOX_ENV=skipexamples @@ -21,10 +21,10 @@ matrix: install: - git submodule update --init --recursive - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi + - pip install -U tox # ChipKIT issue: install 32-bit support for GCC PIC32 - - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi script: - tox -e $TOX_ENV diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index d1efd097..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "python.pythonPath": "${workspaceRoot}/.tox/develop/bin/python", - "python.formatting.provider": "yapf", - "files.exclude": { - "**/*.pyc": true, - "*.egg-info": true, - ".cache": true, - "build": true, - "dist": true - }, - "editor.rulers": [79], - "restructuredtext.builtDocumentationPath": "${workspaceRoot}/docs/_build/html", - "restructuredtext.confPath": "${workspaceRoot}/docs", - "restructuredtext.linter.executablePath": "${workspaceRoot}/.tox/docs/bin/restructuredtext-lint" -} diff --git a/HISTORY.rst b/HISTORY.rst index 2c61d00d..efa87c51 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,78 @@ Release Notes ============= +.. _release_notes_4_0: + +PlatformIO 4.0 +-------------- + +4.0.0 (2019-07-10) +~~~~~~~~~~~~~~~~~~ + +`Migration Guide from 3.0 to 4.0 `__. + +* `PlatformIO Plus Goes Open Source `__ + + - Built-in `PIO Unified Debugger `__ + - Built-in `PIO Unit Testing `__ + +* **Project Configuration** + + - New project configuration parser with a strict options typing (`API `__) + - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) + - Share common (global) options between project environments using `[env] `__ section (`issue #1643 `_) + - Include external configuration files with `extra_configs `__ option (`issue #1590 `_) + - Custom project ``***_dir`` options declared in `platformio `__ section have higher priority than `Environment variables `__ + - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) + - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) + - Added support for `PLATFORMIO_DEFAULT_ENVS `__ system environment variable (`issue #1967 `_) + - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a `PIO Remote `__ machine + +* **Library Management** + + - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` + - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) + - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) + - Use isolated library dependency storage per project build environment (`issue #1696 `_) + - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 `_) + - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields + - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) + - Fixed an issue when `lib_compat_mode = strict `__ does not ignore libraries incompatible with a project framework + +* **Build System** + + - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` + - Switch between `Build Configurations `__ (``release`` and ``debug``) with a new project configuration option `build_type `__ + - Custom `platform_packages `__ per a build environment with an option to override default (`issue #1367 `_) + - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) + - Control a number of parallel build jobs with a new `-j, --jobs `__ option + - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) + - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) + - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir `__ option (`issue #2674 `_) + - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) + +* **Infrastructure** + + - Python 3 support (`issue #895 `_) + - Significantly speedup back-end for PIO Home. It works super fast now! + - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) + - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) + - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) + - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 `_, `issue #1342 `_, `issue #1959 `_, `issue #2100 `_) + +* **Integration** + + - Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file + - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 `_) + - Fixed an issue with Project Generator when an include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) + - Fixed an issue when generating invalid "Eclipse CDT Cross GCC Built-in Compiler Settings" if a custom `PLATFORMIO_CORE_DIR `__ is used (`issue #806 `_) + +* **Miscellaneous** + + - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead + - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) + - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 `_) + PlatformIO 3.0 -------------- diff --git a/Makefile b/Makefile index 6df42c42..c82b7c54 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ - lint: pylint --rcfile=./.pylintrc ./platformio @@ -10,7 +9,7 @@ yapf: yapf --recursive --in-place platformio/ test: - py.test -v -s -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py + py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py before-commit: isort yapf lint test @@ -23,4 +22,9 @@ clean: clean-docs rm -rf .cache rm -rf build rm -rf htmlcov - rm -f .coverage \ No newline at end of file + rm -f .coverage + +profile: + # Usage $ > make PIOARGS="boards" profile + python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS} + snakeviz .tox/.tmp/cprofile.prof diff --git a/README.rst b/README.rst index fa295e95..ee1f6896 100644 --- a/README.rst +++ b/README.rst @@ -78,6 +78,7 @@ Registry Development Platforms --------------------- +* `Aceinna IMU `_ * `Atmel AVR `_ * `Atmel SAM `_ * `Espressif 32 `_ @@ -86,6 +87,7 @@ Development Platforms * `Infineon XMC `_ * `Intel ARC32 `_ * `Intel MCS-51 (8051) `_ +* `Kendryte K210 `_ * `Lattice iCE40 `_ * `Maxim 32 `_ * `Microchip PIC32 `_ @@ -93,9 +95,11 @@ Development Platforms * `Nordic nRF52 `_ * `NXP LPC `_ * `RISC-V `_ +* `RISC-V GAP `_ * `Samsung ARTIK `_ * `Silicon Labs EFM32 `_ * `ST STM32 `_ +* `ST STM8 `_ * `Teensy `_ * `TI MSP430 `_ * `TI Tiva `_ @@ -111,8 +115,11 @@ Frameworks * `ESP-IDF `_ * `ESP8266 Non-OS SDK `_ * `ESP8266 RTOS SDK `_ +* `Freedom E SDK `_ +* `Kendryte Standalone SDK `_ * `libOpenCM3 `_ * `mbed `_ +* `PULP OS `_ * `Pumbaa `_ * `Simba `_ * `SPL `_ diff --git a/docs b/docs index 0c29f967..ae7deefa 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0c29f9671f44b5221e7cac15e93ebb40f9db88c1 +Subproject commit ae7deefa584f8c0fbb98c36b45649cc86bdf46b7 diff --git a/examples b/examples index 9c16f551..70f28968 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 9c16f551d72e37aa4c2f28555487e2fb23408b5d +Subproject commit 70f28968f2fa3f76374d236156581ddc4e2e8670 diff --git a/platformio/__init__.py b/platformio/__init__.py index d1d58c48..9fb531a5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -VERSION = (3, 6, 7) +VERSION = (4, 0, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -33,10 +31,3 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" - -if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0): - msg = ("PlatformIO Core v%s does not run under Python version %s.\n" - "Minimum supported version is 2.7, please upgrade Python.\n" - "Python 3 is not yet supported.\n") - sys.stderr.write(msg % (__version__, sys.version)) - sys.exit(1) diff --git a/platformio/__main__.py b/platformio/__main__.py index 161acac3..d4664935 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,60 +14,22 @@ import os import sys -from os.path import join -from platform import system from traceback import format_exc import click -from platformio import __version__, exception, maintenance -from platformio.util import get_source_dir +from platformio import __version__, exception, maintenance, util +from platformio.commands import PlatformioCLI +from platformio.compat import CYGWIN -class PlatformioCLI(click.MultiCommand): # pylint: disable=R0904 - - def list_commands(self, ctx): - cmds = [] - for filename in os.listdir(join(get_source_dir(), "commands")): - if filename.startswith("__init__"): - continue - if filename.endswith(".py"): - cmds.append(filename[:-3]) - cmds.sort() - return cmds - - def get_command(self, ctx, cmd_name): - mod = None - try: - mod = __import__("platformio.commands." + cmd_name, None, None, - ["cli"]) - except ImportError: - try: - return self._handle_obsolate_command(cmd_name) - except AttributeError: - raise click.UsageError('No such command "%s"' % cmd_name, ctx) - return mod.cli - - @staticmethod - def _handle_obsolate_command(name): - if name == "platforms": - from platformio.commands import platform - return platform.cli - elif name == "serialports": - from platformio.commands import device - return device.cli - raise AttributeError() - - -@click.command( - cls=PlatformioCLI, - context_settings=dict(help_option_names=["-h", "--help"])) +@click.command(cls=PlatformioCLI, + context_settings=dict(help_option_names=["-h", "--help"])) @click.version_option(__version__, prog_name="PlatformIO") -@click.option( - "--force", - "-f", - is_flag=True, - help="Force to accept any confirmation prompts.") +@click.option("--force", + "-f", + is_flag=True, + help="Force to accept any confirmation prompts.") @click.option("--caller", "-c", help="Caller ID (service).") @click.pass_context def cli(ctx, force, caller): @@ -80,8 +42,9 @@ def process_result(ctx, result, force, caller): # pylint: disable=W0613 maintenance.on_platformio_end(ctx, result) +@util.memoized() def configure(): - if "cygwin" in system().lower(): + if CYGWIN: raise exception.CygwinEnvDetected() # https://urllib3.readthedocs.org @@ -114,10 +77,17 @@ def configure(): click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) -def main(): +def main(argv=None): + exit_code = 0 + prev_sys_argv = sys.argv[:] + if argv: + assert isinstance(argv, list) + sys.argv = argv try: configure() cli(None, None, None) + except SystemExit: + pass except Exception as e: # pylint: disable=broad-except if not isinstance(e, exception.ReturnErrorCode): maintenance.on_platformio_exception(e) @@ -143,13 +113,13 @@ An unexpected error occurred. Further steps: ============================================================ """ click.secho(error_str, fg="red", err=True) - return int(str(e)) if str(e).isdigit() else 1 - return 0 + exit_code = int(str(e)) if str(e).isdigit() else 1 + sys.argv = prev_sys_argv + return exit_code def debug_gdb_main(): - sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:] - return main() + return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]) if __name__ == "__main__": diff --git a/platformio/app.py b/platformio/app.py index 497f096e..c2b95967 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,10 +14,8 @@ import codecs import hashlib -import json import os import uuid -from copy import deepcopy from os import environ, getenv, listdir, remove from os.path import abspath, dirname, expanduser, isdir, isfile, join from time import time @@ -25,6 +23,24 @@ from time import time import requests from platformio import exception, lockfile, util +from platformio.compat import (WINDOWS, dump_json_to_unicode, + hashlib_encode_data) +from platformio.proc import is_ci +from platformio.project.helpers import (get_project_cache_dir, + get_project_core_dir) + + +def get_default_projects_dir(): + docs_dir = join(expanduser("~"), "Documents") + try: + assert WINDOWS + import ctypes.wintypes + buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) + docs_dir = buf.value + except: # pylint: disable=bare-except + pass + return join(docs_dir, "PlatformIO", "Projects") def projects_dir_validate(projects_dir): @@ -74,7 +90,7 @@ DEFAULT_SETTINGS = { }, "projects_dir": { "description": "Default location for PlatformIO projects (PIO Home)", - "value": join(expanduser("~"), "Documents", "PlatformIO", "Projects"), + "value": get_default_projects_dir(), "validator": projects_dir_validate }, } @@ -88,28 +104,29 @@ class State(object): self.path = path self.lock = lock if not self.path: - self.path = join(util.get_home_dir(), "appstate.json") - self._state = {} - self._prev_state = {} + self.path = join(get_project_core_dir(), "appstate.json") + self._storage = {} self._lockfile = None + self.modified = False def __enter__(self): try: self._lock_state_file() if isfile(self.path): - self._state = util.load_json(self.path) - except exception.PlatformioException: - self._state = {} - self._prev_state = deepcopy(self._state) - return self._state + self._storage = util.load_json(self.path) + assert isinstance(self._storage, dict) + except (AssertionError, ValueError, UnicodeDecodeError, + exception.InvalidJSONFile): + self._storage = {} + return self def __exit__(self, type_, value, traceback): - if self._prev_state != self._state: + if self.modified: try: - with codecs.open(self.path, "w", encoding="utf8") as fp: - json.dump(self._state, fp) + with open(self.path, "w") as fp: + fp.write(dump_json_to_unicode(self._storage)) except IOError: - raise exception.HomeDirPermissionsError(util.get_home_dir()) + raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() def _lock_state_file(self): @@ -128,6 +145,32 @@ class State(object): def __del__(self): self._unlock_state_file() + # Dictionary Proxy + + def as_dict(self): + return self._storage + + def get(self, key, default=True): + return self._storage.get(key, default) + + def update(self, *args, **kwargs): + self.modified = True + return self._storage.update(*args, **kwargs) + + def __getitem__(self, key): + return self._storage[key] + + def __setitem__(self, key, value): + self.modified = True + self._storage[key] = value + + def __delitem__(self, key): + self.modified = True + del self._storage[key] + + def __contains__(self, item): + return item in self._storage + class ContentCache(object): @@ -136,7 +179,7 @@ class ContentCache(object): self._db_path = None self._lockfile = None - self.cache_dir = cache_dir or util.get_cache_dir() + self.cache_dir = cache_dir or get_project_cache_dir() self._db_path = join(self.cache_dir, "db.data") def __enter__(self): @@ -163,14 +206,16 @@ class ContentCache(object): return True def get_cache_path(self, key): + key = str(key) assert len(key) > 3 return join(self.cache_dir, key[-2:], key) @staticmethod def key_from_args(*args): h = hashlib.md5() - for data in args: - h.update(str(data)) + for arg in args: + if arg: + h.update(hashlib_encode_data(arg)) return h.hexdigest() def get(self, key): @@ -191,7 +236,7 @@ class ContentCache(object): if not isdir(self.cache_dir): os.makedirs(self.cache_dir) tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} - assert valid.endswith(tuple(tdmap.keys())) + assert valid.endswith(tuple(tdmap)) expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) if not self._lock_dbindex(): @@ -230,10 +275,13 @@ class ContentCache(object): if "=" not in line: continue expire, path = line.split("=") - if time() < int(expire) and isfile(path) and \ - path not in paths_for_delete: - newlines.append(line) - continue + try: + if time() < int(expire) and isfile(path) and \ + path not in paths_for_delete: + newlines.append(line) + continue + except ValueError: + pass found = True if isfile(path): try: @@ -280,19 +328,20 @@ def sanitize_setting(name, value): def get_state_item(name, default=None): - with State() as data: - return data.get(name, default) + with State() as state: + return state.get(name, default) def set_state_item(name, value): - with State(lock=True) as data: - data[name] = value + with State(lock=True) as state: + state[name] = value + state.modified = True def delete_state_item(name): - with State(lock=True) as data: - if name in data: - del data[name] + with State(lock=True) as state: + if name in state: + del state[name] def get_setting(name): @@ -300,24 +349,25 @@ def get_setting(name): if _env_name in environ: return sanitize_setting(name, getenv(_env_name)) - with State() as data: - if "settings" in data and name in data['settings']: - return data['settings'][name] + with State() as state: + if "settings" in state and name in state['settings']: + return state['settings'][name] return DEFAULT_SETTINGS[name]['value'] def set_setting(name, value): - with State(lock=True) as data: - if "settings" not in data: - data['settings'] = {} - data['settings'][name] = sanitize_setting(name, value) + with State(lock=True) as state: + if "settings" not in state: + state['settings'] = {} + state['settings'][name] = sanitize_setting(name, value) + state.modified = True def reset_settings(): - with State(lock=True) as data: - if "settings" in data: - del data['settings'] + with State(lock=True) as state: + if "settings" in state: + del state['settings'] def get_session_var(name, default=None): @@ -332,28 +382,29 @@ def set_session_var(name, value): def is_disabled_progressbar(): return any([ get_session_var("force_option"), - util.is_ci(), + is_ci(), getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true" ]) def get_cid(): cid = get_state_item("cid") - if not cid: - _uid = None - if getenv("C9_UID"): - _uid = getenv("C9_UID") - elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): - try: - _uid = requests.get("{api}/user?token={token}".format( - api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), - token=getenv("USER_TOKEN"))).json().get("id") - except: # pylint: disable=bare-except - pass - cid = str( - uuid.UUID( - bytes=hashlib.md5(str( - _uid if _uid else uuid.getnode())).digest())) - if "windows" in util.get_systype() or os.getuid() > 0: - set_state_item("cid", cid) + if cid: + return cid + uid = None + if getenv("C9_UID"): + uid = getenv("C9_UID") + elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): + try: + uid = requests.get("{api}/user?token={token}".format( + api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), + token=getenv("USER_TOKEN"))).json().get("id") + except: # pylint: disable=bare-except + pass + if not uid: + uid = uuid.getnode() + cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) + cid = str(cid) + if WINDOWS or os.getuid() > 0: # yapf: disable pylint: disable=no-member + set_state_item("cid", cid) return cid diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 5f2856ac..8fa012d0 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,108 +12,74 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 -import json -import sys -from os import environ -from os.path import expanduser, join +from os import environ, makedirs +from os.path import isdir, join from time import time -from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, DEFAULT_TARGETS, - AllowSubstExceptions, AlwaysBuild, Default, - DefaultEnvironment, Variables) +import click +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import DEFAULT_TARGETS # pylint: disable=import-error +from SCons.Script import AllowSubstExceptions # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error +from SCons.Script import Default # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error +from SCons.Script import Import # pylint: disable=import-error +from SCons.Script import Variables # pylint: disable=import-error from platformio import util +from platformio.compat import PY2, dump_json_to_unicode +from platformio.managers.platform import PlatformBase +from platformio.proc import get_pythonexe_path +from platformio.project import helpers as project_helpers AllowSubstExceptions(NameError) -# allow common variables from INI file -commonvars = Variables(None) -commonvars.AddVariables( +# append CLI arguments to build environment +clivars = Variables(None) +clivars.AddVariables( ("PLATFORM_MANIFEST",), ("BUILD_SCRIPT",), - ("EXTRA_SCRIPTS",), + ("PROJECT_CONFIG",), ("PIOENV",), - ("PIOTEST",), - ("PIOPLATFORM",), - ("PIOFRAMEWORK",), - - # build options - ("BUILD_FLAGS",), - ("SRC_BUILD_FLAGS",), - ("BUILD_UNFLAGS",), - ("SRC_FILTER",), - - # library options - ("LIB_LDF_MODE",), - ("LIB_COMPAT_MODE",), - ("LIB_DEPS",), - ("LIB_IGNORE",), - ("LIB_EXTRA_DIRS",), - ("LIB_ARCHIVE",), - - # board options - ("BOARD",), - # deprecated options, use board_{object.path} instead - ("BOARD_MCU",), - ("BOARD_F_CPU",), - ("BOARD_F_FLASH",), - ("BOARD_FLASH_MODE",), - # end of deprecated options - - # upload options - ("UPLOAD_PORT",), - ("UPLOAD_PROTOCOL",), - ("UPLOAD_SPEED",), - ("UPLOAD_FLAGS",), - ("UPLOAD_RESETMETHOD",), - - # test options - ("TEST_BUILD_PROJECT_SRC",), - - # debug options - ("DEBUG_TOOL",), - ("DEBUG_SVD_PATH",), - + ("PIOTEST_RUNNING_NAME",), + ("UPLOAD_PORT",) ) # yapf: disable -MULTILINE_VARS = [ - "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS", - "BUILD_UNFLAGS", "UPLOAD_FLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE", - "LIB_EXTRA_DIRS" -] - DEFAULT_ENV_OPTIONS = dict( tools=[ "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform", - "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" - ], # yapf: disable + "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" + ], toolpath=[join(util.get_source_dir(), "builder", "tools")], - variables=commonvars, + variables=clivars, # Propagating External Environment - PIOVARIABLES=commonvars.keys(), ENV=environ, UNIX_TIME=int(time()), - PIOHOME_DIR=util.get_home_dir(), - PROJECT_DIR=util.get_project_dir(), - PROJECTINCLUDE_DIR=util.get_projectinclude_dir(), - PROJECTSRC_DIR=util.get_projectsrc_dir(), - PROJECTTEST_DIR=util.get_projecttest_dir(), - PROJECTDATA_DIR=util.get_projectdata_dir(), - PROJECTBUILD_DIR=util.get_projectbuild_dir(), + PROJECT_DIR=project_helpers.get_project_dir(), + PROJECTCORE_DIR=project_helpers.get_project_core_dir(), + PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(), + PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(), + PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(), + PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(), + PROJECTSRC_DIR=project_helpers.get_project_src_dir(), + PROJECTTEST_DIR=project_helpers.get_project_test_dir(), + PROJECTDATA_DIR=project_helpers.get_project_data_dir(), + PROJECTBUILD_DIR=project_helpers.get_project_build_dir(), + BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"), BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), BUILDSRC_DIR=join("$BUILD_DIR", "src"), BUILDTEST_DIR=join("$BUILD_DIR", "test"), LIBPATH=["$BUILD_DIR"], LIBSOURCE_DIRS=[ - util.get_projectlib_dir(), - util.get_projectlibdeps_dir(), - join("$PIOHOME_DIR", "lib") + project_helpers.get_project_lib_dir(), + join("$PROJECTLIBDEPS_DIR", "$PIOENV"), + project_helpers.get_project_global_lib_dir() ], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), - PYTHONEXE=util.get_pythonexe_path()) + PYTHONEXE=get_pythonexe_path()) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET" @@ -124,12 +90,21 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) -# decode common variables -for k in commonvars.keys(): - if k in env: - env[k] = base64.b64decode(env[k]) - if k in MULTILINE_VARS: - env[k] = util.parse_conf_multi_values(env[k]) +# Load variables from CLI +env.Replace( + **{ + key: PlatformBase.decode_scons_arg(env[key]) + for key in list(clivars.keys()) if key in env + }) + +if env.subst("$BUILDCACHE_DIR"): + if not isdir(env.subst("$BUILDCACHE_DIR")): + makedirs(env.subst("$BUILDCACHE_DIR")) + env.CacheDir("$BUILDCACHE_DIR") + +if int(ARGUMENTS.get("ISATTY", 0)): + # pylint: disable=protected-access + click._compat.isatty = lambda stream: True if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) @@ -137,31 +112,13 @@ if env.GetOption('clean'): elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): print("Verbose mode can be enabled via `-v, --verbose` option") -# Handle custom variables from system environment -for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS", - "UPLOAD_PORT", "UPLOAD_FLAGS", "LIB_EXTRA_DIRS"): - k = "PLATFORMIO_%s" % var - if k not in environ: - continue - if var in ("UPLOAD_PORT", ): - env[var] = environ.get(k) - continue - env.Append(**{var: util.parse_conf_multi_values(environ.get(k))}) - -# Configure extra library source directories for LDF -if util.get_project_optional_dir("lib_extra_dirs"): - env.Prepend( - LIBSOURCE_DIRS=util.parse_conf_multi_values( - util.get_project_optional_dir("lib_extra_dirs"))) -env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", [])) -env['LIBSOURCE_DIRS'] = [ - expanduser(d) if d.startswith("~") else d for d in env['LIBSOURCE_DIRS'] -] - -env.LoadPioPlatform(commonvars) +env.LoadProjectOptions() +env.LoadPioPlatform() env.SConscriptChdir(0) -env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite")) +env.SConsignFile( + join("$PROJECTBUILD_DIR", + ".sconsign.dblite" if PY2 else ".sconsign3.dblite")) for item in env.GetExtraScripts("pre"): env.SConscript(item, exports="env") @@ -170,6 +127,8 @@ env.SConscript("$BUILD_SCRIPT") if "UPLOAD_FLAGS" in env: env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"]) +if env.GetProjectOption("upload_command"): + env.Replace(UPLOADCMD=env.GetProjectOption("upload_command")) for item in env.GetExtraScripts("post"): env.SConscript(item, exports="env") @@ -192,7 +151,6 @@ env.AddPreAction( "Configuring upload protocol...")) AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS)) -AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS)) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) ############################################################################## @@ -202,14 +160,8 @@ if "envdump" in COMMAND_LINE_TARGETS: env.Exit(0) if "idedata" in COMMAND_LINE_TARGETS: - try: - print("\n%s\n" % util.path_to_unicode( - json.dumps(env.DumpIDEData(), ensure_ascii=False))) - env.Exit(0) - except UnicodeDecodeError: - sys.stderr.write( - "\nUnicodeDecodeError: Non-ASCII characters found in build " - "environment\n" - "See explanation in FAQ > Troubleshooting > Building\n" - "https://docs.platformio.org/page/faq.html\n\n") - env.Exit(1) + Import("projenv") + print("\n%s\n" % dump_json_to_unicode( + env.DumpIDEData(projenv) # pylint: disable=undefined-variable + )) + env.Exit(0) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 0e6bf31a..1814f1b9 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -18,17 +18,18 @@ from glob import glob from os import environ from os.path import abspath, isfile, join -from SCons.Defaults import processDefines +from SCons.Defaults import processDefines # pylint: disable=import-error -from platformio import util +from platformio.compat import glob_escape from platformio.managers.core import get_core_package_dir +from platformio.proc import exec_command, where_is_program -def _dump_includes(env): +def _dump_includes(env, projenv): includes = [] - for item in env.get("CPPPATH", []): - includes.append(env.subst(item)) + for item in projenv.get("CPPPATH", []): + includes.append(projenv.subst(item)) # installed libs for lb in env.GetLibBuilders(): @@ -39,7 +40,7 @@ def _dump_includes(env): for name in p.get_installed_packages(): if p.get_package_type(name) != "toolchain": continue - toolchain_dir = util.glob_escape(p.get_package_dir(name)) + toolchain_dir = glob_escape(p.get_package_dir(name)) toolchain_incglobs = [ join(toolchain_dir, "*", "include*"), join(toolchain_dir, "*", "include", "c++", "*"), @@ -71,8 +72,9 @@ def _get_gcc_defines(env): try: sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command( - "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True) + result = exec_command("echo | %s -dM -E -" % env.subst("$CC"), + env=sysenv, + shell=True) except OSError: return items if result['returncode'] != 0: @@ -112,7 +114,7 @@ def _dump_defines(env): def _get_svd_path(env): - svd_path = env.subst("$DEBUG_SVD_PATH") + svd_path = env.GetProjectOption("debug_svd_path") if svd_path: return abspath(svd_path) @@ -133,27 +135,26 @@ def _get_svd_path(env): return None -def DumpIDEData(env): +def DumpIDEData(env, projenv): LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS" LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS" data = { - "libsource_dirs": - [env.subst(l) for l in env.get("LIBSOURCE_DIRS", [])], + "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], "defines": _dump_defines(env), "includes": - _dump_includes(env), + _dump_includes(env, projenv), "cc_flags": env.subst(LINTCCOM), "cxx_flags": env.subst(LINTCXXCOM), "cc_path": - util.where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), + where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), "cxx_path": - util.where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), + where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), "gdb_path": - util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), + where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), "prog_path": env.subst("$PROG_PATH"), "flash_extra_images": [{ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index c628f50c..66f9db57 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -17,41 +17,46 @@ from __future__ import absolute_import +import codecs import hashlib import os import re import sys -from glob import glob -from os.path import (basename, commonprefix, dirname, isdir, isfile, join, +from os.path import (basename, commonprefix, expanduser, isdir, isfile, join, realpath, sep) -import SCons.Scanner -from SCons.Script import ARGUMENTS, COMMAND_LINE_TARGETS, DefaultEnvironment +import click +import SCons.Scanner # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool +from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data, + string_types) from platformio.managers.lib import LibraryManager -from platformio.managers.package import PackageManager class LibBuilderFactory(object): @staticmethod - def new(env, path, verbose=False): + def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" if isfile(join(path, "library.json")): clsname = "PlatformIOLibBuilder" else: used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) - common_frameworks = ( - set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks)) + common_frameworks = (set(env.get("PIOFRAMEWORK", [])) + & set(used_frameworks)) if common_frameworks: clsname = "%sLibBuilder" % list(common_frameworks)[0].title() elif used_frameworks: clsname = "%sLibBuilder" % used_frameworks[0].title() - obj = getattr(sys.modules[__name__], clsname)( - env, path, verbose=verbose) + obj = getattr(sys.modules[__name__], clsname)(env, + path, + verbose=verbose) assert isinstance(obj, LibBuilderBase) return obj @@ -65,8 +70,8 @@ class LibBuilderFactory(object): if isfile(join(path, "module.json")): return ["mbed"] - include_re = re.compile( - r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', flags=re.MULTILINE) + include_re = re.compile(r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', + flags=re.MULTILINE) # check source files for root, _, files in os.walk(path, followlinks=True): @@ -76,19 +81,18 @@ class LibBuilderFactory(object): if not env.IsFileWithExt( fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): continue - with open(join(root, fname)) as f: - content = f.read() - if "Arduino.h" in content and include_re.search(content): - return ["arduino"] - elif "mbed.h" in content and include_re.search(content): - return ["mbed"] + content = get_file_contents(join(root, fname)) + if not content: + continue + if "Arduino.h" in content and include_re.search(content): + return ["arduino"] + if "mbed.h" in content and include_re.search(content): + return ["mbed"] return [] class LibBuilderBase(object): - IS_WINDOWS = "windows" in util.get_systype() - LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"] LDF_MODE_DEFAULT = "chain" @@ -131,9 +135,11 @@ class LibBuilderBase(object): def __contains__(self, path): p1 = self.path p2 = path - if self.IS_WINDOWS: + if WINDOWS: p1 = p1.lower() p2 = p2.lower() + if p1 == p2: + return True return commonprefix((p1 + sep, p2)) == p1 + sep @property @@ -144,13 +150,6 @@ class LibBuilderBase(object): def version(self): return self._manifest.get("version") - @property - def vcs_info(self): - items = glob(join(self.path, ".*", PackageManager.SRC_MANIFEST_NAME)) - if not items: - return None - return util.load_json(items[0]) - @property def dependencies(self): return LibraryManager.normalize_dependencies( @@ -179,16 +178,15 @@ class LibBuilderBase(object): def get_include_dirs(self): items = [] include_dir = self.include_dir - if include_dir and include_dir not in items: + if include_dir: items.append(include_dir) items.append(self.src_dir) return items @property def build_dir(self): - return join("$BUILD_DIR", - "lib%s" % hashlib.sha1(self.path).hexdigest()[:3], - basename(self.path)) + lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] + return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property def build_flags(self): @@ -204,17 +202,18 @@ class LibBuilderBase(object): @property def lib_archive(self): - return self.env.get("LIB_ARCHIVE", "") != "false" + return self.env.GetProjectOption("lib_archive", True) @property def lib_ldf_mode(self): return self.validate_ldf_mode( - self.env.get("LIB_LDF_MODE", self.LDF_MODE_DEFAULT)) + self.env.GetProjectOption("lib_ldf_mode", self.LDF_MODE_DEFAULT)) @property def lib_compat_mode(self): return self.validate_compat_mode( - self.env.get("LIB_COMPAT_MODE", self.COMPAT_MODE_DEFAULT)) + self.env.GetProjectOption("lib_compat_mode", + self.COMPAT_MODE_DEFAULT)) @property def depbuilders(self): @@ -230,7 +229,7 @@ class LibBuilderBase(object): @staticmethod def validate_ldf_mode(mode): - if isinstance(mode, basestring): + if isinstance(mode, string_types): mode = mode.strip().lower() if mode in LibBuilderBase.LDF_MODES: return mode @@ -242,7 +241,7 @@ class LibBuilderBase(object): @staticmethod def validate_compat_mode(mode): - if isinstance(mode, basestring): + if isinstance(mode, string_types): mode = mode.strip().lower() if mode in LibBuilderBase.COMPAT_MODES: return mode @@ -266,51 +265,29 @@ class LibBuilderBase(object): self.env.ProcessFlags(self.build_flags) if self.extra_script: self.env.SConscriptChdir(1) - self.env.SConscript( - realpath(self.extra_script), - exports={ - "env": self.env, - "pio_lib_builder": self - }) + self.env.SConscript(realpath(self.extra_script), + exports={ + "env": self.env, + "pio_lib_builder": self + }) self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): if not self.dependencies: return for item in self.dependencies: - skip = False - for key in ("platforms", "frameworks"): - env_key = "PIO" + key.upper()[:-1] - if env_key not in self.env: - continue - if (key in item and - not util.items_in_list(self.env[env_key], item[key])): - if self.verbose: - sys.stderr.write("Skip %s incompatible dependency %s\n" - % (key[:-1], item)) - skip = True - if skip: - continue - found = False for lb in self.env.GetLibBuilders(): if item['name'] != lb.name: continue - elif "frameworks" in item and \ - not lb.is_frameworks_compatible(item["frameworks"]): - continue - elif "platforms" in item and \ - not lb.is_platforms_compatible(item["platforms"]): - continue found = True - self.depend_recursive(lb) + if lb not in self.depbuilders: + self.depend_recursive(lb) break - if not found: - sys.stderr.write( - "Error: Could not find `%s` dependency for `%s` " - "library\n" % (item['name'], self.name)) - self.env.Exit(1) + if not found and self.verbose: + sys.stderr.write("Warning: Ignored `%s` dependency for `%s` " + "library\n" % (item['name'], self.name)) def get_search_files(self): items = [ @@ -400,9 +377,9 @@ class LibBuilderBase(object): if self != lb: if _already_depends(lb): if self.verbose: - sys.stderr.write( - "Warning! Circular dependencies detected " - "between `%s` and `%s`\n" % (self.path, lb.path)) + sys.stderr.write("Warning! Circular dependencies detected " + "between `%s` and `%s`\n" % + (self.path, lb.path)) self._circular_deps.append(lb) elif lb not in self._depbuilders: self._depbuilders.append(lb) @@ -477,7 +454,8 @@ class ArduinoLibBuilder(LibBuilderBase): manifest = {} if not isfile(join(self.path, "library.properties")): return manifest - with open(join(self.path, "library.properties")) as fp: + manifest_path = join(self.path, "library.properties") + with codecs.open(manifest_path, encoding="utf-8") as fp: for line in fp.readlines(): if "=" not in line: continue @@ -528,22 +506,23 @@ class ArduinoLibBuilder(LibBuilderBase): def is_platforms_compatible(self, platforms): platforms_map = { - "avr": "atmelavr", - "sam": "atmelsam", - "samd": "atmelsam", - "esp8266": "espressif8266", - "esp32": "espressif32", - "arc32": "intel_arc32", - "stm32": "ststm32" + "avr": ["atmelavr"], + "sam": ["atmelsam"], + "samd": ["atmelsam"], + "esp8266": ["espressif8266"], + "esp32": ["espressif32"], + "arc32": ["intel_arc32"], + "stm32": ["ststm32"], + "nrf5": ["nordicnrf51", "nordicnrf52"] } items = [] for arch in self._manifest.get("architectures", "").split(","): - arch = arch.strip() + arch = arch.strip().lower() if arch == "*": items = "*" break if arch in platforms_map: - items.append(platforms_map[arch]) + items.extend(platforms_map[arch]) if not items: return LibBuilderBase.is_platforms_compatible(self, platforms) return util.items_in_list(platforms, items) @@ -643,8 +622,8 @@ class MbedLibBuilder(LibBuilderBase): for key, options in manifest.get("config", {}).items(): if "value" not in options: continue - macros[key] = dict( - name=options.get("macro_name"), value=options.get("value")) + macros[key] = dict(name=options.get("macro_name"), + value=options.get("value")) # overrode items per target for target, options in manifest.get("target_overrides", {}).items(): @@ -664,8 +643,10 @@ class MbedLibBuilder(LibBuilderBase): if "." not in macro['name']: macro['name'] = "%s.%s" % (manifest.get("name"), macro['name']) - macro['name'] = re.sub( - r"[^a-z\d]+", "_", macro['name'], flags=re.I).upper() + macro['name'] = re.sub(r"[^a-z\d]+", + "_", + macro['name'], + flags=re.I).upper() macro['name'] = "MBED_CONF_" + macro['name'] if isinstance(macro['value'], bool): macro['value'] = 1 if macro['value'] else 0 @@ -681,8 +662,8 @@ class MbedLibBuilder(LibBuilderBase): lines.append( "// PlatformIO Library Dependency Finder (LDF)") lines.extend([ - "#define %s %s" % (name, - value if value is not None else "") + "#define %s %s" % + (name, value if value is not None else "") for name, value in macros.items() ]) lines.append("") @@ -716,13 +697,27 @@ class PlatformIOLibBuilder(LibBuilderBase): def _is_arduino_manifest(self): return isfile(join(self.path, "library.properties")) + @property + def include_dir(self): + if "includeDir" in self._manifest.get("build", {}): + with util.cd(self.path): + return realpath(self._manifest.get("build").get("includeDir")) + return LibBuilderBase.include_dir.fget(self) + + @property + def src_dir(self): + if "srcDir" in self._manifest.get("build", {}): + with util.cd(self.path): + return realpath(self._manifest.get("build").get("srcDir")) + return LibBuilderBase.src_dir.fget(self) + @property def src_filter(self): if "srcFilter" in self._manifest.get("build", {}): return self._manifest.get("build").get("srcFilter") - elif self.env['SRC_FILTER']: + if self.env['SRC_FILTER']: return self.env['SRC_FILTER'] - elif self._is_arduino_manifest(): + if self._is_arduino_manifest(): return ArduinoLibBuilder.src_filter.fget(self) return LibBuilderBase.src_filter.fget(self) @@ -788,6 +783,7 @@ class PlatformIOLibBuilder(LibBuilderBase): for path in self.env.get("CPPPATH", []): if path not in self.envorigin.get("CPPPATH", []): include_dirs.append(self.env.subst(path)) + return include_dirs @@ -813,7 +809,9 @@ class ProjectAsLibBuilder(LibBuilderBase): project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR") if isdir(project_include_dir): include_dirs.append(project_include_dir) - include_dirs.extend(LibBuilderBase.get_include_dirs(self)) + for include_dir in LibBuilderBase.get_include_dirs(self): + if include_dir not in include_dirs: + include_dirs.append(include_dir) return include_dirs def get_search_files(self): @@ -841,43 +839,80 @@ class ProjectAsLibBuilder(LibBuilderBase): return (self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self)) + @property + def dependencies(self): + return self.env.GetProjectOption("lib_deps", []) + def process_extra_options(self): # skip for project, options are already processed pass - def process_dependencies(self): # pylint: disable=too-many-branches - uris = self.env.get("LIB_DEPS", []) - if not uris: - return - storage_dirs = [] - for lb in self.env.GetLibBuilders(): - if dirname(lb.path) not in storage_dirs: - storage_dirs.append(dirname(lb.path)) + def install_dependencies(self): + + def _is_builtin(uri): + for lb in self.env.GetLibBuilders(): + if lb.name == uri: + return True + return False + + not_found_uri = [] + for uri in self.dependencies: + # check if built-in library + if _is_builtin(uri): + continue - for uri in uris: found = False - for storage_dir in storage_dirs: + for storage_dir in self.env.GetLibSourceDirs(): + lm = LibraryManager(storage_dir) + if lm.get_package_dir(*lm.parse_pkg_uri(uri)): + found = True + break + if not found: + not_found_uri.append(uri) + + did_install = False + lm = LibraryManager( + self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) + for uri in not_found_uri: + try: + lm.install(uri) + did_install = True + except (exception.LibNotFound, exception.InternetIsOffline) as e: + click.secho("Warning! %s" % e, fg="yellow") + + # reset cache + if did_install: + DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) + + def process_dependencies(self): # pylint: disable=too-many-branches + for uri in self.dependencies: + found = False + for storage_dir in self.env.GetLibSourceDirs(): if found: break lm = LibraryManager(storage_dir) - pkg_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri)) - if not pkg_dir: + lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri)) + if not lib_dir: continue for lb in self.env.GetLibBuilders(): - if lb.path != pkg_dir: + if lib_dir not in lb: continue if lb not in self.depbuilders: self.depend_recursive(lb) found = True break + if found: + continue - if not found: - for lb in self.env.GetLibBuilders(): - if lb.name != uri: - continue - if lb not in self.depbuilders: - self.depend_recursive(lb) - break + # look for built-in libraries by a name + # which don't have package manifest + for lb in self.env.GetLibBuilders(): + if lb.name != uri: + continue + if lb not in self.depbuilders: + self.depend_recursive(lb) + found = True + break def build(self): self._is_built = True # do not build Project now @@ -886,61 +921,69 @@ class ProjectAsLibBuilder(LibBuilderBase): return result +def GetLibSourceDirs(env): + items = env.GetProjectOption("lib_extra_dirs", []) + items.extend(env['LIBSOURCE_DIRS']) + return [ + env.subst(expanduser(item) if item.startswith("~") else item) + for item in items + ] + + +def IsCompatibleLibBuilder(env, + lb, + verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): + compat_mode = lb.lib_compat_mode + if lb.name in env.GetProjectOption("lib_ignore", []): + if verbose: + sys.stderr.write("Ignored library %s\n" % lb.path) + return None + if compat_mode == "strict" and not lb.is_platforms_compatible( + env['PIOPLATFORM']): + if verbose: + sys.stderr.write("Platform incompatible library %s\n" % lb.path) + return False + if (compat_mode in ("soft", "strict") and "PIOFRAMEWORK" in env + and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", []))): + if verbose: + sys.stderr.write("Framework incompatible library %s\n" % lb.path) + return False + return True + + def GetLibBuilders(env): # pylint: disable=too-many-branches + if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None: + return sorted(DefaultEnvironment()['__PIO_LIB_BUILDERS'], + key=lambda lb: 0 if lb.dependent else 1) - if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): - return sorted( - DefaultEnvironment()['__PIO_LIB_BUILDERS'], - key=lambda lb: 0 if lb.dependent else 1) - - items = [] - verbose = int(ARGUMENTS.get("PIOVERBOSE", - 0)) and not env.GetOption('clean') - - def _check_lib_builder(lb): - compat_mode = lb.lib_compat_mode - if lb.name in env.get("LIB_IGNORE", []): - if verbose: - sys.stderr.write("Ignored library %s\n" % lb.path) - return None - if compat_mode == "strict" and not lb.is_platforms_compatible( - env['PIOPLATFORM']): - if verbose: - sys.stderr.write( - "Platform incompatible library %s\n" % lb.path) - return False - if compat_mode == "soft" and "PIOFRAMEWORK" in env and \ - not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])): - if verbose: - sys.stderr.write( - "Framework incompatible library %s\n" % lb.path) - return False - return True + DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[]) + verbose = int(ARGUMENTS.get("PIOVERBOSE", 0)) found_incompat = False - for libs_dir in env['LIBSOURCE_DIRS']: - libs_dir = env.subst(libs_dir) - if not isdir(libs_dir): + + for storage_dir in env.GetLibSourceDirs(): + storage_dir = realpath(storage_dir) + if not isdir(storage_dir): continue - for item in sorted(os.listdir(libs_dir)): - if item == "__cores__" or not isdir(join(libs_dir, item)): + for item in sorted(os.listdir(storage_dir)): + lib_dir = join(storage_dir, item) + if item == "__cores__" or not isdir(lib_dir): continue try: - lb = LibBuilderFactory.new( - env, join(libs_dir, item), verbose=verbose) + lb = LibBuilderFactory.new(env, lib_dir) except exception.InvalidJSONFile: if verbose: - sys.stderr.write("Skip library with broken manifest: %s\n" - % join(libs_dir, item)) + sys.stderr.write( + "Skip library with broken manifest: %s\n" % lib_dir) continue - if _check_lib_builder(lb): - items.append(lb) + if env.IsCompatibleLibBuilder(lb): + DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb]) else: found_incompat = True for lb in env.get("EXTRA_LIB_BUILDERS", []): - if _check_lib_builder(lb): - items.append(lb) + if env.IsCompatibleLibBuilder(lb): + DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb]) else: found_incompat = True @@ -950,13 +993,16 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches "https://docs.platformio.org/page/librarymanager/ldf.html#" "ldf-compat-mode\n") - DefaultEnvironment()['__PIO_LIB_BUILDERS'] = items - return items + return DefaultEnvironment()['__PIO_LIB_BUILDERS'] def ConfigureProjectLibBuilder(env): - def correct_found_libs(lib_builders): + def _get_vcs_info(lb): + path = LibraryManager.get_src_manifest_path(lb.path) + return util.load_json(path) if path else None + + def _correct_found_libs(lib_builders): # build full dependency graph found_lbs = [lb for lb in lib_builders if lb.dependent] for lb in lib_builders: @@ -967,11 +1013,11 @@ def ConfigureProjectLibBuilder(env): if deplb not in found_lbs: lb.depbuilders.remove(deplb) - def print_deps_tree(root, level=0): + def _print_deps_tree(root, level=0): margin = "| " * (level) for lb in root.depbuilders: title = "<%s>" % lb.name - vcs_info = lb.vcs_info + vcs_info = _get_vcs_info(lb) if lb.version: title += " %s" % lb.version if vcs_info and vcs_info.get("version"): @@ -985,27 +1031,29 @@ def ConfigureProjectLibBuilder(env): sys.stdout.write(")") sys.stdout.write("\n") if lb.depbuilders: - print_deps_tree(lb, level + 1) + _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) - print("Library Dependency Finder -> http://bit.ly/configure-pio-ldf") - print("LDF MODES: FINDER(%s) COMPATIBILITY(%s)" % + print("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") + print("LDF Modes: Finder ~ %s, Compatibility ~ %s" % (ldf_mode, project.lib_compat_mode)) + project.install_dependencies() + lib_builders = env.GetLibBuilders() - print("Collected %d compatible libraries" % len(lib_builders)) + print("Found %d compatible libraries" % len(lib_builders)) print("Scanning dependencies...") project.search_deps_recursive() if ldf_mode.startswith("chain") and project.depbuilders: - correct_found_libs(lib_builders) + _correct_found_libs(lib_builders) if project.depbuilders: print("Dependency Graph") - print_deps_tree(project) + _print_deps_tree(project) else: print("No dependencies") @@ -1017,6 +1065,8 @@ def exists(_): def generate(env): + env.AddMethod(GetLibSourceDirs) + env.AddMethod(IsCompatibleLibBuilder) env.AddMethod(GetLibBuilders) env.AddMethod(ConfigureProjectLibBuilder) return env diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 52df8b61..1e01d59e 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -21,11 +21,13 @@ from os import environ, remove, walk from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp -from SCons.Action import Action -from SCons.Script import ARGUMENTS +from SCons.Action import Action # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import util +from platformio.compat import get_file_contents, glob_escape from platformio.managers.core import get_core_package_dir +from platformio.proc import exec_command class InoToCPPConverter(object): @@ -58,7 +60,7 @@ class InoToCPPConverter(object): assert nodes lines = [] for node in nodes: - contents = node.get_text_contents() + contents = get_file_contents(node.get_path()) _lines = [ '# 1 "%s"' % node.get_path().replace("\\", "/"), contents ] @@ -76,8 +78,7 @@ class InoToCPPConverter(object): def process(self, contents): out_file = self._main_ino + ".cpp" assert self._gcc_preprocess(contents, out_file) - with open(out_file) as fp: - contents = fp.read() + contents = get_file_contents(out_file) contents = self._join_multiline_strings(contents) with open(out_file, "w") as fp: fp.write(self.append_prototypes(contents)) @@ -158,9 +159,7 @@ class InoToCPPConverter(object): return total def append_prototypes(self, contents): - prototypes = self._parse_prototypes(contents) - if not prototypes: - return contents + prototypes = self._parse_prototypes(contents) or [] # skip already declared prototypes declared = set( @@ -169,6 +168,9 @@ class InoToCPPConverter(object): m for m in prototypes if m.group(1).strip() not in declared ] + if not prototypes: + return contents + prototype_names = set(m.group(3).strip() for m in prototypes) split_pos = prototypes[0].start() match_ptrs = re.search( @@ -187,9 +189,9 @@ class InoToCPPConverter(object): def ConvertInoToCpp(env): - src_dir = util.glob_escape(env.subst("$PROJECTSRC_DIR")) - ino_nodes = ( - env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde"))) + src_dir = glob_escape(env.subst("$PROJECTSRC_DIR")) + ino_nodes = (env.Glob(join(src_dir, "*.ino")) + + env.Glob(join(src_dir, "*.pde"))) if not ino_nodes: return c = InoToCPPConverter(env) @@ -208,10 +210,12 @@ def _delete_file(path): @util.memoized() def _get_compiler_type(env): + if env.subst("$CC").endswith("-gcc"): + return "gcc" try: sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command([env.subst("$CC"), "-v"], env=sysenv) + result = exec_command([env.subst("$CC"), "-v"], env=sysenv) except OSError: return None if result['returncode'] != 0: @@ -219,7 +223,7 @@ def _get_compiler_type(env): output = "".join([result['out'], result['err']]).lower() if "clang" in output and "LLVM" in output: return "clang" - elif "gcc" in output: + if "gcc" in output: return "gcc" return None @@ -283,10 +287,13 @@ def PioClean(env, clean_dir): if not isdir(clean_dir): print("Build environment is clean") env.Exit(0) + clean_rel_path = relpath(clean_dir) for root, _, files in walk(clean_dir): - for file_ in files: - remove(join(root, file_)) - print("Removed %s" % relpath(join(root, file_))) + for f in files: + dst = join(root, f) + remove(dst) + print("Removed %s" % + (dst if clean_rel_path.startswith(".") else relpath(dst))) print("Done cleaning") util.rmtree_(clean_dir) env.Exit(0) @@ -295,8 +302,8 @@ def PioClean(env, clean_dir): def ProcessDebug(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) - env.Append( - BUILD_FLAGS=list(env['PIODEBUGFLAGS']) + ["-D__PLATFORMIO_DEBUG__"]) + env.Append(BUILD_FLAGS=list(env['PIODEBUGFLAGS']) + + ["-D__PLATFORMIO_BUILD_DEBUG__"]) unflags = ["-Os"] for level in [0, 1, 2]: for flag in ("O", "g", "ggdb"): @@ -305,22 +312,21 @@ def ProcessDebug(env): def ProcessTest(env): - env.Append( - CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], - CPPPATH=[join("$BUILD_DIR", "UnityTestLib")]) - unitylib = env.BuildLibrary( - join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity")) + env.Append(CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], + CPPPATH=[join("$BUILD_DIR", "UnityTestLib")]) + unitylib = env.BuildLibrary(join("$BUILD_DIR", "UnityTestLib"), + get_core_package_dir("tool-unity")) env.Prepend(LIBS=[unitylib]) src_filter = ["+<*.cpp>", "+<*.c>"] - if "PIOTEST" in env: - src_filter.append("+<%s%s>" % (env['PIOTEST'], sep)) + if "PIOTEST_RUNNING_NAME" in env: + src_filter.append("+<%s%s>" % (env['PIOTEST_RUNNING_NAME'], sep)) env.Replace(PIOTEST_SRC_FILTER=src_filter) def GetExtraScripts(env, scope): items = [] - for item in env.get("EXTRA_SCRIPTS", []): + for item in env.GetProjectOption("extra_scripts", []): if scope == "post" and ":" not in item: items.append(item) elif item.startswith("%s:" % scope): diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 580f04a2..09d71287 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -14,35 +14,33 @@ from __future__ import absolute_import -import base64 import sys from os.path import isdir, isfile, join -from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import exception, util +from platformio.compat import WINDOWS from platformio.managers.platform import PlatformFactory +from platformio.project.config import ProjectOptions # pylint: disable=too-many-branches, too-many-locals @util.memoized() -def initPioPlatform(name): - return PlatformFactory.newPlatform(name) - - def PioPlatform(env): - variables = {} - for name in env['PIOVARIABLES']: - if name in env: - variables[name.lower()] = env[name] - p = initPioPlatform(env['PLATFORM_MANIFEST']) + variables = env.GetProjectOptions(as_dict=True) + if "framework" in variables: + # support PIO Core 3.0 dev/platforms + variables['pioframework'] = variables['framework'] + p = PlatformFactory.newPlatform(env['PLATFORM_MANIFEST']) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p def BoardConfig(env, board=None): - p = initPioPlatform(env['PLATFORM_MANIFEST']) + p = env.PioPlatform() try: board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" @@ -62,7 +60,7 @@ def GetFrameworkScript(env, framework): return script_path -def LoadPioPlatform(env, variables): +def LoadPioPlatform(env): p = env.PioPlatform() installed_packages = p.get_installed_packages() @@ -79,7 +77,7 @@ def LoadPioPlatform(env, variables): env.PrependENVPath( "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir) - if ("windows" not in systype and isdir(join(pkg_dir, "lib")) + if (not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain"): env.PrependENVPath( "DYLD_LIBRARY_PATH" @@ -91,92 +89,122 @@ def LoadPioPlatform(env, variables): env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: - # handle _MCU and _F_CPU variables for AVR native - for key, value in variables.UnknownVariables().items(): - if not key.startswith("BOARD_"): - continue - env.Replace(**{ - key.upper().replace("BUILD.", ""): - base64.b64decode(value) - }) return - # update board manifest with a custom data + # update board manifest with overridden data from INI config board_config = env.BoardConfig() - for key, value in variables.UnknownVariables().items(): - if not key.startswith("BOARD_"): - continue - board_config.update(key.lower()[6:], base64.b64decode(value)) + for option, value in env.GetProjectOptions(): + if option.startswith("board_"): + board_config.update(option.lower()[6:], value) - # update default environment variables - for key in variables.keys(): - if key in env or \ - not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]): + # load default variables from board config + for option_meta in ProjectOptions.values(): + if not option_meta.buildenvvar or option_meta.buildenvvar in env: continue - _opt, _val = key.lower().split("_", 1) - if _opt == "board": - _opt = "build" - if _val in board_config.get(_opt): - env.Replace(**{key: board_config.get("%s.%s" % (_opt, _val))}) + data_path = (option_meta.name[6:] + if option_meta.name.startswith("board_") else + option_meta.name.replace("_", ".")) + try: + env[option_meta.buildenvvar] = board_config.get(data_path) + except KeyError: + pass if "build.ldscript" in board_config: env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript")) -def PrintConfiguration(env): +def PrintConfiguration(env): # pylint: disable=too-many-statements platform = env.PioPlatform() - platform_data = ["PLATFORM: %s >" % platform.title] - hardware_data = ["HARDWARE:"] - configuration_data = ["CONFIGURATION:"] - mcu = env.subst("$BOARD_MCU") - f_cpu = env.subst("$BOARD_F_CPU") - if mcu: - hardware_data.append(mcu.upper()) - if f_cpu: - f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()])) - hardware_data.append("%dMHz" % (f_cpu / 1000000)) + board_config = env.BoardConfig() if "BOARD" in env else None - debug_tools = None - if "BOARD" in env: - board_config = env.BoardConfig() - platform_data.append(board_config.get("name")) + def _get_configuration_data(): + return None if not board_config else [ + "CONFIGURATION:", + "https://docs.platformio.org/page/boards/%s/%s.html" % + (platform.name, board_config.id) + ] - debug_tools = board_config.get("debug", {}).get("tools") + def _get_plaform_data(): + data = ["PLATFORM: %s %s" % (platform.title, platform.version)] + src_manifest_path = platform.pm.get_src_manifest_path( + platform.get_dir()) + if src_manifest_path: + src_manifest = util.load_json(src_manifest_path) + if "version" in src_manifest: + data.append("#" + src_manifest['version']) + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + data.append("(%s)" % src_manifest['url']) + if board_config: + data.extend([">", board_config.get("name")]) + return data + + def _get_hardware_data(): + data = ["HARDWARE:"] + mcu = env.subst("$BOARD_MCU") + f_cpu = env.subst("$BOARD_F_CPU") + if mcu: + data.append(mcu.upper()) + if f_cpu: + f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()])) + data.append("%dMHz," % (f_cpu / 1000000)) + if not board_config: + return data ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") - hardware_data.append( - "%s RAM (%s Flash)" % (util.format_filesize(ram), - util.format_filesize(flash))) - configuration_data.append( - "https://docs.platformio.org/page/boards/%s/%s.html" % - (platform.name, board_config.id)) + data.append("%s RAM, %s Flash" % + (util.format_filesize(ram), util.format_filesize(flash))) + return data - for data in (configuration_data, platform_data, hardware_data): - if len(data) > 1: + def _get_debug_data(): + debug_tools = board_config.get( + "debug", {}).get("tools") if board_config else None + if not debug_tools: + return None + data = [ + "DEBUG:", "Current", + "(%s)" % board_config.get_debug_tool_name( + env.GetProjectOption("debug_tool")) + ] + onboard = [] + external = [] + for key, value in debug_tools.items(): + if value.get("onboard"): + onboard.append(key) + else: + external.append(key) + if onboard: + data.extend(["On-board", "(%s)" % ", ".join(sorted(onboard))]) + if external: + data.extend(["External", "(%s)" % ", ".join(sorted(external))]) + return data + + def _get_packages_data(): + data = [] + for name, options in platform.packages.items(): + if options.get("optional"): + continue + pkg_dir = platform.get_package_dir(name) + if not pkg_dir: + continue + manifest = platform.pm.load_manifest(pkg_dir) + original_version = util.get_original_version(manifest['version']) + info = "%s %s" % (manifest['name'], manifest['version']) + extra = [] + if original_version: + extra.append(original_version) + if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)): + extra.append(manifest['__src_url']) + if extra: + info += " (%s)" % ", ".join(extra) + data.append(info) + return ["PACKAGES:", ", ".join(data)] + + for data in (_get_configuration_data(), _get_plaform_data(), + _get_hardware_data(), _get_debug_data(), + _get_packages_data()): + if data and len(data) > 1: print(" ".join(data)) - # Debugging - if not debug_tools: - return - - data = [ - "CURRENT(%s)" % board_config.get_debug_tool_name( - env.subst("$DEBUG_TOOL")) - ] - onboard = [] - external = [] - for key, value in debug_tools.items(): - if value.get("onboard"): - onboard.append(key) - else: - external.append(key) - if onboard: - data.append("ON-BOARD(%s)" % ", ".join(sorted(onboard))) - if external: - data.append("EXTERNAL(%s)" % ", ".join(sorted(external))) - - print("DEBUG: %s" % " ".join(data)) - def exists(_): return True diff --git a/platformio/builder/tools/pioproject.py b/platformio/builder/tools/pioproject.py new file mode 100644 index 00000000..5797755d --- /dev/null +++ b/platformio/builder/tools/pioproject.py @@ -0,0 +1,49 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +from platformio.project.config import ProjectConfig, ProjectOptions + + +def GetProjectConfig(env): + return ProjectConfig.get_instance(env['PROJECT_CONFIG']) + + +def GetProjectOptions(env, as_dict=False): + return env.GetProjectConfig().items(env=env['PIOENV'], as_dict=as_dict) + + +def GetProjectOption(env, option, default=None): + return env.GetProjectConfig().get("env:" + env['PIOENV'], option, default) + + +def LoadProjectOptions(env): + for option, value in env.GetProjectOptions(): + option_meta = ProjectOptions.get("env." + option) + if not option_meta or not option_meta.buildenvvar: + continue + env[option_meta.buildenvvar] = value + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(GetProjectConfig) + env.AddMethod(GetProjectOptions) + env.AddMethod(GetProjectOption) + env.AddMethod(LoadProjectOptions) + return env diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index f9628085..dd03894a 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -22,10 +22,12 @@ from os.path import isfile, join from shutil import copyfile from time import sleep -from SCons.Script import ARGUMENTS +from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException from platformio import exception, util +from platformio.compat import WINDOWS +from platformio.proc import exec_command # pylint: disable=unused-argument @@ -134,8 +136,7 @@ def AutodetectUploadPort(*args, **kwargs): continue port = item['port'] if upload_protocol.startswith("blackmagic"): - if "windows" in util.get_systype() and \ - port.startswith("COM") and len(port) > 4: + if WINDOWS and port.startswith("COM") and len(port) > 4: port = "\\\\.\\%s" % port if "GDB" in item['description']: return port @@ -197,10 +198,9 @@ def CheckUploadSize(_, target, source, env): return def _configure_defaults(): - env.Replace( - SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", - SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", - SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+") + env.Replace(SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", + SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", + SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+") def _get_size_output(): cmd = env.get("SIZECHECKCMD") @@ -211,7 +211,7 @@ def CheckUploadSize(_, target, source, env): cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg] sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command(env.subst(cmd), env=sysenv) + result = exec_command(env.subst(cmd), env=sysenv) if result['returncode'] != 0: return None return result['out'].strip() @@ -250,8 +250,8 @@ def CheckUploadSize(_, target, source, env): if data_max_size and data_size > -1: print("DATA: %s" % _format_availale_bytes(data_size, data_max_size)) if program_size > -1: - print("PROGRAM: %s" % _format_availale_bytes(program_size, - program_max_size)) + print("PROGRAM: %s" % + _format_availale_bytes(program_size, program_max_size)) if int(ARGUMENTS.get("PIOVERBOSE", 0)): print(output) @@ -272,8 +272,8 @@ def PrintUploadInfo(env): configured = env.subst("$UPLOAD_PROTOCOL") available = [configured] if configured else [] if "BOARD" in env: - available.extend(env.BoardConfig().get("upload", {}).get( - "protocols", [])) + available.extend(env.BoardConfig().get("upload", + {}).get("protocols", [])) if available: print("AVAILABLE: %s" % ", ".join(sorted(set(available)))) if configured: diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index 7db4f943..3679897d 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platform import system + +from platformio.compat import WINDOWS, hashlib_encode_data # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options @@ -58,7 +61,8 @@ def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) - tmp_file = join(build_dir, "longcmd-%s" % md5(data).hexdigest()) + tmp_file = join(build_dir, + "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: @@ -71,7 +75,7 @@ def exists(_): def generate(env): - if system() != "Windows": + if not WINDOWS: return None env.Replace(_long_sources_hook=long_sources_hook) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index b93499ae..c584d654 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -20,11 +20,15 @@ from glob import glob from os import sep, walk from os.path import basename, dirname, isdir, join, realpath -from SCons import Builder, Util -from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, - DefaultEnvironment, Export, SConscript) +from SCons import Builder, Util # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error +from SCons.Script import Export # pylint: disable=import-error +from SCons.Script import SConscript # pylint: disable=import-error -from platformio.util import glob_escape, pioversion_to_intstr +from platformio.compat import glob_escape, string_types +from platformio.util import pioversion_to_intstr SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] @@ -65,7 +69,7 @@ def _build_project_deps(env): if is_test: projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER") - if not is_test or env.get("TEST_BUILD_PROJECT_SRC") == "true": + if not is_test or env.GetProjectOption("test_build_project_src", False): projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER")) @@ -93,7 +97,8 @@ def BuildProgram(env): if not Util.case_sensitive_suffixes(".s", ".S"): env.Replace(AS="$CC", ASCOM="$ASPPCOM") - if set(["__debug", "debug"]) & set(COMMAND_LINE_TARGETS): + if ("debug" in COMMAND_LINE_TARGETS + or env.GetProjectOption("build_type") == "debug"): env.ProcessDebug() # process extra flags from board @@ -128,8 +133,8 @@ def BuildProgram(env): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") - program = env.Program( - join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES']) + program = env.Program(join("$BUILD_DIR", env.subst("$PROGNAME")), + env['PIOBUILDFILES']) env.Replace(PIOMAINPROG=program) AlwaysBuild( @@ -189,11 +194,13 @@ def ProcessFlags(env, flags): # pylint: disable=too-many-branches # provided with a -U option // Issue #191 undefines = [ u for u in env.get("CCFLAGS", []) - if isinstance(u, basestring) and u.startswith("-U") + if isinstance(u, string_types) and u.startswith("-U") ] if undefines: for undef in undefines: env['CCFLAGS'].remove(undef) + if undef[2:] in env['CPPDEFINES']: + env['CPPDEFINES'].remove(undef[2:]) env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines)) diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index b0514903..7dc69d58 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -11,3 +11,59 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import os +from os.path import dirname + +import click + + +class PlatformioCLI(click.MultiCommand): + + leftover_args = [] + + @staticmethod + def in_silence(): + args = PlatformioCLI.leftover_args + return args and any([ + args[0] == "debug" and "--interpreter" in " ".join(args), + args[0] == "upgrade", "--json-output" in args, "--version" in args + ]) + + def invoke(self, ctx): + PlatformioCLI.leftover_args = ctx.args + if hasattr(ctx, "protected_args"): + PlatformioCLI.leftover_args = ctx.protected_args + ctx.args + return super(PlatformioCLI, self).invoke(ctx) + + def list_commands(self, ctx): + cmds = [] + for filename in os.listdir(dirname(__file__)): + if filename.startswith("__init__"): + continue + if filename.endswith(".py"): + cmds.append(filename[:-3]) + cmds.sort() + return cmds + + def get_command(self, ctx, cmd_name): + mod = None + try: + mod = __import__("platformio.commands." + cmd_name, None, None, + ["cli"]) + except ImportError: + try: + return self._handle_obsolate_command(cmd_name) + except AttributeError: + raise click.UsageError('No such command "%s"' % cmd_name, ctx) + return mod.cli + + @staticmethod + def _handle_obsolate_command(name): + if name == "platforms": + from platformio.commands import platform + return platform.cli + if name == "serialports": + from platformio.commands import device + return device.cli + raise AttributeError() diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 5764b9f8..6aff1681 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -17,6 +17,7 @@ import json import click from platformio import util +from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformManager @@ -51,24 +52,22 @@ def print_boards(boards): BOARDLIST_TPL = ("{type:<30} {mcu:<14} {frequency:<8} " " {flash:<7} {ram:<6} {name}") click.echo( - BOARDLIST_TPL.format( - type=click.style("ID", fg="cyan"), - mcu="MCU", - frequency="Frequency", - flash="Flash", - ram="RAM", - name="Name")) + BOARDLIST_TPL.format(type=click.style("ID", fg="cyan"), + mcu="MCU", + frequency="Frequency", + flash="Flash", + ram="RAM", + name="Name")) click.echo("-" * terminal_width) for board in boards: click.echo( - BOARDLIST_TPL.format( - type=click.style(board['id'], fg="cyan"), - mcu=board['mcu'], - frequency="%dMHz" % (board['fcpu'] / 1000000), - flash=util.format_filesize(board['rom']), - ram=util.format_filesize(board['ram']), - name=board['name'])) + BOARDLIST_TPL.format(type=click.style(board['id'], fg="cyan"), + mcu=board['mcu'], + frequency="%dMHz" % (board['fcpu'] / 1000000), + flash=util.format_filesize(board['rom']), + ram=util.format_filesize(board['ram']), + name=board['name'])) def _get_boards(installed=False): @@ -84,4 +83,4 @@ def _print_boards_json(query, installed=False): if query.lower() not in search_data.lower(): continue result.append(board) - click.echo(json.dumps(result)) + click.echo(dump_json_to_unicode(result)) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 8fb1bf89..55ef07ad 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -24,7 +24,9 @@ from platformio import app, util from platformio.commands.init import cli as cmd_init from platformio.commands.init import validate_boards from platformio.commands.run import cli as cmd_run +from platformio.compat import glob_escape from platformio.exception import CIBuildEnvsEmpty +from platformio.project.config import ProjectConfig def validate_path(ctx, param, value): # pylint: disable=unused-argument @@ -46,29 +48,31 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.command("ci", short_help="Continuous Integration") @click.argument("src", nargs=-1, callback=validate_path) -@click.option( - "-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY") +@click.option("-l", + "--lib", + multiple=True, + callback=validate_path, + metavar="DIRECTORY") @click.option("--exclude", multiple=True) -@click.option( - "-b", "--board", multiple=True, metavar="ID", callback=validate_boards) -@click.option( - "--build-dir", - default=mkdtemp, - type=click.Path( - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option("-b", + "--board", + multiple=True, + metavar="ID", + callback=validate_boards) +@click.option("--build-dir", + default=mkdtemp, + type=click.Path(file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) @click.option("--keep-build-dir", is_flag=True) -@click.option( - "-C", - "--project-conf", - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("-O", "--project-option", multiple=True) @click.option("-v", "--verbose", is_flag=True) @click.pass_context @@ -106,11 +110,10 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches _exclude_contents(build_dir, exclude) # initialise project - ctx.invoke( - cmd_init, - project_dir=build_dir, - board=board, - project_option=project_option) + ctx.invoke(cmd_init, + project_dir=build_dir, + board=board, + project_option=project_option) # process project ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose) @@ -154,7 +157,7 @@ def _copy_contents(dst_dir, contents): def _exclude_contents(dst_dir, patterns): contents = [] for p in patterns: - contents += glob(join(util.glob_escape(dst_dir), p)) + contents += glob(join(glob_escape(dst_dir), p)) for path in contents: path = abspath(path) if isdir(path): @@ -164,8 +167,7 @@ def _exclude_contents(dst_dir, patterns): def _copy_project_conf(build_dir, project_conf): - config = util.load_project_config(project_conf) + config = ProjectConfig(project_conf, parse_extra=False) if config.has_section("platformio"): config.remove_section("platformio") - with open(join(build_dir, "platformio.ini"), "w") as fp: - config.write(fp) + config.save(join(build_dir, "platformio.ini")) diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py deleted file mode 100644 index e43aeed1..00000000 --- a/platformio/commands/debug.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from os import getcwd - -import click - -from platformio.managers.core import pioplus_call - - -@click.command( - "debug", - context_settings=dict(ignore_unknown_options=True), - short_help="PIO Unified Debugger") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("--environment", "-e", metavar="") -@click.option("--verbose", "-v", is_flag=True) -@click.option("--interface", type=click.Choice(["gdb"])) -@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) diff --git a/platformio/commands/home.py b/platformio/commands/debug/__init__.py similarity index 54% rename from platformio/commands/home.py rename to platformio/commands/debug/__init__.py index cd6b86f6..7fba44c2 100644 --- a/platformio/commands/home.py +++ b/platformio/commands/debug/__init__.py @@ -12,20 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -import click - -from platformio.managers.core import pioplus_call - - -@click.command("home", short_help="PIO Home") -@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") -@click.option( - "--host", - default="127.0.0.1", - help="HTTP host, default=127.0.0.1. " - "You can open PIO Home for inbound connections with --host=0.0.0.0") -@click.option("--no-open", is_flag=True) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) +from platformio.commands.debug.command import cli diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py new file mode 100644 index 00000000..f6d403bb --- /dev/null +++ b/platformio/commands/debug/client.py @@ -0,0 +1,290 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import signal +import time +from hashlib import sha1 +from os.path import abspath, basename, dirname, isdir, join, splitext +from tempfile import mkdtemp + +from twisted.internet import protocol # pylint: disable=import-error +from twisted.internet import reactor # pylint: disable=import-error +from twisted.internet import stdio # pylint: disable=import-error +from twisted.internet import task # pylint: disable=import-error + +from platformio import app, exception, util +from platformio.commands.debug import helpers, initcfgs +from platformio.commands.debug.process import BaseProcess +from platformio.commands.debug.server import DebugServer +from platformio.compat import hashlib_encode_data +from platformio.project.helpers import get_project_cache_dir +from platformio.telemetry import MeasurementProtocol + +LOG_FILE = None + + +class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes + + PIO_SRC_NAME = ".pioinit" + INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" + + def __init__(self, project_dir, args, debug_options, env_options): + self.project_dir = project_dir + self.args = list(args) + self.debug_options = debug_options + self.env_options = env_options + + self._debug_server = DebugServer(debug_options, env_options) + self._session_id = None + + if not isdir(get_project_cache_dir()): + os.makedirs(get_project_cache_dir()) + self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), + prefix=".piodebug-") + + self._target_is_run = False + self._last_server_activity = 0 + self._auto_continue_timer = None + + def spawn(self, gdb_path, prog_path): + session_hash = gdb_path + prog_path + self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest() + self._kill_previous_session() + + patterns = { + "PROJECT_DIR": helpers.escape_path(self.project_dir), + "PROG_PATH": helpers.escape_path(prog_path), + "PROG_DIR": helpers.escape_path(dirname(prog_path)), + "PROG_NAME": basename(splitext(prog_path)[0]), + "DEBUG_PORT": self.debug_options['port'], + "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'], + "INIT_BREAK": self.debug_options['init_break'] or "", + "LOAD_CMDS": "\n".join(self.debug_options['load_cmds'] or []), + } + + self._debug_server.spawn(patterns) + + if not patterns['DEBUG_PORT']: + patterns['DEBUG_PORT'] = self._debug_server.get_debug_port() + self.generate_pioinit(self._gdbsrc_dir, patterns) + + # start GDB client + args = [ + "piogdb", + "-q", + "--directory", self._gdbsrc_dir, + "--directory", self.project_dir, + "-l", "10" + ] # yapf: disable + args.extend(self.args) + if not gdb_path: + raise exception.DebugInvalidOptions("GDB client is not configured") + gdb_data_dir = self._get_data_dir(gdb_path) + if gdb_data_dir: + args.extend(["--data-directory", gdb_data_dir]) + args.append(patterns['PROG_PATH']) + + return reactor.spawnProcess(self, + gdb_path, + args, + path=self.project_dir, + env=os.environ) + + @staticmethod + def _get_data_dir(gdb_path): + if "msp430" in gdb_path: + return None + gdb_data_dir = abspath(join(dirname(gdb_path), "..", "share", "gdb")) + return gdb_data_dir if isdir(gdb_data_dir) else None + + def generate_pioinit(self, dst_dir, patterns): + server_exe = (self.debug_options.get("server") + or {}).get("executable", "").lower() + if "jlink" in server_exe: + cfg = initcfgs.GDB_JLINK_INIT_CONFIG + elif "st-util" in server_exe: + cfg = initcfgs.GDB_STUTIL_INIT_CONFIG + elif "mspdebug" in server_exe: + cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG + elif "qemu" in server_exe: + cfg = initcfgs.GDB_QEMU_INIT_CONFIG + elif self.debug_options['require_debug_port']: + cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG + else: + cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG + commands = cfg.split("\n") + + if self.debug_options['init_cmds']: + commands = self.debug_options['init_cmds'] + commands.extend(self.debug_options['extra_cmds']) + + if not any("define pio_reset_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_target", + " echo Warning! Undefined pio_reset_target command\\n", + " mon reset", + "end" + ] + commands # yapf: disable + if not any("define pio_reset_halt_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_halt_target", + " echo Warning! Undefined pio_reset_halt_target command\\n", + " mon reset halt", + "end" + ] + commands # yapf: disable + if not any("define pio_restart_target" in cmd for cmd in commands): + commands += [ + "define pio_restart_target", + " pio_reset_halt_target", + " $INIT_BREAK", + " %s" % ("continue" if patterns['INIT_BREAK'] else "next"), + "end" + ] # yapf: disable + + banner = [ + "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", + "echo PlatformIO: Initializing remote target...\\n" + ] + footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] + commands = banner + commands + footer + + with open(join(dst_dir, self.PIO_SRC_NAME), "w") as fp: + fp.write("\n".join(self.apply_patterns(commands, patterns))) + + def connectionMade(self): + self._lock_session(self.transport.pid) + + p = protocol.Protocol() + p.dataReceived = self.onStdInData + stdio.StandardIO(p) + + def onStdInData(self, data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + + self._last_server_activity = time.time() + + if b"-exec-run" in data: + if self._target_is_run: + token, _ = data.split(b"-", 1) + self.outReceived(token + b"^running\n") + return + data = data.replace(b"-exec-run", b"-exec-continue") + + if b"-exec-continue" in data: + self._target_is_run = True + if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): + # Allow terminating via SIGINT/CTRL+C + signal.signal(signal.SIGINT, signal.default_int_handler) + self.transport.write(b"pio_reset_target\n") + self.transport.write(data) + + def processEnded(self, reason): # pylint: disable=unused-argument + self._unlock_session() + if self._gdbsrc_dir and isdir(self._gdbsrc_dir): + util.rmtree_(self._gdbsrc_dir) + if self._debug_server: + self._debug_server.terminate() + + reactor.stop() + + def outReceived(self, data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + + self._last_server_activity = time.time() + super(GDBClient, self).outReceived(data) + self._handle_error(data) + # go to init break automatically + if self.INIT_COMPLETED_BANNER.encode() in data: + self._auto_continue_timer = task.LoopingCall( + self._auto_exec_continue) + self._auto_continue_timer.start(0.1) + + def errReceived(self, data): + super(GDBClient, self).errReceived(data) + self._handle_error(data) + + def console_log(self, msg): + if helpers.is_mi_mode(self.args): + self.outReceived(('~"%s\\n"\n' % msg).encode()) + else: + self.outReceived(("%s\n" % msg).encode()) + + def _auto_exec_continue(self): + auto_exec_delay = 0.5 # in seconds + if self._last_server_activity > (time.time() - auto_exec_delay): + return + if self._auto_continue_timer: + self._auto_continue_timer.stop() + self._auto_continue_timer = None + + if not self.debug_options['init_break'] or self._target_is_run: + return + self.console_log( + "PlatformIO: Resume the execution to `debug_init_break = %s`" % + self.debug_options['init_break']) + self.console_log("PlatformIO: More configuration options -> " + "http://bit.ly/pio-debug") + self.transport.write(b"0-exec-continue\n" if helpers. + is_mi_mode(self.args) else b"continue\n") + self._target_is_run = True + + def _handle_error(self, data): + if (self.PIO_SRC_NAME.encode() not in data + or b"Error in sourced" not in data): + return + configuration = {"debug": self.debug_options, "env": self.env_options} + exd = re.sub(r'\\(?!")', "/", json.dumps(configuration)) + exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"', lambda m: '"%s"' % join( + *m.group(1).split("/")[-2:]), exd, re.I | re.M) + mp = MeasurementProtocol() + mp['exd'] = "DebugGDBPioInitError: %s" % exd + mp['exf'] = 1 + mp.send("exception") + self.transport.loseConnection() + + def _kill_previous_session(self): + assert self._session_id + pid = None + with app.ContentCache() as cc: + pid = cc.get(self._session_id) + cc.delete(self._session_id) + if not pid: + return + if "windows" in util.get_systype(): + kill = ["Taskkill", "/PID", pid, "/F"] + else: + kill = ["kill", pid] + try: + util.exec_command(kill) + except: # pylint: disable=bare-except + pass + + def _lock_session(self, pid): + if not self._session_id: + return + with app.ContentCache() as cc: + cc.set(self._session_id, str(pid), "1h") + + def _unlock_session(self): + if not self._session_id: + return + with app.ContentCache() as cc: + cc.delete(self._session_id) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py new file mode 100644 index 00000000..14c4f665 --- /dev/null +++ b/platformio/commands/debug/command.py @@ -0,0 +1,153 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments, too-many-statements +# pylint: disable=too-many-locals, too-many-branches + +import os +import signal +from os.path import isfile, join + +import click + +from platformio import exception, util +from platformio.commands.debug import helpers +from platformio.managers.core import inject_contrib_pysite +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (is_platformio_project, + load_project_ide_data) + + +@click.command("debug", + context_settings=dict(ignore_unknown_options=True), + short_help="PIO Unified Debugger") +@click.option("-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) +@click.option("--environment", "-e", metavar="") +@click.option("--verbose", "-v", is_flag=True) +@click.option("--interface", type=click.Choice(["gdb"])) +@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def cli(ctx, project_dir, project_conf, environment, verbose, interface, + __unprocessed): + # use env variables from Eclipse or CLion + for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): + if is_platformio_project(project_dir): + break + if os.getenv(sysenv): + project_dir = os.getenv(sysenv) + + with util.cd(project_dir): + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(envs=[environment] if environment else None) + + env_name = environment or helpers.get_default_debug_env(config) + env_options = config.items(env=env_name, as_dict=True) + if not set(env_options.keys()) >= set(["platform", "board"]): + raise exception.ProjectEnvsNotAvailable() + debug_options = helpers.validate_debug_options(ctx, env_options) + assert debug_options + + if not interface: + return helpers.predebug_project(ctx, project_dir, env_name, False, + verbose) + + configuration = load_project_ide_data(project_dir, env_name) + if not configuration: + raise exception.DebugInvalidOptions( + "Could not load debug configuration") + + if "--version" in __unprocessed: + result = util.exec_command([configuration['gdb_path'], "--version"]) + if result['returncode'] == 0: + return click.echo(result['out']) + raise exception.PlatformioException("\n".join( + [result['out'], result['err']])) + + try: + util.ensure_udev_rules() + except NameError: + pass + except exception.InvalidUdevRules as e: + for line in str(e).split("\n") + [""]: + click.echo( + ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % + line) + + debug_options['load_cmds'] = helpers.configure_esp32_load_cmds( + debug_options, configuration) + + rebuild_prog = False + preload = debug_options['load_cmds'] == ["preload"] + load_mode = debug_options['load_mode'] + if load_mode == "always": + rebuild_prog = ( + preload + or not helpers.has_debug_symbols(configuration['prog_path'])) + elif load_mode == "modified": + rebuild_prog = ( + helpers.is_prog_obsolete(configuration['prog_path']) + or not helpers.has_debug_symbols(configuration['prog_path'])) + else: + rebuild_prog = not isfile(configuration['prog_path']) + + if preload or (not rebuild_prog and load_mode != "always"): + # don't load firmware through debug server + debug_options['load_cmds'] = [] + + if rebuild_prog: + if helpers.is_mi_mode(__unprocessed): + click.echo('~"Preparing firmware for debugging...\\n"') + output = helpers.GDBBytesIO() + with util.capture_std_streams(output): + helpers.predebug_project(ctx, project_dir, env_name, preload, + verbose) + output.close() + else: + click.echo("Preparing firmware for debugging...") + helpers.predebug_project(ctx, project_dir, env_name, preload, + verbose) + + # save SHA sum of newly created prog + if load_mode == "modified": + helpers.is_prog_obsolete(configuration['prog_path']) + + if not isfile(configuration['prog_path']): + raise exception.DebugInvalidOptions("Program/firmware is missed") + + # run debugging client + inject_contrib_pysite() + from platformio.commands.debug.client import GDBClient, reactor + + client = GDBClient(project_dir, __unprocessed, debug_options, env_options) + client.spawn(configuration['gdb_path'], configuration['prog_path']) + + signal.signal(signal.SIGINT, lambda *args, **kwargs: None) + reactor.run() + + return True diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py new file mode 100644 index 00000000..daaa8d93 --- /dev/null +++ b/platformio/commands/debug/helpers.py @@ -0,0 +1,269 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import time +from fnmatch import fnmatch +from hashlib import sha1 +from io import BytesIO +from os.path import isfile + +from platformio import exception, util +from platformio.commands.platform import \ + platform_install as cmd_platform_install +from platformio.commands.run import cli as cmd_run +from platformio.managers.platform import PlatformFactory +from platformio.project.config import ProjectConfig + + +class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods + + STDOUT = sys.stdout + + def write(self, text): + if "\n" in text: + for line in text.strip().split("\n"): + self.STDOUT.write('~"%s\\n"\n' % line) + else: + self.STDOUT.write('~"%s"' % text) + self.STDOUT.flush() + + +def is_mi_mode(args): + return "--interpreter" in " ".join(args) + + +def escape_path(path): + return path.replace("\\", "/") + + +def get_default_debug_env(config): + default_envs = config.default_envs() + all_envs = config.envs() + for env in default_envs: + if config.get("env:" + env, "build_type") == "debug": + return env + for env in all_envs: + if config.get("env:" + env, "build_type") == "debug": + return env + return default_envs[0] if default_envs else all_envs[0] + + +def predebug_project(ctx, project_dir, env_name, preload, verbose): + ctx.invoke(cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["debug"] + (["upload"] if preload else []), + verbose=verbose) + if preload: + time.sleep(5) + + +def validate_debug_options(cmd_ctx, env_options): + + def _cleanup_cmds(items): + items = ProjectConfig.parse_multi_values(items) + return [ + "$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items + ] + + try: + platform = PlatformFactory.newPlatform(env_options['platform']) + except exception.UnknownPlatform: + cmd_ctx.invoke(cmd_platform_install, + platforms=[env_options['platform']], + skip_default_package=True) + platform = PlatformFactory.newPlatform(env_options['platform']) + + board_config = platform.board_config(env_options['board']) + tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) + tool_settings = board_config.get("debug", {}).get("tools", + {}).get(tool_name, {}) + server_options = None + + # specific server per a system + if isinstance(tool_settings.get("server", {}), list): + for item in tool_settings['server'][:]: + tool_settings['server'] = item + if util.get_systype() in item.get("system", []): + break + + # user overwrites debug server + if env_options.get("debug_server"): + server_options = { + "cwd": None, + "executable": None, + "arguments": env_options.get("debug_server") + } + server_options['executable'] = server_options['arguments'][0] + server_options['arguments'] = server_options['arguments'][1:] + elif "server" in tool_settings: + server_package = tool_settings['server'].get("package") + server_package_dir = platform.get_package_dir( + server_package) if server_package else None + if server_package and not server_package_dir: + platform.install_packages(with_packages=[server_package], + skip_default_package=True, + silent=True) + server_package_dir = platform.get_package_dir(server_package) + server_options = dict( + cwd=server_package_dir if server_package else None, + executable=tool_settings['server'].get("executable"), + arguments=[ + a.replace("$PACKAGE_DIR", escape_path(server_package_dir)) + if server_package_dir else a + for a in tool_settings['server'].get("arguments", []) + ]) + + extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds")) + extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds"))) + result = dict( + tool=tool_name, + upload_protocol=env_options.get( + "upload_protocol", + board_config.get("upload", {}).get("protocol")), + load_cmds=_cleanup_cmds( + env_options.get( + "debug_load_cmds", + tool_settings.get("load_cmds", + tool_settings.get("load_cmd", "load")))), + load_mode=env_options.get("debug_load_mode", + tool_settings.get("load_mode", "always")), + init_break=env_options.get( + "debug_init_break", tool_settings.get("init_break", + "tbreak main")), + init_cmds=_cleanup_cmds( + env_options.get("debug_init_cmds", + tool_settings.get("init_cmds"))), + extra_cmds=extra_cmds, + require_debug_port=tool_settings.get("require_debug_port", False), + port=reveal_debug_port( + env_options.get("debug_port", tool_settings.get("port")), + tool_name, tool_settings), + server=server_options) + return result + + +def configure_esp32_load_cmds(debug_options, configuration): + ignore_conds = [ + debug_options['load_cmds'] != ["load"], + "xtensa-esp32" not in configuration.get("cc_path", ""), + not configuration.get("flash_extra_images"), not all([ + isfile(item['path']) + for item in configuration.get("flash_extra_images") + ]) + ] + if any(ignore_conds): + return debug_options['load_cmds'] + + mon_cmds = [ + 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( + path=escape_path(item['path']), offset=item['offset']) + for item in configuration.get("flash_extra_images") + ] + mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' % + escape_path(configuration['prog_path'][:-4])) + return mon_cmds + + +def has_debug_symbols(prog_path): + if not isfile(prog_path): + return False + matched = { + b".debug_info": False, + b".debug_abbrev": False, + b" -Og": False, + b" -g": False, + b"__PLATFORMIO_BUILD_DEBUG__": False + } + with open(prog_path, "rb") as fp: + last_data = b"" + while True: + data = fp.read(1024) + if not data: + break + for pattern, found in matched.items(): + if found: + continue + if pattern in last_data + data: + matched[pattern] = True + last_data = data + return all(matched.values()) + + +def is_prog_obsolete(prog_path): + prog_hash_path = prog_path + ".sha1" + if not isfile(prog_path): + return True + shasum = sha1() + with open(prog_path, "rb") as fp: + while True: + data = fp.read(1024) + if not data: + break + shasum.update(data) + new_digest = shasum.hexdigest() + old_digest = None + if isfile(prog_hash_path): + with open(prog_hash_path, "r") as fp: + old_digest = fp.read() + if new_digest == old_digest: + return False + with open(prog_hash_path, "w") as fp: + fp.write(new_digest) + return True + + +def reveal_debug_port(env_debug_port, tool_name, tool_settings): + + def _get_pattern(): + if not env_debug_port: + return None + if set(["*", "?", "[", "]"]) & set(env_debug_port): + return env_debug_port + return None + + def _is_match_pattern(port): + pattern = _get_pattern() + if not pattern: + return True + return fnmatch(port, pattern) + + def _look_for_serial_port(hwids): + for item in util.get_serialports(filter_hwid=True): + if not _is_match_pattern(item['port']): + continue + port = item['port'] + if tool_name.startswith("blackmagic"): + if "windows" in util.get_systype() and \ + port.startswith("COM") and len(port) > 4: + port = "\\\\.\\%s" % port + if "GDB" in item['description']: + return port + for hwid in hwids: + hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") + if hwid_str in item['hwid']: + return port + return None + + if env_debug_port and not _get_pattern(): + return env_debug_port + if not tool_settings.get("require_debug_port"): + return None + + debug_port = _look_for_serial_port(tool_settings.get("hwids", [])) + if not debug_port: + raise exception.DebugInvalidOptions( + "Please specify `debug_port` for environment") + return debug_port diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py new file mode 100644 index 00000000..a9d71d32 --- /dev/null +++ b/platformio/commands/debug/initcfgs.py @@ -0,0 +1,124 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GDB_DEFAULT_INIT_CONFIG = """ +define pio_reset_halt_target + monitor reset halt +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMDS +monitor init +pio_reset_halt_target +""" + +GDB_STUTIL_INIT_CONFIG = """ +define pio_reset_halt_target + monitor halt + monitor reset +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMDS +pio_reset_halt_target +""" + +GDB_JLINK_INIT_CONFIG = """ +define pio_reset_halt_target + monitor halt + monitor reset +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMDS +pio_reset_halt_target +""" + +GDB_BLACKMAGIC_INIT_CONFIG = """ +define pio_reset_halt_target + set language c + set *0xE000ED0C = 0x05FA0004 + set $busy = (*0xE000ED0C & 0x4) + while ($busy) + set $busy = (*0xE000ED0C & 0x4) + end + set language auto +end + +define pio_reset_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +monitor swdp_scan +attach 1 +set mem inaccessible-by-default off +$INIT_BREAK +$LOAD_CMDS + +set language c +set *0xE000ED0C = 0x05FA0004 +set $busy = (*0xE000ED0C & 0x4) +while ($busy) + set $busy = (*0xE000ED0C & 0x4) +end +set language auto +""" + +GDB_MSPDEBUG_INIT_CONFIG = """ +define pio_reset_halt_target +end + +define pio_reset_target +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +monitor erase +$LOAD_CMDS +pio_reset_halt_target +""" + +GDB_QEMU_INIT_CONFIG = """ +define pio_reset_halt_target + monitor system_reset +end + +define pio_reset_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +$LOAD_CMDS +pio_reset_halt_target +""" diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py new file mode 100644 index 00000000..98c7cc1a --- /dev/null +++ b/platformio/commands/debug/process.py @@ -0,0 +1,80 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import signal + +import click +from twisted.internet import protocol # pylint: disable=import-error + +from platformio.commands.debug import helpers +from platformio.compat import string_types +from platformio.proc import get_pythonexe_path +from platformio.project.helpers import get_project_core_dir + +LOG_FILE = None + + +class BaseProcess(protocol.ProcessProtocol, object): + + STDOUT_CHUNK_SIZE = 2048 + + COMMON_PATTERNS = { + "PLATFORMIO_HOME_DIR": helpers.escape_path(get_project_core_dir()), + "PLATFORMIO_CORE_DIR": helpers.escape_path(get_project_core_dir()), + "PYTHONEXE": get_pythonexe_path() + } + + def apply_patterns(self, source, patterns=None): + _patterns = self.COMMON_PATTERNS.copy() + _patterns.update(patterns or {}) + + def _replace(text): + for key, value in _patterns.items(): + pattern = "$%s" % key + text = text.replace(pattern, value or "") + return text + + if isinstance(source, string_types): + source = _replace(source) + elif isinstance(source, (list, dict)): + items = enumerate(source) if isinstance(source, + list) else source.items() + for key, value in items: + if isinstance(value, string_types): + source[key] = _replace(value) + elif isinstance(value, (list, dict)): + source[key] = self.apply_patterns(value, patterns) + + return source + + def outReceived(self, data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + while data: + chunk = data[:self.STDOUT_CHUNK_SIZE] + click.echo(chunk, nl=False) + data = data[self.STDOUT_CHUNK_SIZE:] + + @staticmethod + def errReceived(data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + click.echo(data, nl=False, err=True) + + @staticmethod + def processEnded(_): + # Allow terminating via SIGINT/CTRL+C + signal.signal(signal.SIGINT, signal.default_int_handler) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py new file mode 100644 index 00000000..83bba340 --- /dev/null +++ b/platformio/commands/debug/server.py @@ -0,0 +1,123 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from os.path import isdir, isfile, join + +from twisted.internet import error # pylint: disable=import-error +from twisted.internet import reactor # pylint: disable=import-error + +from platformio import exception, util +from platformio.commands.debug import helpers +from platformio.commands.debug.process import BaseProcess +from platformio.proc import where_is_program + + +class DebugServer(BaseProcess): + + def __init__(self, debug_options, env_options): + self.debug_options = debug_options + self.env_options = env_options + + self._debug_port = None + self._transport = None + self._process_ended = False + + def spawn(self, patterns): # pylint: disable=too-many-branches + systype = util.get_systype() + server = self.debug_options.get("server") + if not server: + return None + server = self.apply_patterns(server, patterns) + server_executable = server['executable'] + if not server_executable: + return None + if server['cwd']: + server_executable = join(server['cwd'], server_executable) + if ("windows" in systype and not server_executable.endswith(".exe") + and isfile(server_executable + ".exe")): + server_executable = server_executable + ".exe" + + if not isfile(server_executable): + server_executable = where_is_program(server_executable) + if not isfile(server_executable): + raise exception.DebugInvalidOptions( + "\nCould not launch Debug Server '%s'. Please check that it " + "is installed and is included in a system PATH\n\n" + "See documentation or contact contact@platformio.org:\n" + "http://docs.platformio.org/page/plus/debugging.html\n" % + server_executable) + + self._debug_port = ":3333" + openocd_pipe_allowed = all([ + not self.debug_options['port'], + "openocd" in server_executable + ]) # yapf: disable + if openocd_pipe_allowed: + args = [] + if server['cwd']: + args.extend(["-s", helpers.escape_path(server['cwd'])]) + args.extend([ + "-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled" + ]) + args.extend(server['arguments']) + str_args = " ".join( + [arg if arg.startswith("-") else '"%s"' % arg for arg in args]) + self._debug_port = '| "%s" %s' % ( + helpers.escape_path(server_executable), str_args) + else: + env = os.environ.copy() + # prepend server "lib" folder to LD path + if ("windows" not in systype and server['cwd'] + and isdir(join(server['cwd'], "lib"))): + ld_key = ("DYLD_LIBRARY_PATH" + if "darwin" in systype else "LD_LIBRARY_PATH") + env[ld_key] = join(server['cwd'], "lib") + if os.environ.get(ld_key): + env[ld_key] = "%s:%s" % (env[ld_key], + os.environ.get(ld_key)) + # prepend BIN to PATH + if server['cwd'] and isdir(join(server['cwd'], "bin")): + env['PATH'] = "%s%s%s" % ( + join(server['cwd'], "bin"), os.pathsep, + os.environ.get("PATH", os.environ.get("Path", ""))) + + self._transport = reactor.spawnProcess( + self, + server_executable, [server_executable] + server['arguments'], + path=server['cwd'], + env=env) + if "mspdebug" in server_executable.lower(): + self._debug_port = ":2000" + elif "jlink" in server_executable.lower(): + self._debug_port = ":2331" + elif "qemu" in server_executable.lower(): + self._debug_port = ":1234" + + return self._transport + + def get_debug_port(self): + return self._debug_port + + def processEnded(self, reason): + self._process_ended = True + super(DebugServer, self).processEnded(reason) + + def terminate(self): + if self._process_ended or not self._transport: + return + try: + self._transport.signalProcess("KILL") + except (OSError, error.ProcessExitedAlready): + pass diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 450b52bd..d5ffe030 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -12,14 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import sys +from fnmatch import fnmatch from os import getcwd +from os.path import join import click from serial.tools import miniterm from platformio import exception, util +from platformio.compat import dump_json_to_unicode +from platformio.project.config import ProjectConfig @click.group(short_help="Monitor device or list existing") @@ -44,10 +47,11 @@ def device_list( # pylint: disable=too-many-branches if mdns: data['mdns'] = util.get_mdns_services() - single_key = data.keys()[0] if len(data.keys()) == 1 else None + single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: - return click.echo(json.dumps(data[single_key] if single_key else data)) + return click.echo( + dump_json_to_unicode(data[single_key] if single_key else data)) titles = { "serial": "Serial Ports", @@ -98,81 +102,74 @@ def device_list( # pylint: disable=too-many-branches @cli.command("monitor", short_help="Monitor device (Serial)") @click.option("--port", "-p", help="Port, a number or a device name") @click.option("--baud", "-b", type=int, help="Set baud rate, default=9600") -@click.option( - "--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N") -@click.option( - "--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") -@click.option( - "--xonxoff", - is_flag=True, - help="Enable software flow control, default=Off") -@click.option( - "--rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state") -@click.option( - "--dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state") +@click.option("--parity", + default="N", + type=click.Choice(["N", "E", "O", "S", "M"]), + help="Set parity, default=N") +@click.option("--rtscts", + is_flag=True, + help="Enable RTS/CTS flow control, default=Off") +@click.option("--xonxoff", + is_flag=True, + help="Enable software flow control, default=Off") +@click.option("--rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state") +@click.option("--dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state") @click.option("--echo", is_flag=True, help="Enable local echo, default=Off") -@click.option( - "--encoding", - default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8") +@click.option("--encoding", + default="UTF-8", + help="Set the encoding for the serial port (e.g. hexlify, " + "Latin1, UTF-8), default: UTF-8") @click.option("--filter", "-f", multiple=True, help="Add text transformation") -@click.option( - "--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF") -@click.option( - "--raw", is_flag=True, help="Do not apply any encodings/transformations") -@click.option( - "--exit-char", - type=int, - default=3, - help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)") -@click.option( - "--menu-char", - type=int, - default=20, - help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)") -@click.option( - "--quiet", - is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, resolve_path=True)) +@click.option("--eol", + default="CRLF", + type=click.Choice(["CR", "LF", "CRLF"]), + help="End of line mode, default=CRLF") +@click.option("--raw", + is_flag=True, + help="Do not apply any encodings/transformations") +@click.option("--exit-char", + type=int, + default=3, + help="ASCII code of special character that is used to exit " + "the application, default=3 (Ctrl+C)") +@click.option("--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control miniterm (menu), default=20 (DEC)") +@click.option("--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages, default=Off") +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True)) @click.option( "-e", "--environment", help="Load configuration from `platformio.ini` and specified environment") def device_monitor(**kwargs): # pylint: disable=too-many-branches + env_options = {} try: - project_options = get_project_options(kwargs['project_dir'], - kwargs['environment']) - monitor_options = {k: v for k, v in project_options or []} - if monitor_options: - for k in ("port", "baud", "speed", "rts", "dtr"): - k2 = "monitor_%s" % k - if k == "speed": - k = "baud" - if kwargs[k] is None and k2 in monitor_options: - kwargs[k] = monitor_options[k2] - if k != "port": - kwargs[k] = int(kwargs[k]) + env_options = get_project_options(kwargs['project_dir'], + kwargs['environment']) + for k in ("port", "speed", "rts", "dtr"): + k2 = "monitor_%s" % k + if k == "speed": + k = "baud" + if kwargs[k] is None and k2 in env_options: + kwargs[k] = env_options[k2] + if k != "port": + kwargs[k] = int(kwargs[k]) except exception.NotPlatformIOProject: pass @@ -181,11 +178,13 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches if len(ports) == 1: kwargs['port'] = ports[0]['port'] - sys.argv = ["monitor"] + sys.argv = ["monitor"] + env_options.get("monitor_flags", []) for k, v in kwargs.items(): if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"): continue k = "--" + k.replace("_", "-") + if k in env_options.get("monitor_flags", []): + continue if isinstance(v, bool): if v: sys.argv.append(k) @@ -195,34 +194,28 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches else: sys.argv.extend([k, str(v)]) + if kwargs['port'] and (set(["*", "?", "[", "]"]) & set(kwargs['port'])): + for item in util.get_serial_ports(): + if fnmatch(item['port'], kwargs['port']): + kwargs['port'] = item['port'] + break + try: - miniterm.main( - default_port=kwargs['port'], - default_baudrate=kwargs['baud'] or 9600, - default_rts=kwargs['rts'], - default_dtr=kwargs['dtr']) + miniterm.main(default_port=kwargs['port'], + default_baudrate=kwargs['baud'] or 9600, + default_rts=kwargs['rts'], + default_dtr=kwargs['dtr']) except Exception as e: raise exception.MinitermException(e) -def get_project_options(project_dir, environment): - config = util.load_project_config(project_dir) - if not config.sections(): - return None - - known_envs = [s[4:] for s in config.sections() if s.startswith("env:")] - if environment: - if environment in known_envs: - return config.items("env:%s" % environment) - raise exception.UnknownEnvNames(environment, ", ".join(known_envs)) - - if not known_envs: - return None - - if config.has_option("platformio", "env_default"): - env_default = config.get("platformio", - "env_default").split(", ")[0].strip() - if env_default and env_default in known_envs: - return config.items("env:%s" % env_default) - - return config.items("env:%s" % known_envs[0]) +def get_project_options(project_dir, environment=None): + config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) + config.validate(envs=[environment] if environment else None) + if not environment: + default_envs = config.default_envs() + if default_envs: + environment = default_envs[0] + else: + environment = config.envs()[0] + return config.items(env=environment, as_dict=True) diff --git a/platformio/commands/home/__init__.py b/platformio/commands/home/__init__.py new file mode 100644 index 00000000..a889291e --- /dev/null +++ b/platformio/commands/home/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.home.command import cli diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py new file mode 100644 index 00000000..d1b7d607 --- /dev/null +++ b/platformio/commands/home/command.py @@ -0,0 +1,109 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mimetypes +import socket +from os.path import isdir + +import click + +from platformio import exception +from platformio.managers.core import (get_core_package_dir, + inject_contrib_pysite) + + +@click.command("home", short_help="PIO Home") +@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") +@click.option( + "--host", + default="127.0.0.1", + help="HTTP host, default=127.0.0.1. " + "You can open PIO Home for inbound connections with --host=0.0.0.0") +@click.option("--no-open", is_flag=True) # pylint: disable=too-many-locals +def cli(port, host, no_open): + # import contrib modules + inject_contrib_pysite() + # pylint: disable=import-error + from autobahn.twisted.resource import WebSocketResource + from twisted.internet import reactor + from twisted.web import server + # pylint: enable=import-error + from platformio.commands.home.rpc.handlers.app import AppRPC + from platformio.commands.home.rpc.handlers.ide import IDERPC + from platformio.commands.home.rpc.handlers.misc import MiscRPC + from platformio.commands.home.rpc.handlers.os import OSRPC + from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC + from platformio.commands.home.rpc.handlers.project import ProjectRPC + from platformio.commands.home.rpc.server import JSONRPCServerFactory + from platformio.commands.home.web import WebRoot + + factory = JSONRPCServerFactory() + factory.addHandler(AppRPC(), namespace="app") + factory.addHandler(IDERPC(), namespace="ide") + factory.addHandler(MiscRPC(), namespace="misc") + factory.addHandler(OSRPC(), namespace="os") + factory.addHandler(PIOCoreRPC(), namespace="core") + factory.addHandler(ProjectRPC(), namespace="project") + + contrib_dir = get_core_package_dir("contrib-piohome") + if not isdir(contrib_dir): + raise exception.PlatformioException("Invalid path to PIO Home Contrib") + + # Ensure PIO Home mimetypes are known + mimetypes.add_type("text/html", ".html") + mimetypes.add_type("text/css", ".css") + mimetypes.add_type("application/javascript", ".js") + + root = WebRoot(contrib_dir) + root.putChild(b"wsrpc", WebSocketResource(factory)) + site = server.Site(root) + + # hook for `platformio-node-helpers` + if host == "__do_not_start__": + return + + # if already started + already_started = False + socket.setdefaulttimeout(1) + try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) + already_started = True + except: # pylint: disable=bare-except + pass + + home_url = "http://%s:%d" % (host, port) + if not no_open: + if already_started: + click.launch(home_url) + else: + reactor.callLater(1, lambda: click.launch(home_url)) + + click.echo("\n".join([ + "", + " ___I_", + " /\\-_--\\ PlatformIO Home", + "/ \\_-__\\", + "|[]| [] | %s" % home_url, + "|__|____|______________%s" % ("_" * len(host)), + ])) + click.echo("") + click.echo("Open PIO Home in your browser by this URL => %s" % home_url) + + if already_started: + return + + click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") + + reactor.listenTCP(port, site, interface=host) + reactor.run() diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py new file mode 100644 index 00000000..46c1750c --- /dev/null +++ b/platformio/commands/home/helpers.py @@ -0,0 +1,71 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=keyword-arg-before-vararg, arguments-differ + +import os +import socket + +import requests +from twisted.internet import defer # pylint: disable=import-error +from twisted.internet import reactor # pylint: disable=import-error +from twisted.internet import threads # pylint: disable=import-error + +from platformio import util +from platformio.proc import where_is_program + + +class AsyncSession(requests.Session): + + def __init__(self, n=None, *args, **kwargs): + if n: + pool = reactor.getThreadPool() + pool.adjustPoolsize(0, n) + + super(AsyncSession, self).__init__(*args, **kwargs) + + def request(self, *args, **kwargs): + func = super(AsyncSession, self).request + return threads.deferToThread(func, *args, **kwargs) + + def wrap(self, *args, **kwargs): # pylint: disable=no-self-use + return defer.ensureDeferred(*args, **kwargs) + + +@util.memoized(expire="60s") +def requests_session(): + return AsyncSession(n=5) + + +@util.memoized(expire="60s") +def get_core_fullpath(): + return where_is_program( + "platformio" + (".exe" if "windows" in util.get_systype() else "")) + + +@util.memoized(expire="10s") +def is_twitter_blocked(): + ip = "104.244.42.1" + timeout = 2 + try: + if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): + requests.get("http://%s" % ip, + allow_redirects=False, + timeout=timeout) + else: + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) + return False + except: # pylint: disable=bare-except + pass + return True diff --git a/platformio/commands/home/rpc/__init__.py b/platformio/commands/home/rpc/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/home/rpc/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/home/rpc/handlers/__init__.py b/platformio/commands/home/rpc/handlers/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py new file mode 100644 index 00000000..1666dc17 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/app.py @@ -0,0 +1,71 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +from os.path import expanduser, join + +from platformio import __version__, app, util +from platformio.project.helpers import (get_project_core_dir, + is_platformio_project) + + +class AppRPC(object): + + APPSTATE_PATH = join(get_project_core_dir(), "homestate.json") + + @staticmethod + def load_state(): + with app.State(AppRPC.APPSTATE_PATH, lock=True) as state: + storage = state.get("storage", {}) + + # base data + caller_id = app.get_session_var("caller_id") + storage['cid'] = app.get_cid() + storage['coreVersion'] = __version__ + storage['coreSystype'] = util.get_systype() + storage['coreCaller'] = (str(caller_id).lower() + if caller_id else None) + storage['coreSettings'] = { + name: { + "description": data['description'], + "default_value": data['value'], + "value": app.get_setting(name) + } + for name, data in app.DEFAULT_SETTINGS.items() + } + + storage['homeDir'] = expanduser("~") + storage['projectsDir'] = storage['coreSettings']['projects_dir'][ + 'value'] + + # skip non-existing recent projects + storage['recentProjects'] = [ + p for p in storage.get("recentProjects", []) + if is_platformio_project(p) + ] + + state['storage'] = storage + return state.as_dict() + + @staticmethod + def get_state(): + return AppRPC.load_state() + + @staticmethod + def save_state(state): + with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: + # s.clear() + s.update(state) + return True diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py new file mode 100644 index 00000000..5f8f31a4 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import jsonrpc # pylint: disable=import-error +from twisted.internet import defer # pylint: disable=import-error + + +class IDERPC(object): + + def __init__(self): + self._queue = [] + + def send_command(self, command, params): + if not self._queue: + raise jsonrpc.exceptions.JSONRPCDispatchException( + code=4005, message="PIO Home IDE agent is not started") + while self._queue: + self._queue.pop().callback({ + "id": time.time(), + "method": command, + "params": params + }) + + def listen_commands(self): + self._queue.append(defer.Deferred()) + return self._queue[-1] + + def open_project(self, project_dir): + return self.send_command("open_project", project_dir) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py new file mode 100644 index 00000000..2422da78 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -0,0 +1,54 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import time + +from twisted.internet import defer, reactor # pylint: disable=import-error + +from platformio import app +from platformio.commands.home.rpc.handlers.os import OSRPC + + +class MiscRPC(object): + + def load_latest_tweets(self, username): + cache_key = "piohome_latest_tweets_" + str(username) + cache_valid = "7d" + with app.ContentCache() as cc: + cache_data = cc.get(cache_key) + if cache_data: + cache_data = json.loads(cache_data) + # automatically update cache in background every 12 hours + if cache_data['time'] < (time.time() - (3600 * 12)): + reactor.callLater(5, self._preload_latest_tweets, username, + cache_key, cache_valid) + return cache_data['result'] + + result = self._preload_latest_tweets(username, cache_key, cache_valid) + return result + + @staticmethod + @defer.inlineCallbacks + def _preload_latest_tweets(username, cache_key, cache_valid): + result = yield OSRPC.fetch_content( + "https://api.platformio.org/tweets/" + username) + result = json.loads(result) + with app.ContentCache() as cc: + cc.set(cache_key, + json.dumps({ + "time": int(time.time()), + "result": result + }), cache_valid) + defer.returnValue(result) diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py new file mode 100644 index 00000000..c84f486e --- /dev/null +++ b/platformio/commands/home/rpc/handlers/os.py @@ -0,0 +1,152 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import codecs +import glob +import os +import shutil +from functools import cmp_to_key +from os.path import expanduser, isdir, isfile, join + +import click +from twisted.internet import defer # pylint: disable=import-error + +from platformio import app, util +from platformio.commands.home import helpers +from platformio.compat import PY2, get_filesystem_encoding + + +class OSRPC(object): + + @staticmethod + @defer.inlineCallbacks + def fetch_content(uri, data=None, headers=None, cache_valid=None): + if not headers: + headers = { + "User-Agent": + ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) " + "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 " + "Safari/603.3.8") + } + cache_key = (app.ContentCache.key_from_args(uri, data) + if cache_valid else None) + with app.ContentCache() as cc: + if cache_key: + result = cc.get(cache_key) + if result is not None: + defer.returnValue(result) + + # check internet before and resolve issue with 60 seconds timeout + util.internet_on(raise_exception=True) + + session = helpers.requests_session() + if data: + r = yield session.post(uri, data=data, headers=headers) + else: + r = yield session.get(uri, headers=headers) + + r.raise_for_status() + result = r.text + if cache_valid: + with app.ContentCache() as cc: + cc.set(cache_key, result, cache_valid) + defer.returnValue(result) + + def request_content(self, uri, data=None, headers=None, cache_valid=None): + if uri.startswith('http'): + return self.fetch_content(uri, data, headers, cache_valid) + if not isfile(uri): + return None + with codecs.open(uri, encoding="utf-8") as fp: + return fp.read() + + @staticmethod + def open_url(url): + return click.launch(url) + + @staticmethod + def reveal_file(path): + return click.launch( + path.encode(get_filesystem_encoding()) if PY2 else path, + locate=True) + + @staticmethod + def is_file(path): + return isfile(path) + + @staticmethod + def is_dir(path): + return isdir(path) + + @staticmethod + def make_dirs(path): + return os.makedirs(path) + + @staticmethod + def rename(src, dst): + return os.rename(src, dst) + + @staticmethod + def copy(src, dst): + return shutil.copytree(src, dst) + + @staticmethod + def glob(pathnames, root=None): + if not isinstance(pathnames, list): + pathnames = [pathnames] + result = set() + for pathname in pathnames: + result |= set( + glob.glob(join(root, pathname) if root else pathname)) + return list(result) + + @staticmethod + def list_dir(path): + + def _cmp(x, y): + if x[1] and not y[1]: + return -1 + if not x[1] and y[1]: + return 1 + if x[0].lower() > y[0].lower(): + return 1 + if x[0].lower() < y[0].lower(): + return -1 + return 0 + + items = [] + if path.startswith("~"): + path = expanduser(path) + if not isdir(path): + return items + for item in os.listdir(path): + try: + item_is_dir = isdir(join(path, item)) + if item_is_dir: + os.listdir(join(path, item)) + items.append((item, item_is_dir)) + except OSError: + pass + return sorted(items, key=cmp_to_key(_cmp)) + + @staticmethod + def get_logical_devices(): + items = [] + for item in util.get_logical_devices(): + if item['name']: + item['name'] = item['name'] + items.append(item) + return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py new file mode 100644 index 00000000..a88f495d --- /dev/null +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -0,0 +1,142 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import json +import os +import sys +from io import BytesIO, StringIO + +import click +import jsonrpc # pylint: disable=import-error +from twisted.internet import threads # pylint: disable=import-error +from twisted.internet import utils # pylint: disable=import-error + +from platformio import __main__, __version__, util +from platformio.commands.home import helpers +from platformio.compat import (PY2, get_filesystem_encoding, is_bytes, + string_types) + +try: + from thread import get_ident as thread_get_ident +except ImportError: + from threading import get_ident as thread_get_ident + + +class MultiThreadingStdStream(object): + + def __init__(self, parent_stream): + self._buffers = {thread_get_ident(): parent_stream} + + def __getattr__(self, name): + thread_id = thread_get_ident() + self._ensure_thread_buffer(thread_id) + return getattr(self._buffers[thread_id], name) + + def _ensure_thread_buffer(self, thread_id): + if thread_id not in self._buffers: + self._buffers[thread_id] = BytesIO() if PY2 else StringIO() + + def write(self, value): + thread_id = thread_get_ident() + self._ensure_thread_buffer(thread_id) + return self._buffers[thread_id].write( + value.decode() if is_bytes(value) else value) + + def get_value_and_reset(self): + result = "" + try: + result = self.getvalue() + self.truncate(0) + self.seek(0) + except AttributeError: + pass + return result + + +class PIOCoreRPC(object): + + @staticmethod + def setup_multithreading_std_streams(): + if isinstance(sys.stdout, MultiThreadingStdStream): + return + PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout) + PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr) + sys.stdout = PIOCoreRPC.thread_stdout + sys.stderr = PIOCoreRPC.thread_stderr + + @staticmethod + def call(args, options=None): + PIOCoreRPC.setup_multithreading_std_streams() + cwd = (options or {}).get("cwd") or os.getcwd() + for i, arg in enumerate(args): + if isinstance(arg, string_types): + args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg + else: + args[i] = str(arg) + + def _call_inline(): + with util.cd(cwd): + exit_code = __main__.main(["-c"] + args) + return (PIOCoreRPC.thread_stdout.get_value_and_reset(), + PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code) + + if args and args[0] in ("account", "remote"): + d = utils.getProcessOutputAndValue( + helpers.get_core_fullpath(), + args, + path=cwd, + env={k: v + for k, v in os.environ.items() if "%" not in k}) + else: + d = threads.deferToThread(_call_inline) + + d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) + d.addErrback(PIOCoreRPC._call_errback) + return d + + @staticmethod + def _call_callback(result, json_output=False): + out, err, code = result + text = ("%s\n\n%s" % (out, err)).strip() + if code != 0: + raise Exception(text) + if not json_output: + return text + try: + return json.loads(out) + except ValueError as e: + click.secho("%s => `%s`" % (e, out), fg="red", err=True) + # if PIO Core prints unhandled warnings + for line in out.split("\n"): + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except ValueError: + pass + raise e + + @staticmethod + def _call_errback(failure): + raise jsonrpc.exceptions.JSONRPCDispatchException( + code=4003, + message="PIO Core Call Error", + data=failure.getErrorMessage()) + + @staticmethod + def version(): + return __version__ diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py new file mode 100644 index 00000000..4ca5af63 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/project.py @@ -0,0 +1,277 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os +import shutil +import time +from os.path import (basename, expanduser, getmtime, isdir, isfile, join, + realpath, sep) + +import jsonrpc # pylint: disable=import-error + +from platformio import exception, util +from platformio.commands.home.rpc.handlers.app import AppRPC +from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.compat import PY2, get_filesystem_encoding +from platformio.ide.projectgenerator import ProjectGenerator +from platformio.managers.platform import PlatformManager +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (get_project_libdeps_dir, + get_project_src_dir, + is_platformio_project) + + +class ProjectRPC(object): + + @staticmethod + def _get_projects(project_dirs=None): + + def _get_project_data(project_dir): + data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} + config = ProjectConfig(join(project_dir, "platformio.ini")) + libdeps_dir = get_project_libdeps_dir() + + data['libExtraDirs'].extend( + config.get("platformio", "lib_extra_dirs", [])) + + for section in config.sections(): + if not section.startswith("env:"): + continue + data['envLibdepsDirs'].append(join(libdeps_dir, section[4:])) + if config.has_option(section, "board"): + data['boards'].append(config.get(section, "board")) + data['libExtraDirs'].extend( + config.get(section, "lib_extra_dirs", [])) + + # skip non existing folders and resolve full path + for key in ("envLibdepsDirs", "libExtraDirs"): + data[key] = [ + expanduser(d) if d.startswith("~") else realpath(d) + for d in data[key] if isdir(d) + ] + + return data + + def _path_to_name(path): + return (sep).join(path.split(sep)[-2:]) + + if not project_dirs: + project_dirs = AppRPC.load_state()['storage']['recentProjects'] + + result = [] + pm = PlatformManager() + for project_dir in project_dirs: + data = {} + boards = [] + try: + with util.cd(project_dir): + data = _get_project_data(project_dir) + except exception.PlatformIOProjectException: + continue + + for board_id in data.get("boards", []): + name = board_id + try: + name = pm.board_config(board_id)['name'] + except (exception.UnknownBoard, exception.UnknownPlatform): + pass + boards.append({"id": board_id, "name": name}) + + result.append({ + "path": + project_dir, + "name": + _path_to_name(project_dir), + "modified": + int(getmtime(project_dir)), + "boards": + boards, + "envLibStorages": [{ + "name": basename(d), + "path": d + } for d in data.get("envLibdepsDirs", [])], + "extraLibStorages": [{ + "name": _path_to_name(d), + "path": d + } for d in data.get("libExtraDirs", [])] + }) + return result + + def get_projects(self, project_dirs=None): + return self._get_projects(project_dirs) + + @staticmethod + def get_project_examples(): + result = [] + for manifest in PlatformManager().get_installed(): + examples_dir = join(manifest['__pkg_dir'], "examples") + if not isdir(examples_dir): + continue + items = [] + for project_dir, _, __ in os.walk(examples_dir): + project_description = None + try: + config = ProjectConfig(join(project_dir, "platformio.ini")) + config.validate(silent=True) + project_description = config.get("platformio", + "description") + except exception.PlatformIOProjectException: + continue + + path_tokens = project_dir.split(sep) + items.append({ + "name": + "/".join(path_tokens[path_tokens.index("examples") + 1:]), + "path": + project_dir, + "description": + project_description + }) + result.append({ + "platform": { + "title": manifest['title'], + "version": manifest['version'] + }, + "items": sorted(items, key=lambda item: item['name']) + }) + return sorted(result, key=lambda data: data['platform']['title']) + + def init(self, board, framework, project_dir): + assert project_dir + state = AppRPC.load_state() + if not isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--board", board] + if framework: + args.extend(["--project-option", "framework = %s" % framework]) + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args, options={"cwd": project_dir}) + d.addCallback(self._generate_project_main, project_dir, framework) + return d + + @staticmethod + def _generate_project_main(_, project_dir, framework): + main_content = None + if framework == "arduino": + main_content = "\n".join([ + "#include ", + "", + "void setup() {", + " // put your setup code here, to run once:", + "}", + "", + "void loop() {", + " // put your main code here, to run repeatedly:", + "}" + "" + ]) # yapf: disable + elif framework == "mbed": + main_content = "\n".join([ + "#include ", + "", + "int main() {", + "", + " // put your setup code here, to run once:", + "", + " while(1) {", + " // put your main code here, to run repeatedly:", + " }", + "}", + "" + ]) # yapf: disable + if not main_content: + return project_dir + with util.cd(project_dir): + src_dir = get_project_src_dir() + main_path = join(src_dir, "main.cpp") + if isfile(main_path): + return project_dir + if not isdir(src_dir): + os.makedirs(src_dir) + with open(main_path, "w") as f: + f.write(main_content.strip()) + return project_dir + + def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + board = str(board) + if arduino_project_dir and PY2: + arduino_project_dir = arduino_project_dir.encode( + get_filesystem_encoding()) + # don't import PIO Project + if is_platformio_project(arduino_project_dir): + return arduino_project_dir + + is_arduino_project = any([ + isfile( + join(arduino_project_dir, + "%s.%s" % (basename(arduino_project_dir), ext))) + for ext in ("ino", "pde") + ]) + if not is_arduino_project: + raise jsonrpc.exceptions.JSONRPCDispatchException( + code=4000, + message="Not an Arduino project: %s" % arduino_project_dir) + + state = AppRPC.load_state() + project_dir = join(state['storage']['projectsDir'], + time.strftime("%y%m%d-%H%M%S-") + board) + if not isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--board", board] + args.extend(["--project-option", "framework = arduino"]) + if use_arduino_libs: + args.extend([ + "--project-option", + "lib_extra_dirs = ~/Documents/Arduino/libraries" + ]) + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args, options={"cwd": project_dir}) + d.addCallback(self._finalize_arduino_import, project_dir, + arduino_project_dir) + return d + + @staticmethod + def _finalize_arduino_import(_, project_dir, arduino_project_dir): + with util.cd(project_dir): + src_dir = get_project_src_dir() + if isdir(src_dir): + util.rmtree_(src_dir) + shutil.copytree(arduino_project_dir, src_dir) + return project_dir + + @staticmethod + def import_pio(project_dir): + if not project_dir or not is_platformio_project(project_dir): + raise jsonrpc.exceptions.JSONRPCDispatchException( + code=4001, + message="Not an PlatformIO project: %s" % project_dir) + new_project_dir = join( + AppRPC.load_state()['storage']['projectsDir'], + time.strftime("%y%m%d-%H%M%S-") + basename(project_dir)) + shutil.copytree(project_dir, new_project_dir) + + state = AppRPC.load_state() + args = ["init"] + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args, options={"cwd": new_project_dir}) + d.addCallback(lambda _: new_project_dir) + return d diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py new file mode 100644 index 00000000..36aa1dff --- /dev/null +++ b/platformio/commands/home/rpc/server.py @@ -0,0 +1,77 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=import-error + +import click +import jsonrpc +from autobahn.twisted.websocket import (WebSocketServerFactory, + WebSocketServerProtocol) +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet import defer + +from platformio.compat import PY2, dump_json_to_unicode, is_bytes + + +class JSONRPCServerProtocol(WebSocketServerProtocol): + + def onMessage(self, payload, isBinary): # pylint: disable=unused-argument + # click.echo("> %s" % payload) + response = jsonrpc.JSONRPCResponseManager.handle( + payload, self.factory.dispatcher).data + # if error + if "result" not in response: + self.sendJSONResponse(response) + return None + + d = defer.maybeDeferred(lambda: response['result']) + d.addCallback(self._callback, response) + d.addErrback(self._errback, response) + + return None + + def _callback(self, result, response): + response['result'] = result + self.sendJSONResponse(response) + + def _errback(self, failure, response): + if isinstance(failure.value, JSONRPCDispatchException): + e = failure.value + else: + e = JSONRPCDispatchException(code=4999, + message=failure.getErrorMessage()) + del response["result"] + response['error'] = e.error._data # pylint: disable=protected-access + self.sendJSONResponse(response) + + def sendJSONResponse(self, response): + # click.echo("< %s" % response) + if "error" in response: + click.secho("Error: %s" % response['error'], fg="red", err=True) + response = dump_json_to_unicode(response) + if not PY2 and not is_bytes(response): + response = response.encode("utf-8") + self.sendMessage(response) + + +class JSONRPCServerFactory(WebSocketServerFactory): + + protocol = JSONRPCServerProtocol + + def __init__(self): + super(JSONRPCServerFactory, self).__init__() + self.dispatcher = jsonrpc.Dispatcher() + + def addHandler(self, handler, namespace): + self.dispatcher.build_method_map(handler, prefix="%s." % namespace) diff --git a/platformio/commands/home/web.py b/platformio/commands/home/web.py new file mode 100644 index 00000000..df48b1fa --- /dev/null +++ b/platformio/commands/home/web.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import reactor # pylint: disable=import-error +from twisted.web import static # pylint: disable=import-error + + +class WebRoot(static.File): + + def render_GET(self, request): + if request.args.get("__shutdown__", False): + reactor.stop() + return "Server has been stopped" + + request.setHeader("cache-control", + "no-cache, no-store, must-revalidate") + request.setHeader("pragma", "no-cache") + request.setHeader("expires", "0") + return static.File.render_GET(self, request) diff --git a/platformio/commands/init.py b/platformio/commands/init.py index e7127313..09eddacf 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -16,16 +16,20 @@ from os import getcwd, makedirs from os.path import isdir, isfile, join -from shutil import copyfile import click from platformio import exception, util from platformio.commands.platform import \ platform_install as cli_platform_install -from platformio.commands.run import check_project_envs from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (get_project_include_dir, + get_project_lib_dir, + get_project_src_dir, + get_project_test_dir, + is_platformio_project) def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -40,22 +44,23 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 return value -@click.command( - "init", short_help="Initialize PlatformIO project or update existing") -@click.option( - "--project-dir", - "-d", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option( - "-b", "--board", multiple=True, metavar="ID", callback=validate_boards) -@click.option( - "--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) +@click.command("init", + short_help="Initialize PlatformIO project or update existing") +@click.option("--project-dir", + "-d", + default=getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-b", + "--board", + multiple=True, + metavar="ID", + callback=validate_boards) +@click.option("--ide", + type=click.Choice(ProjectGenerator.get_supported_ides())) @click.option("-O", "--project-option", multiple=True) @click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) @@ -68,38 +73,37 @@ def cli( project_option, env_prefix, silent): - if not silent: if project_dir == getcwd(): - click.secho( - "\nThe current working directory", fg="yellow", nl=False) + click.secho("\nThe current working directory", + fg="yellow", + nl=False) click.secho(" %s " % project_dir, fg="cyan", nl=False) click.secho("will be used for the project.", fg="yellow") click.echo("") click.echo("The next files/directories have been created in %s" % click.style(project_dir, fg="cyan")) - click.echo("%s - Put project header files here" % click.style( - "include", fg="cyan")) + click.echo("%s - Put project header files here" % + click.style("include", fg="cyan")) click.echo("%s - Put here project specific (private) libraries" % click.style("lib", fg="cyan")) - click.echo("%s - Put project source files here" % click.style( - "src", fg="cyan")) - click.echo("%s - Project Configuration File" % click.style( - "platformio.ini", fg="cyan")) + click.echo("%s - Put project source files here" % + click.style("src", fg="cyan")) + click.echo("%s - Project Configuration File" % + click.style("platformio.ini", fg="cyan")) - is_new_project = not util.is_platformio_project(project_dir) - init_base_project(project_dir) + is_new_project = not is_platformio_project(project_dir) + if is_new_project: + init_base_project(project_dir) if board: fill_project_envs(ctx, project_dir, board, project_option, env_prefix, ide is not None) if ide: - env_name = get_best_envname(project_dir, board) - if not env_name: - raise exception.BoardNotDefined() - pg = ProjectGenerator(project_dir, ide, env_name) + pg = ProjectGenerator(project_dir, ide, + get_best_envname(project_dir, board)) pg.generate() if is_new_project: @@ -112,8 +116,8 @@ def cli( if ide: click.secho( "\nProject has been successfully %s including configuration files " - "for `%s` IDE." % ("initialized" if is_new_project else "updated", - ide), + "for `%s` IDE." % + ("initialized" if is_new_project else "updated", ide), fg="green") else: click.secho( @@ -128,38 +132,36 @@ def cli( def get_best_envname(project_dir, boards=None): - config = util.load_project_config(project_dir) - env_default = None - if config.has_option("platformio", "env_default"): - env_default = util.parse_conf_multi_values( - config.get("platformio", "env_default")) - check_project_envs(config, env_default) - if env_default: - return env_default[0] - section = None - for section in config.sections(): - if not section.startswith("env:"): - continue - elif config.has_option(section, "board") and (not boards or config.get( - section, "board") in boards): - break - return section[4:] if section else None + config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) + config.validate() + + envname = None + default_envs = config.default_envs() + if default_envs: + envname = default_envs[0] + if not boards: + return envname + + for env in config.envs(): + if not boards: + return env + if not envname: + envname = env + items = config.items(env=env, as_dict=True) + if "board" in items and items.get("board") in boards: + return env + + return envname def init_base_project(project_dir): - if util.is_platformio_project(project_dir): - return - - copyfile( - join(util.get_source_dir(), "projectconftpl.ini"), - join(project_dir, "platformio.ini")) - + ProjectConfig(join(project_dir, "platformio.ini")).save() with util.cd(project_dir): dir_to_readme = [ - (util.get_projectsrc_dir(), None), - (util.get_projectinclude_dir(), init_include_readme), - (util.get_projectlib_dir(), init_lib_readme), - (util.get_projecttest_dir(), init_test_readme), + (get_project_src_dir(), None), + (get_project_include_dir(), init_include_readme), + (get_project_lib_dir(), init_lib_readme), + (get_project_test_dir(), init_test_readme), ] for (path, cb) in dir_to_readme: if isdir(path): @@ -360,16 +362,14 @@ def init_cvs_ignore(project_dir): if isfile(conf_path): return with open(conf_path, "w") as fp: - fp.writelines([".pio\n", ".pioenvs\n", ".piolibdeps\n"]) + fp.write(".pio\n") def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, force_download): - content = [] + config = ProjectConfig(join(project_dir, "platformio.ini"), + parse_extra=False) used_boards = [] - used_platforms = [] - - config = util.load_project_config(project_dir) for section in config.sections(): cond = [ section.startswith("env:"), @@ -379,12 +379,15 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, used_boards.append(config.get(section, "board")) pm = PlatformManager() + used_platforms = [] + modified = False for id_ in board_ids: board_config = pm.board_config(id_) used_platforms.append(board_config['platform']) if id_ in used_boards: continue used_boards.append(id_) + modified = True envopts = {"platform": board_config['platform'], "board": id_} # find default framework for board @@ -398,20 +401,18 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, _name, _value = item.split("=", 1) envopts[_name.strip()] = _value.strip() - content.append("") - content.append("[env:%s%s]" % (env_prefix, id_)) - for name, value in envopts.items(): - content.append("%s = %s" % (name, value)) + section = "env:%s%s" % (env_prefix, id_) + config.add_section(section) + + for option, value in envopts.items(): + config.set(section, option, value) if force_download and used_platforms: _install_dependent_platforms(ctx, used_platforms) - if not content: - return - - with open(join(project_dir, "platformio.ini"), "a") as f: - content.append("") - f.write("\n".join(content)) + if modified: + config.save() + config.reset_instances() def _install_dependent_platforms(ctx, platforms): @@ -420,6 +421,5 @@ def _install_dependent_platforms(ctx, platforms): ] if set(platforms) <= set(installed_platforms): return - ctx.invoke( - cli_platform_install, - platforms=list(set(platforms) - set(installed_platforms))) + ctx.invoke(cli_platform_install, + platforms=list(set(platforms) - set(installed_platforms))) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index c305dad6..1d75c961 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -14,172 +14,276 @@ # pylint: disable=too-many-branches, too-many-locals -import json import time from os.path import isdir, join -from urllib import quote import click +import semantic_version from platformio import exception, util -from platformio.managers.lib import LibraryManager, get_builtin_libs -from platformio.util import get_api_result +from platformio.commands import PlatformioCLI +from platformio.compat import dump_json_to_unicode +from platformio.managers.lib import (LibraryManager, get_builtin_libs, + is_builtin_lib) +from platformio.proc import is_ci +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (get_project_dir, + get_project_global_lib_dir, + get_project_libdeps_dir, + is_platformio_project) + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + +CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs" +CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments" +CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs" +CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" @click.group(short_help="Library Manager") +@click.option("-d", + "--storage-dir", + multiple=True, + default=None, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True), + help="Manage custom library storage") +@click.option("-g", + "--global", + is_flag=True, + help="Manage global PlatformIO library storage") @click.option( - "-g", - "--global", - is_flag=True, - help="Manage global PlatformIO library storage") -@click.option( - "-d", - "--storage-dir", - default=None, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True), - help="Manage custom library storage") + "-e", + "--environment", + multiple=True, + help=("Manage libraries for the specific project build environments " + "declared in `platformio.ini`")) @click.pass_context def cli(ctx, **options): - non_storage_cmds = ("search", "show", "register", "stats", "builtin") + storage_cmds = ("install", "uninstall", "update", "list") # skip commands that don't need storage folder - if ctx.invoked_subcommand in non_storage_cmds or \ + if ctx.invoked_subcommand not in storage_cmds or \ (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")): return - storage_dir = options['storage_dir'] - if not storage_dir: - if options['global']: - storage_dir = join(util.get_home_dir(), "lib") - elif util.is_platformio_project(): - storage_dir = util.get_projectlibdeps_dir() - elif util.is_ci(): - storage_dir = join(util.get_home_dir(), "lib") + storage_dirs = list(options['storage_dir']) + if options['global']: + storage_dirs.append(get_project_global_lib_dir()) + if not storage_dirs: + if is_platformio_project(): + storage_dirs = [get_project_dir()] + elif is_ci(): + storage_dirs = [get_project_global_lib_dir()] click.secho( "Warning! Global library storage is used automatically. " "Please use `platformio lib --global %s` command to remove " "this warning." % ctx.invoked_subcommand, fg="yellow") - elif util.is_platformio_project(storage_dir): - with util.cd(storage_dir): - storage_dir = util.get_projectlibdeps_dir() - if not storage_dir and not util.is_platformio_project(): - raise exception.NotGlobalLibDir(util.get_project_dir(), - join(util.get_home_dir(), "lib"), + if not storage_dirs: + raise exception.NotGlobalLibDir(get_project_dir(), + get_project_global_lib_dir(), ctx.invoked_subcommand) - ctx.obj = LibraryManager(storage_dir) - if "--json-output" not in ctx.args: - click.echo("Library Storage: " + storage_dir) + in_silence = PlatformioCLI.in_silence() + ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment'] + ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [] + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {} + for storage_dir in storage_dirs: + if not is_platformio_project(storage_dir): + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) + continue + with util.cd(storage_dir): + libdeps_dir = get_project_libdeps_dir() + config = ProjectConfig.get_instance(join(storage_dir, + "platformio.ini")) + config.validate(options['environment'], silent=in_silence) + for env in config.envs(): + if options['environment'] and env not in options['environment']: + continue + storage_dir = join(libdeps_dir, env) + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( + "env:" + env, "lib_deps", []) @cli.command("install", short_help="Install library") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") -# @click.option( -# "--save", -# is_flag=True, -# help="Save installed libraries into the project's platformio.ini " -# "library dependencies") @click.option( - "-s", "--silent", is_flag=True, help="Suppress progress reporting") -@click.option( - "--interactive", + "--save", is_flag=True, - help="Allow to make a choice for all prompts") -@click.option( - "-f", - "--force", - is_flag=True, - help="Reinstall/redownload library if exists") -@click.pass_obj -def lib_install(lm, libraries, silent, interactive, force): - # @TODO: "save" option - for library in libraries: - lm.install( - library, silent=silent, interactive=interactive, force=force) + help="Save installed libraries into the `platformio.ini` dependency list") +@click.option("-s", + "--silent", + is_flag=True, + help="Suppress progress reporting") +@click.option("--interactive", + is_flag=True, + help="Allow to make a choice for all prompts") +@click.option("-f", + "--force", + is_flag=True, + help="Reinstall/redownload library if exists") +@click.pass_context +def lib_install( # pylint: disable=too-many-arguments + ctx, libraries, save, silent, interactive, force): + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) + + installed_manifests = {} + for storage_dir in storage_dirs: + if not silent and (libraries or storage_dir in storage_libdeps): + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + if libraries: + for library in libraries: + pkg_dir = lm.install(library, + silent=silent, + interactive=interactive, + force=force) + installed_manifests[library] = lm.load_manifest(pkg_dir) + elif storage_dir in storage_libdeps: + builtin_lib_storages = None + for library in storage_libdeps[storage_dir]: + try: + pkg_dir = lm.install(library, + silent=silent, + interactive=interactive, + force=force) + installed_manifests[library] = lm.load_manifest(pkg_dir) + except exception.LibNotFound as e: + if builtin_lib_storages is None: + builtin_lib_storages = get_builtin_libs() + if not silent or not is_builtin_lib( + builtin_lib_storages, library): + click.secho("Warning! %s" % e, fg="yellow") + + if not save or not libraries: + return + + input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) + project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] + for input_dir in input_dirs: + config = ProjectConfig.get_instance(join(input_dir, "platformio.ini")) + config.validate(project_environments) + for env in config.envs(): + if project_environments and env not in project_environments: + continue + config.expand_interpolations = False + lib_deps = config.get("env:" + env, "lib_deps", []) + for library in libraries: + if library in lib_deps: + continue + manifest = installed_manifests[library] + try: + assert library.lower() == manifest['name'].lower() + assert semantic_version.Version(manifest['version']) + lib_deps.append("{name}@^{version}".format(**manifest)) + except (AssertionError, ValueError): + lib_deps.append(library) + config.set("env:" + env, "lib_deps", lib_deps) + config.save() @cli.command("uninstall", short_help="Uninstall libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") -@click.pass_obj -def lib_uninstall(lm, libraries): - for library in libraries: - lm.uninstall(library) +@click.pass_context +def lib_uninstall(ctx, libraries): + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + for storage_dir in storage_dirs: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + for library in libraries: + lm.uninstall(library) @cli.command("update", short_help="Update installed libraries") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") -@click.option( - "-c", - "--only-check", - is_flag=True, - help="Do not update, only check for new version") +@click.option("-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead") +@click.option("--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) -@click.pass_obj -def lib_update(lm, libraries, only_check, json_output): - if not libraries: - libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()] +@click.pass_context +def lib_update(ctx, libraries, only_check, dry_run, json_output): + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + only_check = dry_run or only_check + json_result = {} + for storage_dir in storage_dirs: + if not json_output: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) - if only_check and json_output: - result = [] - for library in libraries: - pkg_dir = library if isdir(library) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = lm.parse_pkg_uri(library) - pkg_dir = lm.get_package_dir(name, requirements, url) - if not pkg_dir: - continue - latest = lm.outdated(pkg_dir, requirements) - if not latest: - continue - manifest = lm.load_manifest(pkg_dir) - manifest['versionLatest'] = latest - result.append(manifest) - return click.echo(json.dumps(result)) - else: - for library in libraries: - lm.update(library, only_check=only_check) + _libraries = libraries + if not _libraries: + _libraries = [ + manifest['__pkg_dir'] for manifest in lm.get_installed() + ] + + if only_check and json_output: + result = [] + for library in _libraries: + pkg_dir = library if isdir(library) else None + requirements = None + url = None + if not pkg_dir: + name, requirements, url = lm.parse_pkg_uri(library) + pkg_dir = lm.get_package_dir(name, requirements, url) + if not pkg_dir: + continue + latest = lm.outdated(pkg_dir, requirements) + if not latest: + continue + manifest = lm.load_manifest(pkg_dir) + manifest['versionLatest'] = latest + result.append(manifest) + json_result[storage_dir] = result + else: + for library in _libraries: + lm.update(library, only_check=only_check) + + if json_output: + return click.echo( + dump_json_to_unicode(json_result[storage_dirs[0]] + if len(storage_dirs) == 1 else json_result)) return True -def print_lib_item(item): - click.secho(item['name'], fg="cyan") - click.echo("=" * len(item['name'])) - if "id" in item: - click.secho("#ID: %d" % item['id'], bold=True) - if "description" in item or "url" in item: - click.echo(item.get("description", item.get("url", ""))) - click.echo() - - for key in ("version", "homepage", "license", "keywords"): - if key not in item or not item[key]: - continue - if isinstance(item[key], list): - click.echo("%s: %s" % (key.title(), ", ".join(item[key]))) +@cli.command("list", short_help="List installed libraries") +@click.option("--json-output", is_flag=True) +@click.pass_context +def lib_list(ctx, json_output): + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + json_result = {} + for storage_dir in storage_dirs: + if not json_output: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + items = lm.get_installed() + if json_output: + json_result[storage_dir] = items + elif items: + for item in sorted(items, key=lambda i: i['name']): + print_lib_item(item) else: - click.echo("%s: %s" % (key.title(), item[key])) + click.echo("No items found") - for key in ("frameworks", "platforms"): - if key not in item: - continue - click.echo("Compatible %s: %s" % (key, ", ".join( - [i['title'] if isinstance(i, dict) else i for i in item[key]]))) + if json_output: + return click.echo( + dump_json_to_unicode(json_result[storage_dirs[0]] + if len(storage_dirs) == 1 else json_result)) - if "authors" in item or "authornames" in item: - click.echo("Authors: %s" % ", ".join( - item.get("authornames", - [a.get("name", "") for a in item.get("authors", [])]))) - - if "__src_url" in item: - click.secho("Source: %s" % item['__src_url']) - click.echo() + return True @cli.command("search", short_help="Search for a library") @@ -193,10 +297,9 @@ def print_lib_item(item): @click.option("-f", "--framework", multiple=True) @click.option("-p", "--platform", multiple=True) @click.option("-i", "--header", multiple=True) -@click.option( - "--noninteractive", - is_flag=True, - help="Do not prompt, automatically paginate with delay") +@click.option("--noninteractive", + is_flag=True, + help="Do not prompt, automatically paginate with delay") def lib_search(query, json_output, page, noninteractive, **filters): if not query: query = [] @@ -207,13 +310,12 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) - result = get_api_result( - "/v2/lib/search", - dict(query=" ".join(query), page=page), - cache_valid="1d") + result = util.get_api_result("/v2/lib/search", + dict(query=" ".join(query), page=page), + cache_valid="1d") if json_output: - click.echo(json.dumps(result)) + click.echo(dump_json_to_unicode(result)) return if result['total'] == 0: @@ -232,9 +334,8 @@ def lib_search(query, json_output, page, noninteractive, **filters): fg="cyan") return - click.secho( - "Found %d libraries:\n" % result['total'], - fg="green" if result['total'] else "yellow") + click.secho("Found %d libraries:\n" % result['total'], + fg="green" if result['total'] else "yellow") while True: for item in result['items']: @@ -246,38 +347,18 @@ def lib_search(query, json_output, page, noninteractive, **filters): if noninteractive: click.echo() - click.secho( - "Loading next %d libraries... Press Ctrl+C to stop!" % - result['perpage'], - fg="yellow") + click.secho("Loading next %d libraries... Press Ctrl+C to stop!" % + result['perpage'], + fg="yellow") click.echo() time.sleep(5) elif not click.confirm("Show next libraries?"): break - result = get_api_result( - "/v2/lib/search", { - "query": " ".join(query), - "page": int(result['page']) + 1 - }, - cache_valid="1d") - - -@cli.command("list", short_help="List installed libraries") -@click.option("--json-output", is_flag=True) -@click.pass_obj -def lib_list(lm, json_output): - items = lm.get_installed() - - if json_output: - return click.echo(json.dumps(items)) - - if not items: - return None - - for item in sorted(items, key=lambda i: i['name']): - print_lib_item(item) - - return True + result = util.get_api_result("/v2/lib/search", { + "query": " ".join(query), + "page": int(result['page']) + 1 + }, + cache_valid="1d") @cli.command("builtin", short_help="List built-in libraries") @@ -286,7 +367,7 @@ def lib_list(lm, json_output): def lib_builtin(storage, json_output): items = get_builtin_libs(storage) if json_output: - return click.echo(json.dumps(items)) + return click.echo(dump_json_to_unicode(items)) for storage_ in items: if not storage_['items']: @@ -313,9 +394,9 @@ def lib_show(library, json_output): }, silent=json_output, interactive=not json_output) - lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") + lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: - return click.echo(json.dumps(lib)) + return click.echo(dump_json_to_unicode(lib)) click.secho(lib['name'], fg="cyan") click.echo("=" * len(lib['name'])) @@ -389,21 +470,21 @@ def lib_register(config_url): and not config_url.startswith("https://")): raise exception.InvalidLibConfURL(config_url) - result = get_api_result("/lib/register", data=dict(config_url=config_url)) + result = util.get_api_result("/lib/register", + data=dict(config_url=config_url)) if "message" in result and result['message']: - click.secho( - result['message'], - fg="green" - if "successed" in result and result['successed'] else "red") + click.secho(result['message'], + fg="green" if "successed" in result and result['successed'] + else "red") @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): - result = get_api_result("/lib/stats", cache_valid="1h") + result = util.get_api_result("/lib/stats", cache_valid="1h") if json_output: - return click.echo(json.dumps(result)) + return click.echo(dump_json_to_unicode(result)) printitem_tpl = "{name:<33} {url}" printitemdate_tpl = "{name:<33} {date:23} {url}" @@ -425,10 +506,9 @@ def lib_stats(json_output): date = str( time.strftime("%c", util.parse_date(item['date'])) if "date" in item else "") - url = click.style( - "https://platformio.org/lib/show/%s/%s" % (item['id'], - quote(item['name'])), - fg="blue") + url = click.style("https://platformio.org/lib/show/%s/%s" % + (item['id'], quote(item['name'])), + fg="blue") click.echo( (printitemdate_tpl if "date" in item else printitem_tpl).format( name=click.style(item['name'], fg="cyan"), date=date, url=url)) @@ -437,10 +517,9 @@ def lib_stats(json_output): click.echo( printitem_tpl.format( name=click.style(name, fg="cyan"), - url=click.style( - "https://platformio.org/lib/search?query=" + quote( - "keyword:%s" % name), - fg="blue"))) + url=click.style("https://platformio.org/lib/search?query=" + + quote("keyword:%s" % name), + fg="blue"))) for key in ("updated", "added"): _print_title("Recently " + key) @@ -470,3 +549,44 @@ def lib_stats(json_output): click.echo() return True + + +def print_storage_header(storage_dirs, storage_dir): + if storage_dirs and storage_dirs[0] != storage_dir: + click.echo("") + click.echo( + click.style("Library Storage: ", bold=True) + + click.style(storage_dir, fg="blue")) + + +def print_lib_item(item): + click.secho(item['name'], fg="cyan") + click.echo("=" * len(item['name'])) + if "id" in item: + click.secho("#ID: %d" % item['id'], bold=True) + if "description" in item or "url" in item: + click.echo(item.get("description", item.get("url", ""))) + click.echo() + + for key in ("version", "homepage", "license", "keywords"): + if key not in item or not item[key]: + continue + if isinstance(item[key], list): + click.echo("%s: %s" % (key.title(), ", ".join(item[key]))) + else: + click.echo("%s: %s" % (key.title(), item[key])) + + for key in ("frameworks", "platforms"): + if key not in item: + continue + click.echo("Compatible %s: %s" % (key, ", ".join( + [i['title'] if isinstance(i, dict) else i for i in item[key]]))) + + if "authors" in item or "authornames" in item: + click.echo("Authors: %s" % ", ".join( + item.get("authornames", + [a.get("name", "") for a in item.get("authors", [])]))) + + if "__src_url" in item: + click.secho("Source: %s" % item['__src_url']) + click.echo() diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 017dbc85..5b35c4df 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json from os.path import dirname, isdir import click from platformio import app, exception, util from platformio.commands.boards import print_boards +from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory, PlatformManager @@ -29,9 +29,9 @@ def cli(): def _print_platforms(platforms): for platform in platforms: - click.echo("{name} ~ {title}".format( - name=click.style(platform['name'], fg="cyan"), - title=platform['title'])) + click.echo("{name} ~ {title}".format(name=click.style(platform['name'], + fg="cyan"), + title=platform['title'])) click.echo("=" * (3 + len(platform['name'] + platform['title']))) click.echo(platform['description']) click.echo() @@ -42,7 +42,11 @@ def _print_platforms(platforms): if "packages" in platform: click.echo("Packages: %s" % ", ".join(platform['packages'])) if "version" in platform: - click.echo("Version: " + platform['version']) + if "__src_url" in platform: + click.echo("Version: #%s (%s)" % + (platform['version'], platform['__src_url'])) + else: + click.echo("Version: " + platform['version']) click.echo() @@ -54,18 +58,6 @@ def _get_registry_platforms(): return platforms -def _original_version(version): - if version.count(".") != 2: - return None - _, y = version.split(".")[:2] - if int(y) < 100: - return None - if len(y) % 2 != 0: - y = "0" + y - parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(len(y) / 2)] - return ".".join(parts) - - def _get_platform_data(*args, **kwargs): try: return _get_installed_platform_data(*args, **kwargs) @@ -77,19 +69,18 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): p = PlatformFactory.newPlatform(platform) - data = dict( - name=p.name, - title=p.title, - description=p.description, - version=p.version, - homepage=p.homepage, - repository=p.repository_url, - url=p.vendor_url, - docs=p.docs_url, - license=p.license, - forDesktop=not p.is_embedded(), - frameworks=sorted(p.frameworks.keys() if p.frameworks else []), - packages=p.packages.keys() if p.packages else []) + data = dict(name=p.name, + title=p.title, + description=p.description, + version=p.version, + homepage=p.homepage, + repository=p.repository_url, + url=p.vendor_url, + docs=p.docs_url, + license=p.license, + forDesktop=not p.is_embedded(), + frameworks=sorted(list(p.frameworks) if p.frameworks else []), + packages=list(p.packages) if p.packages else []) # if dump to API # del data['version'] @@ -111,18 +102,17 @@ def _get_installed_platform_data(platform, data['packages'] = [] installed_pkgs = p.get_installed_packages() for name, opts in p.packages.items(): - item = dict( - name=name, - type=p.get_package_type(name), - requirements=opts.get("version"), - optional=opts.get("optional") is True) + item = dict(name=name, + type=p.get_package_type(name), + requirements=opts.get("version"), + optional=opts.get("optional") is True) if name in installed_pkgs: for key, value in installed_pkgs[name].items(): if key not in ("url", "version", "description"): continue item[key] = value if key == "version": - item["originalVersion"] = _original_version(value) + item["originalVersion"] = util.get_original_version(value) data['packages'].append(item) return data @@ -141,18 +131,17 @@ def _get_registry_platform_data( # pylint: disable=unused-argument if not _data: return None - data = dict( - name=_data['name'], - title=_data['title'], - description=_data['description'], - homepage=_data['homepage'], - repository=_data['repository'], - url=_data['url'], - license=_data['license'], - forDesktop=_data['forDesktop'], - frameworks=_data['frameworks'], - packages=_data['packages'], - versions=_data['versions']) + data = dict(name=_data['name'], + title=_data['title'], + description=_data['description'], + homepage=_data['homepage'], + repository=_data['repository'], + url=_data['url'], + license=_data['license'], + forDesktop=_data['forDesktop'], + frameworks=_data['frameworks'], + packages=_data['packages'], + versions=_data['versions']) if with_boards: data['boards'] = [ @@ -171,15 +160,16 @@ def platform_search(query, json_output): for platform in _get_registry_platforms(): if query == "all": query = "" - search_data = json.dumps(platform) + search_data = dump_json_to_unicode(platform) if query and query.lower() not in search_data.lower(): continue platforms.append( - _get_registry_platform_data( - platform['name'], with_boards=False, expose_packages=False)) + _get_registry_platform_data(platform['name'], + with_boards=False, + expose_packages=False)) if json_output: - click.echo(json.dumps(platforms)) + click.echo(dump_json_to_unicode(platforms)) else: _print_platforms(platforms) @@ -192,11 +182,11 @@ def platform_frameworks(query, json_output): for framework in util.get_api_result("/frameworks", cache_valid="7d"): if query == "all": query = "" - search_data = json.dumps(framework) + search_data = dump_json_to_unicode(framework) if query and query.lower() not in search_data.lower(): continue - framework['homepage'] = ( - "https://platformio.org/frameworks/" + framework['name']) + framework['homepage'] = ("https://platformio.org/frameworks/" + + framework['name']) framework['platforms'] = [ platform['name'] for platform in _get_registry_platforms() if framework['name'] in platform['frameworks'] @@ -205,7 +195,7 @@ def platform_frameworks(query, json_output): frameworks = sorted(frameworks, key=lambda manifest: manifest['name']) if json_output: - click.echo(json.dumps(frameworks)) + click.echo(dump_json_to_unicode(frameworks)) else: _print_platforms(frameworks) @@ -217,14 +207,13 @@ def platform_list(json_output): pm = PlatformManager() for manifest in pm.get_installed(): platforms.append( - _get_installed_platform_data( - manifest['__pkg_dir'], - with_boards=False, - expose_packages=False)) + _get_installed_platform_data(manifest['__pkg_dir'], + with_boards=False, + expose_packages=False)) platforms = sorted(platforms, key=lambda manifest: manifest['name']) if json_output: - click.echo(json.dumps(platforms)) + click.echo(dump_json_to_unicode(platforms)) else: _print_platforms(platforms) @@ -237,10 +226,11 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if not data: raise exception.UnknownPlatform(platform) if json_output: - return click.echo(json.dumps(data)) + return click.echo(dump_json_to_unicode(data)) - click.echo("{name} ~ {title}".format( - name=click.style(data['name'], fg="cyan"), title=data['title'])) + click.echo("{name} ~ {title}".format(name=click.style(data['name'], + fg="cyan"), + title=data['title'])) click.echo("=" * (3 + len(data['name'] + data['title']))) click.echo(data['description']) click.echo() @@ -305,17 +295,15 @@ def platform_install(platforms, with_package, without_package, skip_default_package, force): pm = PlatformManager() for platform in platforms: - if pm.install( - name=platform, - with_packages=with_package, - without_packages=without_package, - skip_default_package=skip_default_package, - force=force): - click.secho( - "The platform '%s' has been successfully installed!\n" - "The rest of packages will be installed automatically " - "depending on your build environment." % platform, - fg="green") + if pm.install(name=platform, + with_packages=with_package, + without_packages=without_package, + skip_default_package=skip_default_package, + force=force): + click.secho("The platform '%s' has been successfully installed!\n" + "The rest of packages will be installed automatically " + "depending on your build environment." % platform, + fg="green") @cli.command("uninstall", short_help="Uninstall development platform") @@ -324,26 +312,27 @@ def platform_uninstall(platforms): pm = PlatformManager() for platform in platforms: if pm.uninstall(platform): - click.secho( - "The platform '%s' has been successfully " - "uninstalled!" % platform, - fg="green") + click.secho("The platform '%s' has been successfully " + "uninstalled!" % platform, + fg="green") @cli.command("update", short_help="Update installed development platforms") @click.argument("platforms", nargs=-1, required=False, metavar="[PLATFORM...]") -@click.option( - "-p", - "--only-packages", - is_flag=True, - help="Update only the platform packages") -@click.option( - "-c", - "--only-check", - is_flag=True, - help="Do not update, only check for a new version") +@click.option("-p", + "--only-packages", + is_flag=True, + help="Update only the platform packages") +@click.option("-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead") +@click.option("--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) -def platform_update(platforms, only_packages, only_check, json_output): +def platform_update( # pylint: disable=too-many-locals + platforms, only_packages, only_check, dry_run, json_output): pm = PlatformManager() pkg_dir_to_name = {} if not platforms: @@ -353,6 +342,8 @@ def platform_update(platforms, only_packages, only_check, json_output): pkg_dir_to_name[manifest['__pkg_dir']] = manifest.get( "title", manifest['name']) + only_check = dry_run or only_check + if only_check and json_output: result = [] for platform in platforms: @@ -368,21 +359,22 @@ def platform_update(platforms, only_packages, only_check, json_output): if (not latest and not PlatformFactory.newPlatform( pkg_dir).are_outdated_packages()): continue - data = _get_installed_platform_data( - pkg_dir, with_boards=False, expose_packages=False) + data = _get_installed_platform_data(pkg_dir, + with_boards=False, + expose_packages=False) if latest: data['versionLatest'] = latest result.append(data) - return click.echo(json.dumps(result)) - else: - # cleanup cached board and platform lists - app.clean_cache() - for platform in platforms: - click.echo("Platform %s" % click.style( - pkg_dir_to_name.get(platform, platform), fg="cyan")) - click.echo("--------") - pm.update( - platform, only_packages=only_packages, only_check=only_check) - click.echo() + return click.echo(dump_json_to_unicode(result)) + + # cleanup cached board and platform lists + app.clean_cache() + for platform in platforms: + click.echo( + "Platform %s" % + click.style(pkg_dir_to_name.get(platform, platform), fg="cyan")) + click.echo("--------") + pm.update(platform, only_packages=only_packages, only_check=only_check) + click.echo() return True diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index 9b7ad5fb..8dcdf9a2 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -23,6 +23,7 @@ import click from platformio import exception, util from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.compat import get_file_contents from platformio.managers.core import pioplus_call # pylint: disable=unused-argument @@ -42,12 +43,13 @@ def remote_agent(): @remote_agent.command("start", short_help="Start agent") @click.option("-n", "--name") @click.option("-s", "--share", multiple=True, metavar="E-MAIL") -@click.option( - "-d", - "--working-dir", - envvar="PLATFORMIO_REMOTE_AGENT_DIR", - type=click.Path( - file_okay=False, dir_okay=True, writable=True, resolve_path=True)) +@click.option("-d", + "--working-dir", + envvar="PLATFORMIO_REMOTE_AGENT_DIR", + type=click.Path(file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) def remote_agent_start(**kwargs): pioplus_call(sys.argv[1:]) @@ -62,14 +64,16 @@ def remote_agent_list(): pioplus_call(sys.argv[1:]) -@cli.command( - "update", short_help="Update installed Platforms, Packages and Libraries") -@click.option( - "-c", - "--only-check", - is_flag=True, - help="Do not update, only check for new version") -def remote_update(only_check): +@cli.command("update", + short_help="Update installed Platforms, Packages and Libraries") +@click.option("-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead") +@click.option("--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") +def remote_update(only_check, dry_run): pioplus_call(sys.argv[1:]) @@ -77,16 +81,14 @@ def remote_update(only_check): @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=True, + dir_okay=True, + writable=True, + resolve_path=True)) @click.option("--disable-auto-clean", is_flag=True) @click.option("-r", "--force-remote", is_flag=True) @click.option("-s", "--silent", is_flag=True) @@ -100,16 +102,14 @@ def remote_run(**kwargs): @click.option("--ignore", "-i", multiple=True, metavar="") @click.option("--upload-port") @click.option("--test-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) @click.option("-r", "--force-remote", is_flag=True) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @@ -131,59 +131,55 @@ def device_list(json_output): @remote_device.command("monitor", short_help="Monitor remote device") @click.option("--port", "-p", help="Port, a number or a device name") -@click.option( - "--baud", "-b", type=int, default=9600, help="Set baud rate, default=9600") -@click.option( - "--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N") -@click.option( - "--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") -@click.option( - "--xonxoff", - is_flag=True, - help="Enable software flow control, default=Off") -@click.option( - "--rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state") -@click.option( - "--dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state") +@click.option("--baud", + "-b", + type=int, + default=9600, + help="Set baud rate, default=9600") +@click.option("--parity", + default="N", + type=click.Choice(["N", "E", "O", "S", "M"]), + help="Set parity, default=N") +@click.option("--rtscts", + is_flag=True, + help="Enable RTS/CTS flow control, default=Off") +@click.option("--xonxoff", + is_flag=True, + help="Enable software flow control, default=Off") +@click.option("--rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state") +@click.option("--dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state") @click.option("--echo", is_flag=True, help="Enable local echo, default=Off") -@click.option( - "--encoding", - default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8") +@click.option("--encoding", + default="UTF-8", + help="Set the encoding for the serial port (e.g. hexlify, " + "Latin1, UTF-8), default: UTF-8") @click.option("--filter", "-f", multiple=True, help="Add text transformation") -@click.option( - "--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF") -@click.option( - "--raw", is_flag=True, help="Do not apply any encodings/transformations") -@click.option( - "--exit-char", - type=int, - default=3, - help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)") -@click.option( - "--menu-char", - type=int, - default=20, - help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)") -@click.option( - "--quiet", - is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off") +@click.option("--eol", + default="CRLF", + type=click.Choice(["CR", "LF", "CRLF"]), + help="End of line mode, default=CRLF") +@click.option("--raw", + is_flag=True, + help="Do not apply any encodings/transformations") +@click.option("--exit-char", + type=int, + default=3, + help="ASCII code of special character that is used to exit " + "the application, default=3 (Ctrl+C)") +@click.option("--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control miniterm (menu), default=20 (DEC)") +@click.option("--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages, default=Off") @click.pass_context def device_monitor(ctx, **kwargs): @@ -202,7 +198,7 @@ def device_monitor(ctx, **kwargs): sleep(0.1) if not t.is_alive(): return - kwargs['port'] = open(sock_file).read() + kwargs['port'] = get_file_contents(sock_file) ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: diff --git a/platformio/commands/run.py b/platformio/commands/run.py deleted file mode 100644 index 47f14f81..00000000 --- a/platformio/commands/run.py +++ /dev/null @@ -1,435 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from hashlib import sha1 -from os import getcwd, makedirs, walk -from os.path import getmtime, isdir, isfile, join -from time import time - -import click - -from platformio import __version__, exception, telemetry, util -from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.commands.lib import lib_install as cmd_lib_install -from platformio.commands.platform import \ - platform_install as cmd_platform_install -from platformio.managers.lib import LibraryManager, is_builtin_lib -from platformio.managers.platform import PlatformFactory - -# pylint: disable=too-many-arguments,too-many-locals,too-many-branches - - -@click.command("run", short_help="Process project environments") -@click.option("-e", "--environment", multiple=True) -@click.option("-t", "--target", multiple=True) -@click.option("--upload-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("-s", "--silent", is_flag=True) -@click.option("-v", "--verbose", is_flag=True) -@click.option("--disable-auto-clean", is_flag=True) -@click.pass_context -def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, - disable_auto_clean): - # find project directory on upper level - if isfile(project_dir): - project_dir = util.find_project_dir_above(project_dir) - - if not util.is_platformio_project(project_dir): - raise exception.NotPlatformIOProject(project_dir) - - with util.cd(project_dir): - # clean obsolete build dir - if not disable_auto_clean: - try: - _clean_build_dir(util.get_projectbuild_dir()) - except: # pylint: disable=bare-except - click.secho( - "Can not remove temporary directory `%s`. Please remove " - "it manually to avoid build issues" % - util.get_projectbuild_dir(force=True), - fg="yellow") - - config = util.load_project_config() - env_default = None - if config.has_option("platformio", "env_default"): - env_default = util.parse_conf_multi_values( - config.get("platformio", "env_default")) - - check_project_defopts(config) - check_project_envs(config, environment or env_default) - - results = [] - start_time = time() - for section in config.sections(): - if not section.startswith("env:"): - continue - - envname = section[4:] - skipenv = any([ - environment and envname not in environment, not environment - and env_default and envname not in env_default - ]) - if skipenv: - results.append((envname, None)) - continue - - if not silent and results: - click.echo() - - options = {} - for k, v in config.items(section): - options[k] = v - if "piotest" not in options and "piotest" in ctx.meta: - options['piotest'] = ctx.meta['piotest'] - - ep = EnvironmentProcessor(ctx, envname, options, target, - upload_port, silent, verbose) - result = (envname, ep.process()) - results.append(result) - if result[1] and "monitor" in ep.get_build_targets() and \ - "nobuild" not in ep.get_build_targets(): - ctx.invoke( - cmd_device_monitor, - environment=environment[0] if environment else None) - - found_error = any(status is False for (_, status) in results) - - if (found_error or not silent) and len(results) > 1: - click.echo() - print_summary(results, start_time) - - if found_error: - raise exception.ReturnErrorCode(1) - return True - - -class EnvironmentProcessor(object): - - DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board") - - KNOWN_PLATFORMIO_OPTIONS = [ - "description", "env_default", "home_dir", "lib_dir", "libdeps_dir", - "include_dir", "src_dir", "build_dir", "data_dir", "test_dir", - "boards_dir", "lib_extra_dirs" - ] - - KNOWN_ENV_OPTIONS = [ - "platform", "framework", "board", "build_flags", "src_build_flags", - "build_unflags", "src_filter", "extra_scripts", "targets", - "upload_port", "upload_protocol", "upload_speed", "upload_flags", - "upload_resetmethod", "lib_deps", "lib_ignore", "lib_extra_dirs", - "lib_ldf_mode", "lib_compat_mode", "lib_archive", "piotest", - "test_transport", "test_filter", "test_ignore", "test_port", - "test_speed", "test_build_project_src", "debug_tool", "debug_port", - "debug_init_cmds", "debug_extra_cmds", "debug_server", - "debug_init_break", "debug_load_cmd", "debug_load_mode", - "debug_svd_path", "monitor_port", "monitor_speed", "monitor_rts", - "monitor_dtr" - ] - - IGNORE_BUILD_OPTIONS = [ - "test_transport", "test_filter", "test_ignore", "test_port", - "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds", - "debug_server", "debug_init_break", "debug_load_cmd", - "debug_load_mode", "monitor_port", "monitor_speed", "monitor_rts", - "monitor_dtr" - ] - - REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"} - - RENAMED_OPTIONS = { - "lib_use": "lib_deps", - "lib_force": "lib_deps", - "extra_script": "extra_scripts", - "monitor_baud": "monitor_speed", - "board_mcu": "board_build.mcu", - "board_f_cpu": "board_build.f_cpu", - "board_f_flash": "board_build.f_flash", - "board_flash_mode": "board_build.flash_mode" - } - - RENAMED_PLATFORMS = {"espressif": "espressif8266"} - - def __init__( - self, # pylint: disable=R0913 - cmd_ctx, - name, - options, - targets, - upload_port, - silent, - verbose): - self.cmd_ctx = cmd_ctx - self.name = name - self.options = options - self.targets = targets - self.upload_port = upload_port - self.silent = silent - self.verbose = verbose - - def process(self): - terminal_width, _ = click.get_terminal_size() - start_time = time() - env_dump = [] - - for k, v in self.options.items(): - self.options[k] = self.options[k].strip() - if self.verbose or k in self.DEFAULT_DUMP_OPTIONS: - env_dump.append( - "%s: %s" % (k, ", ".join(util.parse_conf_multi_values(v)))) - - if not self.silent: - click.echo("Processing %s (%s)" % (click.style( - self.name, fg="cyan", bold=True), "; ".join(env_dump))) - click.secho("-" * terminal_width, bold=True) - - self.options = self._validate_options(self.options) - result = self._run() - is_error = result['returncode'] != 0 - - if self.silent and not is_error: - return True - - if is_error or "piotest_processor" not in self.cmd_ctx.meta: - print_header( - "[%s] Took %.2f seconds" % ( - (click.style("ERROR", fg="red", bold=True) if is_error else - click.style("SUCCESS", fg="green", bold=True)), - time() - start_time), - is_error=is_error) - - return not is_error - - def _validate_options(self, options): - result = {} - for k, v in options.items(): - # process obsolete options - if k in self.RENAMED_OPTIONS: - click.secho( - "Warning! `%s` option is deprecated and will be " - "removed in the next release! Please use " - "`%s` instead." % (k, self.RENAMED_OPTIONS[k]), - fg="yellow") - k = self.RENAMED_OPTIONS[k] - # process renamed platforms - if k == "platform" and v in self.RENAMED_PLATFORMS: - click.secho( - "Warning! Platform `%s` is deprecated and will be " - "removed in the next release! Please use " - "`%s` instead." % (v, self.RENAMED_PLATFORMS[v]), - fg="yellow") - v = self.RENAMED_PLATFORMS[v] - - # warn about unknown options - unknown_conditions = [ - k not in self.KNOWN_ENV_OPTIONS, not k.startswith("custom_"), - not k.startswith("board_") - ] - if all(unknown_conditions): - click.secho( - "Detected non-PlatformIO `%s` option in `[env:%s]` section" - % (k, self.name), - fg="yellow") - result[k] = v - return result - - def get_build_variables(self): - variables = {"pioenv": self.name} - if self.upload_port: - variables['upload_port'] = self.upload_port - for k, v in self.options.items(): - if k in self.REMAPED_OPTIONS: - k = self.REMAPED_OPTIONS[k] - if k in self.IGNORE_BUILD_OPTIONS: - continue - if k == "targets" or (k == "upload_port" and self.upload_port): - continue - variables[k] = v - return variables - - def get_build_targets(self): - targets = [] - if self.targets: - targets = [t for t in self.targets] - elif "targets" in self.options: - targets = self.options['targets'].split(", ") - return targets - - def _run(self): - if "platform" not in self.options: - raise exception.UndefinedEnvPlatform(self.name) - - build_vars = self.get_build_variables() - build_targets = self.get_build_targets() - - telemetry.on_run_environment(self.options, build_targets) - - # skip monitor target, we call it above - if "monitor" in build_targets: - build_targets.remove("monitor") - if "nobuild" not in build_targets: - # install dependent libraries - if "lib_install" in self.options: - _autoinstall_libdeps(self.cmd_ctx, [ - int(d.strip()) - for d in self.options['lib_install'].split(",") - if d.strip() - ], self.verbose) - if "lib_deps" in self.options: - _autoinstall_libdeps( - self.cmd_ctx, - util.parse_conf_multi_values(self.options['lib_deps']), - self.verbose) - - try: - p = PlatformFactory.newPlatform(self.options['platform']) - except exception.UnknownPlatform: - self.cmd_ctx.invoke( - cmd_platform_install, - platforms=[self.options['platform']], - skip_default_package=True) - p = PlatformFactory.newPlatform(self.options['platform']) - - return p.run(build_vars, build_targets, self.silent, self.verbose) - - -def _autoinstall_libdeps(ctx, libraries, verbose=False): - if not libraries: - return - storage_dir = util.get_projectlibdeps_dir() - ctx.obj = LibraryManager(storage_dir) - if verbose: - click.echo("Library Storage: " + storage_dir) - for lib in libraries: - try: - ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose) - except exception.LibNotFound as e: - if verbose or not is_builtin_lib(lib): - click.secho("Warning! %s" % e, fg="yellow") - except exception.InternetIsOffline as e: - click.secho(str(e), fg="yellow") - - -def _clean_build_dir(build_dir): - structhash_file = join(build_dir, "structure.hash") - proj_hash = calculate_project_hash() - - # if project's config is modified - if (isdir(build_dir) - and getmtime(join(util.get_project_dir(), - "platformio.ini")) > getmtime(build_dir)): - util.rmtree_(build_dir) - - # check project structure - if isdir(build_dir) and isfile(structhash_file): - with open(structhash_file) as f: - if f.read() == proj_hash: - return - util.rmtree_(build_dir) - - if not isdir(build_dir): - makedirs(build_dir) - - with open(structhash_file, "w") as f: - f.write(proj_hash) - - -def print_header(label, is_error=False): - terminal_width, _ = click.get_terminal_size() - width = len(click.unstyle(label)) - half_line = "=" * ((terminal_width - width - 2) / 2) - click.echo("%s %s %s" % (half_line, label, half_line), err=is_error) - - -def print_summary(results, start_time): - print_header("[%s]" % click.style("SUMMARY")) - - envname_max_len = 0 - for (envname, _) in results: - if len(envname) > envname_max_len: - envname_max_len = len(envname) - - successed = True - for (envname, status) in results: - status_str = click.style("SUCCESS", fg="green") - if status is False: - successed = False - status_str = click.style("ERROR", fg="red") - elif status is None: - status_str = click.style("SKIP", fg="yellow") - - format_str = ( - "Environment {0:<" + str(envname_max_len + 9) + "}\t[{1}]") - click.echo( - format_str.format(click.style(envname, fg="cyan"), status_str), - err=status is False) - - print_header( - "[%s] Took %.2f seconds" % ( - (click.style("SUCCESS", fg="green", bold=True) if successed else - click.style("ERROR", fg="red", bold=True)), time() - start_time), - is_error=not successed) - - -def check_project_defopts(config): - if not config.has_section("platformio"): - return True - unknown = set([k for k, _ in config.items("platformio")]) - set( - EnvironmentProcessor.KNOWN_PLATFORMIO_OPTIONS) - if not unknown: - return True - click.secho( - "Warning! Ignore unknown `%s` option in `[platformio]` section" % - ", ".join(unknown), - fg="yellow") - return False - - -def check_project_envs(config, environments=None): - if not config.sections(): - raise exception.ProjectEnvsNotAvailable() - - known = set([s[4:] for s in config.sections() if s.startswith("env:")]) - unknown = set(environments or []) - known - if unknown: - raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) - return True - - -def calculate_project_hash(): - check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") - chunks = [__version__] - for d in (util.get_projectsrc_dir(), util.get_projectlib_dir()): - if not isdir(d): - continue - for root, _, files in walk(d): - for f in files: - path = join(root, f) - if path.endswith(check_suffixes): - chunks.append(path) - chunks_to_str = ",".join(sorted(chunks)) - if "windows" in util.get_systype(): - # Fix issue with useless project rebuilding for case insensitive FS. - # A case of disk drive can differ... - chunks_to_str = chunks_to_str.lower() - return sha1(chunks_to_str).hexdigest() diff --git a/platformio/commands/run/__init__.py b/platformio/commands/run/__init__.py new file mode 100644 index 00000000..b9e69521 --- /dev/null +++ b/platformio/commands/run/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.run.command import cli +from platformio.commands.run.helpers import print_header diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py new file mode 100644 index 00000000..84d19857 --- /dev/null +++ b/platformio/commands/run/command.py @@ -0,0 +1,128 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from multiprocessing import cpu_count +from os import getcwd +from os.path import isfile, join +from time import time + +import click + +from platformio import exception, util +from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.commands.run.helpers import (clean_build_dir, + handle_legacy_libdeps, + print_summary) +from platformio.commands.run.processor import EnvironmentProcessor +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (find_project_dir_above, + get_project_build_dir) + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches + +try: + DEFAULT_JOB_NUMS = cpu_count() +except NotImplementedError: + DEFAULT_JOB_NUMS = 1 + + +@click.command("run", short_help="Process project environments") +@click.option("-e", "--environment", multiple=True) +@click.option("-t", "--target", multiple=True) +@click.option("--upload-port") +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=True, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) +@click.option("-j", + "--jobs", + type=int, + default=DEFAULT_JOB_NUMS, + help=("Allow N jobs at once. " + "Default is a number of CPUs in a system (N=%d)" % + DEFAULT_JOB_NUMS)) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.option("--disable-auto-clean", is_flag=True) +@click.pass_context +def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, + silent, verbose, disable_auto_clean): + # find project directory on upper level + if isfile(project_dir): + project_dir = find_project_dir_above(project_dir) + + with util.cd(project_dir): + # clean obsolete build dir + if not disable_auto_clean: + try: + clean_build_dir(get_project_build_dir()) + except: # pylint: disable=bare-except + click.secho( + "Can not remove temporary directory `%s`. Please remove " + "it manually to avoid build issues" % + get_project_build_dir(force=True), + fg="yellow") + + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(environment) + + handle_legacy_libdeps(project_dir, config) + + results = [] + start_time = time() + default_envs = config.default_envs() + for envname in config.envs(): + skipenv = any([ + environment and envname not in environment, not environment + and default_envs and envname not in default_envs + ]) + if skipenv: + results.append((envname, None)) + continue + + if not silent and any(status is not None + for (_, status) in results): + click.echo() + + ep = EnvironmentProcessor(ctx, envname, config, target, + upload_port, silent, verbose, jobs) + result = (envname, ep.process()) + results.append(result) + + if result[1] and "monitor" in ep.get_build_targets() and \ + "nobuild" not in ep.get_build_targets(): + ctx.invoke(cmd_device_monitor, + environment=environment[0] if environment else None) + + found_error = any(status is False for (_, status) in results) + + if (found_error or not silent) and len(results) > 1: + click.echo() + print_summary(results, start_time) + + if found_error: + raise exception.ReturnErrorCode(1) + return True diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py new file mode 100644 index 00000000..0c51012b --- /dev/null +++ b/platformio/commands/run/helpers.py @@ -0,0 +1,109 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import makedirs +from os.path import getmtime, isdir, isfile, join +from time import time + +import click + +from platformio import util +from platformio.project.helpers import (calculate_project_hash, + get_project_dir, + get_project_libdeps_dir) + + +def handle_legacy_libdeps(project_dir, config): + legacy_libdeps_dir = join(project_dir, ".piolibdeps") + if (not isdir(legacy_libdeps_dir) + or legacy_libdeps_dir == get_project_libdeps_dir()): + return + if not config.has_section("env"): + config.add_section("env") + lib_extra_dirs = config.get("env", "lib_extra_dirs", []) + lib_extra_dirs.append(legacy_libdeps_dir) + config.set("env", "lib_extra_dirs", lib_extra_dirs) + click.secho( + "DEPRECATED! A legacy library storage `{0}` has been found in a " + "project. \nPlease declare project dependencies in `platformio.ini`" + " file using `lib_deps` option and remove `{0}` folder." + "\nMore details -> http://docs.platformio.org/page/projectconf/" + "section_env_library.html#lib-deps".format(legacy_libdeps_dir), + fg="yellow") + + +def clean_build_dir(build_dir): + # remove legacy ".pioenvs" folder + legacy_build_dir = join(get_project_dir(), ".pioenvs") + if isdir(legacy_build_dir) and legacy_build_dir != build_dir: + util.rmtree_(legacy_build_dir) + + structhash_file = join(build_dir, "structure.hash") + proj_hash = calculate_project_hash() + + # if project's config is modified + if (isdir(build_dir) and getmtime(join( + get_project_dir(), "platformio.ini")) > getmtime(build_dir)): + util.rmtree_(build_dir) + + # check project structure + if isdir(build_dir) and isfile(structhash_file): + with open(structhash_file) as f: + if f.read() == proj_hash: + return + util.rmtree_(build_dir) + + if not isdir(build_dir): + makedirs(build_dir) + + with open(structhash_file, "w") as f: + f.write(proj_hash) + + +def print_header(label, is_error=False, fg=None): + terminal_width, _ = click.get_terminal_size() + width = len(click.unstyle(label)) + half_line = "=" * int((terminal_width - width - 2) / 2) + click.secho("%s %s %s" % (half_line, label, half_line), + fg=fg, + err=is_error) + + +def print_summary(results, start_time): + print_header("[%s]" % click.style("SUMMARY")) + + succeeded_nums = 0 + failed_nums = 0 + envname_max_len = max( + [len(click.style(envname, fg="cyan")) for (envname, _) in results]) + for (envname, status) in results: + if status is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif status is None: + status_str = click.style("IGNORED", fg="yellow") + else: + succeeded_nums += 1 + status_str = click.style("SUCCESS", fg="green") + + format_str = "Environment {0:<%d}\t[{1}]" % envname_max_len + click.echo(format_str.format(click.style(envname, fg="cyan"), + status_str), + err=status is False) + + print_header("%s%d succeeded in %.2f seconds" % + ("%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, time() - start_time), + is_error=failed_nums, + fg="red" if failed_nums else "green") diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py new file mode 100644 index 00000000..9ff1e056 --- /dev/null +++ b/platformio/commands/run/processor.py @@ -0,0 +1,117 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import time + +import click + +from platformio import exception, telemetry +from platformio.commands.platform import \ + platform_install as cmd_platform_install +from platformio.commands.run.helpers import print_header +from platformio.commands.test.processor import (CTX_META_TEST_IS_RUNNING, + CTX_META_TEST_RUNNING_NAME) +from platformio.managers.platform import PlatformFactory + +# pylint: disable=too-many-instance-attributes + + +class EnvironmentProcessor(object): + + DEFAULT_PRINT_OPTIONS = ("platform", "framework", "board") + + def __init__( # pylint: disable=too-many-arguments + self, cmd_ctx, name, config, targets, upload_port, silent, verbose, + jobs): + self.cmd_ctx = cmd_ctx + self.name = name + self.config = config + self.targets = [str(t) for t in targets] + self.upload_port = upload_port + self.silent = silent + self.verbose = verbose + self.jobs = jobs + self.options = config.items(env=name, as_dict=True) + + def process(self): + terminal_width, _ = click.get_terminal_size() + start_time = time() + env_dump = [] + + for k, v in self.options.items(): + if self.verbose or k in self.DEFAULT_PRINT_OPTIONS: + env_dump.append( + "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) + + if not self.silent: + click.echo("Processing %s (%s)" % (click.style( + self.name, fg="cyan", bold=True), "; ".join(env_dump))) + click.secho("-" * terminal_width, bold=True) + + result = self._run_platform() + is_error = result['returncode'] != 0 + + if self.silent and not is_error: + return True + + if is_error or CTX_META_TEST_IS_RUNNING not in self.cmd_ctx.meta: + print_header( + "[%s] Took %.2f seconds" % + ((click.style("ERROR", fg="red", bold=True) if + is_error else click.style("SUCCESS", fg="green", bold=True)), + time() - start_time), + is_error=is_error) + + return not is_error + + def get_build_variables(self): + variables = {"pioenv": self.name, "project_config": self.config.path} + + if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta: + variables['piotest_running_name'] = self.cmd_ctx.meta[ + CTX_META_TEST_RUNNING_NAME] + + if self.upload_port: + # override upload port with a custom from CLI + variables['upload_port'] = self.upload_port + return variables + + def get_build_targets(self): + if self.targets: + return [t for t in self.targets] + return self.config.get("env:" + self.name, "targets", []) + + def _run_platform(self): + if "platform" not in self.options: + raise exception.UndefinedEnvPlatform(self.name) + + build_vars = self.get_build_variables() + build_targets = self.get_build_targets() + + telemetry.on_run_environment(self.options, build_targets) + + # skip monitor target, we call it above + if "monitor" in build_targets: + build_targets.remove("monitor") + + try: + p = PlatformFactory.newPlatform(self.options['platform']) + except exception.UnknownPlatform: + self.cmd_ctx.invoke(cmd_platform_install, + platforms=[self.options['platform']], + skip_default_package=True) + p = PlatformFactory.newPlatform(self.options['platform']) + + return p.run(build_vars, build_targets, self.silent, self.verbose, + self.jobs) diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index a29d3997..5fa20993 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -15,6 +15,7 @@ import click from platformio import app +from platformio.compat import string_types @click.group(short_help="Manage PlatformIO settings") @@ -26,15 +27,14 @@ def cli(): @click.argument("name", required=False) def settings_get(name): - list_tpl = "{name:<40} {value:<35} {description}" + list_tpl = u"{name:<40} {value:<35} {description}" terminal_width, _ = click.get_terminal_size() click.echo( - list_tpl.format( - name=click.style("Name", fg="cyan"), - value=(click.style("Value", fg="green") + click.style( - " [Default]", fg="yellow")), - description="Description")) + list_tpl.format(name=click.style("Name", fg="cyan"), + value=(click.style("Value", fg="green") + + click.style(" [Default]", fg="yellow")), + description="Description")) click.echo("-" * terminal_width) for _name, _data in sorted(app.DEFAULT_SETTINGS.items()): @@ -42,7 +42,8 @@ def settings_get(name): continue _value = app.get_setting(_name) - _value_str = str(_value) + _value_str = (str(_value) + if not isinstance(_value, string_types) else _value) if isinstance(_value, bool): _value_str = "Yes" if _value else "No" _value_str = click.style(_value_str, fg="green") @@ -56,10 +57,9 @@ def settings_get(name): _value_str += click.style(" ", fg="yellow") click.echo( - list_tpl.format( - name=click.style(_name, fg="cyan"), - value=_value_str, - description=_data['description'])) + list_tpl.format(name=click.style(_name, fg="cyan"), + value=_value_str, + description=_data['description'])) @cli.command("set", short_help="Set new value for the setting") diff --git a/platformio/commands/test.py b/platformio/commands/test.py deleted file mode 100644 index 4c6414c9..00000000 --- a/platformio/commands/test.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from os import getcwd - -import click - -from platformio.managers.core import pioplus_call - - -@click.command("test", short_help="Local Unit Testing") -@click.option("--environment", "-e", multiple=True, metavar="") -@click.option( - "--filter", - "-f", - multiple=True, - metavar="", - help="Filter tests by a pattern") -@click.option( - "--ignore", - "-i", - multiple=True, - metavar="", - help="Ignore tests by a pattern") -@click.option("--upload-port") -@click.option("--test-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("--without-building", is_flag=True) -@click.option("--without-uploading", is_flag=True) -@click.option( - "--no-reset", - is_flag=True, - help="Disable software reset via Serial.DTR/RST") -@click.option( - "--monitor-rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state for Serial Monitor") -@click.option( - "--monitor-dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state for Serial Monitor") -@click.option("--verbose", "-v", is_flag=True) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) diff --git a/platformio/commands/test/__init__.py b/platformio/commands/test/__init__.py new file mode 100644 index 00000000..6d4c3e2b --- /dev/null +++ b/platformio/commands/test/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.test.command import cli diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py new file mode 100644 index 00000000..d330a410 --- /dev/null +++ b/platformio/commands/test/command.py @@ -0,0 +1,183 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments, too-many-locals, too-many-branches + +from fnmatch import fnmatch +from os import getcwd, listdir +from os.path import isdir, join +from time import time + +import click + +from platformio import exception, util +from platformio.commands.run.helpers import print_header +from platformio.commands.test.embedded import EmbeddedTestProcessor +from platformio.commands.test.native import NativeTestProcessor +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_test_dir + + +@click.command("test", short_help="Unit Testing") +@click.option("--environment", "-e", multiple=True, metavar="") +@click.option("--filter", + "-f", + multiple=True, + metavar="", + help="Filter tests by a pattern") +@click.option("--ignore", + "-i", + multiple=True, + metavar="", + help="Ignore tests by a pattern") +@click.option("--upload-port") +@click.option("--test-port") +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) +@click.option("--without-building", is_flag=True) +@click.option("--without-uploading", is_flag=True) +@click.option("--without-testing", is_flag=True) +@click.option("--no-reset", is_flag=True) +@click.option("--monitor-rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state for Serial Monitor") +@click.option("--monitor-dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state for Serial Monitor") +@click.option("--verbose", "-v", is_flag=True) +@click.pass_context +def cli( # pylint: disable=redefined-builtin + ctx, environment, ignore, filter, upload_port, test_port, project_dir, + project_conf, without_building, without_uploading, without_testing, + no_reset, monitor_rts, monitor_dtr, verbose): + with util.cd(project_dir): + test_dir = get_project_test_dir() + if not isdir(test_dir): + raise exception.TestDirNotExists(test_dir) + test_names = get_test_names(test_dir) + + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(envs=environment) + + click.echo("Verbose mode can be enabled via `-v, --verbose` option") + click.echo("Collected %d items" % len(test_names)) + + results = [] + start_time = time() + default_envs = config.default_envs() + for testname in test_names: + for envname in config.envs(): + section = "env:%s" % envname + + # filter and ignore patterns + patterns = dict(filter=list(filter), ignore=list(ignore)) + for key in patterns: + patterns[key].extend( + config.get(section, "test_%s" % key, [])) + + skip_conditions = [ + environment and envname not in environment, + not environment and default_envs + and envname not in default_envs, + testname != "*" and patterns['filter'] and + not any([fnmatch(testname, p) + for p in patterns['filter']]), + testname != "*" + and any([fnmatch(testname, p) + for p in patterns['ignore']]), + ] + if any(skip_conditions): + results.append((None, testname, envname)) + continue + + cls = (NativeTestProcessor + if config.get(section, "platform") == "native" else + EmbeddedTestProcessor) + tp = cls( + ctx, testname, envname, + dict(project_config=config, + project_dir=project_dir, + upload_port=upload_port, + test_port=test_port, + without_building=without_building, + without_uploading=without_uploading, + without_testing=without_testing, + no_reset=no_reset, + monitor_rts=monitor_rts, + monitor_dtr=monitor_dtr, + verbose=verbose)) + results.append((tp.process(), testname, envname)) + + if without_testing: + return + + passed_nums = 0 + failed_nums = 0 + testname_max_len = max([len(r[1]) for r in results]) + envname_max_len = max([len(click.style(r[2], fg="cyan")) for r in results]) + + print_header("[%s]" % click.style("TEST SUMMARY")) + click.echo() + + for result in results: + status, testname, envname = result + if status is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif status is None: + status_str = click.style("IGNORED", fg="yellow") + else: + passed_nums += 1 + status_str = click.style("PASSED", fg="green") + + format_str = "test/{:<%d} > {:<%d}\t[{}]" % (testname_max_len, + envname_max_len) + click.echo(format_str.format(testname, click.style(envname, fg="cyan"), + status_str), + err=status is False) + + print_header("%s%d passed in %.2f seconds" % + ("%d failed, " % failed_nums if failed_nums else "", + passed_nums, time() - start_time), + is_error=failed_nums, + fg="red" if failed_nums else "green") + + if failed_nums: + raise exception.ReturnErrorCode(1) + + +def get_test_names(test_dir): + names = [] + for item in sorted(listdir(test_dir)): + if isdir(join(test_dir, item)): + names.append(item) + if not names: + names = ["*"] + return names diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py new file mode 100644 index 00000000..681bfb44 --- /dev/null +++ b/platformio/commands/test/embedded.py @@ -0,0 +1,135 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + +import click +import serial + +from platformio import exception, util +from platformio.commands.test.processor import TestProcessorBase +from platformio.managers.platform import PlatformFactory + + +class EmbeddedTestProcessor(TestProcessorBase): + + SERIAL_TIMEOUT = 600 + + def process(self): + if not self.options['without_building']: + self.print_progress("Building... (1/3)") + target = ["__test"] + if self.options['without_uploading']: + target.append("checkprogsize") + if not self.build_or_upload(target): + return False + + if not self.options['without_uploading']: + self.print_progress("Uploading... (2/3)") + target = ["upload"] + if self.options['without_building']: + target.append("nobuild") + else: + target.append("__test") + if not self.build_or_upload(target): + return False + + if self.options['without_testing']: + return None + + self.print_progress("Testing... (3/3)") + return self.run() + + def run(self): + click.echo("If you don't see any output for the first 10 secs, " + "please reset board (press reset button)") + click.echo() + + try: + ser = serial.Serial(baudrate=self.get_baudrate(), + timeout=self.SERIAL_TIMEOUT) + ser.port = self.get_test_port() + ser.rts = self.options['monitor_rts'] + ser.dtr = self.options['monitor_dtr'] + ser.open() + except serial.SerialException as e: + click.secho(str(e), fg="red", err=True) + return False + + if not self.options['no_reset']: + ser.flushInput() + ser.setDTR(False) + ser.setRTS(False) + sleep(0.1) + ser.setDTR(True) + ser.setRTS(True) + sleep(0.1) + + while True: + line = ser.readline().strip() + + # fix non-ascii output from device + for i, c in enumerate(line[::-1]): + if not isinstance(c, int): + c = ord(c) + if c > 127: + line = line[-i:] + break + + if not line: + continue + if isinstance(line, bytes): + line = line.decode("utf8") + self.on_run_out(line) + if all([l in line for l in ("Tests", "Failures", "Ignored")]): + break + ser.close() + return not self._run_failed + + def get_test_port(self): + # if test port is specified manually or in config + if self.options.get("test_port"): + return self.options.get("test_port") + if self.env_options.get("test_port"): + return self.env_options.get("test_port") + + assert set(["platform", "board"]) & set(self.env_options.keys()) + p = PlatformFactory.newPlatform(self.env_options['platform']) + board_hwids = p.board_config(self.env_options['board']).get( + "build.hwids", []) + port = None + elapsed = 0 + while elapsed < 5 and not port: + for item in util.get_serialports(): + port = item['port'] + for hwid in board_hwids: + hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") + if hwid_str in item['hwid']: + return port + + # check if port is already configured + try: + serial.Serial(port, timeout=self.SERIAL_TIMEOUT).close() + except serial.SerialException: + port = None + + if not port: + sleep(0.25) + elapsed += 0.25 + + if not port: + raise exception.PlatformioException( + "Please specify `test_port` for environment or use " + "global `--test-port` option.") + return port diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py new file mode 100644 index 00000000..7367094f --- /dev/null +++ b/platformio/commands/test/native.py @@ -0,0 +1,43 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os.path import join + +from platformio import util +from platformio.commands.test.processor import TestProcessorBase +from platformio.proc import LineBufferedAsyncPipe +from platformio.project.helpers import get_project_build_dir + + +class NativeTestProcessor(TestProcessorBase): + + def process(self): + if not self.options['without_building']: + self.print_progress("Building... (1/2)") + if not self.build_or_upload(["__test"]): + return False + if self.options['without_testing']: + return None + self.print_progress("Testing... (2/2)") + return self.run() + + def run(self): + with util.cd(self.options['project_dir']): + build_dir = get_project_build_dir() + result = util.exec_command( + [join(build_dir, self.env_name, "program")], + stdout=LineBufferedAsyncPipe(self.on_run_out), + stderr=LineBufferedAsyncPipe(self.on_run_out)) + assert "returncode" in result + return result['returncode'] == 0 and not self._run_failed diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py new file mode 100644 index 00000000..d3029a34 --- /dev/null +++ b/platformio/commands/test/processor.py @@ -0,0 +1,206 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +from os import remove +from os.path import isdir, isfile, join +from string import Template + +import click + +from platformio import exception +from platformio.commands.run.helpers import print_header +from platformio.project.helpers import get_project_test_dir + +TRANSPORT_OPTIONS = { + "arduino": { + "include": "#include ", + "object": "", + "putchar": "Serial.write(c)", + "flush": "Serial.flush()", + "begin": "Serial.begin($baudrate)", + "end": "Serial.end()" + }, + "mbed": { + "include": "#include ", + "object": "Serial pc(USBTX, USBRX);", + "putchar": "pc.putc(c)", + "flush": "", + "begin": "pc.baud($baudrate)", + "end": "" + }, + "energia": { + "include": "#include ", + "object": "", + "putchar": "Serial.write(c)", + "flush": "Serial.flush()", + "begin": "Serial.begin($baudrate)", + "end": "Serial.end()" + }, + "espidf": { + "include": "#include ", + "object": "", + "putchar": "putchar(c)", + "flush": "fflush(stdout)", + "begin": "", + "end": "" + }, + "native": { + "include": "#include ", + "object": "", + "putchar": "putchar(c)", + "flush": "fflush(stdout)", + "begin": "", + "end": "" + }, + "custom": { + "include": '#include "unittest_transport.h"', + "object": "", + "putchar": "unittest_uart_putchar(c)", + "flush": "unittest_uart_flush()", + "begin": "unittest_uart_begin()", + "end": "unittest_uart_end()" + } +} + +CTX_META_TEST_IS_RUNNING = __name__ + ".test_running" +CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name" + + +class TestProcessorBase(object): + + DEFAULT_BAUDRATE = 115200 + + def __init__(self, cmd_ctx, testname, envname, options): + self.cmd_ctx = cmd_ctx + self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True + self.test_name = testname + self.options = options + self.env_name = envname + self.env_options = options['project_config'].items(env=envname, + as_dict=True) + self._run_failed = False + self._outputcpp_generated = False + + def get_transport(self): + if self.env_options.get("platform") == "native": + transport = "native" + elif "framework" in self.env_options: + transport = self.env_options.get("framework")[0] + if "test_transport" in self.env_options: + transport = self.env_options['test_transport'] + if transport not in TRANSPORT_OPTIONS: + raise exception.PlatformioException( + "Unknown Unit Test transport `%s`" % transport) + return transport.lower() + + def get_baudrate(self): + return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE)) + + def print_progress(self, text, is_error=False): + click.echo() + print_header("[test/%s > %s] %s" % + (click.style(self.test_name, fg="yellow"), + click.style(self.env_name, fg="cyan"), text), + is_error=is_error) + + def build_or_upload(self, target): + if not self._outputcpp_generated: + self.generate_outputcpp(get_project_test_dir()) + self._outputcpp_generated = True + + if self.test_name != "*": + self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name + + if not self.options['verbose']: + click.echo("Please wait...") + + try: + from platformio.commands.run import cli as cmd_run + return self.cmd_ctx.invoke(cmd_run, + project_dir=self.options['project_dir'], + upload_port=self.options['upload_port'], + silent=not self.options['verbose'], + environment=[self.env_name], + disable_auto_clean="nobuild" in target, + target=target) + except exception.ReturnErrorCode: + return False + + def process(self): + raise NotImplementedError + + def run(self): + raise NotImplementedError + + def on_run_out(self, line): + line = line.strip() + if line.endswith(":PASS"): + click.echo("%s\t[%s]" % + (line[:-5], click.style("PASSED", fg="green"))) + elif ":FAIL" in line: + self._run_failed = True + click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red"))) + else: + click.echo(line) + + def generate_outputcpp(self, test_dir): + assert isdir(test_dir) + + cpp_tpl = "\n".join([ + "$include", + "#include ", + "", + "$object", + "", + "void output_start(unsigned int baudrate)", + "{", + " $begin;", + "}", + "", + "void output_char(int c)", + "{", + " $putchar;", + "}", + "", + "void output_flush(void)", + "{", + " $flush;", + "}", + "", + "void output_complete(void)", + "{", + " $end;", + "}" + ]) # yapf: disable + + def delete_tmptest_file(file_): + try: + remove(file_) + except: # pylint: disable=bare-except + if isfile(file_): + click.secho( + "Warning: Could not remove temporary file '%s'. " + "Please remove it manually." % file_, + fg="yellow") + + tpl = Template(cpp_tpl).substitute( + TRANSPORT_OPTIONS[self.get_transport()]) + data = Template(tpl).substitute(baudrate=self.get_baudrate()) + + tmp_file = join(test_dir, "output_export.cpp") + with open(tmp_file, "w") as f: + f.write(data) + + atexit.register(delete_tmptest_file, tmp_file) diff --git a/platformio/commands/update.py b/platformio/commands/update.py index 924fb290..dc6ed1c6 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -15,26 +15,32 @@ import click from platformio import app +from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager -@click.command( - "update", short_help="Update installed platforms, packages and libraries") -@click.option( - "--core-packages", is_flag=True, help="Update only the core packages") -@click.option( - "-c", - "--only-check", - is_flag=True, - help="Do not update, only check for new version") +@click.command("update", + short_help="Update installed platforms, packages and libraries") +@click.option("--core-packages", + is_flag=True, + help="Update only the core packages") +@click.option("-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead") +@click.option("--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") @click.pass_context -def cli(ctx, core_packages, only_check): +def cli(ctx, core_packages, only_check, dry_run): # cleanup lib search results, cached board and platform lists app.clean_cache() + only_check = dry_run or only_check + update_core_packages(only_check) if core_packages: @@ -48,5 +54,5 @@ def cli(ctx, core_packages, only_check): click.echo() click.echo("Library Manager") click.echo("===============") - ctx.obj = LibraryManager() + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir] ctx.invoke(cmd_lib_update, only_check=only_check) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 98430483..91286230 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -20,11 +20,14 @@ import click import requests from platformio import VERSION, __version__, exception, util +from platformio.compat import WINDOWS from platformio.managers.core import shutdown_piohome_servers +from platformio.proc import exec_command, get_pythonexe_path +from platformio.project.helpers import get_project_cache_dir -@click.command( - "upgrade", short_help="Upgrade PlatformIO to the latest version") +@click.command("upgrade", + short_help="Upgrade PlatformIO to the latest version") @click.option("--dev", is_flag=True, help="Use development branch") def cli(dev): if not dev and __version__ == get_latest_version(): @@ -43,35 +46,33 @@ def cli(dev): get_pip_package(to_develop)], ["platformio", "--version"]) cmd = None - r = None + r = {} try: for cmd in cmds: - cmd = [util.get_pythonexe_path(), "-m"] + cmd - r = None - r = util.exec_command(cmd) + cmd = [get_pythonexe_path(), "-m"] + cmd + r = exec_command(cmd) # try pip with disabled cache if r['returncode'] != 0 and cmd[2] == "pip": cmd.insert(3, "--no-cache-dir") - r = util.exec_command(cmd) + r = exec_command(cmd) assert r['returncode'] == 0 assert "version" in r['out'] actual_version = r['out'].strip().split("version", 1)[1].strip() - click.secho( - "PlatformIO has been successfully upgraded to %s" % actual_version, - fg="green") + click.secho("PlatformIO has been successfully upgraded to %s" % + actual_version, + fg="green") click.echo("Release notes: ", nl=False) - click.secho( - "https://docs.platformio.org/en/latest/history.html", fg="cyan") + click.secho("https://docs.platformio.org/en/latest/history.html", + fg="cyan") except Exception as e: # pylint: disable=broad-except if not r: raise exception.UpgradeError("\n".join([str(cmd), str(e)])) permission_errors = ("permission denied", "not permitted") if (any(m in r['err'].lower() for m in permission_errors) - and "windows" not in util.get_systype()): - click.secho( - """ + and not WINDOWS): + click.secho(""" ----------------- Permission denied ----------------- @@ -81,12 +82,10 @@ You need the `sudo` permission to install Python packages. Try WARNING! Don't use `sudo` for the rest PlatformIO commands. """, - fg="yellow", - err=True) + fg="yellow", + err=True) raise exception.ReturnErrorCode(1) - else: - raise exception.UpgradeError("\n".join( - [str(cmd), r['out'], r['err']])) + raise exception.UpgradeError("\n".join([str(cmd), r['out'], r['err']])) return True @@ -96,15 +95,15 @@ def get_pip_package(to_develop): return "platformio" dl_url = ("https://github.com/platformio/" "platformio-core/archive/develop.zip") - cache_dir = util.get_cache_dir() + cache_dir = get_project_cache_dir() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") try: with open(pkg_name, "w") as fp: - r = util.exec_command(["curl", "-fsSL", dl_url], - stdout=fp, - universal_newlines=True) + r = exec_command(["curl", "-fsSL", dl_url], + stdout=fp, + universal_newlines=True) assert r['returncode'] == 0 # check ZIP structure with ZipFile(pkg_name) as zp: @@ -150,8 +149,7 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - r = requests.get( - "https://pypi.org/pypi/platformio/json", - headers=util.get_request_defheaders()) + r = requests.get("https://pypi.org/pypi/platformio/json", + headers=util.get_request_defheaders()) r.raise_for_status() return r.json()['info']['version'] diff --git a/platformio/compat.py b/platformio/compat.py new file mode 100644 index 00000000..8b082b4b --- /dev/null +++ b/platformio/compat.py @@ -0,0 +1,108 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-import + +import json +import os +import re +import sys + +PY2 = sys.version_info[0] == 2 +CYGWIN = sys.platform.startswith('cygwin') +WINDOWS = sys.platform.startswith('win') + + +def get_filesystem_encoding(): + return sys.getfilesystemencoding() or sys.getdefaultencoding() + + +if PY2: + # pylint: disable=undefined-variable + string_types = (str, unicode) + + def is_bytes(x): + return isinstance(x, (buffer, bytearray)) + + def path_to_unicode(path): + if isinstance(path, unicode): + return path + return path.decode(get_filesystem_encoding()).encode("utf-8") + + def get_file_contents(path): + with open(path) as f: + return f.read() + + def hashlib_encode_data(data): + if is_bytes(data): + return data + if isinstance(data, unicode): + data = data.encode(get_filesystem_encoding()) + elif not isinstance(data, string_types): + data = str(data) + return data + + def dump_json_to_unicode(obj): + if isinstance(obj, unicode): + return obj + return json.dumps(obj, + encoding=get_filesystem_encoding(), + ensure_ascii=False, + sort_keys=True).encode("utf8") + + _magic_check = re.compile('([*?[])') + _magic_check_bytes = re.compile(b'([*?[])') + + def glob_escape(pathname): + """Escape all special characters.""" + # https://github.com/python/cpython/blob/master/Lib/glob.py#L161 + # Escaping is done by wrapping any of "*?[" between square brackets. + # Metacharacters do not work in the drive part and shouldn't be + # escaped. + drive, pathname = os.path.splitdrive(pathname) + if isinstance(pathname, bytes): + pathname = _magic_check_bytes.sub(br'[\1]', pathname) + else: + pathname = _magic_check.sub(r'[\1]', pathname) + return drive + pathname +else: + from glob import escape as glob_escape # pylint: disable=no-name-in-module + + string_types = (str, ) + + def is_bytes(x): + return isinstance(x, (bytes, memoryview, bytearray)) + + def path_to_unicode(path): + return path + + def get_file_contents(path): + try: + with open(path) as f: + return f.read() + except UnicodeDecodeError: + with open(path, encoding="latin-1") as f: + return f.read() + + def hashlib_encode_data(data): + if is_bytes(data): + return data + if not isinstance(data, string_types): + data = str(data) + return data.encode() + + def dump_json_to_unicode(obj): + if isinstance(obj, string_types): + return obj + return json.dumps(obj, ensure_ascii=False, sort_keys=True) diff --git a/platformio/downloader.py b/platformio/downloader.py index 62c19385..60f0e159 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -15,7 +15,7 @@ from email.utils import parsedate_tz from math import ceil from os.path import getsize, join -from sys import getfilesystemencoding, version_info +from sys import version_info from time import mktime import click @@ -24,6 +24,7 @@ import requests from platformio import util from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, FDUnrecognizedStatusCode) +from platformio.proc import exec_command class FileDownloader(object): @@ -33,11 +34,10 @@ class FileDownloader(object): def __init__(self, url, dest_dir=None): self._request = None # make connection - self._request = requests.get( - url, - stream=True, - headers=util.get_request_defheaders(), - verify=version_info >= (2, 7, 9)) + self._request = requests.get(url, + stream=True, + headers=util.get_request_defheaders(), + verify=version_info >= (2, 7, 9)) if self._request.status_code != 200: raise FDUnrecognizedStatusCode(self._request.status_code, url) @@ -45,14 +45,12 @@ class FileDownloader(object): if disposition and "filename=" in disposition: self._fname = disposition[disposition.index("filename=") + 9:].replace('"', "").replace("'", "") - self._fname = self._fname.encode("utf8") else: self._fname = [p for p in url.split("/") if p][-1] - + self._fname = str(self._fname) self._destination = self._fname if dest_dir: - self.set_destination( - join(dest_dir.decode(getfilesystemencoding()), self._fname)) + self.set_destination(join(dest_dir, self._fname)) def set_destination(self, destination): self._destination = destination @@ -98,24 +96,24 @@ class FileDownloader(object): raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) if not sha1: - return + return None dlsha1 = None try: - result = util.exec_command(["sha1sum", self._destination]) + result = exec_command(["sha1sum", self._destination]) dlsha1 = result['out'] except (OSError, ValueError): try: - result = util.exec_command( - ["shasum", "-a", "1", self._destination]) + result = exec_command(["shasum", "-a", "1", self._destination]) dlsha1 = result['out'] except (OSError, ValueError): pass - - if dlsha1: - dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40] - if sha1 != dlsha1: - raise FDSHASumMismatch(dlsha1, self._fname, sha1) + if not dlsha1: + return None + dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40] + if sha1.lower() != dlsha1.lower(): + raise FDSHASumMismatch(dlsha1, self._fname, sha1) + return True def _preserve_filemtime(self, lmdate): timedata = parsedate_tz(lmdate) diff --git a/platformio/exception.py b/platformio/exception.py index 48f831c9..823f68af 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -19,8 +19,10 @@ class PlatformioException(Exception): def __str__(self): # pragma: no cover if self.MESSAGE: + # pylint: disable=not-an-iterable return self.MESSAGE.format(*self.args) - return Exception.__str__(self) + + return super(PlatformioException, self).__str__() class ReturnErrorCode(PlatformioException): @@ -40,11 +42,16 @@ class UserSideException(PlatformioException): pass -class AbortedByUser(PlatformioException): +class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" +# +# Development Platform +# + + class UnknownPlatform(PlatformioException): MESSAGE = "Unknown development platform '{0}'" @@ -61,13 +68,6 @@ class PlatformNotInstalledYet(PlatformioException): "Use `platformio platform install {0}` command") -class BoardNotDefined(PlatformioException): - - MESSAGE = ( - "You need to specify board ID using `-b` or `--board` option. " - "Supported boards list is available via `platformio boards` command") - - class UnknownBoard(PlatformioException): MESSAGE = "Unknown board ID '{0}'" @@ -83,54 +83,75 @@ class UnknownFramework(PlatformioException): MESSAGE = "Unknown framework '{0}'" -class UnknownPackage(PlatformioException): +# Package Manager + + +class PlatformIOPackageException(PlatformioException): + pass + + +class UnknownPackage(PlatformIOPackageException): MESSAGE = "Detected unknown package '{0}'" -class MissingPackageManifest(PlatformioException): +class MissingPackageManifest(PlatformIOPackageException): MESSAGE = "Could not find one of '{0}' manifest files in the package" -class UndefinedPackageVersion(PlatformioException): +class UndefinedPackageVersion(PlatformIOPackageException): MESSAGE = ("Could not find a version that satisfies the requirement '{0}'" " for your system '{1}'") -class PackageInstallError(PlatformioException): +class PackageInstallError(PlatformIOPackageException): MESSAGE = ("Could not install '{0}' with version requirements '{1}' " "for your system '{2}'.\n\n" "Please try this solution -> http://bit.ly/faq-package-manager") -class ExtractArchiveItemError(PlatformioException): +class ExtractArchiveItemError(PlatformIOPackageException): MESSAGE = ( "Could not extract `{0}` to `{1}`. Try to disable antivirus " "tool or check this solution -> http://bit.ly/faq-package-manager") -class FDUnrecognizedStatusCode(PlatformioException): +class UnsupportedArchiveType(PlatformIOPackageException): + + MESSAGE = "Can not unpack file '{0}'" + + +class FDUnrecognizedStatusCode(PlatformIOPackageException): MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}" -class FDSizeMismatch(PlatformioException): +class FDSizeMismatch(PlatformIOPackageException): MESSAGE = ("The size ({0:d} bytes) of downloaded file '{1}' " "is not equal to remote size ({2:d} bytes)") -class FDSHASumMismatch(PlatformioException): +class FDSHASumMismatch(PlatformIOPackageException): MESSAGE = ("The 'sha1' sum '{0}' of downloaded file '{1}' " "is not equal to remote '{2}'") -class NotPlatformIOProject(PlatformioException): +# +# Project +# + + +class PlatformIOProjectException(PlatformioException): + pass + + +class NotPlatformIOProject(PlatformIOProjectException): MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " @@ -138,26 +159,87 @@ class NotPlatformIOProject(PlatformioException): "please use `platformio init` command") -class UndefinedEnvPlatform(PlatformioException): +class InvalidProjectConf(PlatformIOProjectException): + + MESSAGE = ("Invalid '{0}' (project configuration file): '{1}'") + + +class UndefinedEnvPlatform(PlatformIOProjectException): MESSAGE = "Please specify platform for '{0}' environment" -class UnsupportedArchiveType(PlatformioException): - - MESSAGE = "Can not unpack file '{0}'" - - -class ProjectEnvsNotAvailable(PlatformioException): +class ProjectEnvsNotAvailable(PlatformIOProjectException): MESSAGE = "Please setup environments in `platformio.ini` file" -class UnknownEnvNames(PlatformioException): +class UnknownEnvNames(PlatformIOProjectException): MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" +class ProjectOptionValueError(PlatformIOProjectException): + + MESSAGE = "{0} for option `{1}` in section [{2}]" + + +# +# Library +# + + +class LibNotFound(PlatformioException): + + MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n" + "You can ignore this message, if `{0}` is a built-in library " + "(included in framework, SDK). E.g., SPI, Wire, etc.") + + +class NotGlobalLibDir(UserSideException): + + MESSAGE = ( + "The `{0}` is not a PlatformIO project.\n\n" + "To manage libraries in global storage `{1}`,\n" + "please use `platformio lib --global {2}` or specify custom storage " + "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" + "Check `platformio lib --help` for details.") + + +class InvalidLibConfURL(PlatformioException): + + MESSAGE = "Invalid library config URL '{0}'" + + +# +# UDEV Rules +# + + +class InvalidUdevRules(PlatformioException): + pass + + +class MissedUdevRules(InvalidUdevRules): + + MESSAGE = ( + "Warning! Please install `99-platformio-udev.rules`. \nMode details: " + "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules") + + +class OutdatedUdevRules(InvalidUdevRules): + + MESSAGE = ( + "Warning! Your `{0}` are outdated. Please update or reinstall them." + "\n Mode details: https://docs.platformio.org" + "/en/latest/faq.html#platformio-udev-rules") + + +# +# Misc +# + + class GetSerialPortsError(PlatformioException): MESSAGE = "No implementation for your platform ('{0}') available" @@ -173,7 +255,7 @@ class APIRequestError(PlatformioException): MESSAGE = "[API] {0}" -class InternetIsOffline(PlatformioException): +class InternetIsOffline(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" @@ -181,33 +263,6 @@ class InternetIsOffline(PlatformioException): "to install all dependencies and toolchains.") -class LibNotFound(PlatformioException): - - MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n" - "You can ignore this message, if `{0}` is a built-in library " - "(included in framework, SDK). E.g., SPI, Wire, etc.") - - -class NotGlobalLibDir(PlatformioException): - - MESSAGE = ( - "The `{0}` is not a PlatformIO project.\n\n" - "To manage libraries in global storage `{1}`,\n" - "please use `platformio lib --global {2}` or specify custom storage " - "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" - "Check `platformio lib --help` for details.") - - -class InvalidLibConfURL(PlatformioException): - - MESSAGE = "Invalid library config URL '{0}'" - - -class InvalidProjectConf(PlatformioException): - - MESSAGE = "Invalid `platformio.ini`, project configuration file: '{0}'" - - class BuildScriptNotFound(PlatformioException): MESSAGE = "Invalid path '{0}' to build script" @@ -235,25 +290,6 @@ class CIBuildEnvsEmpty(PlatformioException): "predefined environments using `--project-conf` option") -class InvalidUdevRules(PlatformioException): - pass - - -class MissedUdevRules(InvalidUdevRules): - - MESSAGE = ( - "Warning! Please install `99-platformio-udev.rules`. \nMode details: " - "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules") - - -class OutdatedUdevRules(InvalidUdevRules): - - MESSAGE = ( - "Warning! Your `{0}` are outdated. Please update or reinstall them." - "\n Mode details: https://docs.platformio.org" - "/en/latest/faq.html#platformio-udev-rules") - - class UpgradeError(PlatformioException): MESSAGE = """{0} @@ -282,11 +318,21 @@ class CygwinEnvDetected(PlatformioException): class DebugSupportError(PlatformioException): - MESSAGE = ("Currently, PlatformIO does not support debugging for `{0}`.\n" - "Please contact support@pioplus.com or visit " - "< https://docs.platformio.org/page/plus/debugging.html >") + MESSAGE = ( + "Currently, PlatformIO does not support debugging for `{0}`.\n" + "Please request support at https://github.com/platformio/" + "platformio-core/issues \nor visit -> https://docs.platformio.org" + "/page/plus/debugging.html") class DebugInvalidOptions(PlatformioException): - pass + + +class TestDirNotExists(PlatformioException): + + MESSAGE = "A test folder '{0}' does not exist.\nPlease create 'test' "\ + "directory in project's root and put a test set.\n"\ + "More details about Unit "\ + "Testing: http://docs.platformio.org/page/plus/"\ + "unit-testing.html" diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 8ec43aed..debdeeab 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -12,28 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json +import codecs import os import re import sys from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath import bottle -from click.testing import CliRunner -from platformio import exception, util -from platformio.commands.run import cli as cmd_run +from platformio import util +from platformio.compat import WINDOWS, get_file_contents +from platformio.proc import where_is_program +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (get_project_lib_dir, + get_project_libdeps_dir, + get_project_src_dir, + load_project_ide_data) class ProjectGenerator(object): def __init__(self, project_dir, ide, env_name): self.project_dir = project_dir - self.ide = ide - self.env_name = env_name - - self._tplvars = {} - self._gather_tplvars() + self.ide = str(ide) + self.env_name = str(env_name) @staticmethod def get_supported_ides(): @@ -41,55 +43,44 @@ class ProjectGenerator(object): return sorted( [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) - @util.memoized() - def get_project_env(self): - data = {} - config = util.load_project_config(self.project_dir) - for section in config.sections(): - if not section.startswith("env:"): - continue - if self.env_name != section[4:]: - continue - data = {"env_name": section[4:]} - for k, v in config.items(section): - data[k] = v - return data + def _load_tplvars(self): + tpl_vars = {"env_name": self.env_name} + # default env configuration + tpl_vars.update( + ProjectConfig.get_instance(join( + self.project_dir, "platformio.ini")).items(env=self.env_name, + as_dict=True)) + # build data + tpl_vars.update( + load_project_ide_data(self.project_dir, self.env_name) or {}) - def get_project_build_data(self): - data = { - "defines": [], - "includes": [], - "cxx_path": None, - "prog_path": None - } - envdata = self.get_project_env() - if not envdata: - return data + with util.cd(self.project_dir): + tpl_vars.update({ + "project_name": basename(self.project_dir), + "src_files": self.get_src_files(), + "user_home_dir": abspath(expanduser("~")), + "project_dir": self.project_dir, + "project_src_dir": get_project_src_dir(), + "project_lib_dir": get_project_lib_dir(), + "project_libdeps_dir": join( + get_project_libdeps_dir(), self.env_name), + "systype": util.get_systype(), + "platformio_path": self._fix_os_path( + sys.argv[0] if isfile(sys.argv[0]) + else where_is_program("platformio")), + "env_pathsep": os.pathsep, + "env_path": self._fix_os_path(os.getenv("PATH")) + }) # yapf: disable + return tpl_vars - result = CliRunner().invoke(cmd_run, [ - "--project-dir", self.project_dir, "--environment", - envdata['env_name'], "--target", "idedata" - ]) - - if result.exit_code != 0 and not isinstance(result.exception, - exception.ReturnErrorCode): - raise result.exception - if '"includes":' not in result.output: - raise exception.PlatformioException(result.output) - - for line in result.output.split("\n"): - line = line.strip() - if line.startswith('{"') and line.endswith("}"): - data = json.loads(line) - return data - - def get_project_name(self): - return basename(self.project_dir) + @staticmethod + def _fix_os_path(path): + return (re.sub(r"[\\]+", '\\' * 4, path) if WINDOWS else path) def get_src_files(self): result = [] with util.cd(self.project_dir): - for root, _, files in os.walk(util.get_projectsrc_dir()): + for root, _, files in os.walk(get_project_src_dir()): for f in files: result.append(relpath(join(root, f))) return result @@ -108,6 +99,7 @@ class ProjectGenerator(object): return tpls def generate(self): + tpl_vars = self._load_tplvars() for tpl_relpath, tpl_path in self.get_tpls(): dst_dir = self.project_dir if tpl_relpath: @@ -116,44 +108,16 @@ class ProjectGenerator(object): os.makedirs(dst_dir) file_name = basename(tpl_path)[:-4] - self._merge_contents( - join(dst_dir, file_name), - self._render_tpl(tpl_path).encode("utf8")) + contents = self._render_tpl(tpl_path, tpl_vars) + self._merge_contents(join(dst_dir, file_name), contents) - def _render_tpl(self, tpl_path): - content = "" - with open(tpl_path) as f: - content = f.read() - return bottle.template(content, **self._tplvars) + @staticmethod + def _render_tpl(tpl_path, tpl_vars): + return bottle.template(get_file_contents(tpl_path), **tpl_vars) @staticmethod def _merge_contents(dst_path, contents): if basename(dst_path) == ".gitignore" and isfile(dst_path): return - with open(dst_path, "w") as f: - f.write(contents) - - def _gather_tplvars(self): - self._tplvars.update(self.get_project_env()) - self._tplvars.update(self.get_project_build_data()) - with util.cd(self.project_dir): - self._tplvars.update({ - "project_name": self.get_project_name(), - "src_files": self.get_src_files(), - "user_home_dir": abspath(expanduser("~")), - "project_dir": self.project_dir, - "project_src_dir": util.get_projectsrc_dir(), - "project_lib_dir": util.get_projectlib_dir(), - "project_libdeps_dir": util.get_projectlibdeps_dir(), - "systype": util.get_systype(), - "platformio_path": self._fix_os_path( - sys.argv[0] if isfile(sys.argv[0]) - else util.where_is_program("platformio")), - "env_pathsep": os.pathsep, - "env_path": self._fix_os_path(os.getenv("PATH")) - }) # yapf: disable - - @staticmethod - def _fix_os_path(path): - return (re.sub(r"[\\]+", '\\' * 4, path) - if "windows" in util.get_systype() else path) + with codecs.open(dst_path, "w", encoding="utf8") as fp: + fp.write(contents) diff --git a/platformio/ide/tpls/atom/.gitignore.tpl b/platformio/ide/tpls/atom/.gitignore.tpl index f1520281..bbdd36c7 100644 --- a/platformio/ide/tpls/atom/.gitignore.tpl +++ b/platformio/ide/tpls/atom/.gitignore.tpl @@ -1,5 +1,3 @@ .pio -.pioenvs -.piolibdeps .clang_complete .gcc-flags.json diff --git a/platformio/ide/tpls/clion/.gitignore.tpl b/platformio/ide/tpls/clion/.gitignore.tpl index 4c5921e1..ff1a5181 100644 --- a/platformio/ide/tpls/clion/.gitignore.tpl +++ b/platformio/ide/tpls/clion/.gitignore.tpl @@ -1,4 +1,2 @@ .pio -.pioenvs -.piolibdeps CMakeListsPrivate.txt diff --git a/platformio/ide/tpls/clion/.idea/misc.xml.tpl b/platformio/ide/tpls/clion/.idea/misc.xml.tpl index 9b483ee6..3463fba1 100644 --- a/platformio/ide/tpls/clion/.idea/misc.xml.tpl +++ b/platformio/ide/tpls/clion/.idea/misc.xml.tpl @@ -7,13 +7,10 @@ - + - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl b/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl index b05b34a8..fcf8b23d 100644 --- a/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl +++ b/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl @@ -15,7 +15,7 @@ diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 7b17a983..4680f6b4 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -1,8 +1,19 @@ +# !!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE +# https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags +# +# If you need to override existing CMake configuration or add extra, +# please create `CMakeListsUser.txt` in the root of project. +# The `CMakeListsUser.txt` will not be overwritten by PlatformIO. + cmake_minimum_required(VERSION 3.2) project({{project_name}}) include(CMakeListsPrivate.txt) +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeListsUser.txt) +include(CMakeListsUser.txt) +endif() + add_custom_target( PLATFORMIO_BUILD ALL COMMAND ${PLATFORMIO_CMD} -f -c clion run diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 0f7c053e..b3d84a75 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -1,7 +1,12 @@ -# !!! WARNING !!! -# PLEASE DO NOT MODIFY THIS FILE! -# USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags +# !!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE +# https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags +# +# If you need to override existing CMake configuration or add extra, +# please create `CMakeListsUser.txt` in the root of project. +# The `CMakeListsUser.txt` will not be overwritten by PlatformIO. +% import re +% % def _normalize_path(path): % if project_dir in path: % path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}") @@ -21,9 +26,17 @@ SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") SET(CMAKE_CXX_FLAGS_DISTRIBUTION "{{cxx_flags}}") SET(CMAKE_C_FLAGS_DISTRIBUTION "{{cc_flags}}") -set(CMAKE_CXX_STANDARD 11) -% import re +% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") +% cc_stds = STD_RE.findall(cc_flags) +% cxx_stds = STD_RE.findall(cxx_flags) +% if cc_stds: +SET(CMAKE_C_STANDARD {{ cc_stds[-1] }}) +% end +% if cxx_stds: +set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }}) +% end + % for define in defines: add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}') % end diff --git a/platformio/ide/tpls/eclipse/.cproject.tpl b/platformio/ide/tpls/eclipse/.cproject.tpl index 29b76ebc..8a237f8d 100644 --- a/platformio/ide/tpls/eclipse/.cproject.tpl +++ b/platformio/ide/tpls/eclipse/.cproject.tpl @@ -219,17 +219,17 @@ platformio -f -c eclipse - run -t program + run --target program true - true + false false platformio -f -c eclipse - run -t uploadfs + run --target uploadfs true - true + false false @@ -237,23 +237,39 @@ -f -c eclipse run true - true + false + false + + + platformio + -f -c eclipse + run --verbose + true + false false platformio -f -c eclipse - run -t upload + run --target upload true - true + false + false + + + platformio + -f -c eclipse + run --target upload --verbose + true + false false platformio -f -c eclipse - run -t clean + run --target clean true - true + false false @@ -261,15 +277,15 @@ -f -c eclipse test true - true + false false - + platformio -f -c eclipse - update + remote run --target upload true - true + false false @@ -277,7 +293,39 @@ -f -c eclipse init --ide eclipse true - true + false + false + + + platformio + -f -c eclipse + device list + true + false + false + + + platformio + -f -c eclipse + lib update + true + false + false + + + platformio + -f -c eclipse + update + true + false + false + + + platformio + -f -c eclipse + upgrade + true + false false diff --git a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl index a119c8b2..eefe1a52 100644 --- a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl +++ b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl @@ -3,6 +3,14 @@ % cxx_stds = STD_RE.findall(cxx_flags) % cxx_std = cxx_stds[-1] if cxx_stds else "" % +% if cxx_path.startswith(user_home_dir): +% if "windows" in systype: +% cxx_path = "${USERPROFILE}" + cxx_path.replace(user_home_dir, "") +% else: +% cxx_path = "${HOME}" + cxx_path.replace(user_home_dir, "") +% end +% end +% @@ -10,11 +18,7 @@ - % if "windows" in systype: - - % else: - - % end + @@ -25,11 +29,7 @@ - % if "windows" in systype: - - % else: - - % end + diff --git a/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl index e87cf90b..13b8c382 100644 --- a/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl +++ b/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl @@ -1,11 +1,11 @@ eclipse.preferences.version=1 environment/project/0.910961921/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} environment/project/0.910961921/PATH/operation=replace -environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}} +environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH} environment/project/0.910961921/append=true environment/project/0.910961921/appendContributed=true environment/project/0.910961921.1363900502/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} environment/project/0.910961921.1363900502/PATH/operation=replace -environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}} +environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH} environment/project/0.910961921.1363900502/append=true environment/project/0.910961921.1363900502/appendContributed=true \ No newline at end of file diff --git a/platformio/ide/tpls/emacs/.gitignore.tpl b/platformio/ide/tpls/emacs/.gitignore.tpl index 2c3fccfb..b8e379fa 100644 --- a/platformio/ide/tpls/emacs/.gitignore.tpl +++ b/platformio/ide/tpls/emacs/.gitignore.tpl @@ -1,4 +1,2 @@ .pio -.pioenvs -.piolibdeps .clang_complete diff --git a/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl b/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl index 7c43f565..691c6558 100644 --- a/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl +++ b/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl @@ -11,7 +11,7 @@ nbproject/private/launcher.properties - ^(nbproject|.pio|.pioenvs)$ + ^(nbproject|.pio)$ . diff --git a/platformio/ide/tpls/vim/.gitignore.tpl b/platformio/ide/tpls/vim/.gitignore.tpl index f1520281..bbdd36c7 100644 --- a/platformio/ide/tpls/vim/.gitignore.tpl +++ b/platformio/ide/tpls/vim/.gitignore.tpl @@ -1,5 +1,3 @@ .pio -.pioenvs -.piolibdeps .clang_complete .gcc-flags.json diff --git a/platformio/ide/tpls/vscode/.gitignore.tpl b/platformio/ide/tpls/vscode/.gitignore.tpl index 2de98aba..89cc49cb 100644 --- a/platformio/ide/tpls/vscode/.gitignore.tpl +++ b/platformio/ide/tpls/vscode/.gitignore.tpl @@ -1,6 +1,5 @@ .pio -.pioenvs -.piolibdeps .vscode/.browse.c_cpp.db* .vscode/c_cpp_properties.json .vscode/launch.json +.vscode/ipch diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 7894dbda..56712c25 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -12,26 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import os from os import getenv -from os.path import isdir, join +from os.path import join from time import time import click import semantic_version from platformio import __version__, app, exception, telemetry, util +from platformio.commands import PlatformioCLI +from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib import lib_update as cmd_lib_update -from platformio.commands.platform import \ - platform_install as cmd_platform_install -from platformio.commands.platform import \ - platform_uninstall as cmd_platform_uninstall from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.proc import is_ci, is_container def on_platformio_start(ctx, force, caller): @@ -40,12 +37,12 @@ def on_platformio_start(ctx, force, caller): set_caller(caller) telemetry.on_command() - if not in_silence(ctx): + if not PlatformioCLI.in_silence(): after_upgrade(ctx) -def on_platformio_end(ctx, result): # pylint: disable=W0613 - if in_silence(ctx): +def on_platformio_end(ctx, result): # pylint: disable=unused-argument + if PlatformioCLI.in_silence(): return try: @@ -64,24 +61,13 @@ def on_platformio_exception(e): telemetry.on_exception(e) -def in_silence(ctx=None): - ctx = ctx or app.get_session_var("command_ctx") - if not ctx: - return True - return ctx.args and any([ - ctx.args[0] == "debug" and "--interpreter" in " ".join(ctx.args), - ctx.args[0] == "upgrade", "--json-output" in ctx.args, - "--version" in ctx.args - ]) - - def set_caller(caller=None): if not caller: if getenv("PLATFORMIO_CALLER"): caller = getenv("PLATFORMIO_CALLER") elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): caller = "vscode" - elif util.is_container(): + elif is_container(): if getenv("C9_UID"): caller = "C9" elif getenv("USER") == "cabox": @@ -99,11 +85,7 @@ class Upgrader(object): self.to_version = semantic_version.Version.coerce( util.pepver_to_semver(to_version)) - self._upgraders = [(semantic_version.Version("3.0.0-a.1"), - self._upgrade_to_3_0_0), - (semantic_version.Version("3.0.0-b.11"), - self._upgrade_to_3_0_0b11), - (semantic_version.Version("3.5.0-a.2"), + self._upgraders = [(semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms)] def run(self, ctx): @@ -118,43 +100,6 @@ class Upgrader(object): return all(result) - @staticmethod - def _upgrade_to_3_0_0(ctx): - # convert custom board configuration - boards_dir = join(util.get_home_dir(), "boards") - if isdir(boards_dir): - for item in os.listdir(boards_dir): - if not item.endswith(".json"): - continue - data = util.load_json(join(boards_dir, item)) - if set(["name", "url", "vendor"]) <= set(data.keys()): - continue - os.remove(join(boards_dir, item)) - for key, value in data.items(): - with open(join(boards_dir, "%s.json" % key), "w") as f: - json.dump(value, f, sort_keys=True, indent=2) - - # re-install PlatformIO 2.0 development platforms - installed_platforms = app.get_state_item("installed_platforms", []) - if installed_platforms: - if "espressif" in installed_platforms: - installed_platforms[installed_platforms.index( - "espressif")] = "espressif8266" - ctx.invoke(cmd_platform_install, platforms=installed_platforms) - - return True - - @staticmethod - def _upgrade_to_3_0_0b11(ctx): - current_platforms = [ - m['name'] for m in PlatformManager().get_installed() - ] - if "espressif" not in current_platforms: - return True - ctx.invoke(cmd_platform_install, platforms=["espressif8266"]) - ctx.invoke(cmd_platform_uninstall, platforms=["espressif"]) - return True - @staticmethod def _update_dev_platforms(ctx): ctx.invoke(cmd_platform_update) @@ -173,12 +118,11 @@ def after_upgrade(ctx): last_version)) > semantic_version.Version.coerce( util.pepver_to_semver(__version__)): click.secho("*" * terminal_width, fg="yellow") - click.secho( - "Obsolete PIO Core v%s is used (previous was %s)" % (__version__, - last_version), - fg="yellow") - click.secho( - "Please remove multiple PIO Cores from a system:", fg="yellow") + click.secho("Obsolete PIO Core v%s is used (previous was %s)" % + (__version__, last_version), + fg="yellow") + click.secho("Please remove multiple PIO Cores from a system:", + fg="yellow") click.secho( "https://docs.platformio.org/page/faq.html" "#multiple-pio-cores-in-a-system", @@ -195,22 +139,20 @@ def after_upgrade(ctx): u = Upgrader(last_version, __version__) if u.run(ctx): app.set_state_item("last_version", __version__) - click.secho( - "PlatformIO has been successfully upgraded to %s!\n" % - __version__, - fg="green") - telemetry.on_event( - category="Auto", - action="Upgrade", - label="%s > %s" % (last_version, __version__)) + click.secho("PlatformIO has been successfully upgraded to %s!\n" % + __version__, + fg="green") + telemetry.on_event(category="Auto", + action="Upgrade", + label="%s > %s" % (last_version, __version__)) else: raise exception.UpgradeError("Auto upgrading...") click.echo("") # PlatformIO banner click.echo("*" * terminal_width) - click.echo( - "If you like %s, please:" % (click.style("PlatformIO", fg="cyan"))) + click.echo("If you like %s, please:" % + (click.style("PlatformIO", fg="cyan"))) click.echo("- %s us on Twitter to stay up-to-date " "on the latest project news > %s" % (click.style("follow", fg="cyan"), @@ -224,10 +166,10 @@ def after_upgrade(ctx): "- %s PlatformIO IDE for IoT development > %s" % (click.style("try", fg="cyan"), click.style("https://platformio.org/platformio-ide", fg="cyan"))) - if not util.is_ci(): - click.echo("- %s us with PlatformIO Plus > %s" % (click.style( - "support", fg="cyan"), click.style( - "https://pioplus.com", fg="cyan"))) + if not is_ci(): + click.echo("- %s us with PlatformIO Plus > %s" % + (click.style("support", fg="cyan"), + click.style("https://pioplus.com", fg="cyan"))) click.echo("*" * terminal_width) click.echo("") @@ -257,14 +199,14 @@ def check_platformio_upgrade(): click.echo("") click.echo("*" * terminal_width) - click.secho( - "There is a new version %s of PlatformIO available.\n" - "Please upgrade it via `" % latest_version, - fg="yellow", - nl=False) + click.secho("There is a new version %s of PlatformIO available.\n" + "Please upgrade it via `" % latest_version, + fg="yellow", + nl=False) if getenv("PLATFORMIO_IDE"): - click.secho( - "PlatformIO IDE Menu: Upgrade PlatformIO", fg="cyan", nl=False) + click.secho("PlatformIO IDE Menu: Upgrade PlatformIO", + fg="cyan", + nl=False) click.secho("`.", fg="yellow") elif join("Cellar", "platformio") in util.get_source_dir(): click.secho("brew update && brew upgrade", fg="cyan", nl=False) @@ -275,8 +217,8 @@ def check_platformio_upgrade(): click.secho("pip install -U platformio", fg="cyan", nl=False) click.secho("` command.", fg="yellow") click.secho("Changes: ", fg="yellow", nl=False) - click.secho( - "https://docs.platformio.org/en/latest/history.html", fg="cyan") + click.secho("https://docs.platformio.org/en/latest/history.html", + fg="cyan") click.echo("*" * terminal_width) click.echo("") @@ -312,41 +254,39 @@ def check_internal_updates(ctx, what): click.echo("") click.echo("*" * terminal_width) - click.secho( - "There are the new updates for %s (%s)" % (what, - ", ".join(outdated_items)), - fg="yellow") + click.secho("There are the new updates for %s (%s)" % + (what, ", ".join(outdated_items)), + fg="yellow") if not app.get_setting("auto_update_" + what): click.secho("Please update them via ", fg="yellow", nl=False) - click.secho( - "`platformio %s update`" % - ("lib --global" if what == "libraries" else "platform"), - fg="cyan", - nl=False) + click.secho("`platformio %s update`" % + ("lib --global" if what == "libraries" else "platform"), + fg="cyan", + nl=False) click.secho(" command.\n", fg="yellow") click.secho( "If you want to manually check for the new versions " "without updating, please use ", fg="yellow", nl=False) - click.secho( - "`platformio %s update --only-check`" % - ("lib --global" if what == "libraries" else "platform"), - fg="cyan", - nl=False) + click.secho("`platformio %s update --dry-run`" % + ("lib --global" if what == "libraries" else "platform"), + fg="cyan", + nl=False) click.secho(" command.", fg="yellow") else: click.secho("Please wait while updating %s ..." % what, fg="yellow") if what == "platforms": ctx.invoke(cmd_platform_update, platforms=outdated_items) elif what == "libraries": - ctx.obj = pm + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [pm.package_dir] ctx.invoke(cmd_lib_update, libraries=outdated_items) click.echo() - telemetry.on_event( - category="Auto", action="Update", label=what.title()) + telemetry.on_event(category="Auto", + action="Update", + label=what.title()) click.echo("*" * terminal_width) click.echo("") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index ec48ef07..f93145fb 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,15 +21,18 @@ from time import sleep import requests from platformio import __version__, exception, util +from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager +from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path +from platformio.project.helpers import get_project_packages_dir CORE_PACKAGES = { - "contrib-piohome": "^2.0.1", + "contrib-piohome": "^2.1.0", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.1.4", + "tool-pioplus": "^2.5.2", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" + "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 @@ -40,12 +43,11 @@ PIOPLUS_AUTO_UPDATES_MAX = 100 class CorePackageManager(PackageManager): def __init__(self): - super(CorePackageManager, self).__init__( - join(util.get_home_dir(), "packages"), [ - "https://dl.bintray.com/platformio/dl-packages/manifest.json", - "http%s://dl.platformio.org/packages/manifest.json" % - ("" if sys.version_info < (2, 7, 9) else "s") - ]) + super(CorePackageManager, self).__init__(get_project_packages_dir(), [ + "https://dl.bintray.com/platformio/dl-packages/manifest.json", + "http%s://dl.platformio.org/packages/manifest.json" % + ("" if sys.version_info < (2, 7, 9) else "s") + ]) def install( # pylint: disable=keyword-arg-before-vararg self, @@ -99,7 +101,7 @@ def update_core_packages(only_check=False, silent=False): if not silent or pm.outdated(pkg_dir, requirements): if name == "tool-pioplus" and not only_check: shutdown_piohome_servers() - if "windows" in util.get_systype(): + if WINDOWS: sleep(1) pm.update(name, requirements, only_check=only_check) return True @@ -109,28 +111,38 @@ def shutdown_piohome_servers(): port = 8010 while port < 8050: try: - requests.get( - "http://127.0.0.1:%d?__shutdown__=1" % port, timeout=0.01) + requests.get("http://127.0.0.1:%d?__shutdown__=1" % port, + timeout=0.01) except: # pylint: disable=bare-except pass port += 1 +def inject_contrib_pysite(): + from site import addsitedir + contrib_pysite_dir = get_core_package_dir("contrib-pysite") + if contrib_pysite_dir in sys.path: + return + addsitedir(contrib_pysite_dir) + sys.path.insert(0, contrib_pysite_dir) + + def pioplus_call(args, **kwargs): - if "windows" in util.get_systype() and sys.version_info < (2, 7, 6): + if WINDOWS and sys.version_info < (2, 7, 6): raise exception.PlatformioException( "PlatformIO Core Plus v%s does not run under Python version %s.\n" "Minimum supported version is 2.7.6, please upgrade Python.\n" "Python 3 is not yet supported.\n" % (__version__, sys.version)) pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus") - pythonexe_path = util.get_pythonexe_path() + pythonexe_path = get_pythonexe_path() os.environ['PYTHONEXEPATH'] = pythonexe_path os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite") os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "") - os.environ['PATH'] = (os.pathsep).join( - [dirname(pythonexe_path), os.environ['PATH']]) - util.copy_pythonpath_to_osenv() + if dirname(pythonexe_path) not in os.environ['PATH'].split(os.pathsep): + os.environ['PATH'] = (os.pathsep).join( + [dirname(pythonexe_path), os.environ['PATH']]) + copy_pythonpath_to_osenv() code = subprocess.call([pioplus_path] + args, **kwargs) # handle remote update request diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 9042e624..b6f630c8 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -22,16 +22,20 @@ from os.path import isdir, join import click -from platformio import app, commands, exception, util +from platformio import app, exception, util +from platformio.compat import glob_escape, string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.project.helpers import get_project_global_lib_dir class LibraryManager(BasePkgManager): + FILE_CACHE_VALID = "30d" # 1 month + def __init__(self, package_dir=None): if not package_dir: - package_dir = join(util.get_home_dir(), "lib") + package_dir = get_project_global_lib_dir() super(LibraryManager, self).__init__(package_dir) @property @@ -47,7 +51,7 @@ class LibraryManager(BasePkgManager): return path # if library without manifest, returns first source file - src_dir = join(util.glob_escape(pkg_dir)) + src_dir = join(glob_escape(pkg_dir)) if isdir(join(pkg_dir, "src")): src_dir = join(src_dir, "src") chs_files = glob(join(src_dir, "*.[chS]")) @@ -120,7 +124,7 @@ class LibraryManager(BasePkgManager): # convert listed items via comma to array for key in ("keywords", "frameworks", "platforms"): if key not in manifest or \ - not isinstance(manifest[key], basestring): + not isinstance(manifest[key], string_types): continue manifest[key] = [ i.strip() for i in manifest[key].split(",") if i.strip() @@ -147,7 +151,7 @@ class LibraryManager(BasePkgManager): continue if item[k] == "*": del item[k] - elif isinstance(item[k], basestring): + elif isinstance(item[k], string_types): item[k] = [ i.strip() for i in item[k].split(",") if i.strip() ] @@ -164,7 +168,7 @@ class LibraryManager(BasePkgManager): semver_spec = self.parse_semver_spec( requirements) if requirements else None - item = None + item = {} for v in versions: semver_new = self.parse_semver_version(v['name']) @@ -186,14 +190,12 @@ class LibraryManager(BasePkgManager): def get_latest_repo_version(self, name, requirements, silent=False): item = self.max_satisfying_repo_version( - util.get_api_result( - "/lib/info/%d" % self.search_lib_id( - { - "name": name, - "requirements": requirements - }, - silent=silent), - cache_valid="1h")['versions'], requirements) + util.get_api_result("/lib/info/%d" % self.search_lib_id( + { + "name": name, + "requirements": requirements + }, silent=silent), + cache_valid="1h")['versions'], requirements) return item['name'] if item else None def _install_from_piorepo(self, name, requirements): @@ -202,10 +204,9 @@ class LibraryManager(BasePkgManager): if not version: raise exception.UndefinedPackageVersion(requirements or "latest", util.get_systype()) - dl_data = util.get_api_result( - "/lib/download/" + str(name[3:]), - dict(version=version), - cache_valid="30d") + dl_data = util.get_api_result("/lib/download/" + str(name[3:]), + dict(version=version), + cache_valid="30d") assert dl_data return self._install_from_url( @@ -227,8 +228,8 @@ class LibraryManager(BasePkgManager): # looking in PIO Library Registry if not silent: - click.echo("Looking for %s library in registry" % click.style( - filters['name'], fg="cyan")) + click.echo("Looking for %s library in registry" % + click.style(filters['name'], fg="cyan")) query = [] for key in filters: if key not in ("name", "authors", "frameworks", "platforms"): @@ -241,21 +242,22 @@ class LibraryManager(BasePkgManager): (key[:-1] if key.endswith("s") else key, value)) lib_info = None - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h") + result = util.get_api_result("/v2/lib/search", + dict(query=" ".join(query)), + cache_valid="1h") if result['total'] == 1: lib_info = result['items'][0] elif result['total'] > 1: if silent and not interactive: lib_info = result['items'][0] else: - click.secho( - "Conflict: More than one library has been found " - "by request %s:" % json.dumps(filters), - fg="yellow", - err=True) + click.secho("Conflict: More than one library has been found " + "by request %s:" % json.dumps(filters), + fg="yellow", + err=True) + from platformio.commands.lib import print_lib_item for item in result['items']: - commands.lib.print_lib_item(item) + print_lib_item(item) if not interactive: click.secho( @@ -265,20 +267,20 @@ class LibraryManager(BasePkgManager): err=True) lib_info = result['items'][0] else: - deplib_id = click.prompt( - "Please choose library ID", - type=click.Choice( - [str(i['id']) for i in result['items']])) + deplib_id = click.prompt("Please choose library ID", + type=click.Choice([ + str(i['id']) + for i in result['items'] + ])) for item in result['items']: if item['id'] == int(deplib_id): lib_info = item break if not lib_info: - if filters.keys() == ["name"]: + if list(filters) == ["name"]: raise exception.LibNotFound(filters['name']) - else: - raise exception.LibNotFound(str(filters)) + raise exception.LibNotFound(str(filters)) if not silent: click.echo("Found: %s" % click.style( "https://platformio.org/lib/show/{id}/{name}".format( @@ -303,9 +305,8 @@ class LibraryManager(BasePkgManager): continue if key not in manifest: return None - if not util.items_in_list( - util.items_to_list(filters[key]), - util.items_to_list(manifest[key])): + if not util.items_in_list(util.items_to_list(filters[key]), + util.items_to_list(manifest[key])): return None if "authors" in filters: @@ -336,20 +337,20 @@ class LibraryManager(BasePkgManager): force=False): _name, _requirements, _url = self.parse_pkg_uri(name, requirements) if not _url: - name = "id=%d" % self.search_lib_id({ - "name": _name, - "requirements": _requirements - }, - silent=silent, - interactive=interactive) + name = "id=%d" % self.search_lib_id( + { + "name": _name, + "requirements": _requirements + }, + silent=silent, + interactive=interactive) requirements = _requirements - pkg_dir = BasePkgManager.install( - self, - name, - requirements, - silent=silent, - after_update=after_update, - force=force) + pkg_dir = BasePkgManager.install(self, + name, + requirements, + silent=silent, + after_update=after_update, + force=force) if not pkg_dir: return None @@ -361,6 +362,7 @@ class LibraryManager(BasePkgManager): if not silent: click.secho("Installing dependencies", fg="yellow") + builtin_lib_storages = None for filters in self.normalize_dependencies(manifest['dependencies']): assert "name" in filters @@ -373,39 +375,38 @@ class LibraryManager(BasePkgManager): self.INSTALL_HISTORY.append(history_key) if any(s in filters.get("version", "") for s in ("\\", "/")): - self.install( - "{name}={version}".format(**filters), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install("{name}={version}".format(**filters), + silent=silent, + after_update=after_update, + interactive=interactive, + force=force) else: try: lib_id = self.search_lib_id(filters, silent, interactive) except exception.LibNotFound as e: - if not silent or is_builtin_lib(filters['name']): + if builtin_lib_storages is None: + builtin_lib_storages = get_builtin_libs() + if not silent or is_builtin_lib(builtin_lib_storages, + filters['name']): click.secho("Warning! %s" % e, fg="yellow") continue if filters.get("version"): - self.install( - lib_id, - filters.get("version"), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install(lib_id, + filters.get("version"), + silent=silent, + after_update=after_update, + interactive=interactive, + force=force) else: - self.install( - lib_id, - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install(lib_id, + silent=silent, + after_update=after_update, + interactive=interactive, + force=force) return pkg_dir -@util.memoized() def get_builtin_libs(storage_names=None): items = [] storage_names = storage_names or [] @@ -424,9 +425,8 @@ def get_builtin_libs(storage_names=None): return items -@util.memoized() -def is_builtin_lib(name): - for storage in get_builtin_libs(): +def is_builtin_lib(storages, name): + for storage in storages or []: if any(l.get("name") == name for l in storage['items']): return True return False diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 2f73452d..82043114 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -26,6 +26,7 @@ import requests import semantic_version from platformio import __version__, app, exception, telemetry, util +from platformio.compat import hashlib_encode_data from platformio.downloader import FileDownloader from platformio.lockfile import LockFile from platformio.unpacker import FileUnpacker @@ -36,8 +37,6 @@ from platformio.vcsclient import VCSClientFactory class PackageRepoIterator(object): - _MANIFEST_CACHE = {} - def __init__(self, package, repositories): assert isinstance(repositories, list) self.package = package @@ -49,29 +48,27 @@ class PackageRepoIterator(object): def __next__(self): return self.next() - def next(self): - manifest = {} - repo = next(self.repositories) - if isinstance(repo, dict): - manifest = repo - elif repo in PackageRepoIterator._MANIFEST_CACHE: - manifest = PackageRepoIterator._MANIFEST_CACHE[repo] - else: - r = None - try: - r = requests.get(repo, headers=util.get_request_defheaders()) - r.raise_for_status() - manifest = r.json() - except: # pylint: disable=bare-except - pass - finally: - if r: - r.close() - PackageRepoIterator._MANIFEST_CACHE[repo] = manifest + @staticmethod + @util.memoized(expire="60s") + def load_manifest(url): + r = None + try: + r = requests.get(url, headers=util.get_request_defheaders()) + r.raise_for_status() + return r.json() + except: # pylint: disable=bare-except + pass + finally: + if r: + r.close() + return None - if self.package in manifest: + def next(self): + repo = next(self.repositories) + manifest = repo if isinstance(repo, dict) else self.load_manifest(repo) + if manifest and self.package in manifest: return manifest[self.package] - return self.next() + return next(self) class PkgRepoMixin(object): @@ -91,18 +88,18 @@ class PkgRepoMixin(object): reqspec = None if requirements: try: - reqspec = self.parse_semver_spec( - requirements, raise_exception=True) + reqspec = self.parse_semver_spec(requirements, + raise_exception=True) except ValueError: pass for v in versions: if not self.is_system_compatible(v.get("system")): continue - if "platformio" in v.get("engines", {}): - if PkgRepoMixin.PIO_VERSION not in self.parse_semver_spec( - v['engines']['platformio'], raise_exception=True): - continue + # if "platformio" in v.get("engines", {}): + # if PkgRepoMixin.PIO_VERSION not in self.parse_semver_spec( + # v['engines']['platformio'], raise_exception=True): + # continue specver = semantic_version.Version(v['version']) if reqspec and specver not in reqspec: continue @@ -138,22 +135,19 @@ class PkgInstallerMixin(object): SRC_MANIFEST_NAME = ".piopkgmanager.json" TMP_FOLDER_PREFIX = "_tmp_installing-" - FILE_CACHE_VALID = "1m" # 1 month - FILE_CACHE_MAX_SIZE = 1024 * 1024 + FILE_CACHE_VALID = None # for example, 1 week = "7d" + FILE_CACHE_MAX_SIZE = 1024 * 1024 * 50 # 50 Mb - MEMORY_CACHE = {} + MEMORY_CACHE = {} # cache for package manifests and read dirs - @staticmethod - def cache_get(key, default=None): - return PkgInstallerMixin.MEMORY_CACHE.get(key, default) + def cache_get(self, key, default=None): + return self.MEMORY_CACHE.get(key, default) - @staticmethod - def cache_set(key, value): - PkgInstallerMixin.MEMORY_CACHE[key] = value + def cache_set(self, key, value): + self.MEMORY_CACHE[key] = value - @staticmethod - def cache_reset(): - PkgInstallerMixin.MEMORY_CACHE = {} + def cache_reset(self): + self.MEMORY_CACHE.clear() def read_dirs(self, src_dir): cache_key = "read_dirs-%s" % src_dir @@ -172,11 +166,12 @@ class PkgInstallerMixin(object): cache_key_data = app.ContentCache.key_from_args(url, "data") if self.FILE_CACHE_VALID: with app.ContentCache() as cc: - fname = cc.get(cache_key_fname) + fname = str(cc.get(cache_key_fname)) cache_path = cc.get_cache_path(cache_key_data) if fname and isfile(cache_path): dst_path = join(dest_dir, fname) shutil.copy(cache_path, dst_path) + click.echo("Using cache: %s" % cache_path) return dst_path with_progress = not app.is_disabled_progressbar() @@ -329,14 +324,15 @@ class PkgInstallerMixin(object): name += "_ID%d" % manifest['id'] return str(name) - def get_src_manifest_path(self, pkg_dir): + @classmethod + def get_src_manifest_path(cls, pkg_dir): if not isdir(pkg_dir): return None for item in os.listdir(pkg_dir): if not isdir(join(pkg_dir, item)): continue - if isfile(join(pkg_dir, item, self.SRC_MANIFEST_NAME)): - return join(pkg_dir, item, self.SRC_MANIFEST_NAME) + if isfile(join(pkg_dir, item, cls.SRC_MANIFEST_NAME)): + return join(pkg_dir, item, cls.SRC_MANIFEST_NAME) return None def get_manifest_path(self, pkg_dir): @@ -392,7 +388,7 @@ class PkgInstallerMixin(object): if "version" not in manifest: manifest['version'] = "0.0.0" - manifest['__pkg_dir'] = util.path_to_unicode(pkg_dir) + manifest['__pkg_dir'] = pkg_dir self.cache_set(cache_key, manifest) return manifest @@ -429,8 +425,8 @@ class PkgInstallerMixin(object): try: if requirements and not self.parse_semver_spec( requirements, raise_exception=True).match( - self.parse_semver_version( - manifest['version'], raise_exception=True)): + self.parse_semver_version(manifest['version'], + raise_exception=True)): continue elif not best or (self.parse_semver_version( manifest['version'], raise_exception=True) > @@ -449,7 +445,7 @@ class PkgInstallerMixin(object): def get_package_by_dir(self, pkg_dir): for manifest in self.get_installed(): - if manifest['__pkg_dir'] == util.path_to_unicode(abspath(pkg_dir)): + if manifest['__pkg_dir'] == abspath(pkg_dir): return manifest return None @@ -481,7 +477,7 @@ class PkgInstallerMixin(object): if versions is None: util.internet_on(raise_exception=True) raise exception.UnknownPackage(name) - elif not pkgdata: + if not pkgdata: raise exception.UndefinedPackageVersion(requirements or "latest", util.get_systype()) return pkg_dir @@ -544,7 +540,7 @@ class PkgInstallerMixin(object): def _install_from_tmp_dir( # pylint: disable=too-many-branches self, tmp_dir, requirements=None): tmp_manifest = self.load_manifest(tmp_dir) - assert set(["name", "version"]) <= set(tmp_manifest.keys()) + assert set(["name", "version"]) <= set(tmp_manifest) pkg_dirname = self.get_install_dirname(tmp_manifest) pkg_dir = join(self.package_dir, pkg_dirname) @@ -587,8 +583,10 @@ class PkgInstallerMixin(object): cur_manifest['version']) if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( - pkg_dirname, hashlib.md5( - cur_manifest['__src_url']).hexdigest()) + pkg_dirname, + hashlib.md5( + hashlib_encode_data( + cur_manifest['__src_url'])).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -596,8 +594,10 @@ class PkgInstallerMixin(object): tmp_manifest['version']) if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( - pkg_dirname, hashlib.md5( - tmp_manifest['__src_url']).hexdigest()) + pkg_dirname, + hashlib.md5( + hashlib_encode_data( + tmp_manifest['__src_url'])).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package @@ -645,8 +645,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if "__src_url" in manifest: try: - vcs = VCSClientFactory.newClient( - pkg_dir, manifest['__src_url'], silent=True) + vcs = VCSClientFactory.newClient(pkg_dir, + manifest['__src_url'], + silent=True) except (AttributeError, exception.PlatformioException): return None if not vcs.can_be_updated: @@ -655,8 +656,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): else: try: latest = self.get_latest_repo_version( - "id=%d" % manifest['id'] - if "id" in manifest else manifest['name'], + "id=%d" % + manifest['id'] if "id" in manifest else manifest['name'], requirements, silent=True) except (exception.PlatformioException, ValueError): @@ -668,10 +669,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): up_to_date = False try: assert "__src_url" not in manifest - up_to_date = (self.parse_semver_version( - manifest['version'], raise_exception=True) >= - self.parse_semver_version( - latest, raise_exception=True)) + up_to_date = (self.parse_semver_version(manifest['version'], + raise_exception=True) >= + self.parse_semver_version(latest, + raise_exception=True)) except (AssertionError, ValueError): up_to_date = latest == manifest['version'] @@ -717,8 +718,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): return package_dir if url: - pkg_dir = self._install_from_url( - name, url, requirements, track=True) + pkg_dir = self._install_from_url(name, + url, + requirements, + track=True) else: pkg_dir = self._install_from_piorepo(name, requirements) @@ -730,16 +733,14 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): assert manifest if not after_update: - telemetry.on_event( - category=self.__class__.__name__, - action="Install", - label=manifest['name']) + telemetry.on_event(category=self.__class__.__name__, + action="Install", + label=manifest['name']) - if not silent: - click.secho( - "{name} @ {version} has been successfully installed!". - format(**manifest), - fg="green") + click.secho( + "{name} @ {version} has been successfully installed!".format( + **manifest), + fg="green") return pkg_dir @@ -756,14 +757,13 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPackage( - "%s @ %s" % (package, requirements or "*")) + raise exception.UnknownPackage("%s @ %s" % + (package, requirements or "*")) manifest = self.load_manifest(pkg_dir) - click.echo( - "Uninstalling %s @ %s: \t" % (click.style( - manifest['name'], fg="cyan"), manifest['version']), - nl=False) + click.echo("Uninstalling %s @ %s: \t" % (click.style( + manifest['name'], fg="cyan"), manifest['version']), + nl=False) if islink(pkg_dir): os.unlink(pkg_dir) @@ -782,31 +782,30 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): click.echo("[%s]" % click.style("OK", fg="green")) if not after_update: - telemetry.on_event( - category=self.__class__.__name__, - action="Uninstall", - label=manifest['name']) + telemetry.on_event(category=self.__class__.__name__, + action="Uninstall", + label=manifest['name']) return True def update(self, package, requirements=None, only_check=False): + self.cache_reset() if isdir(package) and self.get_package_by_dir(package): pkg_dir = package else: pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package)) if not pkg_dir: - raise exception.UnknownPackage( - "%s @ %s" % (package, requirements or "*")) + raise exception.UnknownPackage("%s @ %s" % + (package, requirements or "*")) manifest = self.load_manifest(pkg_dir) name = manifest['name'] - click.echo( - "{} {:<40} @ {:<15}".format( - "Checking" if only_check else "Updating", - click.style(manifest['name'], fg="cyan"), manifest['version']), - nl=False) + click.echo("{} {:<40} @ {:<15}".format( + "Checking" if only_check else "Updating", + click.style(manifest['name'], fg="cyan"), manifest['version']), + nl=False) if not util.internet_on(): click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) return None @@ -825,23 +824,20 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if "__src_url" in manifest: vcs = VCSClientFactory.newClient(pkg_dir, manifest['__src_url']) assert vcs.update() - self._update_src_manifest( - dict(version=vcs.get_current_revision()), vcs.storage_dir) + self._update_src_manifest(dict(version=vcs.get_current_revision()), + vcs.storage_dir) else: self.uninstall(pkg_dir, after_update=True) self.install(name, latest, after_update=True) - telemetry.on_event( - category=self.__class__.__name__, - action="Update", - label=manifest['name']) + telemetry.on_event(category=self.__class__.__name__, + action="Update", + label=manifest['name']) return True class PackageManager(BasePkgManager): - FILE_CACHE_VALID = None # disable package caching - @property def manifest_names(self): return ["package.json"] diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index a2c1c795..1a45e433 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -15,23 +15,33 @@ import base64 import os import re +import sys from imp import load_source -from multiprocessing import cpu_count from os.path import basename, dirname, isdir, isfile, join -from urllib import quote import click import semantic_version from platformio import __version__, app, exception, util +from platformio.compat import PY2, hashlib_encode_data, is_bytes from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, + exec_command, get_pythonexe_path) +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (get_project_boards_dir, + get_project_core_dir, + get_project_packages_dir, + get_project_platforms_dir) + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote class PlatformManager(BasePkgManager): - FILE_CACHE_VALID = None # disable platform download caching - def __init__(self, package_dir=None, repositories=None): if not repositories: repositories = [ @@ -39,9 +49,8 @@ class PlatformManager(BasePkgManager): "{0}://dl.platformio.org/platforms/manifest.json".format( "https" if app.get_setting("enable_ssl") else "http") ] - BasePkgManager.__init__( - self, package_dir or join(util.get_home_dir(), "platforms"), - repositories) + BasePkgManager.__init__(self, package_dir + or get_project_platforms_dir(), repositories) @property def manifest_names(self): @@ -66,8 +75,11 @@ class PlatformManager(BasePkgManager): silent=False, force=False, **_): # pylint: disable=too-many-arguments, arguments-differ - platform_dir = BasePkgManager.install( - self, name, requirements, silent=silent, force=force) + platform_dir = BasePkgManager.install(self, + name, + requirements, + silent=silent, + force=force) p = PlatformFactory.newPlatform(platform_dir) # don't cleanup packages or install them after update @@ -75,13 +87,12 @@ class PlatformManager(BasePkgManager): if after_update: return True - p.install_packages( - with_packages, - without_packages, - skip_default_package, - silent=silent, - force=force) - return self.cleanup_packages(p.packages.keys()) + p.install_packages(with_packages, + without_packages, + skip_default_package, + silent=silent, + force=force) + return self.cleanup_packages(list(p.packages)) def uninstall(self, package, requirements=None, after_update=False): if isdir(package): @@ -101,7 +112,7 @@ class PlatformManager(BasePkgManager): if after_update: return True - return self.cleanup_packages(p.packages.keys()) + return self.cleanup_packages(list(p.packages)) def update( # pylint: disable=arguments-differ self, @@ -119,21 +130,21 @@ class PlatformManager(BasePkgManager): raise exception.UnknownPlatform(package) p = PlatformFactory.newPlatform(pkg_dir) - pkgs_before = p.get_installed_packages().keys() + pkgs_before = list(p.get_installed_packages()) missed_pkgs = set() if not only_packages: BasePkgManager.update(self, pkg_dir, requirements, only_check) p = PlatformFactory.newPlatform(pkg_dir) - missed_pkgs = set(pkgs_before) & set(p.packages.keys()) - missed_pkgs -= set(p.get_installed_packages().keys()) + missed_pkgs = set(pkgs_before) & set(p.packages) + missed_pkgs -= set(p.get_installed_packages()) p.update_packages(only_check) - self.cleanup_packages(p.packages.keys()) + self.cleanup_packages(list(p.packages)) if missed_pkgs: - p.install_packages( - with_packages=list(missed_pkgs), skip_default_package=True) + p.install_packages(with_packages=list(missed_pkgs), + skip_default_package=True) return True @@ -147,7 +158,7 @@ class PlatformManager(BasePkgManager): deppkgs[pkgname] = set() deppkgs[pkgname].add(pkgmanifest['version']) - pm = PackageManager(join(util.get_home_dir(), "packages")) + pm = PackageManager(get_project_packages_dir()) for manifest in pm.get_installed(): if manifest['name'] not in names: continue @@ -161,7 +172,7 @@ class PlatformManager(BasePkgManager): self.cache_reset() return True - @util.memoized(expire=5000) + @util.memoized(expire="5s") def get_installed_boards(self): boards = [] for manifest in self.get_installed(): @@ -173,7 +184,6 @@ class PlatformManager(BasePkgManager): return boards @staticmethod - @util.memoized() def get_registered_boards(): return util.get_api_result("/boards", cache_valid="7d") @@ -244,8 +254,8 @@ class PlatformFactory(object): cls.load_module(name, join(platform_dir, "platform.py")), cls.get_clsname(name)) else: - platform_cls = type( - str(cls.get_clsname(name)), (PlatformBase, ), {}) + platform_cls = type(str(cls.get_clsname(name)), (PlatformBase, ), + {}) _instance = platform_cls(join(platform_dir, "platform.json")) assert isinstance(_instance, PlatformBase) @@ -265,7 +275,7 @@ class PlatformPackagesMixin(object): without_packages = set(self.find_pkg_names(without_packages or [])) upkgs = with_packages | without_packages - ppkgs = set(self.packages.keys()) + ppkgs = set(self.packages) if not upkgs.issubset(ppkgs): raise exception.UnknownPackage(", ".join(upkgs - ppkgs)) @@ -276,8 +286,9 @@ class PlatformPackagesMixin(object): elif (name in with_packages or not (skip_default_package or opts.get("optional", False))): if ":" in version: - self.pm.install( - "%s=%s" % (name, version), silent=silent, force=force) + self.pm.install("%s=%s" % (name, version), + silent=silent, + force=force) else: self.pm.install(name, version, silent=silent, force=force) @@ -346,11 +357,27 @@ class PlatformRunMixin(object): LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) - def run(self, variables, targets, silent, verbose): + @staticmethod + def encode_scons_arg(value): + data = base64.urlsafe_b64encode(hashlib_encode_data(value)) + return data.decode() if is_bytes(data) else data + + @staticmethod + def decode_scons_arg(data): + value = base64.urlsafe_b64decode(data) + return value.decode() if is_bytes(value) else value + + def run( # pylint: disable=too-many-arguments + self, variables, targets, silent, verbose, jobs): assert isinstance(variables, dict) assert isinstance(targets, list) - self.configure_default_packages(variables, targets) + config = ProjectConfig.get_instance(variables['project_config']) + options = config.items(env=variables['pioenv'], as_dict=True) + if "framework" in options: + # support PIO Core 3.0 dev/platforms + options['pioframework'] = options['framework'] + self.configure_default_packages(options, targets) self.install_packages(silent=True) self.silent = silent @@ -366,39 +393,53 @@ class PlatformRunMixin(object): if not isfile(variables['build_script']): raise exception.BuildScriptNotFound(variables['build_script']) - result = self._run_scons(variables, targets) + result = self._run_scons(variables, targets, jobs) assert "returncode" in result return result - def _run_scons(self, variables, targets): - cmd = [ - util.get_pythonexe_path(), - join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q", - "-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support", - "-f", - join(util.get_source_dir(), "builder", "main.py") - ] - cmd.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) - cmd += targets + def _run_scons(self, variables, targets, jobs): + args = [ + get_pythonexe_path(), + join(get_core_package_dir("tool-scons"), "script", "scons"), + "-Q", "--warn=no-no-parallel-support", + "--jobs", str(jobs), + "--sconstruct", join(util.get_source_dir(), "builder", "main.py") + ] # yapf: disable + args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + # pylint: disable=protected-access + args.append("ISATTY=%d" % + (1 if click._compat.isatty(sys.stdout) else 0)) + args += targets # encode and append variables for key, value in variables.items(): - cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) + args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - util.copy_pythonpath_to_osenv() - result = util.exec_command( - cmd, - stdout=util.AsyncPipe(self.on_run_out), - stderr=util.AsyncPipe(self.on_run_err)) + def _write_and_flush(stream, data): + try: + stream.write(data) + stream.flush() + except IOError: + pass + + copy_pythonpath_to_osenv() + result = exec_command( + args, + stdout=BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: _write_and_flush(sys.stdout, data)), + stderr=BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data))) return result - def on_run_out(self, line): + def _on_stdout_line(self, line): if "`buildprog' is up to date." in line: return self._echo_line(line, level=1) - def on_run_err(self, line): + def _on_stderr_line(self, line): is_error = self.LINE_ERROR_RE.search(line) is not None self._echo_line(line, level=3 if is_error else 2) @@ -417,7 +458,7 @@ class PlatformRunMixin(object): fg = (None, "yellow", "red")[level - 1] if level == 1 and "is up to date" in line: fg = "green" - click.secho(line, fg=fg, err=level > 1) + click.secho(line, fg=fg, err=level > 1, nl=False) @staticmethod def _echo_missed_dependency(filename): @@ -434,19 +475,12 @@ class PlatformRunMixin(object): """.format(filename=filename, filename_styled=click.style(filename, fg="cyan"), link=click.style( - "https://platformio.org/lib/search?query=header:%s" % quote( - filename, safe=""), + "https://platformio.org/lib/search?query=header:%s" % + quote(filename, safe=""), fg="blue"), dots="*" * (56 + len(filename))) click.echo(banner, err=True) - @staticmethod - def get_job_nums(): - try: - return cpu_count() - except NotImplementedError: - return 1 - class PlatformBase( # pylint: disable=too-many-public-methods PlatformPackagesMixin, PlatformRunMixin): @@ -455,21 +489,22 @@ class PlatformBase( # pylint: disable=too-many-public-methods _BOARDS_CACHE = {} def __init__(self, manifest_path): - self._BOARDS_CACHE = {} self.manifest_path = manifest_path - self._manifest = util.load_json(manifest_path) - - self.pm = PackageManager( - join(util.get_home_dir(), "packages"), self.package_repositories) - self.silent = False self.verbose = False - if self.engines and "platformio" in self.engines: - if self.PIO_VERSION not in semantic_version.Spec( - self.engines['platformio']): - raise exception.IncompatiblePlatform(self.name, - str(self.PIO_VERSION)) + self._BOARDS_CACHE = {} + self._manifest = util.load_json(manifest_path) + self._custom_packages = None + + self.pm = PackageManager(get_project_packages_dir(), + self.package_repositories) + + # if self.engines and "platformio" in self.engines: + # if self.PIO_VERSION not in semantic_version.Spec( + # self.engines['platformio']): + # raise exception.IncompatiblePlatform(self.name, + # str(self.PIO_VERSION)) @property def name(self): @@ -525,9 +560,20 @@ class PlatformBase( # pylint: disable=too-many-public-methods @property def packages(self): - if "packages" not in self._manifest: - self._manifest['packages'] = {} - return self._manifest['packages'] + packages = self._manifest.get("packages", {}) + for item in (self._custom_packages or []): + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 2) + name = name.strip() + if name not in packages: + packages[name] = {} + packages[name].update({ + "version": version.strip(), + "optional": False + }) + return packages def get_dir(self): return dirname(self.manifest_path) @@ -550,15 +596,15 @@ class PlatformBase( # pylint: disable=too-many-public-methods config = PlatformBoardConfig(manifest_path) if "platform" in config and config.get("platform") != self.name: return - elif "platforms" in config \ + if "platforms" in config \ and self.name not in config.get("platforms"): return config.manifest['platform'] = self.name self._BOARDS_CACHE[board_id] = config bdirs = [ - util.get_projectboards_dir(), - join(util.get_home_dir(), "boards"), + get_project_boards_dir(), + join(get_project_core_dir(), "boards"), join(self.get_dir(), "boards"), ] @@ -590,12 +636,12 @@ class PlatformBase( # pylint: disable=too-many-public-methods def get_package_type(self, name): return self.packages[name].get("type") - def configure_default_packages(self, variables, targets): + def configure_default_packages(self, options, targets): + # override user custom packages + self._custom_packages = options.get("platform_packages") + # enable used frameworks - frameworks = variables.get("pioframework", []) - if not isinstance(frameworks, list): - frameworks = frameworks.split(", ") - for framework in frameworks: + for framework in options.get("framework", []): if not self.frameworks: continue framework = framework.lower().strip() @@ -650,7 +696,7 @@ class PlatformBoardConfig(object): self._manifest = util.load_json(manifest_path) except ValueError: raise exception.InvalidBoardManifest(manifest_path) - if not set(["name", "url", "vendor"]) <= set(self._manifest.keys()): + if not set(["name", "url", "vendor"]) <= set(self._manifest): raise exception.PlatformioException( "Please specify name, url and vendor fields for " + manifest_path) @@ -660,12 +706,20 @@ class PlatformBoardConfig(object): value = self._manifest for k in path.split("."): value = value[k] + # pylint: disable=undefined-variable + if PY2 and isinstance(value, unicode): + # cast to plain string from unicode for PY2, resolves issue in + # dev/platform when BoardConfig.get() is used in pair with + # os.path.join(file_encoding, unicode_encoding) + try: + value = value.encode("utf-8") + except UnicodeEncodeError: + pass return value except KeyError: if default is not None: return default - else: - raise KeyError("Invalid board option '%s'" % path) + raise KeyError("Invalid board option '%s'" % path) def update(self, path, value): newdict = None @@ -750,7 +804,7 @@ class PlatformBoardConfig(object): return tool_name raise exception.DebugInvalidOptions( "Unknown debug tool `%s`. Please use one of `%s` or `custom`" % - (tool_name, ", ".join(sorted(debug_tools.keys())))) + (tool_name, ", ".join(sorted(list(debug_tools))))) # automatically select best tool data = {"default": [], "onboard": [], "external": []} diff --git a/platformio/proc.py b/platformio/proc.py new file mode 100644 index 00000000..066813ed --- /dev/null +++ b/platformio/proc.py @@ -0,0 +1,191 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +from os.path import isdir, isfile, join, normpath +from threading import Thread + +from platformio import exception +from platformio.compat import WINDOWS, string_types + + +class AsyncPipeBase(object): + + def __init__(self): + self._fd_read, self._fd_write = os.pipe() + self._pipe_reader = os.fdopen(self._fd_read) + self._buffer = "" + self._thread = Thread(target=self.run) + self._thread.start() + + def get_buffer(self): + return self._buffer + + def fileno(self): + return self._fd_write + + def run(self): + try: + self.do_reading() + except (KeyboardInterrupt, SystemExit, IOError): + self.close() + + def do_reading(self): + raise NotImplementedError() + + def close(self): + self._buffer = "" + os.close(self._fd_write) + self._thread.join() + + +class BuildAsyncPipe(AsyncPipeBase): + + def __init__(self, line_callback, data_callback): + self.line_callback = line_callback + self.data_callback = data_callback + super(BuildAsyncPipe, self).__init__() + + def do_reading(self): + line = "" + print_immediately = False + + for byte in iter(lambda: self._pipe_reader.read(1), ""): + self._buffer += byte + + if line and byte.strip() and line[-3:] == (byte * 3): + print_immediately = True + + if print_immediately: + # leftover bytes + if line: + self.data_callback(line) + line = "" + self.data_callback(byte) + if byte == "\n": + print_immediately = False + else: + line += byte + if byte != "\n": + continue + self.line_callback(line) + line = "" + + self._pipe_reader.close() + + +class LineBufferedAsyncPipe(AsyncPipeBase): + + def __init__(self, line_callback): + self.line_callback = line_callback + super(LineBufferedAsyncPipe, self).__init__() + + def do_reading(self): + for line in iter(self._pipe_reader.readline, ""): + self._buffer += line + self.line_callback(line) + self._pipe_reader.close() + + +def exec_command(*args, **kwargs): + result = {"out": None, "err": None, "returncode": None} + + default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) + default.update(kwargs) + kwargs = default + + p = subprocess.Popen(*args, **kwargs) + try: + result['out'], result['err'] = p.communicate() + result['returncode'] = p.returncode + except KeyboardInterrupt: + raise exception.AbortedByUser() + finally: + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipeBase): + kwargs[s].close() + + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipeBase): + result[s[3:]] = kwargs[s].get_buffer() + + for k, v in result.items(): + if isinstance(result[k], bytes): + try: + result[k] = result[k].decode(sys.getdefaultencoding()) + except UnicodeDecodeError: + result[k] = result[k].decode("latin-1") + if v and isinstance(v, string_types): + result[k] = result[k].strip() + + return result + + +def is_ci(): + return os.getenv("CI", "").lower() == "true" + + +def is_container(): + if not isfile("/proc/1/cgroup"): + return False + with open("/proc/1/cgroup") as fp: + for line in fp: + line = line.strip() + if ":" in line and not line.endswith(":/"): + return True + return False + + +def get_pythonexe_path(): + return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) + + +def copy_pythonpath_to_osenv(): + _PYTHONPATH = [] + if "PYTHONPATH" in os.environ: + _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) + for p in os.sys.path: + conditions = [p not in _PYTHONPATH] + if not WINDOWS: + conditions.append( + isdir(join(p, "click")) or isdir(join(p, "platformio"))) + if all(conditions): + _PYTHONPATH.append(p) + os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH) + + +def where_is_program(program, envpath=None): + env = os.environ + if envpath: + env['PATH'] = envpath + + # try OS's built-in commands + try: + result = exec_command(["where" if WINDOWS else "which", program], + env=env) + if result['returncode'] == 0 and isfile(result['out'].strip()): + return result['out'].strip() + except OSError: + pass + + # look up in $PATH + for bin_dir in env.get("PATH", "").split(os.pathsep): + if isfile(join(bin_dir, program)): + return join(bin_dir, program) + if isfile(join(bin_dir, "%s.exe" % program)): + return join(bin_dir, "%s.exe" % program) + + return program diff --git a/platformio/project/__init__.py b/platformio/project/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/project/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/project/config.py b/platformio/project/config.py new file mode 100644 index 00000000..dc455809 --- /dev/null +++ b/platformio/project/config.py @@ -0,0 +1,314 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +import re +from os.path import isfile + +import click + +from platformio import exception +from platformio.project.options import ProjectOptions + +try: + import ConfigParser as ConfigParser +except ImportError: + import configparser as ConfigParser + +CONFIG_HEADER = """;PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; https://docs.platformio.org/page/projectconf.html + +""" + + +class ProjectConfig(object): + + INLINE_COMMENT_RE = re.compile(r"\s+;.*$") + VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") + + expand_interpolations = True + warnings = [] + + _instances = {} + _parser = None + _parsed = [] + + @staticmethod + def parse_multi_values(items): + result = [] + if not items: + return result + if not isinstance(items, (list, tuple)): + items = items.split("\n" if "\n" in items else ", ") + for item in items: + item = item.strip() + # comment + if not item or item.startswith((";", "#")): + continue + if ";" in item: + item = ProjectConfig.INLINE_COMMENT_RE.sub("", item).strip() + result.append(item) + return result + + @staticmethod + def get_instance(path): + if path not in ProjectConfig._instances: + ProjectConfig._instances[path] = ProjectConfig(path) + return ProjectConfig._instances[path] + + @staticmethod + def reset_instances(): + ProjectConfig._instances = {} + + def __init__(self, path, parse_extra=True, expand_interpolations=True): + self.path = path + self.expand_interpolations = expand_interpolations + self.warnings = [] + self._parsed = [] + self._parser = ConfigParser.ConfigParser() + if isfile(path): + self.read(path, parse_extra) + + def __getattr__(self, name): + return getattr(self._parser, name) + + def read(self, path, parse_extra=True): + if path in self._parsed: + return + self._parsed.append(path) + try: + self._parser.read(path) + except ConfigParser.Error as e: + raise exception.InvalidProjectConf(path, str(e)) + + if not parse_extra: + return + + # load extra configs + for pattern in self.get("platformio", "extra_configs", []): + for item in glob.glob(pattern): + self.read(item) + + self._maintain_renaimed_options() + + def _maintain_renaimed_options(self): + # legacy `lib_extra_dirs` in [platformio] + if (self._parser.has_section("platformio") + and self._parser.has_option("platformio", "lib_extra_dirs")): + if not self._parser.has_section("env"): + self._parser.add_section("env") + self._parser.set("env", "lib_extra_dirs", + self._parser.get("platformio", "lib_extra_dirs")) + self._parser.remove_option("platformio", "lib_extra_dirs") + self.warnings.append( + "`lib_extra_dirs` configuration option is deprecated in " + "section [platformio]! Please move it to global `env` section") + + renamed_options = {} + for option in ProjectOptions.values(): + if option.oldnames: + renamed_options.update( + {name: option.name + for name in option.oldnames}) + + for section in self._parser.sections(): + scope = section.split(":", 1)[0] + if scope not in ("platformio", "env"): + continue + for option in self._parser.options(section): + if option in renamed_options: + self.warnings.append( + "`%s` configuration option in section [%s] is " + "deprecated and will be removed in the next release! " + "Please use `%s` instead" % + (option, section, renamed_options[option])) + # rename on-the-fly + self._parser.set(section, renamed_options[option], + self._parser.get(section, option)) + self._parser.remove_option(section, option) + continue + + # unknown + unknown_conditions = [ + ("%s.%s" % (scope, option)) not in ProjectOptions, + scope != "env" or + not option.startswith(("custom_", "board_")) + ] # yapf: disable + if all(unknown_conditions): + self.warnings.append( + "Ignore unknown configuration option `%s` " + "in section [%s]" % (option, section)) + return True + + def options(self, section=None, env=None): + assert section or env + if not section: + section = "env:" + env + options = self._parser.options(section) + + # handle global options from [env] + if ((env or section.startswith("env:")) + and self._parser.has_section("env")): + for option in self._parser.options("env"): + if option not in options: + options.append(option) + + # handle system environment variables + scope = section.split(":", 1)[0] + for option_meta in ProjectOptions.values(): + if option_meta.scope != scope or option_meta.name in options: + continue + if option_meta.sysenvvar and option_meta.sysenvvar in os.environ: + options.append(option_meta.name) + + return options + + def has_option(self, section, option): + if self._parser.has_option(section, option): + return True + return (section.startswith("env:") and self._parser.has_section("env") + and self._parser.has_option("env", option)) + + def items(self, section=None, env=None, as_dict=False): + assert section or env + if not section: + section = "env:" + env + if as_dict: + return { + option: self.get(section, option) + for option in self.options(section) + } + return [(option, self.get(section, option)) + for option in self.options(section)] + + def set(self, section, option, value): + if isinstance(value, (list, tuple)): + value = "\n".join(value) + if value: + value = "\n" + value # start from a new line + self._parser.set(section, option, value) + + def getraw(self, section, option): + if not self.expand_interpolations: + return self._parser.get(section, option) + + try: + value = self._parser.get(section, option) + except ConfigParser.NoOptionError as e: + if not section.startswith("env:"): + raise e + value = self._parser.get("env", option) + + if "${" not in value or "}" not in value: + return value + return self.VARTPL_RE.sub(self._re_interpolation_handler, value) + + def _re_interpolation_handler(self, match): + section, option = match.group(1), match.group(2) + if section == "sysenv": + return os.getenv(option) + return self.getraw(section, option) + + def get(self, section, option, default=None): + value = None + try: + value = self.getraw(section, option) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + pass # handle value from system environment + except ConfigParser.Error as e: + raise exception.InvalidProjectConf(self.path, str(e)) + + option_meta = ProjectOptions.get("%s.%s" % + (section.split(":", 1)[0], option)) + if not option_meta: + return value or default + + if option_meta.multiple: + value = self.parse_multi_values(value) + + if option_meta.sysenvvar: + envvar_value = os.getenv(option_meta.sysenvvar) + if not envvar_value and option_meta.oldnames: + for oldoption in option_meta.oldnames: + envvar_value = os.getenv("PLATFORMIO_" + oldoption.upper()) + if envvar_value: + break + if envvar_value and option_meta.multiple: + value = value or [] + value.extend(self.parse_multi_values(envvar_value)) + elif envvar_value and not value: + value = envvar_value + + # option is not specified by user + if value is None: + return default + + try: + return self._covert_value(value, option_meta.type) + except click.BadParameter as e: + raise exception.ProjectOptionValueError(e.format_message(), option, + section) + + @staticmethod + def _covert_value(value, to_type): + items = value + if not isinstance(value, (list, tuple)): + items = [value] + items = [ + to_type(item) if isinstance(to_type, click.ParamType) else item + for item in items + ] + return items if isinstance(value, (list, tuple)) else items[0] + + def envs(self): + return [s[4:] for s in self._parser.sections() if s.startswith("env:")] + + def default_envs(self): + return self.get("platformio", "default_envs", []) + + def validate(self, envs=None, silent=False): + if not isfile(self.path): + raise exception.NotPlatformIOProject(self.path) + # check envs + known = set(self.envs()) + if not known: + raise exception.ProjectEnvsNotAvailable() + unknown = set(list(envs or []) + self.default_envs()) - known + if unknown: + raise exception.UnknownEnvNames(", ".join(unknown), + ", ".join(known)) + if not silent: + for warning in self.warnings: + click.secho("Warning! %s" % warning, fg="yellow") + return True + + def to_json(self): + result = {} + for section in self.sections(): + result[section] = self.items(section, as_dict=True) + return json.dumps(result) + + def save(self, path=None): + with open(path or self.path, "w") as fp: + fp.write(CONFIG_HEADER) + self._parser.write(fp) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py new file mode 100644 index 00000000..3c017666 --- /dev/null +++ b/platformio/project/helpers.py @@ -0,0 +1,203 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from hashlib import sha1 +from os import walk +from os.path import (basename, dirname, expanduser, isdir, isfile, join, + realpath, splitdrive) + +from click.testing import CliRunner + +from platformio import __version__, exception +from platformio.compat import WINDOWS, hashlib_encode_data +from platformio.project.config import ProjectConfig + + +def get_project_dir(): + return os.getcwd() + + +def is_platformio_project(project_dir=None): + if not project_dir: + project_dir = get_project_dir() + return isfile(join(project_dir, "platformio.ini")) + + +def find_project_dir_above(path): + if isfile(path): + path = dirname(path) + if is_platformio_project(path): + return path + if isdir(dirname(path)): + return find_project_dir_above(dirname(path)) + return None + + +def get_project_optional_dir(name, default=None): + project_dir = get_project_dir() + config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) + optional_dir = config.get("platformio", name) + + if not optional_dir: + return default + + if "$PROJECT_HASH" in optional_dir: + optional_dir = optional_dir.replace( + "$PROJECT_HASH", "%s-%s" % + (basename(project_dir), sha1( + hashlib_encode_data(project_dir)).hexdigest()[:10])) + + if optional_dir.startswith("~"): + optional_dir = expanduser(optional_dir) + + return realpath(optional_dir) + + +def get_project_core_dir(): + default = join(expanduser("~"), ".platformio") + core_dir = get_project_optional_dir( + "core_dir", get_project_optional_dir("home_dir", default)) + win_core_dir = None + if WINDOWS and core_dir == default: + win_core_dir = splitdrive(core_dir)[0] + "\\.platformio" + if isdir(win_core_dir): + core_dir = win_core_dir + + if not isdir(core_dir): + try: + os.makedirs(core_dir) + except OSError as e: + if win_core_dir: + os.makedirs(win_core_dir) + core_dir = win_core_dir + else: + raise e + + assert isdir(core_dir) + return core_dir + + +def get_project_global_lib_dir(): + return get_project_optional_dir("globallib_dir", + join(get_project_core_dir(), "lib")) + + +def get_project_platforms_dir(): + return get_project_optional_dir("platforms_dir", + join(get_project_core_dir(), "platforms")) + + +def get_project_packages_dir(): + return get_project_optional_dir("packages_dir", + join(get_project_core_dir(), "packages")) + + +def get_project_cache_dir(): + return get_project_optional_dir("cache_dir", + join(get_project_core_dir(), ".cache")) + + +def get_project_workspace_dir(): + return get_project_optional_dir("workspace_dir", + join(get_project_dir(), ".pio")) + + +def get_project_build_dir(force=False): + path = get_project_optional_dir("build_dir", + join(get_project_workspace_dir(), "build")) + try: + if not isdir(path): + os.makedirs(path) + except Exception as e: # pylint: disable=broad-except + if not force: + raise Exception(e) + return path + + +def get_project_libdeps_dir(): + return get_project_optional_dir( + "libdeps_dir", join(get_project_workspace_dir(), "libdeps")) + + +def get_project_lib_dir(): + return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) + + +def get_project_include_dir(): + return get_project_optional_dir("include_dir", + join(get_project_dir(), "include")) + + +def get_project_src_dir(): + return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) + + +def get_project_test_dir(): + return get_project_optional_dir("test_dir", join(get_project_dir(), + "test")) + + +def get_project_boards_dir(): + return get_project_optional_dir("boards_dir", + join(get_project_dir(), "boards")) + + +def get_project_data_dir(): + return get_project_optional_dir("data_dir", join(get_project_dir(), + "data")) + + +def get_project_shared_dir(): + return get_project_optional_dir("shared_dir", + join(get_project_dir(), "shared")) + + +def calculate_project_hash(): + check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") + chunks = [__version__] + for d in (get_project_src_dir(), get_project_lib_dir()): + if not isdir(d): + continue + for root, _, files in walk(d): + for f in files: + path = join(root, f) + if path.endswith(check_suffixes): + chunks.append(path) + chunks_to_str = ",".join(sorted(chunks)) + if WINDOWS: + # Fix issue with useless project rebuilding for case insensitive FS. + # A case of disk drive can differ... + chunks_to_str = chunks_to_str.lower() + return sha1(hashlib_encode_data(chunks_to_str)).hexdigest() + + +def load_project_ide_data(project_dir, env_name): + from platformio.commands.run import cli as cmd_run + result = CliRunner().invoke(cmd_run, [ + "--project-dir", project_dir, "--environment", env_name, "--target", + "idedata" + ]) + if result.exit_code != 0 and not isinstance(result.exception, + exception.ReturnErrorCode): + raise result.exception + if '"includes":' not in result.output: + raise exception.PlatformioException(result.output) + + for line in result.output.split("\n"): + line = line.strip() + if line.startswith('{"') and line.endswith("}"): + return json.loads(line) + return None diff --git a/platformio/project/options.py b/platformio/project/options.py new file mode 100644 index 00000000..bc2f5c3d --- /dev/null +++ b/platformio/project/options.py @@ -0,0 +1,203 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-builtin, too-many-arguments + +from collections import OrderedDict, namedtuple + +import click + +ConfigOptionClass = namedtuple("ConfigOption", [ + "scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames" +]) + + +def ConfigOption(scope, + name, + type=str, + multiple=False, + sysenvvar=None, + buildenvvar=None, + oldnames=None): + return ConfigOptionClass(scope, name, type, multiple, sysenvvar, + buildenvvar, oldnames) + + +def ConfigPlatformioOption(*args, **kwargs): + return ConfigOption("platformio", *args, **kwargs) + + +def ConfigEnvOption(*args, **kwargs): + return ConfigOption("env", *args, **kwargs) + + +ProjectOptions = OrderedDict([ + ("%s.%s" % (option.scope, option.name), option) for option in [ + # + # [platformio] + # + ConfigPlatformioOption(name="description"), + ConfigPlatformioOption(name="default_envs", + oldnames=["env_default"], + multiple=True, + sysenvvar="PLATFORMIO_DEFAULT_ENVS"), + ConfigPlatformioOption(name="extra_configs", multiple=True), + + # Dirs + ConfigPlatformioOption(name="core_dir", + oldnames=["home_dir"], + sysenvvar="PLATFORMIO_CORE_DIR"), + ConfigPlatformioOption(name="globallib_dir", + sysenvvar="PLATFORMIO_GLOBALLIB_DIR"), + ConfigPlatformioOption(name="platforms_dir", + sysenvvar="PLATFORMIO_PLATFORMS_DIR"), + ConfigPlatformioOption(name="packages_dir", + sysenvvar="PLATFORMIO_PACKAGES_DIR"), + ConfigPlatformioOption(name="cache_dir", + sysenvvar="PLATFORMIO_CACHE_DIR"), + ConfigPlatformioOption(name="build_cache_dir", + sysenvvar="PLATFORMIO_BUILD_CACHE_DIR"), + ConfigPlatformioOption(name="workspace_dir", + sysenvvar="PLATFORMIO_WORKSPACE_DIR"), + ConfigPlatformioOption(name="build_dir", + sysenvvar="PLATFORMIO_BUILD_DIR"), + ConfigPlatformioOption(name="libdeps_dir", + sysenvvar="PLATFORMIO_LIBDEPS_DIR"), + ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"), + ConfigPlatformioOption(name="include_dir", + sysenvvar="PLATFORMIO_INCLUDE_DIR"), + ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"), + ConfigPlatformioOption(name="test_dir", + sysenvvar="PLATFORMIO_TEST_DIR"), + ConfigPlatformioOption(name="boards_dir", + sysenvvar="PLATFORMIO_BOARDS_DIR"), + ConfigPlatformioOption(name="data_dir", + sysenvvar="PLATFORMIO_DATA_DIR"), + ConfigPlatformioOption(name="shared_dir", + sysenvvar="PLATFORMIO_SHARED_DIR"), + + # + # [env] + # + + # Generic + ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"), + ConfigEnvOption(name="platform_packages", multiple=True), + ConfigEnvOption( + name="framework", multiple=True, buildenvvar="PIOFRAMEWORK"), + + # Board + ConfigEnvOption(name="board", buildenvvar="BOARD"), + ConfigEnvOption(name="board_build.mcu", + oldnames=["board_mcu"], + buildenvvar="BOARD_MCU"), + ConfigEnvOption(name="board_build.f_cpu", + oldnames=["board_f_cpu"], + buildenvvar="BOARD_F_CPU"), + ConfigEnvOption(name="board_build.f_flash", + oldnames=["board_f_flash"], + buildenvvar="BOARD_F_FLASH"), + ConfigEnvOption(name="board_build.flash_mode", + oldnames=["board_flash_mode"], + buildenvvar="BOARD_FLASH_MODE"), + + # Build + ConfigEnvOption(name="build_type", + type=click.Choice(["release", "debug"])), + ConfigEnvOption(name="build_flags", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_FLAGS", + buildenvvar="BUILD_FLAGS"), + ConfigEnvOption(name="src_build_flags", + multiple=True, + sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS", + buildenvvar="SRC_BUILD_FLAGS"), + ConfigEnvOption(name="build_unflags", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_UNFLAGS", + buildenvvar="BUILD_UNFLAGS"), + ConfigEnvOption(name="src_filter", + multiple=True, + sysenvvar="PLATFORMIO_SRC_FILTER", + buildenvvar="SRC_FILTER"), + ConfigEnvOption(name="targets", multiple=True), + + # Upload + ConfigEnvOption(name="upload_port", + sysenvvar="PLATFORMIO_UPLOAD_PORT", + buildenvvar="UPLOAD_PORT"), + ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"), + ConfigEnvOption( + name="upload_speed", type=click.INT, buildenvvar="UPLOAD_SPEED"), + ConfigEnvOption(name="upload_flags", + multiple=True, + sysenvvar="PLATFORMIO_UPLOAD_FLAGS", + buildenvvar="UPLOAD_FLAGS"), + ConfigEnvOption(name="upload_resetmethod", + buildenvvar="UPLOAD_RESETMETHOD"), + ConfigEnvOption(name="upload_command", buildenvvar="UPLOADCMD"), + + # Monitor + ConfigEnvOption(name="monitor_port"), + ConfigEnvOption(name="monitor_speed", oldnames=["monitor_baud"]), + ConfigEnvOption(name="monitor_rts", type=click.IntRange(0, 1)), + ConfigEnvOption(name="monitor_dtr", type=click.IntRange(0, 1)), + ConfigEnvOption(name="monitor_flags", multiple=True), + + # Library + ConfigEnvOption(name="lib_deps", + oldnames=["lib_use", "lib_force", "lib_install"], + multiple=True), + ConfigEnvOption(name="lib_ignore", multiple=True), + ConfigEnvOption(name="lib_extra_dirs", + multiple=True, + sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), + ConfigEnvOption(name="lib_ldf_mode", + type=click.Choice( + ["off", "chain", "deep", "chain+", "deep+"])), + ConfigEnvOption(name="lib_compat_mode", + type=click.Choice(["off", "soft", "strict"])), + ConfigEnvOption(name="lib_archive", type=click.BOOL), + + # Test + ConfigEnvOption(name="test_filter", multiple=True), + ConfigEnvOption(name="test_ignore", multiple=True), + ConfigEnvOption(name="test_port"), + ConfigEnvOption(name="test_speed", type=click.INT), + ConfigEnvOption(name="test_transport"), + ConfigEnvOption(name="test_build_project_src", type=click.BOOL), + + # Debug + ConfigEnvOption(name="debug_tool"), + ConfigEnvOption(name="debug_init_break"), + ConfigEnvOption(name="debug_init_cmds", multiple=True), + ConfigEnvOption(name="debug_extra_cmds", multiple=True), + ConfigEnvOption(name="debug_load_cmds", + oldnames=["debug_load_cmd"], + multiple=True), + ConfigEnvOption(name="debug_load_mode", + type=click.Choice(["always", "modified", "manual"])), + ConfigEnvOption(name="debug_server", multiple=True), + ConfigEnvOption(name="debug_port"), + ConfigEnvOption(name="debug_svd_path", + type=click.Path( + exists=True, file_okay=True, dir_okay=False)), + + # Other + ConfigEnvOption(name="extra_scripts", + oldnames=["extra_script"], + multiple=True, + sysenvvar="PLATFORMIO_EXTRA_SCRIPTS") + ] +]) diff --git a/platformio/projectconftpl.ini b/platformio/projectconftpl.ini deleted file mode 100644 index ebc510b2..00000000 --- a/platformio/projectconftpl.ini +++ /dev/null @@ -1,9 +0,0 @@ -; PlatformIO Project Configuration File -; -; Build options: build flags, source filter -; Upload options: custom upload port, speed and extra flags -; Library options: dependencies, extra library storages -; Advanced options: extra scripting -; -; Please visit documentation for the other options and examples -; https://docs.platformio.org/page/projectconf.html diff --git a/platformio/telemetry.py b/platformio/telemetry.py index d0f4c468..d35aac40 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -14,7 +14,6 @@ import atexit import platform -import Queue import re import sys import threading @@ -28,6 +27,14 @@ import click import requests from platformio import __version__, app, exception, util +from platformio.commands import PlatformioCLI +from platformio.compat import string_types +from platformio.proc import is_ci, is_container + +try: + import queue +except ImportError: + import Queue as queue class TelemetryBase(object): @@ -78,12 +85,12 @@ class MeasurementProtocol(TelemetryBase): def __getitem__(self, name): if name in self.PARAMS_MAP: name = self.PARAMS_MAP[name] - return TelemetryBase.__getitem__(self, name) + return super(MeasurementProtocol, self).__getitem__(name) def __setitem__(self, name, value): if name in self.PARAMS_MAP: name = self.PARAMS_MAP[name] - TelemetryBase.__setitem__(self, name, value) + super(MeasurementProtocol, self).__setitem__(name, value) def _prefill_appinfo(self): self['av'] = __version__ @@ -117,7 +124,7 @@ class MeasurementProtocol(TelemetryBase): platform.platform()) # self['cd3'] = " ".join(_filter_args(sys.argv[1:])) self['cd4'] = 1 if (not util.is_ci() and - (caller_id or not util.is_container())) else 0 + (caller_id or not is_container())) else 0 if caller_id: self['cd5'] = caller_id.lower() @@ -129,12 +136,15 @@ class MeasurementProtocol(TelemetryBase): return _arg return None - if not app.get_session_var("command_ctx"): - return - ctx_args = app.get_session_var("command_ctx").args - args = [str(s).lower() for s in ctx_args if not str(s).startswith("-")] + args = [] + for arg in PlatformioCLI.leftover_args: + if not isinstance(arg, string_types): + arg = str(arg) + if not arg.startswith("-"): + args.append(arg.lower()) if not args: return + cmd_path = args[:1] if args[0] in ("platform", "platforms", "serialports", "device", "settings", "account"): @@ -182,7 +192,7 @@ class MPDataPusher(object): MAX_WORKERS = 5 def __init__(self): - self._queue = Queue.LifoQueue() + self._queue = queue.LifoQueue() self._failedque = deque() self._http_session = requests.Session() self._http_offline = False @@ -207,7 +217,7 @@ class MPDataPusher(object): try: while True: items.append(self._queue.get_nowait()) - except Queue.Empty: + except queue.Empty: pass return items @@ -268,7 +278,7 @@ def on_command(): mp = MeasurementProtocol() mp.send("screenview") - if util.is_ci(): + if is_ci(): measure_ci() @@ -304,12 +314,15 @@ def measure_ci(): def on_run_environment(options, targets): - opts = [ - "%s=%s" % (opt, value.replace("\n", ", ") if "\n" in value else value) - for opt, value in sorted(options.items()) - ] + non_sensative_values = ["board", "platform", "framework"] + safe_options = [] + for key, value in sorted(options.items()): + if key in non_sensative_values: + safe_options.append("%s=%s" % (key, value)) + else: + safe_options.append(key) targets = [t.title() for t in targets or ["run"]] - on_event("Env", " ".join(targets), "&".join(opts)) + on_event("Env", " ".join(targets), "&".join(safe_options)) def on_event(category, action, label=None, value=None, screen_name=None): @@ -329,21 +342,17 @@ def on_exception(e): def _cleanup_description(text): text = text.replace("Traceback (most recent call last):", "") - text = re.sub( - r'File "([^"]+)"', - lambda m: join(*m.group(1).split(sep)[-2:]), - text, - flags=re.M) + text = re.sub(r'File "([^"]+)"', + lambda m: join(*m.group(1).split(sep)[-2:]), + text, + flags=re.M) text = re.sub(r"\s+", " ", text, flags=re.M) return text.strip() skip_conditions = [ - isinstance(e, cls) - for cls in (IOError, exception.ReturnErrorCode, - exception.AbortedByUser, exception.NotGlobalLibDir, - exception.InternetIsOffline, - exception.NotPlatformIOProject, - exception.UserSideException) + isinstance(e, cls) for cls in (IOError, exception.ReturnErrorCode, + exception.UserSideException, + exception.PlatformIOProjectException) ] try: skip_conditions.append("[API] Account: " in str(e)) @@ -388,7 +397,7 @@ def backup_reports(items): for params in items: # skip static options - for key in params.keys(): + for key in list(params.keys()): if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"): del params[key] diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 11c23814..271b4911 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -68,15 +68,14 @@ class ZIPArchive(ArchiveBase): @staticmethod def preserve_permissions(item, dest_dir): - attrs = item.external_attr >> 16L + attrs = item.external_attr >> 16 if attrs: chmod(join(dest_dir, item.filename), attrs) @staticmethod def preserve_mtime(item, dest_dir): - util.change_filemtime( - join(dest_dir, item.filename), - mktime(list(item.date_time) + [0] * 3)) + util.change_filemtime(join(dest_dir, item.filename), + mktime(tuple(item.date_time) + tuple([0, 0, 0]))) def get_items(self): return self._afo.infolist() diff --git a/platformio/util.py b/platformio/util.py index c04cd949..377160b1 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -18,94 +18,21 @@ import platform import re import socket import stat -import subprocess import sys import time +from contextlib import contextmanager from functools import wraps from glob import glob -from hashlib import sha1 -from os.path import (abspath, basename, dirname, expanduser, isdir, isfile, - join, normpath, splitdrive) +from os.path import abspath, basename, dirname, isfile, join from shutil import rmtree -from threading import Thread import click import requests from platformio import __apiurl__, __version__, exception - -# pylint: disable=wrong-import-order, too-many-ancestors - -try: - import configparser as ConfigParser -except ImportError: - import ConfigParser as ConfigParser - - -class ProjectConfig(ConfigParser.ConfigParser): - - VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") - - def items(self, section, **_): # pylint: disable=arguments-differ - items = [] - for option in ConfigParser.ConfigParser.options(self, section): - items.append((option, self.get(section, option))) - return items - - def get(self, section, option, **kwargs): - try: - value = ConfigParser.ConfigParser.get(self, section, option, - **kwargs) - except ConfigParser.Error as e: - raise exception.InvalidProjectConf(str(e)) - if "${" not in value or "}" not in value: - return value - return self.VARTPL_RE.sub(self._re_sub_handler, value) - - def _re_sub_handler(self, match): - section, option = match.group(1), match.group(2) - if section in ("env", "sysenv") and not self.has_section(section): - if section == "env": - click.secho( - "Warning! Access to system environment variable via " - "`${{env.{0}}}` is deprecated. Please use " - "`${{sysenv.{0}}}` instead".format(option), - fg="yellow") - return os.getenv(option) - return self.get(section, option) - - -class AsyncPipe(Thread): - - def __init__(self, outcallback=None): - super(AsyncPipe, self).__init__() - self.outcallback = outcallback - - self._fd_read, self._fd_write = os.pipe() - self._pipe_reader = os.fdopen(self._fd_read) - self._buffer = [] - - self.start() - - def get_buffer(self): - return self._buffer - - def fileno(self): - return self._fd_write - - def run(self): - for line in iter(self._pipe_reader.readline, ""): - line = line.strip() - self._buffer.append(line) - if self.outcallback: - self.outcallback(line) - else: - print(line) - self._pipe_reader.close() - - def close(self): - os.close(self._fd_write) - self.join() +from platformio.commands import PlatformioCLI +from platformio.compat import PY2, WINDOWS, get_file_contents +from platformio.proc import exec_command, is_ci class cd(object): @@ -124,7 +51,12 @@ class cd(object): class memoized(object): def __init__(self, expire=0): - self.expire = expire / 1000 # milliseconds + expire = str(expire) + if expire.isdigit(): + expire = "%ss" % int((int(expire) / 1000)) + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert expire.endswith(tuple(tdmap)) + self.expire = int(tdmap[expire[-1]] * int(expire[:-1])) self.cache = {} def __call__(self, func): @@ -142,7 +74,7 @@ class memoized(object): return wrapper def _reset(self): - self.cache = {} + self.cache.clear() class throttle(object): @@ -176,8 +108,15 @@ def singleton(cls): return get_instance -def path_to_unicode(path): - return path.decode(sys.getfilesystemencoding()).encode("utf-8") +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr def load_json(file_path): @@ -202,63 +141,6 @@ def pioversion_to_intstr(): return [int(i) for i in vermatch.group(1).split(".")[:3]] -def get_project_optional_dir(name, default=None): - paths = None - var_name = "PLATFORMIO_%s" % name.upper() - if var_name in os.environ: - paths = os.getenv(var_name) - else: - try: - config = load_project_config() - if (config.has_section("platformio") - and config.has_option("platformio", name)): - paths = config.get("platformio", name) - except exception.NotPlatformIOProject: - pass - - if not paths: - return default - - items = [] - for item in paths.split(", "): - if item.startswith("~"): - item = expanduser(item) - items.append(abspath(item)) - paths = ", ".join(items) - - while "$PROJECT_HASH" in paths: - paths = paths.replace("$PROJECT_HASH", - sha1(get_project_dir()).hexdigest()[:10]) - - return paths - - -def get_home_dir(): - home_dir = get_project_optional_dir("home_dir", - join(expanduser("~"), ".platformio")) - win_home_dir = None - if "windows" in get_systype(): - win_home_dir = splitdrive(home_dir)[0] + "\\.platformio" - if isdir(win_home_dir): - home_dir = win_home_dir - - if not isdir(home_dir): - try: - os.makedirs(home_dir) - except: # pylint: disable=bare-except - if win_home_dir: - os.makedirs(win_home_dir) - home_dir = win_home_dir - - assert isdir(home_dir) - return home_dir - - -def get_cache_dir(): - return get_project_optional_dir("cache_dir", join(get_home_dir(), - ".cache")) - - def get_source_dir(): curpath = abspath(__file__) if not isfile(curpath): @@ -269,174 +151,10 @@ def get_source_dir(): return dirname(curpath) -def get_project_dir(): - return os.getcwd() - - -def find_project_dir_above(path): - if isfile(path): - path = dirname(path) - if is_platformio_project(path): - return path - if isdir(dirname(path)): - return find_project_dir_above(dirname(path)) - return None - - -def is_platformio_project(project_dir=None): - if not project_dir: - project_dir = get_project_dir() - return isfile(join(project_dir, "platformio.ini")) - - -def get_projectlib_dir(): - return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) - - -def get_projectlibdeps_dir(): - return get_project_optional_dir("libdeps_dir", - join(get_project_dir(), ".piolibdeps")) - - -def get_projectsrc_dir(): - return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) - - -def get_projectinclude_dir(): - return get_project_optional_dir("include_dir", - join(get_project_dir(), "include")) - - -def get_projecttest_dir(): - return get_project_optional_dir("test_dir", join(get_project_dir(), - "test")) - - -def get_projectboards_dir(): - return get_project_optional_dir("boards_dir", - join(get_project_dir(), "boards")) - - -def get_projectbuild_dir(force=False): - path = get_project_optional_dir("build_dir", - join(get_project_dir(), ".pioenvs")) - try: - if not isdir(path): - os.makedirs(path) - dontmod_path = join(path, "do-not-modify-files-here.url") - if not isfile(dontmod_path): - with open(dontmod_path, "w") as fp: - fp.write(""" -[InternetShortcut] -URL=https://docs.platformio.org/page/projectconf/section_platformio.html#build-dir -""") - except Exception as e: # pylint: disable=broad-except - if not force: - raise Exception(e) - return path - - -# compatibility with PIO Core+ -get_projectpioenvs_dir = get_projectbuild_dir - - -def get_projectdata_dir(): - return get_project_optional_dir("data_dir", join(get_project_dir(), - "data")) - - -def load_project_config(path=None): - if not path or isdir(path): - path = join(path or get_project_dir(), "platformio.ini") - if not isfile(path): - raise exception.NotPlatformIOProject( - dirname(path) if path.endswith("platformio.ini") else path) - cp = ProjectConfig() - try: - cp.read(path) - except ConfigParser.Error as e: - raise exception.InvalidProjectConf(str(e)) - return cp - - -def parse_conf_multi_values(items): - result = [] - if not items: - return result - inline_comment_re = re.compile(r"\s+;.*$") - for item in items.split("\n" if "\n" in items else ", "): - item = item.strip() - # comment - if not item or item.startswith((";", "#")): - continue - if ";" in item: - item = inline_comment_re.sub("", item).strip() - result.append(item) - return result - - def change_filemtime(path, mtime): os.utime(path, (mtime, mtime)) -def is_ci(): - return os.getenv("CI", "").lower() == "true" - - -def is_container(): - if not isfile("/proc/1/cgroup"): - return False - with open("/proc/1/cgroup") as fp: - for line in fp: - line = line.strip() - if ":" in line and not line.endswith(":/"): - return True - return False - - -def exec_command(*args, **kwargs): - result = {"out": None, "err": None, "returncode": None} - - default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) - default.update(kwargs) - kwargs = default - - p = subprocess.Popen(*args, **kwargs) - try: - result['out'], result['err'] = p.communicate() - result['returncode'] = p.returncode - except KeyboardInterrupt: - raise exception.AbortedByUser() - finally: - for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): - kwargs[s].close() - - for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): - result[s[3:]] = "\n".join(kwargs[s].get_buffer()) - - for k, v in result.items(): - if v and isinstance(v, basestring): - result[k].strip() - - return result - - -def copy_pythonpath_to_osenv(): - _PYTHONPATH = [] - if "PYTHONPATH" in os.environ: - _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) - for p in os.sys.path: - conditions = [p not in _PYTHONPATH] - if "windows" not in get_systype(): - conditions.append( - isdir(join(p, "click")) or isdir(join(p, "platformio"))) - if all(conditions): - _PYTHONPATH.append(p) - os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH) - - def get_serial_ports(filter_hwid=False): try: from serial.tools.list_ports import comports @@ -447,8 +165,9 @@ def get_serial_ports(filter_hwid=False): for p, d, h in comports(): if not p: continue - if "windows" in get_systype(): + if WINDOWS and PY2: try: + # pylint: disable=undefined-variable d = unicode(d, errors="ignore") except TypeError: pass @@ -471,11 +190,11 @@ get_serialports = get_serial_ports def get_logical_devices(): items = [] - if "windows" in get_systype(): + if WINDOWS: try: result = exec_command( - ["wmic", "logicaldisk", "get", "name,VolumeName"]).get( - "out", "") + ["wmic", "logicaldisk", "get", + "name,VolumeName"]).get("out", "") devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") for line in result.split("\n"): match = devicenamere.match(line.strip()) @@ -493,17 +212,17 @@ def get_logical_devices(): for device in re.findall(r"[A-Z]:\\", result): items.append({"path": device, "name": None}) return items - else: - result = exec_command(["df"]).get("out") - devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) - for line in result.split("\n"): - match = devicenamere.match(line.strip()) - if not match: - continue - items.append({ - "path": match.group(1), - "name": basename(match.group(1)) - }) + + result = exec_command(["df"]).get("out") + devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) + for line in result.split("\n"): + match = devicenamere.match(line.strip()) + if not match: + continue + items.append({ + "path": match.group(1), + "name": basename(match.group(1)) + }) return items @@ -560,19 +279,31 @@ def get_mdns_services(): time.sleep(3) for service in mdns.get_services(): properties = None - try: - if service.properties: - json.dumps(service.properties) - properties = service.properties - except UnicodeDecodeError: - pass + if service.properties: + try: + properties = { + k.decode("utf8"): + v.decode("utf8") if isinstance(v, bytes) else v + for k, v in service.properties.items() + } + json.dumps(properties) + except UnicodeDecodeError: + properties = None items.append({ - "type": service.type, - "name": service.name, - "ip": ".".join([str(ord(c)) for c in service.address]), - "port": service.port, - "properties": properties + "type": + service.type, + "name": + service.name, + "ip": + ".".join([ + str(c if isinstance(c, int) else ord(c)) + for c in service.address + ]), + "port": + service.port, + "properties": + properties }) return items @@ -582,7 +313,7 @@ def get_request_defheaders(): return {"User-Agent": "PlatformIO/%s CI/%d %s" % data} -@memoized(expire=10000) +@memoized(expire="60s") def _api_request_session(): return requests.Session() @@ -595,7 +326,7 @@ def _get_api_result( auth=None): from platformio.app import get_setting - result = None + result = {} r = None verify_ssl = sys.version_info >= (2, 7, 9) @@ -607,33 +338,30 @@ def _get_api_result( try: if data: - r = _api_request_session().post( - url, - params=params, - data=data, - headers=headers, - auth=auth, - verify=verify_ssl) + r = _api_request_session().post(url, + params=params, + data=data, + headers=headers, + auth=auth, + verify=verify_ssl) else: - r = _api_request_session().get( - url, - params=params, - headers=headers, - auth=auth, - verify=verify_ssl) + r = _api_request_session().get(url, + params=params, + headers=headers, + auth=auth, + verify=verify_ssl) result = r.json() r.raise_for_status() return r.text except requests.exceptions.HTTPError as e: if result and "message" in result: raise exception.APIRequestError(result['message']) - elif result and "errors" in result: + if result and "errors" in result: raise exception.APIRequestError(result['errors'][0]['title']) - else: - raise exception.APIRequestError(e) + raise exception.APIRequestError(e) except ValueError: - raise exception.APIRequestError( - "Invalid response: %s" % r.text.encode("utf-8")) + raise exception.APIRequestError("Invalid response: %s" % + r.text.encode("utf-8")) finally: if r: r.close() @@ -664,9 +392,8 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): return json.loads(result) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - from platformio.maintenance import in_silence total += 1 - if not in_silence(): + if not PlatformioCLI.in_silence(): click.secho( "[API] ConnectionError: {0} (incremented retry: max={1}, " "total={2})".format(e, max_retries, total), @@ -684,18 +411,19 @@ PING_INTERNET_IPS = [ ] -@memoized(expire=5000) +@memoized(expire="5s") def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout) for ip in PING_INTERNET_IPS: try: if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get( - "http://%s" % ip, allow_redirects=False, timeout=timeout) + requests.get("http://%s" % ip, + allow_redirects=False, + timeout=timeout) else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, - 80)) + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect( + (ip, 80)) return True except: # pylint: disable=bare-except pass @@ -709,35 +437,6 @@ def internet_on(raise_exception=False): return result -def get_pythonexe_path(): - return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) - - -def where_is_program(program, envpath=None): - env = os.environ - if envpath: - env['PATH'] = envpath - - # try OS's built-in commands - try: - result = exec_command( - ["where" if "windows" in get_systype() else "which", program], - env=env) - if result['returncode'] == 0 and isfile(result['out'].strip()): - return result['out'].strip() - except OSError: - pass - - # look up in $PATH - for bin_dir in env.get("PATH", "").split(os.pathsep): - if isfile(join(bin_dir, program)): - return join(bin_dir, program) - elif isfile(join(bin_dir, "%s.exe" % program)): - return join(bin_dir, "%s.exe" % program) - - return program - - def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) @@ -791,15 +490,6 @@ def merge_dicts(d1, d2, path=None): return d1 -def get_file_contents(path): - try: - with open(path) as f: - return f.read() - except UnicodeDecodeError: - with open(path, encoding="latin-1") as f: - return f.read() - - def ensure_udev_rules(): def _rules_to_set(rules_path): @@ -831,6 +521,17 @@ def ensure_udev_rules(): return True +def get_original_version(version): + if version.count(".") != 2: + return None + _, raw = version.split(".")[:2] + if int(raw) <= 99: + return None + if int(raw) <= 9999: + return "%s.%s" % (raw[:-2], int(raw[-2:])) + return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) + + def rmtree_(path): def _onerror(_, name, __): @@ -838,33 +539,9 @@ def rmtree_(path): os.chmod(name, stat.S_IWRITE) os.remove(name) except Exception as e: # pylint: disable=broad-except - click.secho( - "%s \nPlease manually remove the file `%s`" % (str(e), name), - fg="red", - err=True) + click.secho("%s \nPlease manually remove the file `%s`" % + (str(e), name), + fg="red", + err=True) return rmtree(path, onerror=_onerror) - - -# -# Glob.Escape from Python 3.4 -# https://github.com/python/cpython/blob/master/Lib/glob.py#L161 -# - -try: - from glob import escape as glob_escape # pylint: disable=unused-import -except ImportError: - magic_check = re.compile('([*?[])') - magic_check_bytes = re.compile(b'([*?[])') - - def glob_escape(pathname): - """Escape all special characters.""" - # Escaping is done by wrapping any of "*?[" between square brackets. - # Metacharacters do not work in the drive part and shouldn't be - # escaped. - drive, pathname = os.path.splitdrive(pathname) - if isinstance(pathname, bytes): - pathname = magic_check_bytes.sub(br'[\1]', pathname) - else: - pathname = magic_check.sub(r'[\1]', pathname) - return drive + pathname diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 6a924370..51f454d0 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -16,10 +16,14 @@ import re from os.path import join from subprocess import CalledProcessError, check_call from sys import modules -from urlparse import urlparse -from platformio import util from platformio.exception import PlatformioException, UserSideException +from platformio.proc import exec_command + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse class VCSClientFactory(object): @@ -38,10 +42,11 @@ class VCSClientFactory(object): if "#" in remote_url: remote_url, tag = remote_url.rsplit("#", 1) if not type_: - raise PlatformioException( - "VCS: Unknown repository type %s" % remote_url) - obj = getattr(modules[__name__], "%sClient" % type_.title())( - src_dir, remote_url, tag, silent) + raise PlatformioException("VCS: Unknown repository type %s" % + remote_url) + obj = getattr(modules[__name__], + "%sClient" % type_.title())(src_dir, remote_url, tag, + silent) assert isinstance(obj, VCSClientBase) return obj @@ -98,14 +103,14 @@ class VCSClientBase(object): check_call(args, **kwargs) return True except CalledProcessError as e: - raise PlatformioException( - "VCS: Could not process command %s" % e.cmd) + raise PlatformioException("VCS: Could not process command %s" % + e.cmd) def get_cmd_output(self, args, **kwargs): args = [self.command] + args if "cwd" not in kwargs: kwargs['cwd'] = self.src_dir - result = util.exec_command(args, **kwargs) + result = exec_command(args, **kwargs) if result['returncode'] == 0: return result['out'].strip() raise PlatformioException( diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 99755db1..51a4c9eb 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -63,10 +63,10 @@ SUBSYSTEMS=="usb", ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE="0666 SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666" # Teensy boards -ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", ENV{ID_MM_DEVICE_IGNORE}="1" -ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", ENV{MTP_NO_PROBE}="1" -SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", MODE:="0666" -KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", MODE:="0666" +ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1" +SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MODE:="0666" +KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666" #TI Stellaris Launchpad SUBSYSTEMS=="usb", ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666" @@ -84,176 +84,176 @@ SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server" SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port" # opendous and estick -ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666" # Original FT232/FT245 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="0666" # Original FT2232 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="0666" # Original FT4232 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="0666" # Original FT232H VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="0666" # DISTORTEC JTAG-lock-pick Tiny 2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666" # TUMPA, TUMPA Lite -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="0666" # XDS100v2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666" # Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE) -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="0666" # TI/Luminary Stellaris Evaluation Board FTDI (several) -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="0666" # TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="0666" # egnite Turtelizer 2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666" # Section5 ICEbear -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="0666" # Amontec JTAGkey and JTAGkey-tiny -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666" # TI ICDI -ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666" # STLink v1 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="0666" # STLink v2 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="0666" # STLink v2-1 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="0666" # Hilscher NXHX Boards -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666" # Hitex STR9-comStick -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="0666" # Hitex STM32-PerformanceStick -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="0666" # Altera USB Blaster -ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666" # Amontec JTAGkey-HiSpeed -ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666" # SEGGER J-Link -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="0666" # Raisonance RLink -ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666" # Debug Board for Neo1973 -ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666" # Olimex ARM-USB-OCD -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="0666" # Olimex ARM-USB-OCD-TINY -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="0666" # Olimex ARM-JTAG-EW -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="0666" # Olimex ARM-USB-OCD-TINY-H -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="0666" # Olimex ARM-USB-OCD-H -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="0666" # USBprog with OpenOCD firmware -ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666" # TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board -ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666" # Marvell Sheevaplug -ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666" # Keil Software, Inc. ULink -ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666" # CMSIS-DAP compatible adapters -ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{product}=="*CMSIS-DAP*", MODE="0666" #SEGGER J-LIK -ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="0666" diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 721d39da..b4e5a67f 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -690,7 +690,7 @@ Uploading --------- %s supports the next uploading protocols: """ % board['name']) - for protocol in upload_protocols: + for protocol in sorted(upload_protocols): lines.append("* ``%s``" % protocol) lines.append(""" Default protocol is ``%s``""" % variables['upload_protocol']) diff --git a/scripts/get-platformio.py b/scripts/get-platformio.py index 90ad5fea..ec201d62 100644 --- a/scripts/get-platformio.py +++ b/scripts/get-platformio.py @@ -90,7 +90,7 @@ def exec_python_cmd(args): def install_pip(): r = exec_python_cmd(["-m", "pip", "--version"]) if r['returncode'] == 0: - print r['out'] + print(r['out']) return try: from urllib2 import urlopen @@ -143,7 +143,7 @@ def main(): try: s[1]() print("[SUCCESS]") - except Exception, e: + except Exception as e: is_error = True print(str(e)) print("[FAILURE]") diff --git a/scripts/install_devplatforms.py b/scripts/install_devplatforms.py index 434ab7cd..a7a2668e 100644 --- a/scripts/install_devplatforms.py +++ b/scripts/install_devplatforms.py @@ -21,7 +21,7 @@ from platformio import util def main(): platforms = json.loads( subprocess.check_output( - ["platformio", "platform", "search", "--json-output"])) + ["platformio", "platform", "search", "--json-output"]).decode()) for platform in platforms: if platform['forDesktop']: continue @@ -34,7 +34,7 @@ def main(): and platform['name'] == "aceinna_imu"): continue subprocess.check_call( - ["platformio", "platform", "install", platform['repository']]) + ["platformio", "platform", "install", platform['name']]) if __name__ == "__main__": diff --git a/setup.py b/setup.py index dc9b7bf4..eb667d72 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ from platformio import (__author__, __description__, __email__, __license__, install_requires = [ "bottle<0.13", - "click>=5,<6", + "click>=5,<8", "colorama", "pyserial>=3,<4,!=3.3", "requests>=2.4.0,<3", @@ -35,12 +35,12 @@ setup( author_email=__email__, url=__url__, license=__license__, - python_requires='>=2.7, <3', + python_requires=", ".join([ + ">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"]), install_requires=install_requires, packages=find_packages() + ["scripts"], package_data={ "platformio": [ - "projectconftpl.ini", "ide/tpls/*/.*.tpl", "ide/tpls/*/*.tpl", "ide/tpls/*/*/*.tpl", @@ -65,6 +65,7 @@ setup( "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python", + "Programming Language :: Python :: 3", "Topic :: Software Development", "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Compilers" diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 67ff4420..bce43700 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -20,7 +20,7 @@ from platformio.commands.lib import cli as cmd_lib def test_ci_empty(clirunner): result = clirunner.invoke(cmd_ci) - assert result.exit_code == 2 + assert result.exit_code != 0 assert "Invalid value: Missing argument 'src'" in result.output diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index eecb25f3..7c2e4a48 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -16,9 +16,10 @@ import json from os import getcwd, makedirs from os.path import getsize, isdir, isfile, join -from platformio import exception, util +from platformio import exception from platformio.commands.boards import cli as cmd_boards from platformio.commands.init import cli as cmd_init +from platformio.project.config import ProjectConfig def validate_pioproject(pioproject_dir): @@ -50,15 +51,16 @@ def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir): result = clirunner.invoke(cmd_init, ["-b", "uno", "-b", "uno"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - config = util.load_project_config() + config = ProjectConfig(join(getcwd(), "platformio.ini")) + config.validate() assert set(config.sections()) == set(["env:uno"]) def test_init_ide_without_board(clirunner, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke(cmd_init, ["--ide", "atom"]) - assert result.exit_code == -1 - assert isinstance(result.exception, exception.BoardNotDefined) + assert result.exit_code != 0 + assert isinstance(result.exception, exception.ProjectEnvsNotAvailable) def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): @@ -105,14 +107,15 @@ def test_init_special_board(clirunner, validate_cliresult): validate_cliresult(result) boards = json.loads(result.output) - config = util.load_project_config() - expected_result = [("platform", str(boards[0]['platform'])), - ("framework", - str(boards[0]['frameworks'][0])), ("board", "uno")] + config = ProjectConfig(join(getcwd(), "platformio.ini")) + config.validate() + expected_result = dict(platform=str(boards[0]['platform']), + board="uno", + framework=[str(boards[0]['frameworks'][0])]) assert config.has_section("env:uno") - assert not set(expected_result).symmetric_difference( - set(config.items("env:uno"))) + assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( + expected_result.items()) def test_init_enable_auto_uploading(clirunner, validate_cliresult): @@ -121,12 +124,15 @@ def test_init_enable_auto_uploading(clirunner, validate_cliresult): cmd_init, ["-b", "uno", "--project-option", "targets=upload"]) validate_cliresult(result) validate_pioproject(getcwd()) - config = util.load_project_config() - expected_result = [("platform", "atmelavr"), ("framework", "arduino"), - ("board", "uno"), ("targets", "upload")] + config = ProjectConfig(join(getcwd(), "platformio.ini")) + config.validate() + expected_result = dict(targets=["upload"], + platform="atmelavr", + board="uno", + framework=["arduino"]) assert config.has_section("env:uno") - assert not set(expected_result).symmetric_difference( - set(config.items("env:uno"))) + assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( + expected_result.items()) def test_init_custom_framework(clirunner, validate_cliresult): @@ -135,12 +141,15 @@ def test_init_custom_framework(clirunner, validate_cliresult): cmd_init, ["-b", "teensy31", "--project-option", "framework=mbed"]) validate_cliresult(result) validate_pioproject(getcwd()) - config = util.load_project_config() - expected_result = [("platform", "teensy"), ("framework", "mbed"), - ("board", "teensy31")] + config = ProjectConfig(join(getcwd(), "platformio.ini")) + config.validate() + expected_result = dict(platform="teensy", + board="teensy31", + framework=["mbed"]) assert config.has_section("env:teensy31") - assert not set(expected_result).symmetric_difference( - set(config.items("env:teensy31"))) + assert sorted(config.items(env="teensy31", + as_dict=True).items()) == sorted( + expected_result.items()) def test_init_incorrect_board(clirunner): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index b67cec39..4b75d81b 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -16,8 +16,11 @@ import json import re from platformio import exception +from platformio.commands import PlatformioCLI from platformio.commands.lib import cli as cmd_lib +PlatformioCLI.leftover_args = ["--json-output"] # hook for click + def test_search(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) @@ -58,7 +61,6 @@ def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, [ "-g", "install", - "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz", @@ -74,10 +76,7 @@ def test_global_install_archive(clirunner, validate_cliresult, assert result.exit_code != 0 items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = [ - "RadioHead-1.62", "ArduinoJson", "SomeLib_ID54", - "OneWire_ID1", "ESP32WebServer" - ] + items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire_ID1", "ESP32WebServer"] assert set(items1) >= set(items2) @@ -123,7 +122,7 @@ def test_install_duplicates(clirunner, validate_cliresult, without_internet): # archive result = clirunner.invoke(cmd_lib, [ "-g", "install", - "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip" + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip" ]) validate_cliresult(result) assert "is already installed" in result.output @@ -145,7 +144,7 @@ def test_global_lib_list(clirunner, validate_cliresult): ("Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", "Version: 5.10.1", "Source: git+https://github.com/gioblu/PJON.git#3.0", - "Version: 1fb26fd", "RadioHead-1.62") + "Version: 1fb26fd") ]) result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) @@ -158,10 +157,9 @@ def test_global_lib_list(clirunner, validate_cliresult): items1 = [i['name'] for i in json.loads(result.output)] items2 = [ "ESP32WebServer", "ArduinoJson", "ArduinoJson", "ArduinoJson", - "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "SomeLib", - "ESPAsyncTCP", "NeoPixelBus", "OneWire", "PJON", "PJON", - "PubSubClient", "RFcontrol", "RadioHead-1.62", "platformio-libmirror", - "rs485-nodeproto" + "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "SomeLib", "ESPAsyncTCP", + "NeoPixelBus", "OneWire", "PJON", "PJON", "PubSubClient", "RFcontrol", + "platformio-libmirror", "rs485-nodeproto" ] assert sorted(items1) == sorted(items2) @@ -169,9 +167,9 @@ def test_global_lib_list(clirunner, validate_cliresult): "{name}@{version}".format(**item) for item in json.loads(result.output) ] versions2 = [ - 'ArduinoJson@5.8.2', 'ArduinoJson@5.10.1', 'AsyncMqttClient@0.8.2', - 'NeoPixelBus@2.2.4', 'PJON@07fe9aa', 'PJON@1fb26fd', - 'PubSubClient@bef5814', 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0' + "ArduinoJson@5.8.2", "ArduinoJson@5.10.1", "AsyncMqttClient@0.8.2", + "NeoPixelBus@2.2.4", "PJON@07fe9aa", "PJON@1fb26fd", + "PubSubClient@bef5814", "RFcontrol@77d4eb3f8a" ] assert set(versions1) >= set(versions2) @@ -202,7 +200,7 @@ def test_global_lib_update(clirunner, validate_cliresult): # update rest libraries result = clirunner.invoke(cmd_lib, ["-g", "update"]) validate_cliresult(result) - assert result.output.count("[Detached]") == 6 + assert result.output.count("[Detached]") == 5 assert result.output.count("[Up-to-date]") == 11 assert "Uninstalling RFcontrol @ 77d4eb3f8a" in result.output @@ -232,10 +230,10 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "RadioHead-1.62", "rs485-nodeproto", "platformio-libmirror", + "rs485-nodeproto", "platformio-libmirror", "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "ESPAsyncTCP_ID305", "SomeLib_ID54", "NeoPixelBus_ID547", - "PJON", "AsyncMqttClient_ID346", "ArduinoJson_ID64", + "ESPAsyncTCP_ID305", "SomeLib_ID54", "NeoPixelBus_ID547", "PJON", + "AsyncMqttClient_ID346", "ArduinoJson_ID64", "PJON@src-79de467ebe19de18287becff0a1fb42d", "ESP32WebServer" ] assert set(items1) == set(items2) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index d327ef37..72941574 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -38,14 +38,14 @@ def test_search_raw_output(clirunner, validate_cliresult): def test_install_unknown_version(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) - assert result.exit_code == -1 + assert result.exit_code != 0 assert isinstance(result.exception, exception.UndefinedPackageVersion) def test_install_unknown_from_registry(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) - assert result.exit_code == -1 + assert result.exit_code != 0 assert isinstance(result.exception, exception.UnknownPackage) diff --git a/tests/conftest.py b/tests/conftest.py index 02d16069..c61166bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,8 +24,10 @@ from platformio import util def validate_cliresult(): def decorator(result): - assert result.exit_code == 0, result.output - assert not result.exception, result.output + assert result.exit_code == 0, "{} => {}".format( + result.exception, result.output) + assert not result.exception, "{} => {}".format(result.exception, + result.output) return decorator @@ -38,10 +40,10 @@ def clirunner(): @pytest.fixture(scope="module") def isolated_pio_home(request, tmpdir_factory): home_dir = tmpdir_factory.mktemp(".platformio") - os.environ['PLATFORMIO_HOME_DIR'] = str(home_dir) + os.environ['PLATFORMIO_CORE_DIR'] = str(home_dir) def fin(): - del os.environ['PLATFORMIO_HOME_DIR'] + del os.environ['PLATFORMIO_CORE_DIR'] request.addfinalizer(fin) return home_dir diff --git a/tests/test_examples.py b/tests/test_examples.py index 0727da75..3cb365d6 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -15,12 +15,14 @@ import random from glob import glob from os import listdir, walk -from os.path import dirname, getsize, isdir, isfile, join, normpath +from os.path import basename, dirname, getsize, isdir, isfile, join, normpath import pytest from platformio import util from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_build_dir def pytest_generate_tests(metafunc): @@ -34,11 +36,13 @@ def pytest_generate_tests(metafunc): # dev/platforms for manifest in PlatformManager().get_installed(): p = PlatformFactory.newPlatform(manifest['__pkg_dir']) - if not p.is_embedded(): - continue - # issue with "version `CXXABI_1.3.9' not found (required by sdcc)" - if "linux" in util.get_systype() and p.name in ("intel_mcs51", - "ststm8"): + ignore_conds = [ + not p.is_embedded(), + p.name == "ststm8", + # issue with "version `CXXABI_1.3.9' not found (required by sdcc)" + "linux" in util.get_systype() and p.name == "intel_mcs51" + ] + if any(ignore_conds): continue examples_dir = join(p.get_dir(), "examples") assert isdir(examples_dir) @@ -46,36 +50,39 @@ def pytest_generate_tests(metafunc): project_dirs = [] for examples_dir in examples_dirs: - platform_examples = [] + candidates = {} for root, _, files in walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue - platform_examples.append(root) + group = basename(root) + if "-" in group: + group = group.split("-", 1)[0] + if group not in candidates: + candidates[group] = [] + candidates[group].append(root) - # test random 3 examples - random.shuffle(platform_examples) - project_dirs.extend(platform_examples[:3]) - project_dirs.sort() - metafunc.parametrize("pioproject_dir", project_dirs) + project_dirs.extend([ + random.choice(examples) for examples in candidates.values() + if examples + ]) + + metafunc.parametrize("pioproject_dir", sorted(project_dirs)) @pytest.mark.examples def test_run(pioproject_dir): with util.cd(pioproject_dir): - build_dir = util.get_projectbuild_dir() + build_dir = get_project_build_dir() if isdir(build_dir): util.rmtree_(build_dir) - env_names = [] - for section in util.load_project_config().sections(): - if section.startswith("env:"): - env_names.append(section[4:]) - + env_names = ProjectConfig(join(pioproject_dir, + "platformio.ini")).envs() result = util.exec_command( ["platformio", "run", "-e", random.choice(env_names)]) if result['returncode'] != 0: - pytest.fail(result) + pytest.fail(str(result)) assert isdir(build_dir) diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index ec9d749a..d7b4dea0 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -22,48 +22,7 @@ from platformio.commands import upgrade as cmd_upgrade from platformio.managers.platform import PlatformManager -def test_after_upgrade_2_to_3(clirunner, validate_cliresult, - isolated_pio_home): - app.set_state_item("last_version", "2.11.2") - app.set_state_item("installed_platforms", ["native"]) - - # generate PlatformIO 2.0 boards - boards = isolated_pio_home.mkdir("boards") - board_ids = set() - for prefix in ("foo", "bar"): - data = {} - for i in range(3): - board_id = "board_%s_%d" % (prefix, i) - board_ids.add(board_id) - data[board_id] = { - "name": "Board %s #%d" % (prefix, i), - "url": "", - "vendor": "" - } - boards.join(prefix + ".json").write(json.dumps(data)) - - result = clirunner.invoke(cli_pio, ["settings", "get"]) - validate_cliresult(result) - assert "upgraded to 3" in result.output - - # check PlatformIO 3.0 boards - assert board_ids == set([p.basename[:-5] for p in boards.listdir()]) - - result = clirunner.invoke(cli_pio, - ["boards", "--installed", "--json-output"]) - validate_cliresult(result) - assert board_ids == set([b['id'] for b in json.loads(result.output)]) - - -def test_after_upgrade_silence(clirunner, validate_cliresult): - app.set_state_item("last_version", "2.11.2") - result = clirunner.invoke(cli_pio, ["boards", "--json-output"]) - validate_cliresult(result) - boards = json.loads(result.output) - assert any([b['id'] == "uno" for b in boards]) - - -def test_check_pio_upgrade(clirunner, validate_cliresult): +def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): def _patch_pio_version(version): maintenance.__version__ = version @@ -93,7 +52,7 @@ def test_check_pio_upgrade(clirunner, validate_cliresult): _patch_pio_version(origin_version) -def test_check_lib_updates(clirunner, validate_cliresult): +def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): # install obsolete library result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<5.7"]) @@ -110,7 +69,8 @@ def test_check_lib_updates(clirunner, validate_cliresult): result.output) -def test_check_and_update_libraries(clirunner, validate_cliresult): +def test_check_and_update_libraries(clirunner, isolated_pio_home, + validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_libraries", "Yes"]) @@ -141,8 +101,8 @@ def test_check_and_update_libraries(clirunner, validate_cliresult): assert prev_data[0]['version'] != json.loads(result.output)[0]['version'] -def test_check_platform_updates(clirunner, validate_cliresult, - isolated_pio_home): +def test_check_platform_updates(clirunner, isolated_pio_home, + validate_cliresult): # install obsolete platform result = clirunner.invoke(cli_pio, ["platform", "install", "native"]) validate_cliresult(result) @@ -164,7 +124,8 @@ def test_check_platform_updates(clirunner, validate_cliresult, assert "There are the new updates for platforms (native)" in result.output -def test_check_and_update_platforms(clirunner, validate_cliresult): +def test_check_and_update_platforms(clirunner, isolated_pio_home, + validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_platforms", "Yes"]) diff --git a/tests/test_managers.py b/tests/test_managers.py index ee2d5df1..f2946f12 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -15,8 +15,8 @@ import json from os.path import join -from platformio import util from platformio.managers.package import PackageManager +from platformio.project.helpers import get_project_core_dir def test_pkg_input_parser(): @@ -28,16 +28,16 @@ def test_pkg_input_parser(): ["id=13", ("id=13", None, None)], ["id=13@~1.2.3", ("id=13", "~1.2.3", None)], [ - util.get_home_dir(), - (".platformio", None, "file://" + util.get_home_dir()) + get_project_core_dir(), + (".platformio", None, "file://" + get_project_core_dir()) ], [ - "LocalName=" + util.get_home_dir(), - ("LocalName", None, "file://" + util.get_home_dir()) + "LocalName=" + get_project_core_dir(), + ("LocalName", None, "file://" + get_project_core_dir()) ], [ - "LocalName=%s@>2.3.0" % util.get_home_dir(), - ("LocalName", ">2.3.0", "file://" + util.get_home_dir()) + "LocalName=%s@>2.3.0" % get_project_core_dir(), + ("LocalName", ">2.3.0", "file://" + get_project_core_dir()) ], [ "https://github.com/user/package.git", @@ -130,7 +130,8 @@ def test_pkg_input_parser(): ], [ "LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0", - ("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0") + ("LocalName", "~1.2.0", + "git+git@github.com:user/package.git#v1.2.0") ], [ "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", @@ -164,15 +165,18 @@ def test_install_packages(isolated_pio_home, tmpdir): dict(id=1, name="name_1", version="1.2"), dict(id=1, name="name_1", version="1.0.0"), dict(name="name_2", version="1.0.0"), - dict(name="name_2", version="2.0.0", + dict(name="name_2", + version="2.0.0", __src_url="git+https://github.com"), - dict(name="name_2", version="3.0.0", + dict(name="name_2", + version="3.0.0", __src_url="git+https://github2.com"), - dict(name="name_2", version="4.0.0", + dict(name="name_2", + version="4.0.0", __src_url="git+https://github2.com") ] - pm = PackageManager(join(util.get_home_dir(), "packages")) + pm = PackageManager(join(get_project_core_dir(), "packages")) for package in packages: tmp_dir = tmpdir.mkdir("tmp-package") tmp_dir.join("package.json").write(json.dumps(package)) @@ -182,36 +186,44 @@ def test_install_packages(isolated_pio_home, tmpdir): assert len(pm.get_installed()) == len(packages) - 1 pkg_dirnames = [ - 'name_1_ID1', 'name_1_ID1@1.0.0', 'name_1_ID1@1.2', - 'name_1_ID1@2.0.0', 'name_1_ID1@shasum', 'name_2', + 'name_1_ID1', 'name_1_ID1@1.0.0', 'name_1_ID1@1.2', 'name_1_ID1@2.0.0', + 'name_1_ID1@shasum', 'name_2', 'name_2@src-177cbce1f0705580d17790fda1cc2ef5', 'name_2@src-f863b537ab00f4c7b5011fc44b120e1f' ] - assert set([p.basename for p in isolated_pio_home.join( - "packages").listdir()]) == set(pkg_dirnames) + assert set([ + p.basename for p in isolated_pio_home.join("packages").listdir() + ]) == set(pkg_dirnames) def test_get_package(): tests = [ [("unknown", ), None], [("1", ), None], - [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")], - [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("name_1", "<2"), dict(id=1, name="name_1", version="1.2")], + [("id=1", "shasum"), + dict(id=1, name="name_1", version="shasum")], + [("id=1", "*"), + dict(id=1, name="name_1", version="2.1.0")], + [("id=1", "^1"), + dict(id=1, name="name_1", version="1.2")], + [("id=1", "^1"), + dict(id=1, name="name_1", version="1.2")], + [("name_1", "<2"), + dict(id=1, name="name_1", version="1.2")], [("name_1", ">2"), None], [("name_1", "2-0-0"), None], [("name_2", ), dict(name="name_2", version="4.0.0")], [("url_has_higher_priority", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", + dict(name="name_2", + version="2.0.0", __src_url="git+https://github.com")], [("name_2", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", + dict(name="name_2", + version="2.0.0", __src_url="git+https://github.com")], ] - pm = PackageManager(join(util.get_home_dir(), "packages")) + pm = PackageManager(join(get_project_core_dir(), "packages")) for test in tests: manifest = pm.get_package(*test[0]) if test[1] is None: diff --git a/tests/test_misc.py b/tests/test_misc.py index b918767b..8f6fccf2 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -18,6 +18,12 @@ import requests from platformio import exception, util +def test_platformio_cli(): + result = util.exec_command(["pio", "--help"]) + assert result['returncode'] == 0 + assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result['out'] + + def test_ping_internet_ips(): for ip in util.PING_INTERNET_IPS: requests.get("http://%s" % ip, allow_redirects=False, timeout=2) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 92da706f..e0875a16 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -28,7 +28,7 @@ def test_packages(): "https://dl.bintray.com/platformio/dl-packages/manifest.json").json() assert isinstance(pkgs_manifest, dict) items = [] - for _, variants in pkgs_manifest.iteritems(): + for _, variants in pkgs_manifest.items(): for item in variants: items.append(item) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py new file mode 100644 index 00000000..1ad8b2cd --- /dev/null +++ b/tests/test_projectconf.py @@ -0,0 +1,197 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from platformio.exception import UnknownEnvNames +from platformio.project.config import ConfigParser, ProjectConfig + +BASE_CONFIG = """ +[platformio] +env_default = base, extra_2 +extra_configs = + extra_envs.ini + extra_debug.ini + +# global options per [env:*] +[env] +monitor_speed = 115200 +lib_deps = + Lib1 + Lib2 +lib_ignore = ${custom.lib_ignore} + +[custom] +debug_flags = -D RELEASE +lib_flags = -lc -lm +extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS} +lib_ignore = LibIgnoreCustom + +[env:base] +build_flags = ${custom.debug_flags} ${custom.extra_flags} +targets = +""" + +EXTRA_ENVS_CONFIG = """ +[env:extra_1] +build_flags = ${custom.lib_flags} ${custom.debug_flags} +lib_install = 574 + +[env:extra_2] +build_flags = ${custom.debug_flags} ${custom.extra_flags} +lib_ignore = ${env.lib_ignore}, Lib3 +upload_port = /dev/extra_2/port +""" + +EXTRA_DEBUG_CONFIG = """ +# Override original "custom.debug_flags" +[custom] +debug_flags = -D DEBUG=1 + +[env:extra_2] +build_flags = -Og +""" + + +def test_real_config(tmpdir): + tmpdir.join("platformio.ini").write(BASE_CONFIG) + tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) + tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) + + config = None + with tmpdir.as_cwd(): + config = ProjectConfig(tmpdir.join("platformio.ini").strpath) + assert config + assert len(config.warnings) == 2 + assert "lib_install" in config.warnings[1] + + config.validate(["extra_2", "base"], silent=True) + with pytest.raises(UnknownEnvNames): + config.validate(["non-existing-env"]) + + # unknown section + with pytest.raises(ConfigParser.NoSectionError): + config.getraw("unknown_section", "unknown_option") + # unknown option + with pytest.raises(ConfigParser.NoOptionError): + config.getraw("custom", "unknown_option") + # unknown option even if exists in [env] + with pytest.raises(ConfigParser.NoOptionError): + config.getraw("platformio", "monitor_speed") + + # sections + assert config.sections() == [ + "platformio", "env", "custom", "env:base", "env:extra_1", "env:extra_2" + ] + + # envs + assert config.envs() == ["base", "extra_1", "extra_2"] + assert config.default_envs() == ["base", "extra_2"] + + # options + assert config.options(env="base") == [ + "build_flags", "targets", "monitor_speed", "lib_deps", "lib_ignore" + ] + + # has_option + assert config.has_option("env:base", "monitor_speed") + assert not config.has_option("custom", "monitor_speed") + assert not config.has_option("env:extra_1", "lib_install") + + # sysenv + assert config.get("custom", "extra_flags") is None + assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] + assert config.get("env:base", "upload_port") is None + assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" + os.environ["PLATFORMIO_BUILD_FLAGS"] = "-DSYSENVDEPS1 -DSYSENVDEPS2" + os.environ["PLATFORMIO_UPLOAD_PORT"] = "/dev/sysenv/port" + os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] = "-L /usr/local/lib" + assert config.get("custom", "extra_flags") == "-L /usr/local/lib" + assert config.get("env:base", "build_flags") == [ + "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2" + ] + assert config.get("env:base", "upload_port") == "/dev/sysenv/port" + assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" + + # getraw + assert config.getraw("env:base", "targets") == "" + assert config.getraw("env:extra_1", "lib_deps") == "574" + assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" + + # get + assert config.get("custom", "debug_flags") == "-D DEBUG=1" + assert config.get("env:extra_1", "build_flags") == [ + "-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2" + ] + assert config.get("env:extra_2", "build_flags") == [ + "-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"] + assert config.get("env:extra_2", "monitor_speed") == "115200" + assert config.get("env:base", "build_flags") == ([ + "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2" + ]) + + # items + assert config.items("custom") == [ + ("debug_flags", "-D DEBUG=1"), + ("lib_flags", "-lc -lm"), + ("extra_flags", "-L /usr/local/lib"), + ("lib_ignore", "LibIgnoreCustom") + ] # yapf: disable + assert config.items(env="base") == [ + ("build_flags", [ + "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), + ("targets", []), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ("upload_port", "/dev/sysenv/port") + ] # yapf: disable + assert config.items(env="extra_1") == [ + ("build_flags", ["-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), + ("lib_deps", ["574"]), + ("monitor_speed", "115200"), + ("lib_ignore", ["LibIgnoreCustom"]), + ("upload_port", "/dev/sysenv/port") + ] # yapf: disable + assert config.items(env="extra_2") == [ + ("build_flags", ["-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), + ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), + ("upload_port", "/dev/extra_2/port"), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]) + ] # yapf: disable + + # cleanup system environment variables + del os.environ["PLATFORMIO_BUILD_FLAGS"] + del os.environ["PLATFORMIO_UPLOAD_PORT"] + del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] + + +def test_empty_config(): + config = ProjectConfig("/non/existing/platformio.ini") + + # unknown section + with pytest.raises(ConfigParser.NoSectionError): + config.getraw("unknown_section", "unknown_option") + + assert config.sections() == [] + assert config.get("section", "option") is None + assert config.get("section", "option", 13) == 13 + + # sysenv + os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir" + assert config.get("platformio", "core_dir") == "/custom/core/dir" + del os.environ["PLATFORMIO_HOME_DIR"] diff --git a/tox.ini b/tox.ini index 26327fd7..597c7b8f 100644 --- a/tox.ini +++ b/tox.ini @@ -13,10 +13,10 @@ # limitations under the License. [tox] -envlist = py27, docs, lint +envlist = py27, py35, py36, py37, docs -[testenv:develop] -basepython = python2.7 +[testenv] +passenv = * usedevelop = True deps = isort @@ -24,10 +24,15 @@ deps = pylint pytest pytest-xdist -commands = python --version +commands = + {envpython} --version + pylint --rcfile=./.pylintrc ./platformio + {envpython} -c "print('travis_fold:start:install_devplatforms')" + {envpython} scripts/install_devplatforms.py + {envpython} -c "print('travis_fold:end:install_devplatforms')" + py.test -v --basetemp="{envtmpdir}" tests [testenv:docs] -basepython = python2.7 deps = sphinx sphinx_rtd_theme @@ -37,44 +42,23 @@ commands = sphinx-build -W -b latex -d {envtmpdir}/doctrees docs docs/_build/latex [testenv:docslinkcheck] -basepython = python2.7 deps = sphinx sphinx_rtd_theme commands = sphinx-build -W -b linkcheck docs docs/_build/html -[testenv:lint] -basepython = python2.7 -deps = - pylint -commands = - pylint --rcfile=./.pylintrc ./platformio - -[testenv] -basepython = python2.7 -passenv = * -deps = - pytest -commands = - {envpython} --version - {envpython} -c "print 'travis_fold:start:install_devplatforms'" - {envpython} scripts/install_devplatforms.py - {envpython} -c "print 'travis_fold:end:install_devplatforms'" - py.test -v --basetemp="{envtmpdir}" tests - [testenv:skipexamples] -basepython = python2.7 deps = pytest commands = py.test -v --basetemp="{envtmpdir}" tests --ignore tests/test_examples.py -[testenv:coverage] -basepython = python2.7 -passenv = * -deps = - pytest - pytest-cov -commands = - py.test --cov=platformio --cov-report term --cov-report xml --ignore=tests/test_examples.py --ignore=tests/test_pkgmanifest.py -v tests +; [testenv:coverage] +; basepython = python2 +; passenv = * +; deps = +; pytest +; pytest-cov +; commands = +; py.test --cov=platformio --cov-report term --cov-report xml --ignore=tests/test_examples.py --ignore=tests/test_pkgmanifest.py -v tests