From e269c91d26e5fe78973b9e232182793b84c13966 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jun 2019 13:30:35 +0300 Subject: [PATCH] Improve compatibility with hashlib Py2/Py3 --- platformio/app.py | 10 ++++------ platformio/builder/main.py | 12 ++++++------ platformio/builder/tools/piolib.py | 6 +++--- platformio/builder/tools/piowinhooks.py | 7 +++---- platformio/commands/debug/client.py | 5 ++--- platformio/compat.py | 16 ++++++++++++++++ platformio/managers/package.py | 10 ++++++---- platformio/managers/platform.py | 20 +++++++++++++------- platformio/proc.py | 6 +++--- platformio/project/helpers.py | 9 ++++----- platformio/telemetry.py | 12 ++++++++---- 11 files changed, 68 insertions(+), 45 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index c0923bf1..49f24625 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,7 +25,7 @@ from time import time import requests from platformio import exception, lockfile, util -from platformio.compat import PY2, WINDOWS +from platformio.compat import WINDOWS, hashlib_encode_data from platformio.proc import is_ci from platformio.project.helpers import (get_project_cache_dir, get_project_core_dir) @@ -174,10 +174,8 @@ class ContentCache(object): def key_from_args(*args): h = hashlib.md5() for arg in args: - if not arg: - continue - arg = str(arg) - h.update(arg if PY2 else arg.encode()) + if arg: + h.update(hashlib_encode_data(arg)) return h.hexdigest() def get(self, key): @@ -363,7 +361,7 @@ def get_cid(): pass if not uid: uid = uuid.getnode() - cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest()) + cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) cid = str(cid) if WINDOWS or os.getuid() > 0: # yapf: disable pylint: disable=no-member set_state_item("cid", cid) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index bf1b001e..1fd76b24 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 import json import sys from os import environ @@ -31,6 +30,7 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import util from platformio.compat import PY2, path_to_unicode +from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project import helpers as project_helpers @@ -91,11 +91,11 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) # Load variables from CLI -for key in list(clivars.keys()): - if key in env: - env[key] = base64.b64decode(env[key]) - if isinstance(env[key], bytes): - env[key] = env[key].decode() +env.Replace( + **{ + key: PlatformBase.decode_scons_arg(env[key]) + for key in list(clivars.keys()) if key in env + }) if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 29389e38..b9855411 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -32,7 +32,8 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool -from platformio.compat import PY2, WINDOWS, get_file_contents, string_types +from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data, + string_types) from platformio.managers.lib import LibraryManager @@ -181,8 +182,7 @@ class LibBuilderBase(object): @property def build_dir(self): - lib_hash = hashlib.sha1( - self.path if PY2 else self.path.encode()).hexdigest()[:3] + lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index 1105091f..3679897d 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -18,7 +18,7 @@ from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platformio.compat import PY2, WINDOWS +from platformio.compat import WINDOWS, hashlib_encode_data # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options @@ -61,9 +61,8 @@ def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) - tmp_file = join( - build_dir, - "longcmd-%s" % md5(data if PY2 else data.encode()).hexdigest()) + tmp_file = join(build_dir, + "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index e7945284..03daed25 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -29,7 +29,7 @@ from platformio import app, exception, util from platformio.commands.debug import helpers, initcfgs from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer -from platformio.compat import PY2 +from platformio.compat import hashlib_encode_data from platformio.project.helpers import get_project_cache_dir from platformio.telemetry import MeasurementProtocol @@ -61,8 +61,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path - self._session_id = sha1( - session_hash if PY2 else session_hash.encode()).hexdigest() + self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest() self._kill_previous_session() patterns = { diff --git a/platformio/compat.py b/platformio/compat.py index 3c3afc52..975400fe 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -35,12 +35,21 @@ if PY2: return isinstance(x, (buffer, bytearray)) def path_to_unicode(path): + if isinstance(path, unicode): + return path return path.decode(get_filesystem_encoding()).encode("utf-8") def get_file_contents(path): with open(path) as f: return f.read() + def hashlib_encode_data(data): + if is_bytes(data): + return data + if not isinstance(data, string_types): + data = str(data) + return data + _magic_check = re.compile('([*?[])') _magic_check_bytes = re.compile(b'([*?[])') @@ -74,3 +83,10 @@ else: except UnicodeDecodeError: with open(path, encoding="latin-1") as f: return f.read() + + def hashlib_encode_data(data): + if is_bytes(data): + return data + if not isinstance(data, string_types): + data = str(data) + return data.encode() diff --git a/platformio/managers/package.py b/platformio/managers/package.py index fc3f4105..56a33f0b 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -26,7 +26,7 @@ import requests import semantic_version from platformio import __version__, app, exception, telemetry, util -from platformio.compat import path_to_unicode +from platformio.compat import hashlib_encode_data, path_to_unicode from platformio.downloader import FileDownloader from platformio.lockfile import LockFile from platformio.unpacker import FileUnpacker @@ -173,7 +173,7 @@ class PkgInstallerMixin(object): cache_key_data = app.ContentCache.key_from_args(url, "data") if self.FILE_CACHE_VALID: with app.ContentCache() as cc: - fname = cc.get(cache_key_fname) + fname = str(cc.get(cache_key_fname)) cache_path = cc.get_cache_path(cache_key_data) if fname and isfile(cache_path): dst_path = join(dest_dir, fname) @@ -591,7 +591,8 @@ class PkgInstallerMixin(object): target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5( - cur_manifest['__src_url'].encode()).hexdigest()) + hashlib_encode_data( + cur_manifest['__src_url'])).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -601,7 +602,8 @@ class PkgInstallerMixin(object): target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5( - tmp_manifest['__src_url'].encode()).hexdigest()) + hashlib_encode_data( + tmp_manifest['__src_url'])).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 7aa153f8..e7a29027 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -24,7 +24,7 @@ import click import semantic_version from platformio import __version__, app, exception, util -from platformio.compat import PY2 +from platformio.compat import get_filesystem_encoding, string_types from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, @@ -359,6 +359,17 @@ class PlatformRunMixin(object): LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) + @staticmethod + def encode_scons_arg(value): + if isinstance(value, string_types): + value = value.encode(get_filesystem_encoding()) + return base64.urlsafe_b64encode(value).decode() + + @staticmethod + def decode_scons_arg(value): + return base64.urlsafe_b64decode(value).decode( + get_filesystem_encoding()) + def run(self, variables, targets, silent, verbose): assert isinstance(variables, dict) assert isinstance(targets, list) @@ -402,12 +413,7 @@ class PlatformRunMixin(object): # encode and append variables for key, value in variables.items(): - if PY2: - cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) - else: - cmd.append( - "%s=%s" % - (key.upper(), base64.b64encode(value.encode()).decode())) + cmd.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) def _write_and_flush(stream, data): stream.write(data) diff --git a/platformio/proc.py b/platformio/proc.py index 9ffad954..7808fa42 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -19,7 +19,7 @@ from os.path import isdir, isfile, join, normpath from threading import Thread from platformio import exception -from platformio.compat import PY2, WINDOWS, string_types +from platformio.compat import WINDOWS, get_filesystem_encoding, string_types class AsyncPipeBase(object): @@ -123,8 +123,8 @@ def exec_command(*args, **kwargs): result[s[3:]] = kwargs[s].get_buffer() for k, v in result.items(): - if not PY2 and isinstance(result[k], bytes): - result[k] = result[k].decode() + if isinstance(result[k], bytes): + result[k] = result[k].decode(get_filesystem_encoding()) if v and isinstance(v, string_types): result[k] = result[k].strip() diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index bf63f8b0..71f73649 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -19,7 +19,7 @@ from os.path import (basename, dirname, expanduser, isdir, isfile, join, realpath, splitdrive) from platformio import __version__ -from platformio.compat import PY2, WINDOWS +from platformio.compat import WINDOWS, hashlib_encode_data from platformio.project.config import ProjectConfig @@ -54,9 +54,8 @@ def get_project_optional_dir(name, default=None): if "$PROJECT_HASH" in optional_dir: optional_dir = optional_dir.replace( "$PROJECT_HASH", "%s-%s" % - (basename(project_dir), - sha1(project_dir if PY2 else project_dir.encode()).hexdigest() - [:10])) + (basename(project_dir), sha1( + hashlib_encode_data(project_dir)).hexdigest()[:10])) if optional_dir.startswith("~"): optional_dir = expanduser(optional_dir) @@ -179,4 +178,4 @@ def calculate_project_hash(): # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() - return sha1(chunks_to_str if PY2 else chunks_to_str.encode()).hexdigest() + return sha1(hashlib_encode_data(chunks_to_str)).hexdigest() diff --git a/platformio/telemetry.py b/platformio/telemetry.py index f697a241..d35aac40 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -28,6 +28,7 @@ import requests from platformio import __version__, app, exception, util from platformio.commands import PlatformioCLI +from platformio.compat import string_types from platformio.proc import is_ci, is_container try: @@ -135,12 +136,15 @@ class MeasurementProtocol(TelemetryBase): return _arg return None - args = [ - str(arg).lower() for arg in PlatformioCLI.leftover_args - if not str(arg).startswith("-") - ] + args = [] + for arg in PlatformioCLI.leftover_args: + if not isinstance(arg, string_types): + arg = str(arg) + if not arg.startswith("-"): + args.append(arg.lower()) if not args: return + cmd_path = args[:1] if args[0] in ("platform", "platforms", "serialports", "device", "settings", "account"):