Improve compatibility with hashlib Py2/Py3

platformio/app.py

@@ -25,7 +25,7 @@ from time import time
 import requests
 
 from platformio import exception, lockfile, util
-from platformio.compat import PY2, WINDOWS
+from platformio.compat import WINDOWS, hashlib_encode_data
 from platformio.proc import is_ci
 from platformio.project.helpers import (get_project_cache_dir,
                                         get_project_core_dir)
@@ -174,10 +174,8 @@ class ContentCache(object):
     def key_from_args(*args):
         h = hashlib.md5()
         for arg in args:
-            if not arg:
+            if arg:
-                continue
+                h.update(hashlib_encode_data(arg))
-            arg = str(arg)
-            h.update(arg if PY2 else arg.encode())
         return h.hexdigest()
 
     def get(self, key):
@@ -363,7 +361,7 @@ def get_cid():
             pass
     if not uid:
         uid = uuid.getnode()
-    cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest())
+    cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())
     cid = str(cid)
     if WINDOWS or os.getuid() > 0:  # yapf: disable pylint: disable=no-member
         set_state_item("cid", cid)

platformio/builder/main.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import base64
 import json
 import sys
 from os import environ
@@ -31,6 +30,7 @@ from SCons.Script import Variables  # pylint: disable=import-error
 
 from platformio import util
 from platformio.compat import PY2, path_to_unicode
+from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
 from platformio.project import helpers as project_helpers
 
@@ -91,11 +91,11 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
 env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
 
 # Load variables from CLI
-for key in list(clivars.keys()):
+env.Replace(
-    if key in env:
+    **{
-        env[key] = base64.b64decode(env[key])
+        key: PlatformBase.decode_scons_arg(env[key])
-        if isinstance(env[key], bytes):
+        for key in list(clivars.keys()) if key in env
-            env[key] = env[key].decode()
+    })
 
 if env.GetOption('clean'):
     env.PioClean(env.subst("$BUILD_DIR"))

platformio/builder/tools/piolib.py

@@ -32,7 +32,8 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, util
 from platformio.builder.tools import platformio as piotool
-from platformio.compat import PY2, WINDOWS, get_file_contents, string_types
+from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data,
+                               string_types)
 from platformio.managers.lib import LibraryManager
 
 
@@ -181,8 +182,7 @@ class LibBuilderBase(object):
 
     @property
     def build_dir(self):
-        lib_hash = hashlib.sha1(
+        lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3]
-            self.path if PY2 else self.path.encode()).hexdigest()[:3]
         return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path))
 
     @property

platformio/builder/tools/piowinhooks.py

@@ -18,7 +18,7 @@ from hashlib import md5
 from os import makedirs
 from os.path import isdir, isfile, join
 
-from platformio.compat import PY2, WINDOWS
+from platformio.compat import WINDOWS, hashlib_encode_data
 
 # Windows CLI has limit with command length to 8192
 # Leave 2000 chars for flags and other options
@@ -61,9 +61,8 @@ def _file_long_data(env, data):
     build_dir = env.subst("$BUILD_DIR")
     if not isdir(build_dir):
         makedirs(build_dir)
-    tmp_file = join(
+    tmp_file = join(build_dir,
-        build_dir,
+                    "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest())
-        "longcmd-%s" % md5(data if PY2 else data.encode()).hexdigest())
     if isfile(tmp_file):
         return tmp_file
     with open(tmp_file, "w") as fp:

platformio/commands/debug/client.py

@@ -29,7 +29,7 @@ from platformio import app, exception, util
 from platformio.commands.debug import helpers, initcfgs
 from platformio.commands.debug.process import BaseProcess
 from platformio.commands.debug.server import DebugServer
-from platformio.compat import PY2
+from platformio.compat import hashlib_encode_data
 from platformio.project.helpers import get_project_cache_dir
 from platformio.telemetry import MeasurementProtocol
 
@@ -61,8 +61,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
 
     def spawn(self, gdb_path, prog_path):
         session_hash = gdb_path + prog_path
-        self._session_id = sha1(
+        self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest()
-            session_hash if PY2 else session_hash.encode()).hexdigest()
         self._kill_previous_session()
 
         patterns = {

platformio/compat.py

@@ -35,12 +35,21 @@ if PY2:
         return isinstance(x, (buffer, bytearray))
 
     def path_to_unicode(path):
+        if isinstance(path, unicode):
+            return path
         return path.decode(get_filesystem_encoding()).encode("utf-8")
 
     def get_file_contents(path):
         with open(path) as f:
             return f.read()
 
+    def hashlib_encode_data(data):
+        if is_bytes(data):
+            return data
+        if not isinstance(data, string_types):
+            data = str(data)
+        return data
+
     _magic_check = re.compile('([*?[])')
     _magic_check_bytes = re.compile(b'([*?[])')
 
@@ -74,3 +83,10 @@ else:
         except UnicodeDecodeError:
             with open(path, encoding="latin-1") as f:
                 return f.read()
+
+    def hashlib_encode_data(data):
+        if is_bytes(data):
+            return data
+        if not isinstance(data, string_types):
+            data = str(data)
+        return data.encode()

platformio/managers/package.py

@@ -26,7 +26,7 @@ import requests
 import semantic_version
 
 from platformio import __version__, app, exception, telemetry, util
-from platformio.compat import path_to_unicode
+from platformio.compat import hashlib_encode_data, path_to_unicode
 from platformio.downloader import FileDownloader
 from platformio.lockfile import LockFile
 from platformio.unpacker import FileUnpacker
@@ -173,7 +173,7 @@ class PkgInstallerMixin(object):
         cache_key_data = app.ContentCache.key_from_args(url, "data")
         if self.FILE_CACHE_VALID:
             with app.ContentCache() as cc:
-                fname = cc.get(cache_key_fname)
+                fname = str(cc.get(cache_key_fname))
                 cache_path = cc.get_cache_path(cache_key_data)
                 if fname and isfile(cache_path):
                     dst_path = join(dest_dir, fname)
@@ -591,7 +591,8 @@ class PkgInstallerMixin(object):
                     target_dirname = "%s@src-%s" % (
                         pkg_dirname,
                         hashlib.md5(
-                            cur_manifest['__src_url'].encode()).hexdigest())
+                            hashlib_encode_data(
+                                cur_manifest['__src_url'])).hexdigest())
                 shutil.move(pkg_dir, join(self.package_dir, target_dirname))
             # fix to a version
             elif action == 2:
@@ -601,7 +602,8 @@ class PkgInstallerMixin(object):
                     target_dirname = "%s@src-%s" % (
                         pkg_dirname,
                         hashlib.md5(
-                            tmp_manifest['__src_url'].encode()).hexdigest())
+                            hashlib_encode_data(
+                                tmp_manifest['__src_url'])).hexdigest())
                 pkg_dir = join(self.package_dir, target_dirname)
 
         # remove previous/not-satisfied package

platformio/managers/platform.py

@@ -24,7 +24,7 @@ import click
 import semantic_version
 
 from platformio import __version__, app, exception, util
-from platformio.compat import PY2
+from platformio.compat import get_filesystem_encoding, string_types
 from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
 from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv,
@@ -359,6 +359,17 @@ class PlatformRunMixin(object):
 
     LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I)
 
+    @staticmethod
+    def encode_scons_arg(value):
+        if isinstance(value, string_types):
+            value = value.encode(get_filesystem_encoding())
+        return base64.urlsafe_b64encode(value).decode()
+
+    @staticmethod
+    def decode_scons_arg(value):
+        return base64.urlsafe_b64decode(value).decode(
+            get_filesystem_encoding())
+
     def run(self, variables, targets, silent, verbose):
         assert isinstance(variables, dict)
         assert isinstance(targets, list)
@@ -402,12 +413,7 @@ class PlatformRunMixin(object):
 
         # encode and append variables
         for key, value in variables.items():
-            if PY2:
+            cmd.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
-                cmd.append("%s=%s" % (key.upper(), base64.b64encode(value)))
-            else:
-                cmd.append(
-                    "%s=%s" %
-                    (key.upper(), base64.b64encode(value.encode()).decode()))
 
         def _write_and_flush(stream, data):
             stream.write(data)

platformio/proc.py

@@ -19,7 +19,7 @@ from os.path import isdir, isfile, join, normpath
 from threading import Thread
 
 from platformio import exception
-from platformio.compat import PY2, WINDOWS, string_types
+from platformio.compat import WINDOWS, get_filesystem_encoding, string_types
 
 
 class AsyncPipeBase(object):
@@ -123,8 +123,8 @@ def exec_command(*args, **kwargs):
             result[s[3:]] = kwargs[s].get_buffer()
 
     for k, v in result.items():
-        if not PY2 and isinstance(result[k], bytes):
+        if isinstance(result[k], bytes):
-            result[k] = result[k].decode()
+            result[k] = result[k].decode(get_filesystem_encoding())
         if v and isinstance(v, string_types):
             result[k] = result[k].strip()
 

platformio/project/helpers.py

@@ -19,7 +19,7 @@ from os.path import (basename, dirname, expanduser, isdir, isfile, join,
                      realpath, splitdrive)
 
 from platformio import __version__
-from platformio.compat import PY2, WINDOWS
+from platformio.compat import WINDOWS, hashlib_encode_data
 from platformio.project.config import ProjectConfig
 
 
@@ -54,9 +54,8 @@ def get_project_optional_dir(name, default=None):
     if "$PROJECT_HASH" in optional_dir:
         optional_dir = optional_dir.replace(
             "$PROJECT_HASH", "%s-%s" %
-            (basename(project_dir),
+            (basename(project_dir), sha1(
-             sha1(project_dir if PY2 else project_dir.encode()).hexdigest()
+                hashlib_encode_data(project_dir)).hexdigest()[:10]))
-             [:10]))
 
     if optional_dir.startswith("~"):
         optional_dir = expanduser(optional_dir)
@@ -179,4 +178,4 @@ def calculate_project_hash():
         # Fix issue with useless project rebuilding for case insensitive FS.
         # A case of disk drive can differ...
         chunks_to_str = chunks_to_str.lower()
-    return sha1(chunks_to_str if PY2 else chunks_to_str.encode()).hexdigest()
+    return sha1(hashlib_encode_data(chunks_to_str)).hexdigest()

platformio/telemetry.py

@@ -28,6 +28,7 @@ import requests
 
 from platformio import __version__, app, exception, util
 from platformio.commands import PlatformioCLI
+from platformio.compat import string_types
 from platformio.proc import is_ci, is_container
 
 try:
@@ -135,12 +136,15 @@ class MeasurementProtocol(TelemetryBase):
                     return _arg
             return None
 
-        args = [
+        args = []
-            str(arg).lower() for arg in PlatformioCLI.leftover_args
+        for arg in PlatformioCLI.leftover_args:
-            if not str(arg).startswith("-")
+            if not isinstance(arg, string_types):
-        ]
+                arg = str(arg)
+            if not arg.startswith("-"):
+                args.append(arg.lower())
         if not args:
             return
+
         cmd_path = args[:1]
         if args[0] in ("platform", "platforms", "serialports", "device",
                        "settings", "account"):