Implement memory usage profiling RPC

platformio/builder/main.py

@@ -15,7 +15,7 @@
 import json
 import os
 import sys
-from time import time
+import time
 
 import click
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
@@ -61,7 +61,7 @@ DEFAULT_ENV_OPTIONS = dict(
         "piotarget",
         "piolib",
         "pioupload",
-        "piosize",
+        "piomemusage",
         "pioino",
         "piomisc",
         "piointegration",
@@ -71,7 +71,7 @@ DEFAULT_ENV_OPTIONS = dict(
     variables=clivars,
     # Propagating External Environment
     ENV=os.environ,
-    UNIX_TIME=int(time()),
+    UNIX_TIME=int(time.time()),
     PYTHONEXE=get_pythonexe_path(),
 )
 
@@ -183,7 +183,7 @@ env.SConscript(env.GetExtraScripts("post"), exports="env")
 
 # Checking program size
 if env.get("SIZETOOL") and not (
-    set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS)
+    set(["nobuild", "__memusage"]) & set(COMMAND_LINE_TARGETS)
 ):
     env.Depends("upload", "checkprogsize")
     # Replace platform's "size" target with our
@@ -235,16 +235,16 @@ if env.IsIntegrationDump():
     )
     env.Exit(0)
 
-if "sizedata" in COMMAND_LINE_TARGETS:
+if "__memusage" in COMMAND_LINE_TARGETS:
     AlwaysBuild(
         env.Alias(
-            "sizedata",
+            "__memusage",
             DEFAULT_TARGETS,
-            env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."),
+            env.VerboseAction(env.DumpMemoryUsage, "Generating memory usage report..."),
         )
     )
 
-    Default("sizedata")
+    Default("__memusage")
 
 # issue #4604: process targets sequentially
 for index, target in enumerate(

platformio/builder/tools/piomemusage.py

@@ -14,33 +14,33 @@
 
 # pylint: disable=too-many-locals
 
-import json
+import os
 import sys
-from os import environ, makedirs, remove
+import time
-from os.path import isdir, join, splitdrive
 
 from elftools.elf.descriptions import describe_sh_flags
 from elftools.elf.elffile import ELFFile
 
 from platformio.compat import IS_WINDOWS
 from platformio.proc import exec_command
+from platformio.project.memusage import save_report
 
 
 def _run_tool(cmd, env, tool_args):
-    sysenv = environ.copy()
+    sysenv = os.environ.copy()
     sysenv["PATH"] = str(env["ENV"]["PATH"])
 
     build_dir = env.subst("$BUILD_DIR")
-    if not isdir(build_dir):
+    if not os.path.isdir(build_dir):
-        makedirs(build_dir)
+        os.makedirs(build_dir)
-    tmp_file = join(build_dir, "size-data-longcmd.txt")
+    tmp_file = os.path.join(build_dir, "size-data-longcmd.txt")
 
     with open(tmp_file, mode="w", encoding="utf8") as fp:
         fp.write("\n".join(tool_args))
 
     cmd.append("@" + tmp_file)
     result = exec_command(cmd, env=sysenv)
-    remove(tmp_file)
+    os.remove(tmp_file)
 
     return result
 
@@ -92,8 +92,8 @@ def _collect_sections_info(env, elffile):
         }
 
         sections[section.name] = section_data
-        sections[section.name]["in_flash"] = env.pioSizeIsFlashSection(section_data)
+        sections[section.name]["in_flash"] = env.memusageIsFlashSection(section_data)
-        sections[section.name]["in_ram"] = env.pioSizeIsRamSection(section_data)
+        sections[section.name]["in_ram"] = env.memusageIsRamSection(section_data)
 
     return sections
 
@@ -106,7 +106,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?")
         env.Exit(1)
 
-    sysenv = environ.copy()
+    sysenv = os.environ.copy()
     sysenv["PATH"] = str(env["ENV"]["PATH"])
 
     symbol_addrs = []
@@ -117,7 +117,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         symbol_size = s["st_size"]
         symbol_type = symbol_info["type"]
 
-        if not env.pioSizeIsValidSymbol(s.name, symbol_type, symbol_addr):
+        if not env.memusageIsValidSymbol(s.name, symbol_type, symbol_addr):
             continue
 
         symbol = {
@@ -126,7 +126,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
             "name": s.name,
             "type": symbol_type,
             "size": symbol_size,
-            "section": env.pioSizeDetermineSection(sections, symbol_addr),
+            "section": env.memusageDetermineSection(sections, symbol_addr),
         }
 
         if s.name.startswith("_Z"):
@@ -144,8 +144,8 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         if not location or "?" in location:
             continue
         if IS_WINDOWS:
-            drive, tail = splitdrive(location)
+            drive, tail = os.path.splitdrive(location)
-            location = join(drive.upper(), tail)
+            location = os.path.join(drive.upper(), tail)
         symbol["file"] = location
         symbol["line"] = 0
         if ":" in location:
@@ -156,7 +156,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
     return symbols
 
 
-def pioSizeDetermineSection(_, sections, symbol_addr):
+def memusageDetermineSection(_, sections, symbol_addr):
     for section, info in sections.items():
         if not info.get("in_flash", False) and not info.get("in_ram", False):
             continue
@@ -165,22 +165,22 @@ def pioSizeDetermineSection(_, sections, symbol_addr):
     return "unknown"
 
 
-def pioSizeIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
+def memusageIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
     return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
 
 
-def pioSizeIsRamSection(_, section):
+def memusageIsRamSection(_, section):
     return (
         section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
         and section.get("flags", "") == "WA"
     )
 
 
-def pioSizeIsFlashSection(_, section):
+def memusageIsFlashSection(_, section):
     return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
 
 
-def pioSizeCalculateFirmwareSize(_, sections):
+def memusageCalculateFirmwareSize(_, sections):
     flash_size = ram_size = 0
     for section_info in sections.values():
         if section_info.get("in_flash", False):
@@ -191,8 +191,8 @@ def pioSizeCalculateFirmwareSize(_, sections):
     return ram_size, flash_size
 
 
-def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
+def DumpMemoryUsage(_, target, source, env):  # pylint: disable=unused-argument
-    data = {"device": {}, "memory": {}, "version": 1}
+    data = {"version": 1, "timestamp": int(time.time()), "device": {}, "memory": {}}
 
     board = env.BoardConfig()
     if board:
@@ -216,7 +216,7 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
             env.Exit(1)
 
         sections = _collect_sections_info(env, elffile)
-        firmware_ram, firmware_flash = env.pioSizeCalculateFirmwareSize(sections)
+        firmware_ram, firmware_flash = env.memusageCalculateFirmwareSize(sections)
         data["memory"]["total"] = {
             "ram_size": firmware_ram,
             "flash_size": firmware_flash,
@@ -225,7 +225,7 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
 
         files = {}
         for symbol in _collect_symbols_info(env, elffile, elf_path, sections):
-            file_path = symbol.get("file") or "unknown"
+            file_path = symbol.pop("file", "unknown")
             if not files.get(file_path, {}):
                 files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0}
 
@@ -246,10 +246,10 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
             file_data.update(v)
             data["memory"]["files"].append(file_data)
 
-    with open(
+    print(
-        join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8"
+        "Memory usage report has been saved to the following location: "
-    ) as fp:
+        f"\"{save_report(os.getcwd(), env['PIOENV'], data)}\""
-        fp.write(json.dumps(data))
+    )
 
 
 def exists(_):
@@ -257,10 +257,10 @@ def exists(_):
 
 
 def generate(env):
-    env.AddMethod(pioSizeIsRamSection)
+    env.AddMethod(memusageIsRamSection)
-    env.AddMethod(pioSizeIsFlashSection)
+    env.AddMethod(memusageIsFlashSection)
-    env.AddMethod(pioSizeCalculateFirmwareSize)
+    env.AddMethod(memusageCalculateFirmwareSize)
-    env.AddMethod(pioSizeDetermineSection)
+    env.AddMethod(memusageDetermineSection)
-    env.AddMethod(pioSizeIsValidSymbol)
+    env.AddMethod(memusageIsValidSymbol)
-    env.AddMethod(DumpSizeData)
+    env.AddMethod(DumpMemoryUsage)
     return env

platformio/home/rpc/handlers/memusage.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+
+from platformio.home.rpc.handlers.base import BaseRPCHandler
+from platformio.project import memusage
+
+
+class MemUsageRPC(BaseRPCHandler):
+    NAMESPACE = "memusage"
+
+    async def summary(self, project_dir, env, options=None):
+        options = options or {}
+        existing_reports = memusage.list_reports(project_dir, env)
+        current_report = previous_report = None
+        if options.get("cached") and existing_reports:
+            current_report = memusage.read_report(existing_reports[-1])
+            if len(existing_reports) > 1:
+                previous_report = memusage.read_report(existing_reports[-2])
+        else:
+            if existing_reports:
+                previous_report = memusage.read_report(existing_reports[-1])
+            await self.factory.manager.dispatcher["core.exec"](
+                ["run", "-d", project_dir, "-e", env, "-t", "__memusage"],
+                options=options.get("exec"),
+                raise_exception=True,
+            )
+            current_report = memusage.read_report(
+                memusage.list_reports(project_dir, env)[-1]
+            )
+
+        max_top_items = 10
+        return dict(
+            timestamp=dict(
+                current=current_report["timestamp"],
+                previous=previous_report["timestamp"] if previous_report else None,
+            ),
+            device=current_report["device"],
+            trend=dict(
+                current=current_report["memory"]["total"],
+                previous=previous_report["memory"]["total"]
+                if previous_report
+                else None,
+            ),
+            top=dict(
+                files=self._calculate_top_files(current_report["memory"]["files"])[
+                    0:max_top_items
+                ],
+                symbols=self._calculate_top_symbols(current_report["memory"]["files"])[
+                    0:max_top_items
+                ],
+                sections=sorted(
+                    current_report["memory"]["total"]["sections"].values(),
+                    key=lambda item: item["size"],
+                    reverse=True,
+                )[0:max_top_items],
+            ),
+        )
+
+    @staticmethod
+    def _calculate_top_files(items):
+        return [
+            {"path": item["path"], "ram": item["ram_size"], "flash": item["flash_size"]}
+            for item in sorted(
+                items,
+                key=lambda item: item["ram_size"] + item["flash_size"],
+                reverse=True,
+            )
+        ]
+
+    @staticmethod
+    def _calculate_top_symbols(files):
+        symbols = functools.reduce(
+            lambda result, filex: result
+            + [
+                {
+                    "name": s["name"],
+                    "type": s["type"],
+                    "size": s["size"],
+                    "file": filex["path"],
+                    "line": s.get("line"),
+                }
+                for s in filex["symbols"]
+            ],
+            files,
+            [],
+        )
+        return sorted(symbols, key=lambda item: item["size"], reverse=True)

platformio/home/run.py

@@ -30,6 +30,7 @@ from platformio.home.rpc.handlers.account import AccountRPC
 from platformio.home.rpc.handlers.app import AppRPC
 from platformio.home.rpc.handlers.core import CoreRPC
 from platformio.home.rpc.handlers.ide import IDERPC
+from platformio.home.rpc.handlers.memusage import MemUsageRPC
 from platformio.home.rpc.handlers.misc import MiscRPC
 from platformio.home.rpc.handlers.os import OSRPC
 from platformio.home.rpc.handlers.platform import PlatformRPC
@@ -70,6 +71,7 @@ def run_server(host, port, no_open, shutdown_timeout, home_url):
     ws_rpc_factory.add_object_handler(AccountRPC())
     ws_rpc_factory.add_object_handler(AppRPC())
     ws_rpc_factory.add_object_handler(IDERPC())
+    ws_rpc_factory.add_object_handler(MemUsageRPC())
     ws_rpc_factory.add_object_handler(MiscRPC())
     ws_rpc_factory.add_object_handler(OSRPC())
     ws_rpc_factory.add_object_handler(CoreRPC())

platformio/project/helpers.py

@@ -143,8 +143,7 @@ def get_build_type(config, env, run_targets=None):
     run_targets = run_targets or []
     declared_build_type = config.get(f"env:{env}", "build_type")
     if (
-        set(["__debug", "sizedata"])  # sizedata = for memory inspection
+        set(["__debug", "__memusage"]) & set(run_targets)
-        & set(run_targets)
         or declared_build_type == "debug"
     ):
         types.append("debug")

platformio/project/memusage.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gzip
+import json
+import os
+import time
+
+from platformio import fs
+from platformio.project.config import ProjectConfig
+
+
+def get_report_dir(project_dir, env):
+    with fs.cd(project_dir):
+        return os.path.join(
+            ProjectConfig.get_instance().get("platformio", "memusage_dir"), env
+        )
+
+
+def list_reports(project_dir, env):
+    report_dir = get_report_dir(project_dir, env)
+    if not os.path.isdir(report_dir):
+        return []
+    return [os.path.join(report_dir, item) for item in sorted(os.listdir(report_dir))]
+
+
+def read_report(path):
+    with gzip.open(path, mode="rt", encoding="utf8") as fp:
+        return json.load(fp)
+
+
+def save_report(project_dir, env, data):
+    report_dir = get_report_dir(project_dir, env)
+    if not os.path.isdir(report_dir):
+        os.makedirs(report_dir)
+    report_path = os.path.join(report_dir, f"{int(time.time())}.json.gz")
+    with gzip.open(report_path, mode="wt", encoding="utf8") as fp:
+        json.dump(data, fp)
+    rotate_reports(report_dir)
+    return report_path
+
+
+def rotate_reports(report_dir, max_reports=100):
+    reports = os.listdir(report_dir)
+    if len(reports) < max_reports:
+        return
+    for fname in sorted(reports)[0 : len(reports) - max_reports]:
+        os.remove(os.path.join(report_dir, fname))

platformio/project/options.py

@@ -83,7 +83,7 @@ def ConfigEnvOption(*args, **kwargs):
 def calculate_path_hash(path):
     return "%s-%s" % (
         os.path.basename(path),
-        hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10],
+        hashlib.sha1(hashlib_encode_data(path.lower())).hexdigest()[:10],
     )
 
 
@@ -266,6 +266,17 @@ ProjectOptions = OrderedDict(
                 default=os.path.join("${platformio.workspace_dir}", "libdeps"),
                 validate=validate_dir,
             ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="memusage_dir",
+                description=(
+                    "A location where PlatformIO Core will store "
+                    "project memory usage reports"
+                ),
+                sysenvvar="PLATFORMIO_MEMUSAGE_DIR",
+                default=os.path.join("${platformio.workspace_dir}", "memusage"),
+                validate=validate_dir,
+            ),
             ConfigPlatformioOption(
                 group="directory",
                 name="include_dir",