diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 4e579741..cfd0180d 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -15,7 +15,7 @@ import json import os import sys -from time import time +import time import click from SCons.Script import ARGUMENTS # pylint: disable=import-error @@ -61,7 +61,7 @@ DEFAULT_ENV_OPTIONS = dict( "piotarget", "piolib", "pioupload", - "piosize", + "piomemusage", "pioino", "piomisc", "piointegration", @@ -71,7 +71,7 @@ DEFAULT_ENV_OPTIONS = dict( variables=clivars, # Propagating External Environment ENV=os.environ, - UNIX_TIME=int(time()), + UNIX_TIME=int(time.time()), PYTHONEXE=get_pythonexe_path(), ) @@ -183,7 +183,7 @@ env.SConscript(env.GetExtraScripts("post"), exports="env") # Checking program size if env.get("SIZETOOL") and not ( - set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS) + set(["nobuild", "__memusage"]) & set(COMMAND_LINE_TARGETS) ): env.Depends("upload", "checkprogsize") # Replace platform's "size" target with our @@ -235,16 +235,16 @@ if env.IsIntegrationDump(): ) env.Exit(0) -if "sizedata" in COMMAND_LINE_TARGETS: +if "__memusage" in COMMAND_LINE_TARGETS: AlwaysBuild( env.Alias( - "sizedata", + "__memusage", DEFAULT_TARGETS, - env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."), + env.VerboseAction(env.DumpMemoryUsage, "Generating memory usage report..."), ) ) - Default("sizedata") + Default("__memusage") # issue #4604: process targets sequentially for index, target in enumerate( diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piomemusage.py similarity index 80% rename from platformio/builder/tools/piosize.py rename to platformio/builder/tools/piomemusage.py index 3ac24311..0adab756 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piomemusage.py @@ -14,33 +14,33 @@ # pylint: disable=too-many-locals -import json +import os import sys -from os import environ, makedirs, remove -from os.path import isdir, join, splitdrive +import time from elftools.elf.descriptions import describe_sh_flags from elftools.elf.elffile import ELFFile from platformio.compat import IS_WINDOWS from platformio.proc import exec_command +from platformio.project.memusage import save_report def _run_tool(cmd, env, tool_args): - sysenv = environ.copy() + sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) build_dir = env.subst("$BUILD_DIR") - if not isdir(build_dir): - makedirs(build_dir) - tmp_file = join(build_dir, "size-data-longcmd.txt") + if not os.path.isdir(build_dir): + os.makedirs(build_dir) + tmp_file = os.path.join(build_dir, "size-data-longcmd.txt") with open(tmp_file, mode="w", encoding="utf8") as fp: fp.write("\n".join(tool_args)) cmd.append("@" + tmp_file) result = exec_command(cmd, env=sysenv) - remove(tmp_file) + os.remove(tmp_file) return result @@ -92,8 +92,8 @@ def _collect_sections_info(env, elffile): } sections[section.name] = section_data - sections[section.name]["in_flash"] = env.pioSizeIsFlashSection(section_data) - sections[section.name]["in_ram"] = env.pioSizeIsRamSection(section_data) + sections[section.name]["in_flash"] = env.memusageIsFlashSection(section_data) + sections[section.name]["in_ram"] = env.memusageIsRamSection(section_data) return sections @@ -106,7 +106,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections): sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?") env.Exit(1) - sysenv = environ.copy() + sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) symbol_addrs = [] @@ -117,7 +117,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections): symbol_size = s["st_size"] symbol_type = symbol_info["type"] - if not env.pioSizeIsValidSymbol(s.name, symbol_type, symbol_addr): + if not env.memusageIsValidSymbol(s.name, symbol_type, symbol_addr): continue symbol = { @@ -126,7 +126,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections): "name": s.name, "type": symbol_type, "size": symbol_size, - "section": env.pioSizeDetermineSection(sections, symbol_addr), + "section": env.memusageDetermineSection(sections, symbol_addr), } if s.name.startswith("_Z"): @@ -144,8 +144,8 @@ def _collect_symbols_info(env, elffile, elf_path, sections): if not location or "?" in location: continue if IS_WINDOWS: - drive, tail = splitdrive(location) - location = join(drive.upper(), tail) + drive, tail = os.path.splitdrive(location) + location = os.path.join(drive.upper(), tail) symbol["file"] = location symbol["line"] = 0 if ":" in location: @@ -156,7 +156,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections): return symbols -def pioSizeDetermineSection(_, sections, symbol_addr): +def memusageDetermineSection(_, sections, symbol_addr): for section, info in sections.items(): if not info.get("in_flash", False) and not info.get("in_ram", False): continue @@ -165,22 +165,22 @@ def pioSizeDetermineSection(_, sections, symbol_addr): return "unknown" -def pioSizeIsValidSymbol(_, symbol_name, symbol_type, symbol_address): +def memusageIsValidSymbol(_, symbol_name, symbol_type, symbol_address): return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE" -def pioSizeIsRamSection(_, section): +def memusageIsRamSection(_, section): return ( section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS") and section.get("flags", "") == "WA" ) -def pioSizeIsFlashSection(_, section): +def memusageIsFlashSection(_, section): return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "") -def pioSizeCalculateFirmwareSize(_, sections): +def memusageCalculateFirmwareSize(_, sections): flash_size = ram_size = 0 for section_info in sections.values(): if section_info.get("in_flash", False): @@ -191,8 +191,8 @@ def pioSizeCalculateFirmwareSize(_, sections): return ram_size, flash_size -def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument - data = {"device": {}, "memory": {}, "version": 1} +def DumpMemoryUsage(_, target, source, env): # pylint: disable=unused-argument + data = {"version": 1, "timestamp": int(time.time()), "device": {}, "memory": {}} board = env.BoardConfig() if board: @@ -216,7 +216,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument env.Exit(1) sections = _collect_sections_info(env, elffile) - firmware_ram, firmware_flash = env.pioSizeCalculateFirmwareSize(sections) + firmware_ram, firmware_flash = env.memusageCalculateFirmwareSize(sections) data["memory"]["total"] = { "ram_size": firmware_ram, "flash_size": firmware_flash, @@ -225,7 +225,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument files = {} for symbol in _collect_symbols_info(env, elffile, elf_path, sections): - file_path = symbol.get("file") or "unknown" + file_path = symbol.pop("file", "unknown") if not files.get(file_path, {}): files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0} @@ -246,10 +246,10 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument file_data.update(v) data["memory"]["files"].append(file_data) - with open( - join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8" - ) as fp: - fp.write(json.dumps(data)) + print( + "Memory usage report has been saved to the following location: " + f"\"{save_report(os.getcwd(), env['PIOENV'], data)}\"" + ) def exists(_): @@ -257,10 +257,10 @@ def exists(_): def generate(env): - env.AddMethod(pioSizeIsRamSection) - env.AddMethod(pioSizeIsFlashSection) - env.AddMethod(pioSizeCalculateFirmwareSize) - env.AddMethod(pioSizeDetermineSection) - env.AddMethod(pioSizeIsValidSymbol) - env.AddMethod(DumpSizeData) + env.AddMethod(memusageIsRamSection) + env.AddMethod(memusageIsFlashSection) + env.AddMethod(memusageCalculateFirmwareSize) + env.AddMethod(memusageDetermineSection) + env.AddMethod(memusageIsValidSymbol) + env.AddMethod(DumpMemoryUsage) return env diff --git a/platformio/home/rpc/handlers/memusage.py b/platformio/home/rpc/handlers/memusage.py new file mode 100644 index 00000000..a63d01de --- /dev/null +++ b/platformio/home/rpc/handlers/memusage.py @@ -0,0 +1,100 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools + +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.project import memusage + + +class MemUsageRPC(BaseRPCHandler): + NAMESPACE = "memusage" + + async def summary(self, project_dir, env, options=None): + options = options or {} + existing_reports = memusage.list_reports(project_dir, env) + current_report = previous_report = None + if options.get("cached") and existing_reports: + current_report = memusage.read_report(existing_reports[-1]) + if len(existing_reports) > 1: + previous_report = memusage.read_report(existing_reports[-2]) + else: + if existing_reports: + previous_report = memusage.read_report(existing_reports[-1]) + await self.factory.manager.dispatcher["core.exec"]( + ["run", "-d", project_dir, "-e", env, "-t", "__memusage"], + options=options.get("exec"), + raise_exception=True, + ) + current_report = memusage.read_report( + memusage.list_reports(project_dir, env)[-1] + ) + + max_top_items = 10 + return dict( + timestamp=dict( + current=current_report["timestamp"], + previous=previous_report["timestamp"] if previous_report else None, + ), + device=current_report["device"], + trend=dict( + current=current_report["memory"]["total"], + previous=previous_report["memory"]["total"] + if previous_report + else None, + ), + top=dict( + files=self._calculate_top_files(current_report["memory"]["files"])[ + 0:max_top_items + ], + symbols=self._calculate_top_symbols(current_report["memory"]["files"])[ + 0:max_top_items + ], + sections=sorted( + current_report["memory"]["total"]["sections"].values(), + key=lambda item: item["size"], + reverse=True, + )[0:max_top_items], + ), + ) + + @staticmethod + def _calculate_top_files(items): + return [ + {"path": item["path"], "ram": item["ram_size"], "flash": item["flash_size"]} + for item in sorted( + items, + key=lambda item: item["ram_size"] + item["flash_size"], + reverse=True, + ) + ] + + @staticmethod + def _calculate_top_symbols(files): + symbols = functools.reduce( + lambda result, filex: result + + [ + { + "name": s["name"], + "type": s["type"], + "size": s["size"], + "file": filex["path"], + "line": s.get("line"), + } + for s in filex["symbols"] + ], + files, + [], + ) + return sorted(symbols, key=lambda item: item["size"], reverse=True) diff --git a/platformio/home/run.py b/platformio/home/run.py index 194fbef5..780e4ccb 100644 --- a/platformio/home/run.py +++ b/platformio/home/run.py @@ -30,6 +30,7 @@ from platformio.home.rpc.handlers.account import AccountRPC from platformio.home.rpc.handlers.app import AppRPC from platformio.home.rpc.handlers.core import CoreRPC from platformio.home.rpc.handlers.ide import IDERPC +from platformio.home.rpc.handlers.memusage import MemUsageRPC from platformio.home.rpc.handlers.misc import MiscRPC from platformio.home.rpc.handlers.os import OSRPC from platformio.home.rpc.handlers.platform import PlatformRPC @@ -70,6 +71,7 @@ def run_server(host, port, no_open, shutdown_timeout, home_url): ws_rpc_factory.add_object_handler(AccountRPC()) ws_rpc_factory.add_object_handler(AppRPC()) ws_rpc_factory.add_object_handler(IDERPC()) + ws_rpc_factory.add_object_handler(MemUsageRPC()) ws_rpc_factory.add_object_handler(MiscRPC()) ws_rpc_factory.add_object_handler(OSRPC()) ws_rpc_factory.add_object_handler(CoreRPC()) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 97bdbbb8..f8a4e492 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -143,8 +143,7 @@ def get_build_type(config, env, run_targets=None): run_targets = run_targets or [] declared_build_type = config.get(f"env:{env}", "build_type") if ( - set(["__debug", "sizedata"]) # sizedata = for memory inspection - & set(run_targets) + set(["__debug", "__memusage"]) & set(run_targets) or declared_build_type == "debug" ): types.append("debug") diff --git a/platformio/project/memusage.py b/platformio/project/memusage.py new file mode 100644 index 00000000..4bd11132 --- /dev/null +++ b/platformio/project/memusage.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import json +import os +import time + +from platformio import fs +from platformio.project.config import ProjectConfig + + +def get_report_dir(project_dir, env): + with fs.cd(project_dir): + return os.path.join( + ProjectConfig.get_instance().get("platformio", "memusage_dir"), env + ) + + +def list_reports(project_dir, env): + report_dir = get_report_dir(project_dir, env) + if not os.path.isdir(report_dir): + return [] + return [os.path.join(report_dir, item) for item in sorted(os.listdir(report_dir))] + + +def read_report(path): + with gzip.open(path, mode="rt", encoding="utf8") as fp: + return json.load(fp) + + +def save_report(project_dir, env, data): + report_dir = get_report_dir(project_dir, env) + if not os.path.isdir(report_dir): + os.makedirs(report_dir) + report_path = os.path.join(report_dir, f"{int(time.time())}.json.gz") + with gzip.open(report_path, mode="wt", encoding="utf8") as fp: + json.dump(data, fp) + rotate_reports(report_dir) + return report_path + + +def rotate_reports(report_dir, max_reports=100): + reports = os.listdir(report_dir) + if len(reports) < max_reports: + return + for fname in sorted(reports)[0 : len(reports) - max_reports]: + os.remove(os.path.join(report_dir, fname)) diff --git a/platformio/project/options.py b/platformio/project/options.py index 2e13815a..51681ee6 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -83,7 +83,7 @@ def ConfigEnvOption(*args, **kwargs): def calculate_path_hash(path): return "%s-%s" % ( os.path.basename(path), - hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10], + hashlib.sha1(hashlib_encode_data(path.lower())).hexdigest()[:10], ) @@ -266,6 +266,17 @@ ProjectOptions = OrderedDict( default=os.path.join("${platformio.workspace_dir}", "libdeps"), validate=validate_dir, ), + ConfigPlatformioOption( + group="directory", + name="memusage_dir", + description=( + "A location where PlatformIO Core will store " + "project memory usage reports" + ), + sysenvvar="PLATFORMIO_MEMUSAGE_DIR", + default=os.path.join("${platformio.workspace_dir}", "memusage"), + validate=validate_dir, + ), ConfigPlatformioOption( group="directory", name="include_dir",