feedc0de/platformio-core
1
0
Fork 0
forked from platformio/platformio-core
Code Pull Requests Activity

Implement memory usage profiling RPC

Browse Source
This commit is contained in:
Ivan Kravets
2023-07-25 12:31:25 +03:00
parent 65b31c69b0
commit a3ad3103ef
7 changed files with 216 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
platformio/builder/main.py
View File

@ -15,7 +15,7 @@
import json
import os
import sys
from time import time
import time
import click
from SCons.Script import ARGUMENTS # pylint: disable=import-error
@ -61,7 +61,7 @@ DEFAULT_ENV_OPTIONS = dict(
"piotarget",
"piolib",
"pioupload",
"piosize",
"piomemusage",
"pioino",
"piomisc",
"piointegration",
@ -71,7 +71,7 @@ DEFAULT_ENV_OPTIONS = dict(
variables=clivars,
# Propagating External Environment
ENV=os.environ,
UNIX_TIME=int(time()),
UNIX_TIME=int(time.time()),
PYTHONEXE=get_pythonexe_path(),
)
@ -183,7 +183,7 @@ env.SConscript(env.GetExtraScripts("post"), exports="env")
# Checking program size
if env.get("SIZETOOL") and not (
set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS)
set(["nobuild", "__memusage"]) & set(COMMAND_LINE_TARGETS)
):
env.Depends("upload", "checkprogsize")
# Replace platform's "size" target with our
@ -235,16 +235,16 @@ if env.IsIntegrationDump():
)
env.Exit(0)
if "sizedata" in COMMAND_LINE_TARGETS:
if "__memusage" in COMMAND_LINE_TARGETS:
AlwaysBuild(
env.Alias(
"sizedata",
"__memusage",
DEFAULT_TARGETS,
env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."),
env.VerboseAction(env.DumpMemoryUsage, "Generating memory usage report..."),
)
)
Default("sizedata")
Default("__memusage")
# issue #4604: process targets sequentially
for index, target in enumerate(

68
platformio/builder/tools/piosize.py → platformio/builder/tools/piomemusage.py
View File

@ -14,33 +14,33 @@
# pylint: disable=too-many-locals
import json
import os
import sys
from os import environ, makedirs, remove
from os.path import isdir, join, splitdrive
import time
from elftools.elf.descriptions import describe_sh_flags
from elftools.elf.elffile import ELFFile
from platformio.compat import IS_WINDOWS
from platformio.proc import exec_command
from platformio.project.memusage import save_report
def _run_tool(cmd, env, tool_args):
sysenv = environ.copy()
sysenv = os.environ.copy()
sysenv["PATH"] = str(env["ENV"]["PATH"])
build_dir = env.subst("$BUILD_DIR")
if not isdir(build_dir):
makedirs(build_dir)
tmp_file = join(build_dir, "size-data-longcmd.txt")
if not os.path.isdir(build_dir):
os.makedirs(build_dir)
tmp_file = os.path.join(build_dir, "size-data-longcmd.txt")
with open(tmp_file, mode="w", encoding="utf8") as fp:
fp.write("\n".join(tool_args))
cmd.append("@" + tmp_file)
result = exec_command(cmd, env=sysenv)
remove(tmp_file)
os.remove(tmp_file)
return result
@ -92,8 +92,8 @@ def _collect_sections_info(env, elffile):
}
sections[section.name] = section_data
sections[section.name]["in_flash"] = env.pioSizeIsFlashSection(section_data)
sections[section.name]["in_ram"] = env.pioSizeIsRamSection(section_data)
sections[section.name]["in_flash"] = env.memusageIsFlashSection(section_data)
sections[section.name]["in_ram"] = env.memusageIsRamSection(section_data)
return sections
@ -106,7 +106,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?")
env.Exit(1)
sysenv = environ.copy()
sysenv = os.environ.copy()
sysenv["PATH"] = str(env["ENV"]["PATH"])
symbol_addrs = []
@ -117,7 +117,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
symbol_size = s["st_size"]
symbol_type = symbol_info["type"]
if not env.pioSizeIsValidSymbol(s.name, symbol_type, symbol_addr):
if not env.memusageIsValidSymbol(s.name, symbol_type, symbol_addr):
continue
symbol = {
@ -126,7 +126,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
"name": s.name,
"type": symbol_type,
"size": symbol_size,
"section": env.pioSizeDetermineSection(sections, symbol_addr),
"section": env.memusageDetermineSection(sections, symbol_addr),
}
if s.name.startswith("_Z"):
@ -144,8 +144,8 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
if not location or "?" in location:
continue
if IS_WINDOWS:
drive, tail = splitdrive(location)
location = join(drive.upper(), tail)
drive, tail = os.path.splitdrive(location)
location = os.path.join(drive.upper(), tail)
symbol["file"] = location
symbol["line"] = 0
if ":" in location:
@ -156,7 +156,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
return symbols
def pioSizeDetermineSection(_, sections, symbol_addr):
def memusageDetermineSection(_, sections, symbol_addr):
for section, info in sections.items():
if not info.get("in_flash", False) and not info.get("in_ram", False):
continue
@ -165,22 +165,22 @@ def pioSizeDetermineSection(_, sections, symbol_addr):
return "unknown"
def pioSizeIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
def memusageIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
def pioSizeIsRamSection(_, section):
def memusageIsRamSection(_, section):
return (
section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
and section.get("flags", "") == "WA"
)
def pioSizeIsFlashSection(_, section):
def memusageIsFlashSection(_, section):
return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
def pioSizeCalculateFirmwareSize(_, sections):
def memusageCalculateFirmwareSize(_, sections):
flash_size = ram_size = 0
for section_info in sections.values():
if section_info.get("in_flash", False):
@ -191,8 +191,8 @@ def pioSizeCalculateFirmwareSize(_, sections):
return ram_size, flash_size
def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument
data = {"device": {}, "memory": {}, "version": 1}
def DumpMemoryUsage(_, target, source, env): # pylint: disable=unused-argument
data = {"version": 1, "timestamp": int(time.time()), "device": {}, "memory": {}}
board = env.BoardConfig()
if board:
@ -216,7 +216,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument
env.Exit(1)
sections = _collect_sections_info(env, elffile)
firmware_ram, firmware_flash = env.pioSizeCalculateFirmwareSize(sections)
firmware_ram, firmware_flash = env.memusageCalculateFirmwareSize(sections)
data["memory"]["total"] = {
"ram_size": firmware_ram,
"flash_size": firmware_flash,
@ -225,7 +225,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument
files = {}
for symbol in _collect_symbols_info(env, elffile, elf_path, sections):
file_path = symbol.get("file") or "unknown"
file_path = symbol.pop("file", "unknown")
if not files.get(file_path, {}):
files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0}
@ -246,10 +246,10 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument
file_data.update(v)
data["memory"]["files"].append(file_data)
with open(
join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8"
) as fp:
fp.write(json.dumps(data))
print(
"Memory usage report has been saved to the following location: "
f"\"{save_report(os.getcwd(), env['PIOENV'], data)}\""
)
def exists(_):
@ -257,10 +257,10 @@ def exists(_):
def generate(env):
env.AddMethod(pioSizeIsRamSection)
env.AddMethod(pioSizeIsFlashSection)
env.AddMethod(pioSizeCalculateFirmwareSize)
env.AddMethod(pioSizeDetermineSection)
env.AddMethod(pioSizeIsValidSymbol)
env.AddMethod(DumpSizeData)
env.AddMethod(memusageIsRamSection)
env.AddMethod(memusageIsFlashSection)
env.AddMethod(memusageCalculateFirmwareSize)
env.AddMethod(memusageDetermineSection)
env.AddMethod(memusageIsValidSymbol)
env.AddMethod(DumpMemoryUsage)
return env

100
platformio/home/rpc/handlers/memusage.py Normal file
View File

@ -0,0 +1,100 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
from platformio.home.rpc.handlers.base import BaseRPCHandler
from platformio.project import memusage
class MemUsageRPC(BaseRPCHandler):
NAMESPACE = "memusage"
async def summary(self, project_dir, env, options=None):
options = options or {}
existing_reports = memusage.list_reports(project_dir, env)
current_report = previous_report = None
if options.get("cached") and existing_reports:
current_report = memusage.read_report(existing_reports[-1])
if len(existing_reports) > 1:
previous_report = memusage.read_report(existing_reports[-2])
else:
if existing_reports:
previous_report = memusage.read_report(existing_reports[-1])
await self.factory.manager.dispatcher["core.exec"](
["run", "-d", project_dir, "-e", env, "-t", "__memusage"],
options=options.get("exec"),
raise_exception=True,
)
current_report = memusage.read_report(
memusage.list_reports(project_dir, env)[-1]
)
max_top_items = 10
return dict(
timestamp=dict(
current=current_report["timestamp"],
previous=previous_report["timestamp"] if previous_report else None,
),
device=current_report["device"],
trend=dict(
current=current_report["memory"]["total"],
previous=previous_report["memory"]["total"]
if previous_report
else None,
),
top=dict(
files=self._calculate_top_files(current_report["memory"]["files"])[
0:max_top_items
],
symbols=self._calculate_top_symbols(current_report["memory"]["files"])[
0:max_top_items
],
sections=sorted(
current_report["memory"]["total"]["sections"].values(),
key=lambda item: item["size"],
reverse=True,
)[0:max_top_items],
),
)
@staticmethod
def _calculate_top_files(items):
return [
{"path": item["path"], "ram": item["ram_size"], "flash": item["flash_size"]}
for item in sorted(
items,
key=lambda item: item["ram_size"] + item["flash_size"],
reverse=True,
)
]
@staticmethod
def _calculate_top_symbols(files):
symbols = functools.reduce(
lambda result, filex: result
+ [
{
"name": s["name"],
"type": s["type"],
"size": s["size"],
"file": filex["path"],
"line": s.get("line"),
}
for s in filex["symbols"]
],
files,
[],
)
return sorted(symbols, key=lambda item: item["size"], reverse=True)

2
platformio/home/run.py
View File

@ -30,6 +30,7 @@ from platformio.home.rpc.handlers.account import AccountRPC
from platformio.home.rpc.handlers.app import AppRPC
from platformio.home.rpc.handlers.core import CoreRPC
from platformio.home.rpc.handlers.ide import IDERPC
from platformio.home.rpc.handlers.memusage import MemUsageRPC
from platformio.home.rpc.handlers.misc import MiscRPC
from platformio.home.rpc.handlers.os import OSRPC
from platformio.home.rpc.handlers.platform import PlatformRPC
@ -70,6 +71,7 @@ def run_server(host, port, no_open, shutdown_timeout, home_url):
ws_rpc_factory.add_object_handler(AccountRPC())
ws_rpc_factory.add_object_handler(AppRPC())
ws_rpc_factory.add_object_handler(IDERPC())
ws_rpc_factory.add_object_handler(MemUsageRPC())
ws_rpc_factory.add_object_handler(MiscRPC())
ws_rpc_factory.add_object_handler(OSRPC())
ws_rpc_factory.add_object_handler(CoreRPC())

3
platformio/project/helpers.py
View File

@ -143,8 +143,7 @@ def get_build_type(config, env, run_targets=None):
run_targets = run_targets or []
declared_build_type = config.get(f"env:{env}", "build_type")
if (
set(["__debug", "sizedata"]) # sizedata = for memory inspection
& set(run_targets)
set(["__debug", "__memusage"]) & set(run_targets)
or declared_build_type == "debug"
):
types.append("debug")

59
platformio/project/memusage.py Normal file
View File

@ -0,0 +1,59 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import gzip
import json
import os
import time
from platformio import fs
from platformio.project.config import ProjectConfig
def get_report_dir(project_dir, env):
with fs.cd(project_dir):
return os.path.join(
ProjectConfig.get_instance().get("platformio", "memusage_dir"), env
)
def list_reports(project_dir, env):
report_dir = get_report_dir(project_dir, env)
if not os.path.isdir(report_dir):
return []
return [os.path.join(report_dir, item) for item in sorted(os.listdir(report_dir))]
def read_report(path):
with gzip.open(path, mode="rt", encoding="utf8") as fp:
return json.load(fp)
def save_report(project_dir, env, data):
report_dir = get_report_dir(project_dir, env)
if not os.path.isdir(report_dir):
os.makedirs(report_dir)
report_path = os.path.join(report_dir, f"{int(time.time())}.json.gz")
with gzip.open(report_path, mode="wt", encoding="utf8") as fp:
json.dump(data, fp)
rotate_reports(report_dir)
return report_path
def rotate_reports(report_dir, max_reports=100):
reports = os.listdir(report_dir)
if len(reports) < max_reports:
return
for fname in sorted(reports)[0 : len(reports) - max_reports]:
os.remove(os.path.join(report_dir, fname))

13
platformio/project/options.py
View File

@ -83,7 +83,7 @@ def ConfigEnvOption(*args, **kwargs):
def calculate_path_hash(path):
return "%s-%s" % (
os.path.basename(path),
hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10],
hashlib.sha1(hashlib_encode_data(path.lower())).hexdigest()[:10],
)
@ -266,6 +266,17 @@ ProjectOptions = OrderedDict(
default=os.path.join("${platformio.workspace_dir}", "libdeps"),
validate=validate_dir,
),
ConfigPlatformioOption(
group="directory",
name="memusage_dir",
description=(
"A location where PlatformIO Core will store "
"project memory usage reports"
),
sysenvvar="PLATFORMIO_MEMUSAGE_DIR",
default=os.path.join("${platformio.workspace_dir}", "memusage"),
validate=validate_dir,
),
ConfigPlatformioOption(
group="directory",
name="include_dir",