diff --git a/tools/ldgen/entity.py b/tools/ldgen/entity.py new file mode 100644 index 0000000000..38e526b460 --- /dev/null +++ b/tools/ldgen/entity.py @@ -0,0 +1,222 @@ +# +# Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import collections +import fnmatch +import os +from enum import Enum +from functools import total_ordering + +from pyparsing import (Group, Literal, OneOrMore, ParseException, SkipTo, Suppress, White, Word, ZeroOrMore, alphas, + nums, restOfLine) + + +@total_ordering +class Entity(): + """ + Definition of an entity which can be placed or excluded + from placement. + """ + + ALL = '*' + + class Specificity(Enum): + NONE = 0 + ARCHIVE = 1 + OBJ = 2 + SYMBOL = 3 + + def __init__(self, archive=None, obj=None, symbol=None): + archive_spec = archive and archive != Entity.ALL + obj_spec = obj and obj != Entity.ALL + symbol_spec = symbol and symbol != Entity.ALL + + if not archive_spec and not obj_spec and not symbol_spec: + self.specificity = Entity.Specificity.NONE + elif archive_spec and not obj_spec and not symbol_spec: + self.specificity = Entity.Specificity.ARCHIVE + elif archive_spec and obj_spec and not symbol_spec: + self.specificity = Entity.Specificity.OBJ + elif archive_spec and obj_spec and symbol_spec: + self.specificity = Entity.Specificity.SYMBOL + else: + raise ValueError("Invalid arguments '(%s, %s, %s)'" % (archive, obj, symbol)) + + self.archive = archive + self.obj = obj + self.symbol = symbol + + def __eq__(self, other): + return (self.specificity.value == other.specificity.value and + self.archive == other.archive and + self.obj == other.obj and + self.symbol == other.symbol) + + def __lt__(self, other): + res = False + if self.specificity.value < other.specificity.value: + res = True + elif self.specificity == other.specificity: + for s in Entity.Specificity: + a = self[s] if self[s] else '' + b = other[s] if other[s] else '' + + if a != b: + res = a < b + break + else: + res = False + return res + + def __hash__(self): + return hash(self.__repr__()) + + def __str__(self): + return '%s:%s %s' % self.__repr__() + + def __repr__(self): + return (self.archive, self.obj, self.symbol) + + def __getitem__(self, spec): + res = None + if spec == Entity.Specificity.ARCHIVE: + res = self.archive + elif spec == Entity.Specificity.OBJ: + res = self.obj + elif spec == Entity.Specificity.SYMBOL: + res = self.symbol + else: + res = None + return res + + +class EntityDB(): + """ + Encapsulates an output of objdump. Contains information about the static library sections + and names + """ + + __info = collections.namedtuple('__info', 'filename content') + + def __init__(self): + self.sections = dict() + + def add_sections_info(self, sections_info_dump): + first_line = sections_info_dump.readline() + + archive_path = (Literal('In archive').suppress() + + White().suppress() + + # trim the colon and line ending characters from archive_path + restOfLine.setResultsName('archive_path').setParseAction(lambda s, loc, toks: s.rstrip(':\n\r '))) + parser = archive_path + + results = None + + try: + results = parser.parseString(first_line, parseAll=True) + except ParseException as p: + raise ParseException('Parsing sections info for library ' + sections_info_dump.name + ' failed. ' + p.msg) + + archive = os.path.basename(results.archive_path) + self.sections[archive] = EntityDB.__info(sections_info_dump.name, sections_info_dump.read()) + + def _get_infos_from_file(self, info): + # {object}: file format elf32-xtensa-le + object_line = SkipTo(':').setResultsName('object') + Suppress(restOfLine) + + # Sections: + # Idx Name ... + section_start = Suppress(Literal('Sections:')) + section_header = Suppress(OneOrMore(Word(alphas))) + + # 00 {section} 0000000 ... + # CONTENTS, ALLOC, .... + section_entry = Suppress(Word(nums)) + SkipTo(' ') + Suppress(restOfLine) + \ + Suppress(ZeroOrMore(Word(alphas) + Literal(',')) + Word(alphas)) + + content = Group(object_line + section_start + section_header + Group(OneOrMore(section_entry)).setResultsName('sections')) + parser = Group(ZeroOrMore(content)).setResultsName('contents') + + results = None + + try: + results = parser.parseString(info.content, parseAll=True) + except ParseException as p: + raise ParseException('Unable to parse section info file ' + info.filename + '. ' + p.msg) + + return results + + def _process_archive(self, archive): + stored = self.sections[archive] + + # Parse the contents of the sections file on-demand, + # save the result for later + if not isinstance(stored, dict): + parsed = self._get_infos_from_file(stored) + stored = dict() + for content in parsed.contents: + sections = list(map(lambda s: s, content.sections)) + stored[content.object] = sections + self.sections[archive] = stored + + def get_archives(self): + return self.sections.keys() + + def get_objects(self, archive): + try: + self._process_archive(archive) + except KeyError: + return [] + + return self.sections[archive].keys() + + def _match_obj(self, archive, obj): + objs = self.get_objects(archive) + match_objs = fnmatch.filter(objs, obj + '.o') + fnmatch.filter(objs, obj + '.*.obj') + fnmatch.filter(objs, obj + '.obj') + + if len(match_objs) > 1: + raise ValueError("Multiple matches for object: '%s: %s': %s" % (archive, obj, str(match_objs))) + + try: + return match_objs[0] + except IndexError: + return None + + def get_sections(self, archive, obj): + obj = self._match_obj(archive, obj) + res = [] + if obj: + res = self.sections[archive][obj] + return res + + def _match_symbol(self, archive, obj, symbol): + sections = self.get_sections(archive, obj) + return [s for s in sections if s.endswith(symbol)] + + def check_exists(self, entity): + res = True + + if entity.specificity != Entity.Specificity.NONE: + if entity.specificity == Entity.Specificity.ARCHIVE: + res = entity.archive in self.get_archives() + elif entity.specificity == Entity.Specificity.OBJ: + res = self._match_obj(entity.archive, entity.obj) is not None + elif entity.specificity == Entity.Specificity.SYMBOL: + res = len(self._match_symbol(entity.archive, entity.obj, entity.symbol)) > 0 + else: + res = False + + return res diff --git a/tools/ldgen/generation.py b/tools/ldgen/generation.py index a2d0267fb6..d27f40e442 100644 --- a/tools/ldgen/generation.py +++ b/tools/ldgen/generation.py @@ -17,242 +17,256 @@ import collections import fnmatch import itertools -import os +from collections import namedtuple -from fragments import Fragment, Mapping, Scheme, Sections +from entity import Entity +from fragments import Mapping, Scheme, Sections from ldgen_common import LdGenFailure -from pyparsing import (Group, Literal, OneOrMore, ParseException, SkipTo, Suppress, White, Word, ZeroOrMore, alphas, - nums, restOfLine) +from output_commands import InputSectionDesc -class PlacementRule(): - """ - Encapsulates a generated placement rule placed under a target - """ +class RuleNode(): - DEFAULT_SPECIFICITY = 0 - ARCHIVE_SPECIFICITY = 1 - OBJECT_SPECIFICITY = 2 - SYMBOL_SPECIFICITY = 3 + class Section(): - class __container(): - def __init__(self, content): - self.content = content + def __init__(self, target, exclusions, explicit=False): + self.target = target + self.exclusions = set(exclusions) - __metadata = collections.namedtuple('__metadata', 'excludes expansions expanded') + # Indicate whether this node has been created explicitly from a mapping, + # or simply just to create a path to the explicitly created node. + # + # For example, + # + # lib.a + # obj:sym (scheme) + # + # Nodes for lib.a and obj will be created, but only the node for + # sym will have been created explicitly. + # + # This is used in deciding whether or not an output command should + # be emitted for this node, or for exclusion rule generation. + self.explicit = explicit - def __init__(self, archive, obj, symbol, sections, target): - if archive == '*': - archive = None + def __init__(self, parent, name, sections): + self.children = [] + self.parent = parent + self.name = name + self.child_node = None + self.entity = None - if obj == '*': - obj = None - - self.archive = archive - self.obj = obj - self.symbol = symbol - self.target = target self.sections = dict() - self.specificity = 0 - self.specificity += 1 if self.archive else 0 - self.specificity += 1 if (self.obj and not self.obj == '*') else 0 - self.specificity += 1 if self.symbol else 0 + # A node inherits the section -> target entries from + # its parent. This is to simplify logic, avoiding + # going up the parental chain to try a 'basis' rule + # in creating exclusions. This relies on the fact that + # the mappings must be inserted from least to most specific. + # This sort is done in generate_rules(). + if sections: + for (s, v) in sections.items(): + self.sections[s] = RuleNode.Section(v.target, [], []) - for section in sections: - section_data = Sections.get_section_data_from_entry(section, self.symbol) + def add_exclusion(self, sections, exclusion): + self.sections[sections].exclusions.add(exclusion) - if not self.symbol: - for s in section_data: - metadata = self.__metadata(self.__container([]), self.__container([]), self.__container(False)) - self.sections[s] = metadata + # Recursively create exclusions in parents + if self.parent: + self.exclude_from_parent(sections) + + def add_sections(self, sections, target): + try: + _sections = self.sections[sections] + if not _sections.explicit: + _sections.target = target + _sections.explicit = True else: - (section, expansion) = section_data - if expansion: - metadata = self.__metadata(self.__container([]), self.__container([expansion]), self.__container(True)) - self.sections[section] = metadata + if target != _sections.target: + raise GenerationException('Sections mapped to multiple targets') + except KeyError: + self.sections[sections] = RuleNode.Section(target, [], True) - def get_section_names(self): - return self.sections.keys() + def exclude_from_parent(self, sections): + self.parent.add_exclusion(sections, self.entity) - def add_exclusion(self, other, sections_infos=None): - # Utility functions for this method - def do_section_expansion(rule, section): - if section in rule.get_section_names(): - sections_in_obj = sections_infos.get_obj_sections(rule.archive, rule.obj) - expansions = fnmatch.filter(sections_in_obj, section) - return expansions + def add_child(self, entity): + child_specificity = self.entity.specificity.value + 1 + assert(child_specificity <= Entity.Specificity.SYMBOL.value) + name = entity[Entity.Specificity(child_specificity)] + assert(name and name != Entity.ALL) - def remove_section_expansions(rule, section, expansions): - existing_expansions = self.sections[section].expansions.content - self.sections[section].expansions.content = [e for e in existing_expansions if e not in expansions] + child = [c for c in self.children if c.name == name] + assert(len(child) <= 1) - # Exit immediately if the exclusion to be added is more general than this rule. - if not other.is_more_specific_rule_of(self): - return - - for section in self.get_sections_intersection(other): - if(other.specificity == PlacementRule.SYMBOL_SPECIFICITY): - # If this sections has not been expanded previously, expand now and keep track. - previously_expanded = self.sections[section].expanded.content - if not previously_expanded: - expansions = do_section_expansion(self, section) - if expansions: - self.sections[section].expansions.content = expansions - self.sections[section].expanded.content = True - previously_expanded = True - - # Remove the sections corresponding to the symbol name - remove_section_expansions(self, section, other.sections[section].expansions.content) - - # If it has been expanded previously but now the expansions list is empty, - # it means adding exclusions has exhausted the list. Remove the section entirely. - if previously_expanded and not self.sections[section].expanded.content: - del self.sections[section] - else: - # A rule section can have multiple rule sections excluded from it. Get the - # most specific rule from the list, and if an even more specific rule is found, - # replace it entirely. Otherwise, keep appending. - exclusions = self.sections[section].excludes - exclusions_list = exclusions.content if exclusions.content is not None else [] - exclusions_to_remove = filter(lambda r: r.is_more_specific_rule_of(other), exclusions_list) - - remaining_exclusions = [e for e in exclusions_list if e not in exclusions_to_remove] - remaining_exclusions.append(other) - - self.sections[section].excludes.content = remaining_exclusions - - def get_sections_intersection(self, other): - return set(self.sections.keys()).intersection(set(other.sections.keys())) - - def is_more_specific_rule_of(self, other): - if (self.specificity <= other.specificity): - return False - - # Compare archive, obj and target - for entity_index in range(1, other.specificity + 1): - if self[entity_index] != other[entity_index] and other[entity_index] is not None: - return False - - return True - - def maps_same_entities_as(self, other): - if self.specificity != other.specificity: - return False - - # Compare archive, obj and target - for entity_index in range(1, other.specificity + 1): - if self[entity_index] != other[entity_index] and other[entity_index] is not None: - return False - - return True - - def __getitem__(self, key): - if key == PlacementRule.ARCHIVE_SPECIFICITY: - return self.archive - elif key == PlacementRule.OBJECT_SPECIFICITY: - return self.obj - elif key == PlacementRule.SYMBOL_SPECIFICITY: - return self.symbol + if not child: + child = self.child_node(self, name, self.sections) + self.children.append(child) else: - return None + child = child[0] - def __str__(self): - sorted_sections = sorted(self.get_section_names()) + return child - sections_string = list() + def get_output_commands(self): + commands = collections.defaultdict(list) - for section in sorted_sections: - exclusions = self.sections[section].excludes.content + def process_commands(cmds): + for (target, commands_list) in cmds.items(): + commands[target].extend(commands_list) - exclusion_string = None + # Process the commands generated from this node + node_commands = self.get_node_output_commands() + process_commands(node_commands) - if exclusions: - exclusion_string = ' '.join(map(lambda e: '*' + e.archive + (':' + e.obj + '.*' if e.obj else ''), exclusions)) - exclusion_string = 'EXCLUDE_FILE(' + exclusion_string + ')' - else: - exclusion_string = '' + # Process the commands generated from this node's children + # recursively + for child in sorted(self.children, key=lambda c: c.name): + children_commands = child.get_output_commands() + process_commands(children_commands) - section_string = None - exclusion_section_string = None + return commands - section_expansions = self.sections[section].expansions.content - section_expanded = self.sections[section].expanded.content + def add_node_child(self, entity, sections, target, sections_db): + child = self.add_child(entity) + child.insert(entity, sections, target, sections_db) - if section_expansions and section_expanded: - section_string = ' '.join(section_expansions) - exclusion_section_string = section_string - else: - section_string = section - exclusion_section_string = exclusion_string + ' ' + section_string + def get_node_output_commands(self): + commands = collections.defaultdict(list) - sections_string.append(exclusion_section_string) + for sections in self.get_section_keys(): + info = self.sections[sections] + if info.exclusions or info.explicit: + command = InputSectionDesc(self.entity, sections, info.exclusions) + commands[info.target].append(command) - sections_string = ' '.join(sections_string) + return commands - archive = str(self.archive) if self.archive else '' - obj = (str(self.obj) + ('.*' if self.obj else '')) if self.obj else '' - - # Handle output string generation based on information available - if self.specificity == PlacementRule.DEFAULT_SPECIFICITY: - rule_string = '*(%s)' % (sections_string) - elif self.specificity == PlacementRule.ARCHIVE_SPECIFICITY: - rule_string = '*%s:(%s)' % (archive, sections_string) + def insert(self, entity, sections, target, sections_db): + if self.entity.specificity == entity.specificity: + if self.parent.sections[sections].target != target: + self.add_sections(sections, target) + self.exclude_from_parent(sections) else: - rule_string = '*%s:%s(%s)' % (archive, obj, sections_string) + self.add_node_child(entity, sections, target, sections_db) - return rule_string - - def __eq__(self, other): - if id(self) == id(other): - return True - - def exclusions_set(exclusions): - exclusions_set = {(e.archive, e.obj, e.symbol, e.target) for e in exclusions} - return exclusions_set - - if self.archive != other.archive: - return False - - if self.obj != other.obj: - return False - - if self.symbol != other.symbol: - return False - - if set(self.sections.keys()) != set(other.sections.keys()): - return False - - for (section, metadata) in self.sections.items(): - - self_meta = metadata - other_meta = other.sections[section] - - if exclusions_set(self_meta.excludes.content) != exclusions_set(other_meta.excludes.content): - return False - - if set(self_meta.expansions.content) != set(other_meta.expansions.content): - return False - - return True - - def __ne__(self, other): - return not self.__eq__(other) - - def __iter__(self): - yield self.archive - yield self.obj - yield self.symbol - raise StopIteration + def get_section_keys(self): + return sorted(self.sections.keys(), key=' '.join) -class GenerationModel: +class SymbolNode(RuleNode): + + def __init__(self, parent, name, sections): + RuleNode.__init__(self, parent, name, sections) + self.entity = Entity(self.parent.parent.name, self.parent.name, self.name) + + def insert(self, entity, sections, target, sections_db): + self.add_sections(sections, target) + + def get_node_output_commands(self): + commands = collections.defaultdict(list) + + for sections in self.get_section_keys(): + info = self.sections[sections] + if info.explicit: + command = InputSectionDesc(Entity(self.parent.parent.name, self.parent.name), sections, []) + commands[info.target].append(command) + + return commands + + +class ObjectNode(RuleNode): + + def __init__(self, parent, name, sections): + RuleNode.__init__(self, parent, name, sections) + self.child_node = SymbolNode + self.expanded_sections = dict() + self.entity = Entity(self.parent.name, self.name) + + def add_node_child(self, entity, sections, target, sections_db): + if self.sections[sections].target != target: + symbol = entity.symbol + match_sections = None + + obj_sections = sections_db.get_sections(self.parent.name, self.name) + + try: + match_sections = self.expanded_sections[sections] + except KeyError: + match_sections = [] + for s in sections: + match_sections.extend(fnmatch.filter(obj_sections, s)) + + if match_sections: + remove_sections = [s.replace('.*', '.%s' % symbol) for s in sections if '.*' in s] + filtered_sections = [s for s in match_sections if s not in remove_sections] + + if set(filtered_sections) != set(match_sections): # some sections removed + child = self.add_child(entity) + child.insert(entity, frozenset(remove_sections), target, obj_sections) + + # Remember the result for node command generation + self.expanded_sections[sections] = filtered_sections + self.exclude_from_parent(sections) + + def get_node_output_commands(self): + commands = collections.defaultdict(list) + + for sections in self.get_section_keys(): + info = self.sections[sections] + + try: + match_sections = self.expanded_sections[sections] + except KeyError: + match_sections = [] + + if match_sections or info.explicit: + command_sections = match_sections if match_sections else sections + command = InputSectionDesc(self.entity, command_sections, []) + commands[info.target].append(command) + + return commands + + def exclude_from_parent(self, sections): + # Check if there is an explicit emmission for the parent node, which is an archive node. + # If there is, make the exclusion there. If not, make the exclusion on the root node. + # This is to avoid emitting unecessary command and exclusions for the archive node and + # from the root node, respectively. + if self.parent.sections[sections].explicit: + self.parent.add_exclusion(sections, self.entity) + else: + self.parent.parent.add_exclusion(sections, self.entity) + + +class ArchiveNode(RuleNode): + + def __init__(self, parent, name, sections): + RuleNode.__init__(self, parent, name, sections) + self.child_node = ObjectNode + self.entity = Entity(self.name) + + +class RootNode(RuleNode): + def __init__(self): + RuleNode.__init__(self, None, Entity.ALL, None) + self.child_node = ArchiveNode + self.entity = Entity('*') + + def insert(self, entity, sections, target, sections_db): + if self.entity.specificity == entity.specificity: + self.add_sections(sections, target) + else: + self.add_node_child(entity, sections, target, sections_db) + + +class Generation: """ Implements generation of placement rules based on collected sections, scheme and mapping fragment. """ DEFAULT_SCHEME = 'default' + # Processed mapping, scheme and section entries + EntityMapping = namedtuple('EntityMapping', 'entity sections_group target') + def __init__(self, check_mappings=False, check_mapping_exceptions=None): self.schemes = {} self.sections = {} @@ -265,23 +279,6 @@ class GenerationModel: else: self.check_mapping_exceptions = [] - def _add_mapping_rules(self, archive, obj, symbol, scheme_name, scheme_dict, rules): - # Use an ordinary dictionary to raise exception on non-existing keys - temp_dict = dict(scheme_dict) - - sections_bucket = temp_dict[scheme_name] - - for (target, sections) in sections_bucket.items(): - section_entries = [] - - for section in sections: - section_entries.extend(section.entries) - - rule = PlacementRule(archive, obj, symbol, section_entries, target) - - if rule not in rules: - rules.append(rule) - def _build_scheme_dictionary(self): scheme_dictionary = collections.defaultdict(dict) @@ -297,7 +294,7 @@ class GenerationModel: try: sections = self.sections[sections_name] except KeyError: - message = GenerationException.UNDEFINED_REFERENCE + " to sections '" + sections + "'." + message = GenerationException.UNDEFINED_REFERENCE + " to sections '" + sections_name + "'." raise GenerationException(message, scheme) sections_in_bucket.append(sections) @@ -327,147 +324,69 @@ class GenerationModel: return scheme_dictionary - def generate_rules(self, sections_infos): - scheme_dictionary = self._build_scheme_dictionary() + def get_section_strs(self, section): + s_list = [Sections.get_section_data_from_entry(s) for s in section.entries] + return frozenset([item for sublist in s_list for item in sublist]) - # Generate default rules - default_rules = list() - self._add_mapping_rules(None, None, None, GenerationModel.DEFAULT_SCHEME, scheme_dictionary, default_rules) + def _generate_entity_mappings(self, scheme_dictionary, entities): + entity_mappings = [] - all_mapping_rules = collections.defaultdict(list) - - # Generate rules based on mapping fragments for mapping in self.mappings.values(): archive = mapping.archive - mapping_rules = all_mapping_rules[archive] + for (obj, symbol, scheme_name) in mapping.entries: - try: - if not (obj == Mapping.MAPPING_ALL_OBJECTS and symbol is None and - scheme_name == GenerationModel.DEFAULT_SCHEME): - if self.check_mappings and mapping.name not in self.check_mapping_exceptions: - if not obj == Mapping.MAPPING_ALL_OBJECTS: - obj_sections = sections_infos.get_obj_sections(archive, obj) - if not obj_sections: - message = "'%s:%s' not found" % (archive, obj) - raise GenerationException(message, mapping) + entity = Entity(archive, obj, symbol) - if symbol: - obj_sym = fnmatch.filter(obj_sections, '*%s' % symbol) - if not obj_sym: - message = "'%s:%s %s' not found" % (archive, obj, symbol) - raise GenerationException(message, mapping) + # Check the entity exists + if (self.check_mappings and + entity.specificity.value > Entity.Specificity.ARCHIVE.value and + mapping.name not in self.check_mapping_exceptions): + if not entities.check_exists(entity): + message = "'%s' not found" % str(entity) + raise GenerationException(message, mapping) - self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules) - except KeyError: - message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'." - raise GenerationException(message, mapping) + # Create placement rule for each 'section -> target' in the scheme. + # + # For example. for the mapping entry: + # + # obj (scheme) + # + # The enumrated to: + # + # obj (section1 -> target1) + # obj (section2 -> target2) + # ... + for (target, sections) in scheme_dictionary[scheme_name].items(): + for section in sections: + entity_mappings.append(Generation.EntityMapping(entity, self.get_section_strs(section), target)) - # Detect rule conflicts - for mapping_rules in all_mapping_rules.items(): - self._detect_conflicts(mapping_rules) + return entity_mappings - # Add exclusions - for mapping_rules in all_mapping_rules.values(): - self._create_exclusions(mapping_rules, default_rules, sections_infos) + def generate_rules(self, entities): + scheme_dictionary = self._build_scheme_dictionary() - placement_rules = collections.defaultdict(list) + entity_mappings = self._generate_entity_mappings(scheme_dictionary, entities) - # Add the default rules grouped by target - for default_rule in default_rules: - existing_rules = placement_rules[default_rule.target] - if default_rule.get_section_names(): - existing_rules.append(default_rule) + entity_mappings.sort(key=lambda m: m.entity) - archives = sorted(all_mapping_rules.keys()) + # Create root nodes dictionary for the default scheme, whose + # key is the target name and value is a list of the root nodes for that target. + root_node = RootNode() + for (target, sections) in scheme_dictionary['default'].items(): + for section in sections: + root_node.insert(Entity(), self.get_section_strs(section), target, entities) - for archive in archives: - # Add the mapping rules grouped by target - mapping_rules = sorted(all_mapping_rules[archive], key=lambda m: (m.specificity, str(m))) - for mapping_rule in mapping_rules: - existing_rules = placement_rules[mapping_rule.target] - if mapping_rule.get_section_names(): - existing_rules.append(mapping_rule) + for mapping in entity_mappings: + (entity, sections, target) = mapping + try: + root_node.insert(entity, sections, target, entities) + except ValueError as e: + raise GenerationException(str(e)) - return placement_rules + # Traverse the tree, creating the rules + commands = root_node.get_output_commands() - def _detect_conflicts(self, rules): - (archive, rules_list) = rules - - for specificity in range(0, PlacementRule.OBJECT_SPECIFICITY + 1): - rules_with_specificity = filter(lambda r: r.specificity == specificity, rules_list) - - for rule_a, rule_b in itertools.combinations(rules_with_specificity, 2): - intersections = rule_a.get_sections_intersection(rule_b) - - if intersections and rule_a.maps_same_entities_as(rule_b): - rules_string = str([str(rule_a), str(rule_b)]) - message = 'Rules ' + rules_string + ' map sections ' + str(list(intersections)) + ' into multiple targets.' - raise GenerationException(message) - - def _create_extra_rules(self, rules): - # This function generates extra rules for symbol specific rules. The reason for generating extra rules is to isolate, - # as much as possible, rules that require expansion. Particularly, object specific extra rules are generated. - rules_to_process = sorted(rules, key=lambda r: r.specificity) - symbol_specific_rules = list(filter(lambda r: r.specificity == PlacementRule.SYMBOL_SPECIFICITY, rules_to_process)) - - extra_rules = dict() - - for symbol_specific_rule in symbol_specific_rules: - extra_rule_candidate = {s: None for s in symbol_specific_rule.get_section_names()} - - super_rules = filter(lambda r: symbol_specific_rule.is_more_specific_rule_of(r), rules_to_process) - - # Take a look at the existing rules that are more general than the current symbol-specific rule. - # Only generate an extra rule if there is no existing object specific rule for that section - for super_rule in super_rules: - intersections = symbol_specific_rule.get_sections_intersection(super_rule) - for intersection in intersections: - if super_rule.specificity != PlacementRule.OBJECT_SPECIFICITY: - extra_rule_candidate[intersection] = super_rule - else: - extra_rule_candidate[intersection] = None - - # Generate the extra rules for the symbol specific rule section, keeping track of the generated extra rules - for (section, section_rule) in extra_rule_candidate.items(): - if section_rule: - extra_rule = None - extra_rules_key = (symbol_specific_rule.archive, symbol_specific_rule.obj, section_rule.target) - - try: - extra_rule = extra_rules[extra_rules_key] - - if section not in extra_rule.get_section_names(): - new_rule = PlacementRule(extra_rule.archive, extra_rule.obj, extra_rule.symbol, - list(extra_rule.get_section_names()) + [section], extra_rule.target) - extra_rules[extra_rules_key] = new_rule - except KeyError: - extra_rule = PlacementRule(symbol_specific_rule.archive, symbol_specific_rule.obj, None, [section], section_rule.target) - extra_rules[extra_rules_key] = extra_rule - - return extra_rules.values() - - def _create_exclusions(self, mapping_rules, default_rules, sections_info): - rules = list(default_rules) - rules.extend(mapping_rules) - - extra_rules = self._create_extra_rules(rules) - - mapping_rules.extend(extra_rules) - rules.extend(extra_rules) - - # Sort the rules by means of how specific they are. Sort by specificity from lowest to highest - # * -> lib:* -> lib:obj -> lib:obj:symbol - sorted_rules = sorted(rules, key=lambda r: r.specificity) - - # Now that the rules have been sorted, loop through each rule, and then loop - # through rules below it (higher indeces), adding exclusions whenever appropriate. - for general_rule in sorted_rules: - for specific_rule in reversed(sorted_rules): - if (specific_rule.specificity > general_rule.specificity and - specific_rule.specificity != PlacementRule.SYMBOL_SPECIFICITY) or \ - (specific_rule.specificity == PlacementRule.SYMBOL_SPECIFICITY and - general_rule.specificity == PlacementRule.OBJECT_SPECIFICITY): - general_rule.add_exclusion(specific_rule, sections_info) + return commands def add_fragments_from_file(self, fragment_file): for fragment in fragment_file.fragments: @@ -493,79 +412,6 @@ class GenerationModel: dict_to_append_to[fragment.name] = fragment -class TemplateModel: - """ - Encapsulates a linker script template file. Finds marker syntax and handles replacement to generate the - final output. - """ - - Marker = collections.namedtuple('Marker', 'target indent rules') - - def __init__(self, template_file): - self.members = [] - self.file = os.path.realpath(template_file.name) - - self._generate_members(template_file) - - def _generate_members(self, template_file): - lines = template_file.readlines() - - target = Fragment.IDENTIFIER - reference = Suppress('mapping') + Suppress('[') + target.setResultsName('target') + Suppress(']') - pattern = White(' \t').setResultsName('indent') + reference - - # Find the markers in the template file line by line. If line does not match marker grammar, - # set it as a literal to be copied as is to the output file. - for line in lines: - try: - parsed = pattern.parseString(line) - - indent = parsed.indent - target = parsed.target - - marker = TemplateModel.Marker(target, indent, []) - - self.members.append(marker) - except ParseException: - # Does not match marker syntax - self.members.append(line) - - def fill(self, mapping_rules): - for member in self.members: - target = None - try: - target = member.target - rules = member.rules - - del rules[:] - - rules.extend(mapping_rules[target]) - except KeyError: - message = GenerationException.UNDEFINED_REFERENCE + " to target '" + target + "'." - raise GenerationException(message) - except AttributeError: - pass - - def write(self, output_file): - # Add information that this is a generated file. - output_file.write('/* Automatically generated file; DO NOT EDIT */\n') - output_file.write('/* Espressif IoT Development Framework Linker Script */\n') - output_file.write('/* Generated from: %s */\n' % self.file) - output_file.write('\n') - - # Do the text replacement - for member in self.members: - try: - indent = member.indent - rules = member.rules - - for rule in rules: - generated_line = ''.join([indent, str(rule), '\n']) - output_file.write(generated_line) - except AttributeError: - output_file.write(member) - - class GenerationException(LdGenFailure): """ Exception for linker script generation failures such as undefined references/ failure to @@ -583,90 +429,3 @@ class GenerationException(LdGenFailure): return "%s\nIn fragment '%s' defined in '%s'." % (self.message, self.fragment.name, self.fragment.path) else: return self.message - - -class SectionsInfo(dict): - """ - Encapsulates an output of objdump. Contains information about the static library sections - and names - """ - - __info = collections.namedtuple('__info', 'filename content') - - def __init__(self): - self.sections = dict() - - def add_sections_info(self, sections_info_dump): - first_line = sections_info_dump.readline() - - archive_path = (Literal('In archive').suppress() + - White().suppress() + - # trim the colon and line ending characters from archive_path - restOfLine.setResultsName('archive_path').setParseAction(lambda s, loc, toks: s.rstrip(':\n\r '))) - parser = archive_path - - results = None - - try: - results = parser.parseString(first_line, parseAll=True) - except ParseException as p: - raise ParseException('Parsing sections info for library ' + sections_info_dump.name + ' failed. ' + p.msg) - - archive = os.path.basename(results.archive_path) - self.sections[archive] = SectionsInfo.__info(sections_info_dump.name, sections_info_dump.read()) - - def _get_infos_from_file(self, info): - # {object}: file format elf32-xtensa-le - object_line = SkipTo(':').setResultsName('object') + Suppress(restOfLine) - - # Sections: - # Idx Name ... - section_start = Suppress(Literal('Sections:')) - section_header = Suppress(OneOrMore(Word(alphas))) - - # 00 {section} 0000000 ... - # CONTENTS, ALLOC, .... - section_entry = Suppress(Word(nums)) + SkipTo(' ') + Suppress(restOfLine) + \ - Suppress(ZeroOrMore(Word(alphas) + Literal(',')) + Word(alphas)) - - content = Group(object_line + section_start + section_header + Group(OneOrMore(section_entry)).setResultsName('sections')) - parser = Group(ZeroOrMore(content)).setResultsName('contents') - - results = None - - try: - results = parser.parseString(info.content, parseAll=True) - except ParseException as p: - raise ParseException('Unable to parse section info file ' + info.filename + '. ' + p.msg) - - return results - - def get_obj_sections(self, archive, obj): - res = [] - try: - stored = self.sections[archive] - - # Parse the contents of the sections file on-demand, - # save the result for later - if not isinstance(stored, dict): - parsed = self._get_infos_from_file(stored) - stored = dict() - for content in parsed.contents: - sections = list(map(lambda s: s, content.sections)) - stored[content.object] = sections - self.sections[archive] = stored - - try: - res = stored[obj + '.o'] - except KeyError: - try: - res = stored[obj + '.c.obj'] - except KeyError: - try: - res = stored[obj + '.cpp.obj'] - except KeyError: - res = stored[obj + '.S.obj'] - except KeyError: - pass - - return res diff --git a/tools/ldgen/ldgen.py b/tools/ldgen/ldgen.py index df1a74af92..85e99ea59a 100755 --- a/tools/ldgen/ldgen.py +++ b/tools/ldgen/ldgen.py @@ -24,9 +24,11 @@ import sys import tempfile from io import StringIO +from entity import EntityDB from fragments import FragmentFile -from generation import GenerationModel, SectionsInfo, TemplateModel +from generation import Generation from ldgen_common import LdGenFailure +from linker_script import LinkerScript from pyparsing import ParseException, ParseFatalException from sdkconfig import SDKConfig @@ -125,7 +127,7 @@ def main(): check_mapping_exceptions = None try: - sections_infos = SectionsInfo() + sections_infos = EntityDB() for library in libraries_file: library = library.strip() if library: @@ -133,7 +135,7 @@ def main(): dump.name = library sections_infos.add_sections_info(dump) - generation_model = GenerationModel(check_mapping, check_mapping_exceptions) + generation_model = Generation(check_mapping, check_mapping_exceptions) _update_environment(args) # assign args.env and args.env_file to os.environ @@ -151,7 +153,7 @@ def main(): mapping_rules = generation_model.generate_rules(sections_infos) - script_model = TemplateModel(input_file) + script_model = LinkerScript(input_file) script_model.fill(mapping_rules) with tempfile.TemporaryFile('w+') as output: diff --git a/tools/ldgen/linker_script.py b/tools/ldgen/linker_script.py new file mode 100644 index 0000000000..d7d3dfc7e3 --- /dev/null +++ b/tools/ldgen/linker_script.py @@ -0,0 +1,95 @@ +# +# Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import collections +import os + +from fragments import Fragment +from generation import GenerationException +from pyparsing import ParseException, Suppress, White + + +class LinkerScript: + """ + Encapsulates a linker script template file. Finds marker syntax and handles replacement to generate the + final output. + """ + + Marker = collections.namedtuple('Marker', 'target indent rules') + + def __init__(self, template_file): + self.members = [] + self.file = os.path.realpath(template_file.name) + + self._generate_members(template_file) + + def _generate_members(self, template_file): + lines = template_file.readlines() + + target = Fragment.IDENTIFIER + reference = Suppress('mapping') + Suppress('[') + target.setResultsName('target') + Suppress(']') + pattern = White(' \t').setResultsName('indent') + reference + + # Find the markers in the template file line by line. If line does not match marker grammar, + # set it as a literal to be copied as is to the output file. + for line in lines: + try: + parsed = pattern.parseString(line) + + indent = parsed.indent + target = parsed.target + + marker = LinkerScript.Marker(target, indent, []) + + self.members.append(marker) + except ParseException: + # Does not match marker syntax + self.members.append(line) + + def fill(self, mapping_rules): + for member in self.members: + target = None + try: + target = member.target + rules = member.rules + + del rules[:] + + rules.extend(mapping_rules[target]) + except KeyError: + message = GenerationException.UNDEFINED_REFERENCE + " to target '" + target + "'." + raise GenerationException(message) + except AttributeError: + pass + + def write(self, output_file): + # Add information that this is a generated file. + output_file.write('/* Automatically generated file; DO NOT EDIT */\n') + output_file.write('/* Espressif IoT Development Framework Linker Script */\n') + output_file.write('/* Generated from: %s */\n' % self.file) + output_file.write('\n') + + # Do the text replacement + for member in self.members: + try: + indent = member.indent + rules = member.rules + + for rule in rules: + generated_line = ''.join([indent, str(rule), '\n']) + output_file.write(generated_line) + except AttributeError: + output_file.write(member)