Merge pull request #20 from Bagira80/develop

Fix for issue 11224 ("Pre-processing / Pre-generating MPL-containers stopped working")
This commit is contained in:
Edward Diener
2015-05-18 10:42:49 -04:00
9 changed files with 492 additions and 65 deletions

65
preprocessed/README.txt Normal file
View File

@ -0,0 +1,65 @@
Pre-processing of MPL-containers
--------------------------------
Pre-processing of MPL-containers can be accomplished using the script
"boost_mpl_preprocess.py". In the simplest case call it with a single
argument which is the path to the source-directory of Boost.
python boost_mpl_preprocess.py <path-to-boost-sourcedir>
This will pre-process all four MPL-container types (vector, list, set,
map) and makes them able to hold up to 100 elements. They can be used
either in their 'numbered' or their 'variadic' form.
Additionally, the script also allows more fine-grained pre-processing.
The maximal number of elements an MPL-container type is able to hold can
be different from the one of other MPL-container types and it can also
differ between its 'numbered' and 'variadic' form.
To see all options, call the script like this:
python boost_mpl_preprocess.py --help
Fixing pre-processing of MPL-containers
---------------------------------------
Sadly, pre-processing of MPL-containers might fail, if the source-files
used as input are missing some header-comments required during the pre-
processing step.
However, the script "boost_mpl_preprocess.py" makes sure to patch these
input source-files prior to pre-processing (by implicitly calling script
"fix_boost_mpl_preprocess.py" with the chosen settings). It only patches
the source-files needed for pre-processing the selected MPL-container
types and their selected form ('numbered' or 'variadic').
If calling it with a single argument (as in the former section) all input
source-files will be patched automatically.
Instead of fixing the input-files implicitly during pre-processing one
can also fix them explicitly by calling "fix_boost_mpl_preprocess.py"
directly.
If you just want to test if any fixing is needed call it like this:
python fix_boost_mpl_preprocess.py --check-only <path-to-boost-sourcedir>
This will tell you if any fixing is needed. In such a case call the script
"fix_boost_mpl_preprocess.py" like this:
python fix_boost_mpl_preprocess.py <path-to-boost-sourcedir>
This will fix the header-comments of all the source-files needed during
pre-processing. Calling "boost_mpl_preprocess.py" afterwards should then
successfully pre-process the MPL-containers (without the need of implicitly
fixing any files again).
Note:
Failure of pre-processing can be checked by examining at least one of the
following directories in which automatically generated files will be put
during pre-processing. If at least one file in these directories (or sub-
directories therein) has a size of zero bytes, fixing is needed.
<path-to-boost-sourcedir>/boost/mpl/vector/aux_/preprocessed/
<path-to-boost-sourcedir>/boost/mpl/list/aux_/preprocessed/
<path-to-boost-sourcedir>/boost/mpl/set/aux_/preprocessed/
<path-to-boost-sourcedir>/boost/mpl/map/aux_/preprocessed/
<path-to-boost-sourcedir>/boost/mpl/aux_/preprocessed/

View File

@ -0,0 +1,219 @@
# Copyright Deniz Bahadir 2015
#
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# http://www.boost.org/LICENSE_1_0.txt)
#
# See http://www.boost.org/libs/mpl for documentation.
# See http://stackoverflow.com/a/20660264/3115457 for further information.
# See http://stackoverflow.com/a/29627158/3115457 for further information.
import fix_boost_mpl_preprocess as fixmpl
import argparse
import sys
import os
import os.path
import re
import fileinput
import shutil
def create_more_container_files(sourceDir, suffix, maxElements, containers, containers2):
"""Creates additional files for the individual MPL-containers."""
# Create files for each MPL-container with 20 to 'maxElements' elements
# which will be used during generation.
for container in containers:
for i in range(20, maxElements, 10):
# Create copy of "template"-file.
newFile = os.path.join( sourceDir, container, container + str(i+10) + suffix )
shutil.copyfile( os.path.join( sourceDir, container, container + "20" + suffix ), newFile )
# Adjust copy of "template"-file accordingly.
for line in fileinput.input( newFile, inplace=1, mode="rU" ):
line = re.sub(r'20', re.escape(str(i+10)), line.rstrip())
line = re.sub(r'11', re.escape(str(i + 1)), line.rstrip())
line = re.sub(r'10(?![0-9])', re.escape(str(i)), line.rstrip())
print(line)
for container in containers2:
for i in range(20, maxElements, 10):
# Create copy of "template"-file.
newFile = os.path.join( sourceDir, container, container + str(i+10) + "_c" + suffix )
shutil.copyfile( os.path.join( sourceDir, container, container + "20_c" + suffix ), newFile )
# Adjust copy of "template"-file accordingly.
for line in fileinput.input( newFile, inplace=1, mode="rU" ):
line = re.sub(r'20', re.escape(str(i+10)), line.rstrip())
line = re.sub(r'11', re.escape(str(i + 1)), line.rstrip())
line = re.sub(r'10(?![0-9])', re.escape(str(i)), line.rstrip())
print(line)
def create_input_for_numbered_sequences(headerDir, sourceDir, containers, maxElements):
"""Creates additional source- and header-files for the numbered sequence MPL-containers."""
# Create additional container-list without "map".
containersWithoutMap = containers[:]
try:
containersWithoutMap.remove('map')
except ValueError:
# We can safely ignore if "map" is not contained in 'containers'!
pass
# Create header/source-files.
create_more_container_files(headerDir, ".hpp", maxElements, containers, containersWithoutMap)
create_more_container_files(sourceDir, ".cpp", maxElements, containers, containersWithoutMap)
def adjust_container_limits_for_variadic_sequences(headerDir, containers, maxElements):
"""Adjusts the limits of variadic sequence MPL-containers."""
for container in containers:
headerFile = os.path.join( headerDir, "limits", container + ".hpp" )
regexMatch = r'(define\s+BOOST_MPL_LIMIT_' + container.upper() + r'_SIZE\s+)[0-9]+'
regexReplace = r'\g<1>' + re.escape( str(maxElements) )
for line in fileinput.input( headerFile, inplace=1, mode="rU" ):
line = re.sub(regexMatch, regexReplace, line.rstrip())
print(line)
def to_positive_multiple_of_10(string):
"""Converts a string into its encoded positive integer (greater zero) or throws an exception."""
try:
value = int(string)
except ValueError:
msg = '"%r" is not a positive multiple of 10 (greater zero).' % string
raise argparse.ArgumentTypeError(msg)
if value <= 0 or value % 10 != 0:
msg = '"%r" is not a positive multiple of 10 (greater zero).' % string
raise argparse.ArgumentTypeError(msg)
return value
def to_existing_absolute_path(string):
"""Converts a path into its absolute path and verifies that it exists or throws an exception."""
value = os.path.abspath(string)
if not os.path.exists( value ) or not os.path.isdir( value ):
msg = '"%r" is not a valid path to a directory.' % string
raise argparse.ArgumentTypeError(msg)
return value
def main():
"""The main function."""
# Prepare and run cmdline-parser.
cmdlineParser = argparse.ArgumentParser(description="A generator-script for pre-processed Boost.MPL headers.")
cmdlineParser.add_argument("-v", "--verbose", dest='verbose', action='store_true',
help="Be a little bit more verbose.")
cmdlineParser.add_argument("-s", "--sequence-type", dest='seqType', choices=['variadic', 'numbered', 'both'],
default='both',
help="Only update pre-processed headers for the selected sequence types, "
"either 'numbered' sequences, 'variadic' sequences or 'both' sequence "
"types. (Default=both)")
cmdlineParser.add_argument("--no-vector", dest='want_vector', action='store_false',
help="Do not update pre-processed headers for Boost.MPL Vector.")
cmdlineParser.add_argument("--no-list", dest='want_list', action='store_false',
help="Do not update pre-processed headers for Boost.MPL List.")
cmdlineParser.add_argument("--no-set", dest='want_set', action='store_false',
help="Do not update pre-processed headers for Boost.MPL Set.")
cmdlineParser.add_argument("--no-map", dest='want_map', action='store_false',
help="Do not update pre-processed headers for Boost.MPL Map.")
cmdlineParser.add_argument("--num-elements", dest='numElements', metavar="<num-elements>",
type=to_positive_multiple_of_10, default=100,
help="The maximal number of elements per container sequence. (Default=100)")
cmdlineParser.add_argument(dest='sourceDir', metavar="<source-dir>",
type=to_existing_absolute_path,
help="The source-directory of Boost.")
args = cmdlineParser.parse_args()
# Some verbose debug output.
if args.verbose:
print "Arguments extracted from command-line:"
print " verbose = ", args.verbose
print " source directory = ", args.sourceDir
print " num elements = ", args.numElements
print " sequence type = ", args.seqType
print " want: vector = ", args.want_vector
print " want: list = ", args.want_list
print " want: set = ", args.want_set
print " want: map = ", args.want_map
# The directories for header- and source files of Boost.MPL.
# NOTE: Assuming 'args.sourceDir' is the source-directory of the entire boost project.
headerDir = os.path.join( args.sourceDir, "boost", "mpl" )
sourceDir = os.path.join( args.sourceDir, "libs", "mpl", "preprocessed" )
# Check that the header/source-directories exist.
if not os.path.exists( headerDir ) or not os.path.exists( sourceDir ):
# Maybe 'args.sourceDir' is not the source-directory of the entire boost project
# but instead of the Boost.MPL git-directory, only?
headerDir = os.path.join( args.sourceDir, "include", "boost", "mpl" )
sourceDir = os.path.join( args.sourceDir, "preprocessed" )
if not os.path.exists( headerDir ) or not os.path.exists( sourceDir ):
cmdlineParser.print_usage()
print "error: Cannot find Boost.MPL header/source files in given Boost source-directory!"
sys.exit(0)
# Some verbose debug output.
if args.verbose:
print "Chosen header-directory: ", headerDir
print "Chosen source-directory: ", sourceDir
# Create list of containers for which files shall be pre-processed.
containers = []
if args.want_vector:
containers.append('vector')
if args.want_list:
containers.append('list')
if args.want_set:
containers.append('set')
if args.want_map:
containers.append('map')
if containers == []:
print "Nothing to do."
print "(Why did you prevent generating pre-processed headers for all Boost.MPL container types?)"
sys.exit(0)
# Possibly fix the header-comments of input-files needed for pre-processing.
if args.verbose:
print "Checking if prior to pre-processing some input-files need fixing."
needFixing = fixmpl.check_input_files(headerDir, sourceDir, containers, args.seqType, args.verbose)
if needFixing:
if args.verbose:
print "Fixing of some input-files prior to pre-processing is needed."
print "Will fix them now!"
fixmpl.fix_input_files(headerDir, sourceDir, containers, args.seqType, args.verbose)
# Some verbose debug output.
if args.verbose:
print "Containers for which to pre-process headers: ", containers
# Create (additional) input files for generating pre-processed headers of numbered sequence MPL containers.
if args.seqType == "both" or args.seqType == "numbered":
create_input_for_numbered_sequences(headerDir, sourceDir, containers, args.numElements)
# Modify settings for generating pre-processed headers of variadic sequence MPL containers.
if args.seqType == "both" or args.seqType == "variadic":
adjust_container_limits_for_variadic_sequences(headerDir, containers, args.numElements)
# Generate MPL-preprocessed files.
os.chdir( sourceDir )
if args.seqType == "both" or args.seqType == "numbered":
if args.want_vector:
if args.verbose:
print "Pre-process headers for Boost.MPL numbered vectors."
os.system( "python " + os.path.join( sourceDir, "preprocess_vector.py" ) + " all " + args.sourceDir )
if args.want_list:
if args.verbose:
print "Pre-process headers for Boost.MPL numbered lists."
os.system( "python " + os.path.join( sourceDir, "preprocess_list.py" ) + " all " + args.sourceDir )
if args.want_set:
if args.verbose:
print "Pre-process headers for Boost.MPL numbered sets."
os.system( "python " + os.path.join( sourceDir, "preprocess_set.py" ) + " all " + args.sourceDir )
if args.want_map:
if args.verbose:
print "Pre-process headers for Boost.MPL numbered maps."
os.system( "python " + os.path.join( sourceDir, "preprocess_map.py" ) + " all " + args.sourceDir )
if args.seqType == "both" or args.seqType == "variadic":
if args.verbose:
print "Pre-process headers for Boost.MPL variadic containers."
os.system( "python " + os.path.join( sourceDir, "preprocess.py" ) + " all " + args.sourceDir )
if __name__ == '__main__':
main()

View File

@ -0,0 +1,205 @@
# Copyright Deniz Bahadir 2015
#
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or copy at
# http://www.boost.org/LICENSE_1_0.txt)
#
# See http://www.boost.org/libs/mpl for documentation.
# See http://stackoverflow.com/a/29627158/3115457 for further information.
import argparse
import sys
import os.path
import re
import fileinput
import datetime
import glob
def check_header_comment(filename):
"""Checks if the header-comment of the given file needs fixing."""
# Check input file.
name = os.path.basename( filename )
# Read content of input file.
sourcefile = open( filename, "rU" )
content = sourcefile.read()
sourcefile.close()
# Search content for '$Id$'.
match = re.search(r'\$Id\$', content)
if match == None:
# Make sure that the correct value for '$Id$' was already set.
match = re.search(r'\$Id: ' + name + r'\s+[^$]+\$', content)
if match != None:
# The given file needs no fixing.
return False
# The given file needs fixing.
return True
def check_input_files_for_variadic_seq(headerDir, sourceDir):
"""Checks if files, used as input when pre-processing MPL-containers in their variadic form, need fixing."""
# Check input files in include/source-directories.
files = glob.glob( os.path.join( headerDir, "*.hpp" ) )
files += glob.glob( os.path.join( headerDir, "aux_", "*.hpp" ) )
files += glob.glob( os.path.join( sourceDir, "src", "*" ) )
for currentFile in sorted( files ):
if check_header_comment( currentFile ):
return True
return False
def check_input_files_for_numbered_seq(sourceDir, suffix, containers):
"""Check if files, used as input when pre-processing MPL-containers in their numbered form, need fixing."""
# Check input files for each MPL-container type.
for container in containers:
files = glob.glob( os.path.join( sourceDir, container, container + '*' + suffix ) )
for currentFile in sorted( files ):
if check_header_comment( currentFile ):
return True
return False
def check_input_files(headerDir, sourceDir, containers=['vector', 'list', 'set', 'map'],
seqType='both', verbose=False):
"""Checks if source- and header-files, used as input when pre-processing MPL-containers, need fixing."""
# Check the input files for containers in their variadic form.
result1 = False
if seqType == "both" or seqType == "variadic":
if verbose:
print "Check if input files for pre-processing Boost.MPL variadic containers need fixing."
result1 = check_input_files_for_variadic_seq(headerDir, sourceDir)
if verbose:
if result1:
print " At least one input file needs fixing!"
else:
print " No input file needs fixing!"
# Check the input files for containers in their numbered form.
result2 = False
result3 = False
if seqType == "both" or seqType == "numbered":
if verbose:
print "Check input files for pre-processing Boost.MPL numbered containers."
result2 = check_input_files_for_numbered_seq(headerDir, ".hpp", containers)
result3 = check_input_files_for_numbered_seq(sourceDir, ".cpp", containers)
if verbose:
if result2 or result3:
print " At least one input file needs fixing!"
else:
print " No input file needs fixing!"
# Return result.
return result1 or result2 or result3
def fix_header_comment(filename, timestamp):
"""Fixes the header-comment of the given file."""
# Fix input file.
name = os.path.basename( filename )
for line in fileinput.input( filename, inplace=1, mode="rU" ):
# If header-comment already contains anything for '$Id$', remove it.
line = re.sub(r'\$Id:[^$]+\$', r'$Id$', line.rstrip())
# Replace '$Id$' by a string containing the file's name (and a timestamp)!
line = re.sub(re.escape(r'$Id$'), r'$Id: ' + name + r' ' + timestamp.isoformat() + r' $', line.rstrip())
print(line)
def fix_input_files_for_variadic_seq(headerDir, sourceDir, timestamp):
"""Fixes files used as input when pre-processing MPL-containers in their variadic form."""
# Fix files in include/source-directories.
files = glob.glob( os.path.join( headerDir, "*.hpp" ) )
files += glob.glob( os.path.join( headerDir, "aux_", "*.hpp" ) )
files += glob.glob( os.path.join( sourceDir, "src", "*" ) )
for currentFile in sorted( files ):
fix_header_comment( currentFile, timestamp )
def fix_input_files_for_numbered_seq(sourceDir, suffix, timestamp, containers):
"""Fixes files used as input when pre-processing MPL-containers in their numbered form."""
# Fix input files for each MPL-container type.
for container in containers:
files = glob.glob( os.path.join( sourceDir, container, container + '*' + suffix ) )
for currentFile in sorted( files ):
fix_header_comment( currentFile, timestamp )
def fix_input_files(headerDir, sourceDir, containers=['vector', 'list', 'set', 'map'],
seqType='both', verbose=False):
"""Fixes source- and header-files used as input when pre-processing MPL-containers."""
# The new modification time.
timestamp = datetime.datetime.now();
# Fix the input files for containers in their variadic form.
if seqType == "both" or seqType == "variadic":
if verbose:
print "Fix input files for pre-processing Boost.MPL variadic containers."
fix_input_files_for_variadic_seq(headerDir, sourceDir, timestamp)
# Fix the input files for containers in their numbered form.
if seqType == "both" or seqType == "numbered":
if verbose:
print "Fix input files for pre-processing Boost.MPL numbered containers."
fix_input_files_for_numbered_seq(headerDir, ".hpp", timestamp, containers)
fix_input_files_for_numbered_seq(sourceDir, ".cpp", timestamp, containers)
def to_existing_absolute_path(string):
"""Converts a path into its absolute path and verifies that it exists or throws an exception."""
value = os.path.abspath(string)
if not os.path.exists( value ) or not os.path.isdir( value ):
msg = '"%r" is not a valid path to a directory.' % string
raise argparse.ArgumentTypeError(msg)
return value
def main():
"""The main function."""
# Prepare and run cmdline-parser.
cmdlineParser = argparse.ArgumentParser(
description="Fixes the input files used for pre-processing of Boost.MPL headers.")
cmdlineParser.add_argument("-v", "--verbose", dest='verbose', action='store_true',
help="Be a little bit more verbose.")
cmdlineParser.add_argument("--check-only", dest='checkonly', action='store_true',
help="Only checks if fixing is required.")
cmdlineParser.add_argument(dest='sourceDir', metavar="<source-dir>",
type=to_existing_absolute_path,
help="The source-directory of Boost.")
args = cmdlineParser.parse_args()
# Some verbose debug output.
if args.verbose:
print "Arguments extracted from command-line:"
print " verbose = ", args.verbose
print " check-only = ", args.checkonly
print " source directory = ", args.sourceDir
# The directories for header- and source files of Boost.MPL.
# NOTE: Assuming 'args.sourceDir' is the source-directory of the entire boost project.
headerDir = os.path.join( args.sourceDir, "boost", "mpl" )
sourceDir = os.path.join( args.sourceDir, "libs", "mpl", "preprocessed" )
# Check that the header/source-directories exist.
if not os.path.exists( headerDir ) or not os.path.exists( sourceDir ):
# Maybe 'args.sourceDir' is not the source-directory of the entire boost project
# but instead of the Boost.MPL git-directory, only?
headerDir = os.path.join( args.sourceDir, "include", "boost", "mpl" )
sourceDir = os.path.join( args.sourceDir, "preprocessed" )
if not os.path.exists( headerDir ) or not os.path.exists( sourceDir ):
cmdlineParser.print_usage()
print "error: Cannot find Boost.MPL header/source files in given Boost source-directory!"
sys.exit(0)
# Some verbose debug output.
if args.verbose:
print "Chosen header-directory: ", headerDir
print "Chosen source-directory: ", sourceDir
if args.checkonly:
# Check input files for generating pre-processed headers.
result = check_input_files(headerDir, sourceDir, verbose = args.verbose)
if result:
print "Fixing the input-files used for pre-processing of Boost.MPL headers IS required."
else:
print "Fixing the input-files used for pre-processing of Boost.MPL headers is NOT required."
else:
# Fix input files for generating pre-processed headers.
fix_input_files(headerDir, sourceDir, verbose = args.verbose)
if __name__ == '__main__':
main()

View File

@ -1,26 +0,0 @@
// Copyright Aleksey Gurtovoy 2003-2004
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//
// See http://www.boost.org/libs/mpl for documentation.
// $Id$
// $Date$
// $Revision$
#define BOOST_NO_CONFIG
#define BOOST_MSVC 1200
#define BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION
#define BOOST_NO_INCLASS_MEMBER_INITIALIZATION
#define BOOST_NO_TEMPLATE_TEMPLATES
#define BOOST_NO_CV_VOID_SPECIALIZATIONS
#define BOOST_NO_INTRINSIC_WCHAR_T
#define BOOST_NO_STD_ALLOCATOR
#define BOOST_MPL_CFG_NO_BIND_TEMPLATE
#define BOOST_MPL_CFG_NO_APPLY_TEMPLATE
#define BOOST_MPL_CFG_MSVC_60_ETI_BUG
#define BOOST_MPL_CFG_MSVC_ETI_BUG
#define BOOST_MPL_CFG_NO_NESTED_VALUE_ARITHMETIC

View File

@ -1,21 +0,0 @@
// Copyright Aleksey Gurtovoy 2003-2004
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//
// See http://www.boost.org/libs/mpl for documentation.
// $Id$
// $Date$
// $Revision$
#define BOOST_NO_CONFIG
#define BOOST_MSVC 1300
#define BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION
#define BOOST_NO_INCLASS_MEMBER_INITIALIZATION
#define BOOST_MPL_CFG_NO_BIND_TEMPLATE
#define BOOST_MPL_CFG_NO_APPLY_TEMPLATE
#define BOOST_MPL_CFG_MSVC_ETI_BUG
#define BOOST_MPL_CFG_NO_NESTED_VALUE_ARITHMETIC

View File

@ -1,15 +0,0 @@
// Copyright Aleksey Gurtovoy 2003-2004
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//
// See http://www.boost.org/libs/mpl for documentation.
// $Id$
// $Date$
// $Revision$
#define BOOST_NO_CONFIG
#define BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION

View File

@ -82,7 +82,7 @@ def main( all_modes, src_dir, dst_dir ):
if __name__ == '__main__':
main(
["bcc", "bcc551", "gcc", "msvc60", "msvc70", "mwcw", "dmc", "no_ctps", "no_ttp", "plain"]
["bcc", "bcc551", "gcc", "mwcw", "dmc", "no_ttp", "plain"]
, "src"
, os.path.join( "boost", "mpl", "aux_", "preprocessed" )
)

View File

@ -15,7 +15,7 @@ import preprocess
import os.path
preprocess.main(
[ "plain", "typeof_based", "no_ctps" ]
[ "plain", "typeof_based" ]
, "map"
, os.path.join( "boost", "mpl", "map", "aux_", "preprocessed" )
)

View File

@ -15,7 +15,7 @@ import preprocess
import os.path
preprocess.main(
[ "no_ctps", "plain", "typeof_based" ]
[ "plain", "typeof_based" ]
, "vector"
, os.path.join( "boost", "mpl", "vector", "aux_", "preprocessed" )
)