Rework the sketch builder to match the IDE (#3146)

* Rework the sketch builder to match the IDE

* Link the board to the home folder

* Rename files for clarity

* move ci files to own subfolder

* Update Github CI to use the new script locations

tools/build.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-if [ ! -z "$TRAVIS_TAG" ]; then
-    # zip the package if tagged build
-    tools/build-release.sh -a$ESP32_GITHUB_TOKEN
-#else
-    # run cmake and sketch tests
-    #tools/build-tests.sh
-fi

tools/ci/build-sketch.py

@@ -29,28 +29,30 @@ import subprocess
 import tempfile
 import shutil
 
-def compile(tmp_dir, sketch, tools_dir, hardware_dir, ide_path, f, args):
+def compile(tmp_dir, cache_dir, sketch, ide_path, f, args):
     cmd = ide_path + '/arduino-builder '
     cmd += '-compile -logger=human '
-    cmd += '-build-path "' + tmp_dir + '" '
+    cmd += '-hardware "' + ide_path + '/hardware" '
+    if args.usr_path:
+        cmd += '-hardware "' + args.usr_path + '/hardware" '
+    if args.hardware_path:
+        for hw_dir in args.hardware_path:
+            cmd += '-hardware "' + hw_dir + '" '
     cmd += '-tools "' +  ide_path + '/tools-builder" '
+    if args.tools_path:
+        for tools_dir in args.tools_path:
+            cmd += '-tools "' + tools_dir + '" '
+    cmd += '-built-in-libraries "' +  ide_path + '/libraries" '
+    if args.usr_path:
+        cmd += '-libraries "' + args.usr_path + '/libraries" '
     if args.library_path:
         for lib_dir in args.library_path:
             cmd += '-libraries "' + lib_dir + '" '
-    cmd += '-hardware "' + ide_path + '/hardware" '
-    if args.hardware_dir:
-        for hw_dir in args.hardware_dir:
-            cmd += '-hardware "' + hw_dir + '" '
-    else:
-        cmd += '-hardware "' + hardware_dir + '" '
-    # Debug=Serial,DebugLevel=Core____
-    cmd += '-fqbn=espressif:esp32:{board_name}:' \
-            'FlashFreq={flash_freq},' \
-            'PartitionScheme=huge_app,' \
-            'UploadSpeed=921600'.format(**vars(args))
-    cmd += ' '
-    cmd += '-ide-version=10607 '
+    cmd += '-fqbn={fqbn} '.format(**vars(args))
+    cmd += '-ide-version=10810 '
+    cmd += '-build-path "' + tmp_dir + '" '
     cmd += '-warnings={warnings} '.format(**vars(args))
+    cmd += '-build-cache "' + cache_dir + '" '
     if args.verbose:
         cmd += '-verbose '
     cmd += sketch
@@ -65,22 +67,18 @@ def compile(tmp_dir, sketch, tools_dir, hardware_dir, ide_path, f, args):
 
 def parse_args():
     parser = argparse.ArgumentParser(description='Sketch build helper')
-    parser.add_argument('-v', '--verbose', help='Enable verbose output',
-                        action='store_true')
+    parser.add_argument('-v', '--verbose', help='Enable verbose output', action='store_true')
     parser.add_argument('-i', '--ide_path', help='Arduino IDE path')
-    parser.add_argument('-p', '--build_path', help='Build directory')
-    parser.add_argument('-l', '--library_path', help='Additional library path',
-                        action='append')
-    parser.add_argument('-d', '--hardware_dir', help='Additional hardware path',
-                        action='append')
-    parser.add_argument('-b', '--board_name', help='Board name', default='esp32')
-    parser.add_argument('-w', '--warnings', help='Compilation warnings level',
-                        default='none', choices=['none', 'all', 'more'])
+    parser.add_argument('-b', '--build_path', help='Build directory')
+    parser.add_argument('-c', '--build_cache', help='Core Cache directory')
+    parser.add_argument('-u', '--usr_path', help='Arduino Home directory (holds your sketches, libraries and hardware)')
+    parser.add_argument('-f', '--fqbn', help='Arduino Board FQBN')
+    parser.add_argument('-l', '--library_path', help='Additional library path', action='append')
+    parser.add_argument('-d', '--hardware_path', help='Additional hardware path', action='append')
+    parser.add_argument('-t', '--tools_path', help='Additional tools path', action='append')
+    parser.add_argument('-w', '--warnings', help='Compilation warnings level', default='none', choices=['none', 'all', 'more', 'default'])
     parser.add_argument('-o', '--output_binary', help='File name for output binary')
-    parser.add_argument('-k', '--keep', action='store_true',
-                        help='Don\'t delete temporary build directory')
-    parser.add_argument('--flash_freq', help='Flash frequency', default=40,
-                        type=int, choices=[40, 80])
+    parser.add_argument('-k', '--keep', action='store_true', help='Don\'t delete temporary build directory')
     parser.add_argument('sketch_path', help='Sketch file path')
     return parser.parse_args()
 
@@ -95,21 +93,30 @@ def main():
                   "or ARDUINO_IDE_PATH environment variable.", file=sys.stderr)
             return 2
 
+    if not args.fqbn:
+        print("Please specify Arduino Board FQBN using the --fqbn option", file=sys.stderr)
+        return 3
+
     sketch_path = args.sketch_path
+    
     tmp_dir = args.build_path
     created_tmp_dir = False
     if not tmp_dir:
         tmp_dir = tempfile.mkdtemp()
         created_tmp_dir = True
 
-    tools_dir = os.path.dirname(os.path.realpath(__file__)) + '/../tools'
-    # this is not the correct hardware folder to add.
-    hardware_dir = os.path.dirname(os.path.realpath(__file__)) + '/../cores'
-
+    cache_dir = args.build_cache
+    created_cache_dir = False
+    if not cache_dir:
+        cache_dir = tempfile.mkdtemp()
+        created_cache_dir = True
+    
     output_name = tmp_dir + '/' + os.path.basename(sketch_path) + '.bin'
+    
     if args.verbose:
         print("Sketch: ", sketch_path)
         print("Build dir: ", tmp_dir)
+        print("Cache dir: ", cache_dir)
         print("Output: ", output_name)
 
     if args.verbose:
@@ -117,7 +124,7 @@ def main():
     else:
         f = open(tmp_dir + '/build.log', 'w')
 
-    res = compile(tmp_dir, sketch_path, tools_dir, hardware_dir, ide_path, f, args)
+    res = compile(tmp_dir, cache_dir, sketch_path, ide_path, f, args)
     if res != 0:
         return res
 
@@ -127,5 +134,8 @@ def main():
     if created_tmp_dir and not args.keep:
         shutil.rmtree(tmp_dir, ignore_errors=True)
 
+    if created_cache_dir and not args.keep:
+        shutil.rmtree(cache_dir, ignore_errors=True)
+
 if __name__ == '__main__':
     sys.exit(main())

tools/ci/build-tests.sh

@@ -29,7 +29,7 @@ fi
 # CMake Test
 if [ "$CHUNK_INDEX" -eq 0 ]; then
 	echo -e "travis_fold:start:check_cmakelists"
-	tools/check_cmakelists.sh
+	tools/ci/check-cmakelists.sh
 	if [ $? -ne 0 ]; then exit 1; fi
 	echo -e "travis_fold:end:check_cmakelists"
 fi
@@ -37,12 +37,12 @@ fi
 if [ "$BUILD_PIO" -eq 0 ]; then
 	# ArduinoIDE Test
 	echo -e "travis_fold:start:prep_arduino_ide"
-	tools/prep-arduino-ide.sh
+	tools/ci/prep-arduino-ide.sh
 	if [ $? -ne 0 ]; then exit 1; fi
 	echo -e "travis_fold:end:prep_arduino_ide"
 
 	echo -e "travis_fold:start:test_arduino_ide"
-	tools/test-arduino-ide.sh $CHUNK_INDEX $CHUNKS_CNT
+	tools/ci/test-arduino-ide.sh $CHUNK_INDEX $CHUNKS_CNT
 	if [ $? -ne 0 ]; then exit 1; fi
 	echo -e "travis_fold:end:test_arduino_ide"
 
@@ -53,12 +53,12 @@ else
 	# PlatformIO Test
 	echo -e "travis_fold:start:prep_platformio"
 	cd tools && python get.py && cd ..
-	tools/prep-platformio.sh
+	tools/ci/prep-platformio.sh
 	if [ $? -ne 0 ]; then exit 1; fi
 	echo -e "travis_fold:end:prep_platformio"
 
 	echo -e "travis_fold:start:test_platformio"
-	tools/test-platformio.sh
+	tools/ci/test-platformio.sh
 	if [ $? -ne 0 ]; then exit 1; fi
 	echo -e "travis_fold:end:test_platformio"
 fi

tools/ci/prep-arduino-ide.sh

@@ -5,10 +5,9 @@ wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-li
 tar xf arduino.tar.xz
 mv arduino-nightly $HOME/arduino_ide
 mkdir -p $HOME/Arduino/libraries
+mkdir -p $HOME/Arduino/hardware/espressif
+cd $HOME/Arduino/hardware/espressif
 
-cd $HOME/arduino_ide/hardware
-mkdir espressif
-cd espressif
 ln -s $TRAVIS_BUILD_DIR esp32
 cd esp32/tools
 python get.py

tools/ci/test-arduino-ide.sh

@@ -16,11 +16,15 @@ if [ "$CHUNK_INDEX" -ge "$CHUNKS_CNT" ]; then
 	exit 1
 fi
 
+export ARDUINO_BOARD_FQBN="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app,CPUFreq=240,FlashMode=qio,FlashFreq=80,FlashSize=4M,UploadSpeed=921600,DebugLevel=none"
+
 export ARDUINO_IDE_PATH=$HOME/arduino_ide
-export ARDUINO_LIB_PATH=$HOME/Arduino/libraries
+export ARDUINO_USR_PATH=$HOME/Arduino
+
 export EXAMPLES_PATH=$TRAVIS_BUILD_DIR/libraries
-export EXAMPLES_BUILD_DIR=$TRAVIS_BUILD_DIR/build.tmp
-export EXAMPLES_BUILD_CMD="python tools/build.py -b esp32 -v -k -p $EXAMPLES_BUILD_DIR -l $ARDUINO_LIB_PATH "
+export EXAMPLES_BUILD_DIR=$HOME/build.tmp
+export EXAMPLES_CACHE_DIR=$HOME/cache.tmp
+export EXAMPLES_BUILD_CMD="python $TRAVIS_BUILD_DIR/tools/ci/build-sketch.py -v -k -b $EXAMPLES_BUILD_DIR -c $EXAMPLES_CACHE_DIR -u $ARDUINO_USR_PATH -f $ARDUINO_BOARD_FQBN "
 export EXAMPLES_SIZE_BIN=$TRAVIS_BUILD_DIR/tools/xtensa-esp32-elf/bin/xtensa-esp32-elf-size
 
 function print_size_info()
@@ -93,6 +97,10 @@ function count_sketches()
 function build_sketches()
 {
     mkdir -p $EXAMPLES_BUILD_DIR
+    mkdir -p $EXAMPLES_CACHE_DIR
+    mkdir -p $ARDUINO_USR_PATH/libraries
+    mkdir -p $ARDUINO_USR_PATH/hardware
+    
     local chunk_idex=$1
     local chunks_num=$2
     count_sketches