| Message ID | 1321918844-19597-2-git-send-email-afleming@freescale.com |
|---|---|
| State | Superseded, archived |
| Delegated to: | Wolfgang Denk |
| Headers | show |
Hi Andy, This is great, thank you for doing it. I just have a few fairly trivial comments. On Mon, Nov 21, 2011 at 3:40 PM, Andy Fleming <afleming@freescale.com> wrote: > The MAKEALL script cleverly runs make with the appropriate options > to use all of the cores on the system, but your average U-Boot build > can't make much use of more than a few cores. If you happen to have > a many-core server, your builds will leave most of the system idle. > > In order to make full use of such a system, we need to build multiple > targets in parallel, and this requires directing make output into > multiple directories. We add a BUILD_NBUILDS variable, which allows > users to specify how many builds to run in parallel. > When BUILD_NBUILDS is set greater than 1, we redefine BUILD_DIR for > each build to be ${BUILD_DIR}/${target}. Also, we make "./build" the > default BUILD_DIR when BUILD_NBUILDS is greater than 1. > > MAKEALL now tracks which builds are still running, and when one > finishes, it starts a new build. > > Once each build finishes, we run "make tidy" on its directory, to reduce > the footprint. > > As a result, we are left with a build directory with all of the built > targets still there for use, which means anyone who wanted to use > MAKEALL as part of a test harness can now do so. > > Signed-off-by: Andy Fleming <afleming@freescale.com> Tested-by: Simon Glass <sjg@chromium.org> > --- > v2: - Update to keep BUILD_NBUILDS builds in flight, rather than batching > - Clean up style things > - Defer error output until build completion to make output *slightly* > more readable > > Performance tests (highly unscientific): > > ./MAKEALL 83xx (original) > real 6m27.123s > user 12m59.342s > sys 2m2.528s > > BUILD_NBUILDS=2 ./MAKEALL 83xx > real 4m54.854s > user 12m10.336s > sys 1m55.957s > > BUILD_NBUILDS=4 ./MAKEALL 83xx > real 3m17.039s > user 11m9.596s > sys 1m40.327s > > BUILD_NBUILDS=8 ./MAKEALL 83xx > real 2m44.366s > user 10m38.658s > sys 1m32.425s > > Amusingly, this was the best configuration I found so far: > > BUILD_NBUILDS=50 BUILD_NCPUS=1 ./MAKEALL 83xx > real 2m1.252s > user 10m49.617s > sys 1m36.176s > > > MAKEALL | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++++------- > 1 files changed, 102 insertions(+), 13 deletions(-) > > diff --git a/MAKEALL b/MAKEALL > index 95b7cd3..c14a1da 100755 > --- a/MAKEALL > +++ b/MAKEALL > @@ -31,6 +31,7 @@ usage() > CROSS_COMPILE cross-compiler toolchain prefix (default: "") > MAKEALL_LOGDIR output all logs to here (default: ./LOG/) > BUILD_DIR output build directory (default: ./) > + BUILD_NBUILDS number of parallel targets (default: 1) > > Examples: > - build all Power Architecture boards: > @@ -160,11 +161,20 @@ else > LOG_DIR="LOG" > fi > > -if [ ! "${BUILD_DIR}" ] ; then > - BUILD_DIR="." > +: ${BUILD_NBUILDS:=1} > + > +if [ "${BUILD_NBUILDS}" -gt 1 ] ; then Perhaps create a new variable like BUILD_MANY to mean that BUILD_NBUILDS > 1 > + : ${BUILD_DIR:=./build} > + mkdir -p "${BUILD_DIR}/ERR" > + find "${BUILD_DIR}/ERR/" -type f -exec rm -f {} + > fi > > -[ -d ${LOG_DIR} ] || mkdir ${LOG_DIR} || exit 1 > +: ${BUILD_DIR:=.} > + > +OUTPUT_PREFIX="${BUILD_DIR}" > + > +[ -d ${LOG_DIR} ] || mkdir "${LOG_DIR}" || exit 1 > +find "${LOG_DIR}/" -type f -exec rm -f {} + > > LIST="" > > @@ -172,6 +182,8 @@ LIST="" > ERR_CNT=0 > ERR_LIST="" > TOTAL_CNT=0 > +CURRENT_COUNT=0 > +OLDEST_IDX=1 > RC=0 > > # Helper funcs for parsing boards.cfg > @@ -485,31 +497,56 @@ LIST_nds32="$(boards_by_arch nds32)" > > #----------------------------------------------------------------------- > > +DONE_PREFIX="${LOG_DIR}/._build_" > +SKIP_PREFIX="${LOG_DIR}/._skip_" Comment on what these vars are for? Lower case, and perhaps just done and skip? I don't suppose it would be easier to use an environment variable to hold a list of active builds and the bash word count feature to see how many there are... > + > build_target() { > target=$1 > + build_idx=$2 > + > + if [ "$BUILD_NBUILDS" -gt 1 ] ; then > + output_dir="${OUTPUT_PREFIX}/${target}" > + mkdir -p "${output_dir}" > + else > + output_dir="${OUTPUT_PREFIX}" > + fi > + > + export BUILD_DIR="${output_dir}" > > ${MAKE} distclean >/dev/null > ${MAKE} -s ${target}_config > > - ${MAKE} ${JOBS} all 2>&1 >${LOG_DIR}/$target.MAKELOG \ > - | tee ${LOG_DIR}/$target.ERR > + ${MAKE} ${JOBS} all \ > + >${LOG_DIR}/$target.MAKELOG 2> ${LOG_DIR}/$target.ERR > > # Check for 'make' errors > if [ ${PIPESTATUS[0]} -ne 0 ] ; then > RC=1 > fi > > - if [ -s ${LOG_DIR}/$target.ERR ] ; then > - ERR_CNT=$((ERR_CNT + 1)) > - ERR_LIST="${ERR_LIST} $target" > + if [ "$BUILD_NBUILDS" -gt 1 ] ; then > + ${MAKE} tidy > + > + if [ -s ${LOG_DIR}/${target}.ERR ] ; then > + touch ${OUTPUT_PREFIX}/ERR/${target} > + else > + rm ${LOG_DIR}/${target}.ERR > + fi > else > - rm ${LOG_DIR}/$target.ERR > + if [ -s ${LOG_DIR}/${target}.ERR ] ; then > + : $(( ERR_CNT += 1 )) > + ERR_LIST="${ERR_LIST} $target" > + else > + rm ${LOG_DIR}/${target}.ERR > + fi > fi > > - TOTAL_CNT=$((TOTAL_CNT + 1)) > - > - ${CROSS_COMPILE}size ${BUILD_DIR}/u-boot \ > + ${CROSS_COMPILE}size ${output_dir}/u-boot \ > | tee -a ${LOG_DIR}/$target.MAKELOG > + > + [ -e "${LOG_DIR}/${target}.ERR" ] && cat "${LOG_DIR}/${target}.ERR" > + > + touch "${DONE_PREFIX}${build_idx}" > } > build_targets() { > for t in "$@" ; do > @@ -523,7 +560,49 @@ build_targets() { > if [ -n "${list}" ] ; then > build_targets ${list} > else > - build_target ${t} > + : $((TOTAL_CNT += 1)) > + : $((CURRENT_COUNT += 1)) Should TOTAL_CNT and CURRENT_COUNT have more similar names...and perhaps lower case? > + rm -f "${DONE_PREFIX}${TOTAL_CNT}" > + rm -f "${SKIP_PREFIX}${TOTAL_CNT}" > + build_target ${t} ${TOTAL_CNT} & > + fi > + > + # We maintain a running count of all the builds we have done. > + # Each finished build will have a file called ${DONE_PREFIX}${n}, > + # where n is the index of the build. Each build > + # we've already noted as finished will have ${SKIP_PREFIX}${n}. > + # We track the current index via TOTAL_CNT, and the oldest > + # index. When we exceed the maximum number of parallel builds, > + # We look from oldest to current for builds that have completed, > + # and update the current count and oldest index as appropriate. > + # If we've gone through the entire list, wait a second, and > + # reprocess the entire list until we find a build that has > + # completed > + if [ ${CURRENT_COUNT} -ge ${BUILD_NBUILDS} ] ; then I suggest putting the command and everything after the 'if' into a separate function. > + search_idx=${OLDEST_IDX} > + while true; do > + if [ -e "${DONE_PREFIX}${search_idx}" ] ; then > + : $(( CURRENT_COUNT-- )) > + [ ${OLDEST_IDX} -eq ${search_idx} ] && > + : $(( OLDEST_IDX++ )) > + > + # Only want to count it once > + rm -f "${DONE_PREFIX}${search_idx}" > + touch "${SKIP_PREFIX}${search_idx}" > + elif [ -e "${SKIP_PREFIX}${search_idx}" ] ; then > + [ ${OLDEST_IDX} -eq ${search_idx} ] && > + : $(( OLDEST_IDX++ )) > + fi > + if [ ${search_idx} -ge ${TOTAL_CNT} ] ; then > + if [ ${CURRENT_COUNT} -ge ${BUILD_NBUILDS} ] ; then > + search_idx=${OLDEST_IDX} > + sleep 1 > + else > + break > + fi > + fi > + : $(( search_idx++ )) > + done > fi > done > } > @@ -531,6 +610,15 @@ build_targets() { > #----------------------------------------------------------------------- > > print_stats() { > + for ((build_num=1; ${build_num} <= ${TOTAL_CNT} ; build_num++)) ; do > + rm -f "${DONE_PREFIX}${build_num}" > + rm -f "${SKIP_PREFIX}${build_num}" > + done Perhaps just use a wildcard to delete them all, like rm -f $(DONE_PREFIX}* ${SKIP_PREFIX}* > + > + if [ "$BUILD_NBUILDS" -gt 1 ] ; then > + ERR_LIST=$(ls ${OUTPUT_PREFIX}/ERR/) > + ERR_CNT=`ls -1 ${OUTPUT_PREFIX}/ERR/ | wc | awk '{print $1}'` > + fi > echo "" > echo "--------------------- SUMMARY ----------------------------" > echo "Boards compiled: ${TOTAL_CNT}" > @@ -549,3 +637,4 @@ set -- ${SELECTED} "$@" > # run PowerPC by default > [ $# = 0 ] && set -- powerpc > build_targets "$@" > +wait > -- > 1.7.3.4 > > > _______________________________________________ > U-Boot mailing list > U-Boot@lists.denx.de > http://lists.denx.de/mailman/listinfo/u-boot > Regards, SImon
On Nov 21, 2011, at 5:40 PM, Andy Fleming wrote: > The MAKEALL script cleverly runs make with the appropriate options > to use all of the cores on the system, but your average U-Boot build > can't make much use of more than a few cores. If you happen to have > a many-core server, your builds will leave most of the system idle. > > In order to make full use of such a system, we need to build multiple > targets in parallel, and this requires directing make output into > multiple directories. We add a BUILD_NBUILDS variable, which allows > users to specify how many builds to run in parallel. > When BUILD_NBUILDS is set greater than 1, we redefine BUILD_DIR for > each build to be ${BUILD_DIR}/${target}. Also, we make "./build" the > default BUILD_DIR when BUILD_NBUILDS is greater than 1. > > MAKEALL now tracks which builds are still running, and when one > finishes, it starts a new build. > > Once each build finishes, we run "make tidy" on its directory, to reduce > the footprint. > > As a result, we are left with a build directory with all of the built > targets still there for use, which means anyone who wanted to use > MAKEALL as part of a test harness can now do so. > > Signed-off-by: Andy Fleming <afleming@freescale.com> > --- > v2: - Update to keep BUILD_NBUILDS builds in flight, rather than batching > - Clean up style things > - Defer error output until build completion to make output *slightly* > more readable Can you re-fresh patch against top of tree. - k
diff --git a/MAKEALL b/MAKEALL index 95b7cd3..c14a1da 100755 --- a/MAKEALL +++ b/MAKEALL @@ -31,6 +31,7 @@ usage() CROSS_COMPILE cross-compiler toolchain prefix (default: "") MAKEALL_LOGDIR output all logs to here (default: ./LOG/) BUILD_DIR output build directory (default: ./) + BUILD_NBUILDS number of parallel targets (default: 1) Examples: - build all Power Architecture boards: @@ -160,11 +161,20 @@ else LOG_DIR="LOG" fi -if [ ! "${BUILD_DIR}" ] ; then - BUILD_DIR="." +: ${BUILD_NBUILDS:=1} + +if [ "${BUILD_NBUILDS}" -gt 1 ] ; then + : ${BUILD_DIR:=./build} + mkdir -p "${BUILD_DIR}/ERR" + find "${BUILD_DIR}/ERR/" -type f -exec rm -f {} + fi -[ -d ${LOG_DIR} ] || mkdir ${LOG_DIR} || exit 1 +: ${BUILD_DIR:=.} + +OUTPUT_PREFIX="${BUILD_DIR}" + +[ -d ${LOG_DIR} ] || mkdir "${LOG_DIR}" || exit 1 +find "${LOG_DIR}/" -type f -exec rm -f {} + LIST="" @@ -172,6 +182,8 @@ LIST="" ERR_CNT=0 ERR_LIST="" TOTAL_CNT=0 +CURRENT_COUNT=0 +OLDEST_IDX=1 RC=0 # Helper funcs for parsing boards.cfg @@ -485,31 +497,56 @@ LIST_nds32="$(boards_by_arch nds32)" #----------------------------------------------------------------------- +DONE_PREFIX="${LOG_DIR}/._build_" +SKIP_PREFIX="${LOG_DIR}/._skip_" + build_target() { target=$1 + build_idx=$2 + + if [ "$BUILD_NBUILDS" -gt 1 ] ; then + output_dir="${OUTPUT_PREFIX}/${target}" + mkdir -p "${output_dir}" + else + output_dir="${OUTPUT_PREFIX}" + fi + + export BUILD_DIR="${output_dir}" ${MAKE} distclean >/dev/null ${MAKE} -s ${target}_config - ${MAKE} ${JOBS} all 2>&1 >${LOG_DIR}/$target.MAKELOG \ - | tee ${LOG_DIR}/$target.ERR + ${MAKE} ${JOBS} all \ + >${LOG_DIR}/$target.MAKELOG 2> ${LOG_DIR}/$target.ERR # Check for 'make' errors if [ ${PIPESTATUS[0]} -ne 0 ] ; then RC=1 fi - if [ -s ${LOG_DIR}/$target.ERR ] ; then - ERR_CNT=$((ERR_CNT + 1)) - ERR_LIST="${ERR_LIST} $target" + if [ "$BUILD_NBUILDS" -gt 1 ] ; then + ${MAKE} tidy + + if [ -s ${LOG_DIR}/${target}.ERR ] ; then + touch ${OUTPUT_PREFIX}/ERR/${target} + else + rm ${LOG_DIR}/${target}.ERR + fi else - rm ${LOG_DIR}/$target.ERR + if [ -s ${LOG_DIR}/${target}.ERR ] ; then + : $(( ERR_CNT += 1 )) + ERR_LIST="${ERR_LIST} $target" + else + rm ${LOG_DIR}/${target}.ERR + fi fi - TOTAL_CNT=$((TOTAL_CNT + 1)) - - ${CROSS_COMPILE}size ${BUILD_DIR}/u-boot \ + ${CROSS_COMPILE}size ${output_dir}/u-boot \ | tee -a ${LOG_DIR}/$target.MAKELOG + + [ -e "${LOG_DIR}/${target}.ERR" ] && cat "${LOG_DIR}/${target}.ERR" + + touch "${DONE_PREFIX}${build_idx}" } build_targets() { for t in "$@" ; do @@ -523,7 +560,49 @@ build_targets() { if [ -n "${list}" ] ; then build_targets ${list} else - build_target ${t} + : $((TOTAL_CNT += 1)) + : $((CURRENT_COUNT += 1)) + rm -f "${DONE_PREFIX}${TOTAL_CNT}" + rm -f "${SKIP_PREFIX}${TOTAL_CNT}" + build_target ${t} ${TOTAL_CNT} & + fi + + # We maintain a running count of all the builds we have done. + # Each finished build will have a file called ${DONE_PREFIX}${n}, + # where n is the index of the build. Each build + # we've already noted as finished will have ${SKIP_PREFIX}${n}. + # We track the current index via TOTAL_CNT, and the oldest + # index. When we exceed the maximum number of parallel builds, + # We look from oldest to current for builds that have completed, + # and update the current count and oldest index as appropriate. + # If we've gone through the entire list, wait a second, and + # reprocess the entire list until we find a build that has + # completed + if [ ${CURRENT_COUNT} -ge ${BUILD_NBUILDS} ] ; then + search_idx=${OLDEST_IDX} + while true; do + if [ -e "${DONE_PREFIX}${search_idx}" ] ; then + : $(( CURRENT_COUNT-- )) + [ ${OLDEST_IDX} -eq ${search_idx} ] && + : $(( OLDEST_IDX++ )) + + # Only want to count it once + rm -f "${DONE_PREFIX}${search_idx}" + touch "${SKIP_PREFIX}${search_idx}" + elif [ -e "${SKIP_PREFIX}${search_idx}" ] ; then + [ ${OLDEST_IDX} -eq ${search_idx} ] && + : $(( OLDEST_IDX++ )) + fi + if [ ${search_idx} -ge ${TOTAL_CNT} ] ; then + if [ ${CURRENT_COUNT} -ge ${BUILD_NBUILDS} ] ; then + search_idx=${OLDEST_IDX} + sleep 1 + else + break + fi + fi + : $(( search_idx++ )) + done fi done } @@ -531,6 +610,15 @@ build_targets() { #----------------------------------------------------------------------- print_stats() { + for ((build_num=1; ${build_num} <= ${TOTAL_CNT} ; build_num++)) ; do + rm -f "${DONE_PREFIX}${build_num}" + rm -f "${SKIP_PREFIX}${build_num}" + done + + if [ "$BUILD_NBUILDS" -gt 1 ] ; then + ERR_LIST=$(ls ${OUTPUT_PREFIX}/ERR/) + ERR_CNT=`ls -1 ${OUTPUT_PREFIX}/ERR/ | wc | awk '{print $1}'` + fi echo "" echo "--------------------- SUMMARY ----------------------------" echo "Boards compiled: ${TOTAL_CNT}" @@ -549,3 +637,4 @@ set -- ${SELECTED} "$@" # run PowerPC by default [ $# = 0 ] && set -- powerpc build_targets "$@" +wait
The MAKEALL script cleverly runs make with the appropriate options to use all of the cores on the system, but your average U-Boot build can't make much use of more than a few cores. If you happen to have a many-core server, your builds will leave most of the system idle. In order to make full use of such a system, we need to build multiple targets in parallel, and this requires directing make output into multiple directories. We add a BUILD_NBUILDS variable, which allows users to specify how many builds to run in parallel. When BUILD_NBUILDS is set greater than 1, we redefine BUILD_DIR for each build to be ${BUILD_DIR}/${target}. Also, we make "./build" the default BUILD_DIR when BUILD_NBUILDS is greater than 1. MAKEALL now tracks which builds are still running, and when one finishes, it starts a new build. Once each build finishes, we run "make tidy" on its directory, to reduce the footprint. As a result, we are left with a build directory with all of the built targets still there for use, which means anyone who wanted to use MAKEALL as part of a test harness can now do so. Signed-off-by: Andy Fleming <afleming@freescale.com> --- v2: - Update to keep BUILD_NBUILDS builds in flight, rather than batching - Clean up style things - Defer error output until build completion to make output *slightly* more readable Performance tests (highly unscientific): ./MAKEALL 83xx (original) real 6m27.123s user 12m59.342s sys 2m2.528s BUILD_NBUILDS=2 ./MAKEALL 83xx real 4m54.854s user 12m10.336s sys 1m55.957s BUILD_NBUILDS=4 ./MAKEALL 83xx real 3m17.039s user 11m9.596s sys 1m40.327s BUILD_NBUILDS=8 ./MAKEALL 83xx real 2m44.366s user 10m38.658s sys 1m32.425s Amusingly, this was the best configuration I found so far: BUILD_NBUILDS=50 BUILD_NCPUS=1 ./MAKEALL 83xx real 2m1.252s user 10m49.617s sys 1m36.176s MAKEALL | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 files changed, 102 insertions(+), 13 deletions(-)