* [Buildroot] [PATCH 1/1] package/jack2: fix build with python 3.12
@ 2024-02-11 22:02 Fabrice Fontaine
2024-07-12 13:01 ` Thomas Petazzoni via buildroot
0 siblings, 1 reply; 2+ messages in thread
From: Fabrice Fontaine @ 2024-02-11 22:02 UTC (permalink / raw)
To: buildroot; +Cc: Wojciech M . Zabolotny, Fabrice Fontaine
Fix the following build failure with python 3.12 (which removed imp
module):
Traceback (most recent call last):
File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/./waf", line 166, in <module>
from waflib import Scripting
File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/waflib/Scripting.py", line 10, in <module>
from waflib import Utils, Configure, Logs, Options, ConfigSet, Context, Errors, Build, Node
File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/waflib/Configure.py", line 16, in <module>
from waflib import ConfigSet, Utils, Options, Logs, Context, Build, Errors
File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/waflib/Options.py", line 14, in <module>
from waflib import Logs, Utils, Context, Errors
File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/waflib/Context.py", line 9, in <module>
import os, re, imp, sys
ModuleNotFoundError: No module named 'imp'
Fixes: 36e635d2d5c0166476858aa239ccbe78e8f2af14
- http://autobuild.buildroot.org/results/1bfe34e10ffdab80647ac01863165e93bcc9b0d8
Signed-off-by: Fabrice Fontaine <fontaine.fabrice@gmail.com>
---
package/jack2/0001-Update-to-waf-2-0-26.patch | 3480 +++++++++++++++++
1 file changed, 3480 insertions(+)
create mode 100644 package/jack2/0001-Update-to-waf-2-0-26.patch
diff --git a/package/jack2/0001-Update-to-waf-2-0-26.patch b/package/jack2/0001-Update-to-waf-2-0-26.patch
new file mode 100644
index 0000000000..8471de09bc
--- /dev/null
+++ b/package/jack2/0001-Update-to-waf-2-0-26.patch
@@ -0,0 +1,3480 @@
+From 553ca1e649293ef87e96dd3e7621fd87e0b59986 Mon Sep 17 00:00:00 2001
+From: Nils Philippsen <nils@tiptoe.de>
+Date: Tue, 29 Aug 2023 12:41:46 +0200
+Subject: [PATCH] Update to waf 2.0.26
+
+This makes waf compatible with Python 3.12 again.
+
+Also, apply modifications needed for MacOS and add as a patch file (see
+commits 0f2e3b2 and dc6c995).
+
+Signed-off-by: Nils Philippsen <nils@tiptoe.de>
+
+Upstream: https://github.com/jackaudio/jack2/pull/953/commits/553ca1e649293ef87e96dd3e7621fd87e0b59986
+Signed-off-by: Fabrice Fontaine <fontaine.fabrice@gmail.com>
+---
+ waf | 7 +-
+ waflib-macos-mods.patch | 18 +
+ waflib/Build.py | 47 +-
+ waflib/ConfigSet.py | 2 +-
+ waflib/Configure.py | 46 +-
+ waflib/Context.py | 24 +-
+ waflib/Logs.py | 9 +-
+ waflib/Node.py | 3 +-
+ waflib/Options.py | 31 +-
+ waflib/Runner.py | 27 +-
+ waflib/Scripting.py | 27 +-
+ waflib/Task.py | 48 ++-
+ waflib/TaskGen.py | 16 +-
+ waflib/Tools/c_aliases.py | 10 +-
+ waflib/Tools/c_config.py | 37 +-
+ waflib/Tools/c_preproc.py | 6 +-
+ waflib/Tools/c_tests.py | 18 +-
+ waflib/Tools/ccroot.py | 20 +-
+ waflib/Tools/compiler_c.py | 25 +-
+ waflib/Tools/compiler_cxx.py | 25 +-
+ waflib/Tools/irixcc.py | 14 +-
+ waflib/Tools/msvc.py | 45 +-
+ waflib/Tools/waf_unit_test.py | 14 +-
+ waflib/Utils.py | 60 ++-
+ waflib/ansiterm.py | 2 +-
+ waflib/extras/clang_cross.py | 92 ++++
+ waflib/extras/clang_cross_common.py | 113 +++++
+ waflib/extras/clangxx_cross.py | 106 +++++
+ waflib/extras/classic_runner.py | 68 +++
+ waflib/extras/color_msvc.py | 59 +++
+ waflib/extras/fc_fujitsu.py | 52 +++
+ waflib/extras/fc_nfort.py | 52 +++
+ waflib/extras/genpybind.py | 194 +++++++++
+ waflib/extras/haxe.py | 154 +++++++
+ waflib/extras/msvc_pdb.py | 46 ++
+ waflib/extras/sphinx.py | 120 ++++++
+ waflib/extras/wafcache.py | 648 ++++++++++++++++++++++++++++
+ waflib/extras/xcode6.py | 18 +-
+ waflib/fixpy2.py | 2 +-
+ waflib/processor.py | 4 +
+ 40 files changed, 2114 insertions(+), 195 deletions(-)
+ create mode 100644 waflib-macos-mods.patch
+ create mode 100644 waflib/extras/clang_cross.py
+ create mode 100644 waflib/extras/clang_cross_common.py
+ create mode 100644 waflib/extras/clangxx_cross.py
+ create mode 100644 waflib/extras/classic_runner.py
+ create mode 100644 waflib/extras/color_msvc.py
+ create mode 100644 waflib/extras/fc_fujitsu.py
+ create mode 100644 waflib/extras/fc_nfort.py
+ create mode 100644 waflib/extras/genpybind.py
+ create mode 100644 waflib/extras/haxe.py
+ create mode 100644 waflib/extras/msvc_pdb.py
+ create mode 100644 waflib/extras/sphinx.py
+ create mode 100644 waflib/extras/wafcache.py
+
+diff --git a/waf b/waf
+index 845fba5e9..38b2c9106 100755
+--- a/waf
++++ b/waf
+@@ -1,4 +1,4 @@
+-#!/usr/bin/python3
++#!/usr/bin/env python
+ # encoding: latin-1
+ # Thomas Nagy, 2005-2018
+ #
+@@ -32,7 +32,7 @@ POSSIBILITY OF SUCH DAMAGE.
+
+ import os, sys, inspect
+
+-VERSION="2.0.12"
++VERSION="2.0.26"
+ REVISION="x"
+ GIT="x"
+ INSTALL="x"
+@@ -142,6 +142,9 @@ def find_lib():
+ if name.endswith('waf-light'):
+ w = test(base)
+ if w: return w
++ for dir in sys.path:
++ if test(dir):
++ return dir
+ err('waf-light requires waflib -> export WAFDIR=/folder')
+
+ dirname = '%s-%s-%s' % (WAF, VERSION, REVISION)
+diff --git a/waflib-macos-mods.patch b/waflib-macos-mods.patch
+new file mode 100644
+index 000000000..9e2c8a3de
+--- /dev/null
++++ b/waflib-macos-mods.patch
+@@ -0,0 +1,18 @@
++diff --git a/waflib/Tools/ccroot.py b/waflib/Tools/ccroot.py
++index cfef8bf5..484846f5 100644
++--- a/waflib/Tools/ccroot.py
+++++ b/waflib/Tools/ccroot.py
++@@ -575,12 +575,10 @@ def apply_vnum(self):
++
++ cnum = getattr(self, 'cnum', str(nums[0]))
++ cnums = cnum.split('.')
++- if len(cnums)>len(nums) or nums[0:len(cnums)] != cnums:
++- raise Errors.WafError('invalid compatibility version %s' % cnum)
++
++ libname = node.name
++ if libname.endswith('.dylib'):
++- name3 = libname.replace('.dylib', '.%s.dylib' % self.vnum)
+++ name3 = libname.replace('.dylib', '.%s.dylib' % cnums[0])
++ name2 = libname.replace('.dylib', '.%s.dylib' % cnum)
++ else:
++ name3 = libname + '.' + self.vnum
+diff --git a/waflib/Build.py b/waflib/Build.py
+index c9661df15..b49dd8302 100644
+--- a/waflib/Build.py
++++ b/waflib/Build.py
+@@ -104,7 +104,7 @@ def __init__(self, **kw):
+ """Amount of jobs to run in parallel"""
+
+ self.targets = Options.options.targets
+- """List of targets to build (default: \*)"""
++ """List of targets to build (default: \\*)"""
+
+ self.keep = Options.options.keep
+ """Whether the build should continue past errors"""
+@@ -753,10 +753,12 @@ def tgpost(tg):
+ else:
+ ln = self.launch_node()
+ if ln.is_child_of(self.bldnode):
+- Logs.warn('Building from the build directory, forcing --targets=*')
++ if Logs.verbose > 1:
++ Logs.warn('Building from the build directory, forcing --targets=*')
+ ln = self.srcnode
+ elif not ln.is_child_of(self.srcnode):
+- Logs.warn('CWD %s is not under %s, forcing --targets=* (run distclean?)', ln.abspath(), self.srcnode.abspath())
++ if Logs.verbose > 1:
++ Logs.warn('CWD %s is not under %s, forcing --targets=* (run distclean?)', ln.abspath(), self.srcnode.abspath())
+ ln = self.srcnode
+
+ def is_post(tg, ln):
+@@ -1054,7 +1056,7 @@ def post_run(self):
+ def get_install_path(self, destdir=True):
+ """
+ Returns the destination path where files will be installed, pre-pending `destdir`.
+-
++
+ Relative paths will be interpreted relative to `PREFIX` if no `destdir` is given.
+
+ :rtype: string
+@@ -1062,11 +1064,11 @@ def get_install_path(self, destdir=True):
+ if isinstance(self.install_to, Node.Node):
+ dest = self.install_to.abspath()
+ else:
+- dest = Utils.subst_vars(self.install_to, self.env)
++ dest = os.path.normpath(Utils.subst_vars(self.install_to, self.env))
+ if not os.path.isabs(dest):
+- dest = os.path.join(self.env.PREFIX, dest)
++ dest = os.path.join(self.env.PREFIX, dest)
+ if destdir and Options.options.destdir:
+- dest = os.path.join(Options.options.destdir, os.path.splitdrive(dest)[1].lstrip(os.sep))
++ dest = Options.options.destdir.rstrip(os.sep) + os.sep + os.path.splitdrive(dest)[1].lstrip(os.sep)
+ return dest
+
+ def copy_fun(self, src, tgt):
+@@ -1160,11 +1162,19 @@ def do_install(self, src, tgt, lbl, **kw):
+ # same size and identical timestamps -> make no copy
+ if st1.st_mtime + 2 >= st2.st_mtime and st1.st_size == st2.st_size:
+ if not self.generator.bld.progress_bar:
+- Logs.info('- install %s (from %s)', tgt, lbl)
++
++ c1 = Logs.colors.NORMAL
++ c2 = Logs.colors.BLUE
++
++ Logs.info('%s- install %s%s%s (from %s)', c1, c2, tgt, c1, lbl)
+ return False
+
+ if not self.generator.bld.progress_bar:
+- Logs.info('+ install %s (from %s)', tgt, lbl)
++
++ c1 = Logs.colors.NORMAL
++ c2 = Logs.colors.BLUE
++
++ Logs.info('%s+ install %s%s%s (from %s)', c1, c2, tgt, c1, lbl)
+
+ # Give best attempt at making destination overwritable,
+ # like the 'install' utility used by 'make install' does.
+@@ -1221,14 +1231,18 @@ def do_link(self, src, tgt, **kw):
+ """
+ if os.path.islink(tgt) and os.readlink(tgt) == src:
+ if not self.generator.bld.progress_bar:
+- Logs.info('- symlink %s (to %s)', tgt, src)
++ c1 = Logs.colors.NORMAL
++ c2 = Logs.colors.BLUE
++ Logs.info('%s- symlink %s%s%s (to %s)', c1, c2, tgt, c1, src)
+ else:
+ try:
+ os.remove(tgt)
+ except OSError:
+ pass
+ if not self.generator.bld.progress_bar:
+- Logs.info('+ symlink %s (to %s)', tgt, src)
++ c1 = Logs.colors.NORMAL
++ c2 = Logs.colors.BLUE
++ Logs.info('%s+ symlink %s%s%s (to %s)', c1, c2, tgt, c1, src)
+ os.symlink(src, tgt)
+ self.fix_perms(tgt)
+
+@@ -1237,7 +1251,9 @@ def do_uninstall(self, src, tgt, lbl, **kw):
+ See :py:meth:`waflib.Build.inst.do_install`
+ """
+ if not self.generator.bld.progress_bar:
+- Logs.info('- remove %s', tgt)
++ c1 = Logs.colors.NORMAL
++ c2 = Logs.colors.BLUE
++ Logs.info('%s- remove %s%s%s', c1, c2, tgt, c1)
+
+ #self.uninstall.append(tgt)
+ try:
+@@ -1257,7 +1273,9 @@ def do_unlink(self, src, tgt, **kw):
+ """
+ try:
+ if not self.generator.bld.progress_bar:
+- Logs.info('- remove %s', tgt)
++ c1 = Logs.colors.NORMAL
++ c2 = Logs.colors.BLUE
++ Logs.info('%s- remove %s%s%s', c1, c2, tgt, c1)
+ os.remove(tgt)
+ except OSError:
+ pass
+@@ -1318,7 +1336,8 @@ def build(bld):
+ lst = []
+ for env in self.all_envs.values():
+ lst.extend(self.root.find_or_declare(f) for f in env[CFG_FILES])
+- for n in self.bldnode.ant_glob('**/*', excl='.lock* *conf_check_*/** config.log c4che/*', quiet=True):
++ excluded_dirs = '.lock* *conf_check_*/** config.log %s/*' % CACHE_DIR
++ for n in self.bldnode.ant_glob('**/*', excl=excluded_dirs, quiet=True):
+ if n in lst:
+ continue
+ n.delete()
+diff --git a/waflib/ConfigSet.py b/waflib/ConfigSet.py
+index 84736c9c8..901fba6c0 100644
+--- a/waflib/ConfigSet.py
++++ b/waflib/ConfigSet.py
+@@ -11,7 +11,7 @@
+
+ import copy, re, os
+ from waflib import Logs, Utils
+-re_imp = re.compile('^(#)*?([^#=]*?)\ =\ (.*?)$', re.M)
++re_imp = re.compile(r'^(#)*?([^#=]*?)\ =\ (.*?)$', re.M)
+
+ class ConfigSet(object):
+ """
+diff --git a/waflib/Configure.py b/waflib/Configure.py
+index d0a4793a8..f6fdc4e94 100644
+--- a/waflib/Configure.py
++++ b/waflib/Configure.py
+@@ -125,7 +125,7 @@ def init_dirs(self):
+ self.bldnode.mkdir()
+
+ if not os.path.isdir(self.bldnode.abspath()):
+- conf.fatal('Could not create the build directory %s' % self.bldnode.abspath())
++ self.fatal('Could not create the build directory %s' % self.bldnode.abspath())
+
+ def execute(self):
+ """
+@@ -180,6 +180,7 @@ def execute(self):
+ env.hash = self.hash
+ env.files = self.files
+ env.environ = dict(self.environ)
++ env.launch_dir = Context.launch_dir
+
+ if not (self.env.NO_LOCK_IN_RUN or env.environ.get('NO_LOCK_IN_RUN') or getattr(Options.options, 'no_lock_in_run')):
+ env.store(os.path.join(Context.run_dir, Options.lockfile))
+@@ -438,7 +439,7 @@ def find_program(self, filename, **kw):
+
+ var = kw.get('var', '')
+ if not var:
+- var = re.sub(r'[-.]', '_', filename[0].upper())
++ var = re.sub(r'\W', '_', filename[0].upper())
+
+ path_list = kw.get('path_list', '')
+ if path_list:
+@@ -507,23 +508,27 @@ def find_binary(self, filenames, exts, paths):
+ @conf
+ def run_build(self, *k, **kw):
+ """
+- Create a temporary build context to execute a build. A reference to that build
+- context is kept on self.test_bld for debugging purposes, and you should not rely
+- on it too much (read the note on the cache below).
+- The parameters given in the arguments to this function are passed as arguments for
+- a single task generator created in the build. Only three parameters are obligatory:
++ Create a temporary build context to execute a build. A temporary reference to that build
++ context is kept on self.test_bld for debugging purposes.
++ The arguments to this function are passed to a single task generator for that build.
++ Only three parameters are mandatory:
+
+ :param features: features to pass to a task generator created in the build
+ :type features: list of string
+ :param compile_filename: file to create for the compilation (default: *test.c*)
+ :type compile_filename: string
+- :param code: code to write in the filename to compile
++ :param code: input file contents
+ :type code: string
+
+- Though this function returns *0* by default, the build may set an attribute named *retval* on the
++ Though this function returns *0* by default, the build may bind attribute named *retval* on the
+ build context object to return a particular value. See :py:func:`waflib.Tools.c_config.test_exec_fun` for example.
+
+- This function also provides a limited cache. To use it, provide the following option::
++ The temporary builds creates a temporary folder; the name of that folder is calculated
++ by hashing input arguments to this function, with the exception of :py:class:`waflib.ConfigSet.ConfigSet`
++ objects which are used for both reading and writing values.
++
++ This function also features a cache which is disabled by default; that cache relies
++ on the hash value calculated as indicated above::
+
+ def options(opt):
+ opt.add_option('--confcache', dest='confcache', default=0,
+@@ -534,10 +539,24 @@ def options(opt):
+ $ waf configure --confcache
+
+ """
+- lst = [str(v) for (p, v) in kw.items() if p != 'env']
+- h = Utils.h_list(lst)
++ buf = []
++ for key in sorted(kw.keys()):
++ v = kw[key]
++ if isinstance(v, ConfigSet.ConfigSet):
++ # values are being written to, so they are excluded from contributing to the hash
++ continue
++ elif hasattr(v, '__call__'):
++ buf.append(Utils.h_fun(v))
++ else:
++ buf.append(str(v))
++ h = Utils.h_list(buf)
+ dir = self.bldnode.abspath() + os.sep + (not Utils.is_win32 and '.' or '') + 'conf_check_' + Utils.to_hex(h)
+
++ cachemode = kw.get('confcache', getattr(Options.options, 'confcache', None))
++
++ if not cachemode and os.path.exists(dir):
++ shutil.rmtree(dir)
++
+ try:
+ os.makedirs(dir)
+ except OSError:
+@@ -548,7 +567,6 @@ def options(opt):
+ except OSError:
+ self.fatal('cannot use the configuration test folder %r' % dir)
+
+- cachemode = getattr(Options.options, 'confcache', None)
+ if cachemode == 1:
+ try:
+ proj = ConfigSet.ConfigSet(os.path.join(dir, 'cache_run_build'))
+@@ -588,7 +606,7 @@ def options(opt):
+ else:
+ ret = getattr(bld, 'retval', 0)
+ finally:
+- if cachemode == 1:
++ if cachemode:
+ # cache the results each time
+ proj = ConfigSet.ConfigSet()
+ proj['cache_run_build'] = ret
+diff --git a/waflib/Context.py b/waflib/Context.py
+index 761b521f5..369664819 100644
+--- a/waflib/Context.py
++++ b/waflib/Context.py
+@@ -6,20 +6,30 @@
+ Classes and functions enabling the command system
+ """
+
+-import os, re, imp, sys
++import os, re, sys
+ from waflib import Utils, Errors, Logs
+ import waflib.Node
+
++if sys.hexversion > 0x3040000:
++ import types
++ class imp(object):
++ new_module = lambda x: types.ModuleType(x)
++else:
++ import imp
++
+ # the following 3 constants are updated on each new release (do not touch)
+-HEXVERSION=0x2000c00
++HEXVERSION=0x2001a00
+ """Constant updated on new releases"""
+
+-WAFVERSION="2.0.12"
++WAFVERSION="2.0.26"
+ """Constant updated on new releases"""
+
+-WAFREVISION="54841218840ffa34fddf834680a5a17db69caa12"
++WAFREVISION="0fb985ce1932c6f3e7533f435e4ee209d673776e"
+ """Git revision when the waf version is updated"""
+
++WAFNAME="waf"
++"""Application name displayed on --help"""
++
+ ABI = 20
+ """Version of the build data cache file format (used in :py:const:`waflib.Context.DBFILE`)"""
+
+@@ -134,7 +144,7 @@ def foo(ctx):
+ :type fun: string
+
+ .. inheritance-diagram:: waflib.Context.Context waflib.Build.BuildContext waflib.Build.InstallContext waflib.Build.UninstallContext waflib.Build.StepContext waflib.Build.ListContext waflib.Configure.ConfigurationContext waflib.Scripting.Dist waflib.Scripting.DistCheck waflib.Build.CleanContext
+-
++ :top-classes: waflib.Context.Context
+ """
+
+ errors = Errors
+@@ -613,7 +623,7 @@ def load_special_tools(self, var, ban=[]):
+ is typically called once for a programming language group, see for
+ example :py:mod:`waflib.Tools.compiler_c`
+
+- :param var: glob expression, for example 'cxx\_\*.py'
++ :param var: glob expression, for example 'cxx\\_\\*.py'
+ :type var: string
+ :param ban: list of exact file names to exclude
+ :type ban: list of string
+@@ -678,7 +688,7 @@ def load_module(path, encoding=None):
+
+ def load_tool(tool, tooldir=None, ctx=None, with_sys_path=True):
+ """
+- Importx a Waf tool as a python module, and stores it in the dict :py:const:`waflib.Context.Context.tools`
++ Imports a Waf tool as a python module, and stores it in the dict :py:const:`waflib.Context.Context.tools`
+
+ :type tool: string
+ :param tool: Name of the tool
+diff --git a/waflib/Logs.py b/waflib/Logs.py
+index 2a475169b..298411db5 100644
+--- a/waflib/Logs.py
++++ b/waflib/Logs.py
+@@ -237,7 +237,10 @@ def format(self, rec):
+ if rec.levelno >= logging.INFO:
+ # the goal of this is to format without the leading "Logs, hour" prefix
+ if rec.args:
+- return msg % rec.args
++ try:
++ return msg % rec.args
++ except UnicodeDecodeError:
++ return msg.encode('utf-8') % rec.args
+ return msg
+
+ rec.msg = msg
+@@ -276,9 +279,9 @@ def error(*k, **kw):
+
+ def warn(*k, **kw):
+ """
+- Wraps logging.warn
++ Wraps logging.warning
+ """
+- log.warn(*k, **kw)
++ log.warning(*k, **kw)
+
+ def info(*k, **kw):
+ """
+diff --git a/waflib/Node.py b/waflib/Node.py
+index 4ac1ea8a0..2ad184669 100644
+--- a/waflib/Node.py
++++ b/waflib/Node.py
+@@ -73,7 +73,7 @@ def ant_matcher(s, ignorecase):
+ if k == '**':
+ accu.append(k)
+ else:
+- k = k.replace('.', '[.]').replace('*','.*').replace('?', '.').replace('+', '\\+')
++ k = k.replace('.', '[.]').replace('*', '.*').replace('?', '.').replace('+', '\\+')
+ k = '^%s$' % k
+ try:
+ exp = re.compile(k, flags=reflags)
+@@ -595,7 +595,6 @@ def ant_iter(self, accept=None, maxdepth=25, pats=[], dir=False, src=True, remov
+ :rtype: iterator
+ """
+ dircont = self.listdir()
+- dircont.sort()
+
+ try:
+ lst = set(self.children.keys())
+diff --git a/waflib/Options.py b/waflib/Options.py
+index ad802d4b9..d4104917c 100644
+--- a/waflib/Options.py
++++ b/waflib/Options.py
+@@ -44,7 +44,7 @@ class opt_parser(optparse.OptionParser):
+ """
+ def __init__(self, ctx, allow_unknown=False):
+ optparse.OptionParser.__init__(self, conflict_handler='resolve', add_help_option=False,
+- version='waf %s (%s)' % (Context.WAFVERSION, Context.WAFREVISION))
++ version='%s %s (%s)' % (Context.WAFNAME, Context.WAFVERSION, Context.WAFREVISION))
+ self.formatter.width = Logs.get_term_cols()
+ self.ctx = ctx
+ self.allow_unknown = allow_unknown
+@@ -62,6 +62,21 @@ def _process_args(self, largs, rargs, values):
+ else:
+ self.error(str(e))
+
++ def _process_long_opt(self, rargs, values):
++ # --custom-option=-ftxyz is interpreted as -f -t... see #2280
++ if self.allow_unknown:
++ back = [] + rargs
++ try:
++ optparse.OptionParser._process_long_opt(self, rargs, values)
++ except optparse.BadOptionError:
++ while rargs:
++ rargs.pop()
++ rargs.extend(back)
++ rargs.pop(0)
++ raise
++ else:
++ optparse.OptionParser._process_long_opt(self, rargs, values)
++
+ def print_usage(self, file=None):
+ return self.print_help(file)
+
+@@ -96,11 +111,11 @@ def get_usage(self):
+ lst.sort()
+ ret = '\n'.join(lst)
+
+- return '''waf [commands] [options]
++ return '''%s [commands] [options]
+
+-Main commands (example: ./waf build -j4)
++Main commands (example: ./%s build -j4)
+ %s
+-''' % ret
++''' % (Context.WAFNAME, Context.WAFNAME, ret)
+
+
+ class OptionsContext(Context.Context):
+@@ -141,9 +156,9 @@ def __init__(self, **kw):
+ gr.add_option('-o', '--out', action='store', default='', help='build dir for the project', dest='out')
+ gr.add_option('-t', '--top', action='store', default='', help='src dir for the project', dest='top')
+
+- gr.add_option('--no-lock-in-run', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_run')
+- gr.add_option('--no-lock-in-out', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_out')
+- gr.add_option('--no-lock-in-top', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_top')
++ gr.add_option('--no-lock-in-run', action='store_true', default=os.environ.get('NO_LOCK_IN_RUN', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_run')
++ gr.add_option('--no-lock-in-out', action='store_true', default=os.environ.get('NO_LOCK_IN_OUT', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_out')
++ gr.add_option('--no-lock-in-top', action='store_true', default=os.environ.get('NO_LOCK_IN_TOP', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_top')
+
+ default_prefix = getattr(Context.g_module, 'default_prefix', os.environ.get('PREFIX'))
+ if not default_prefix:
+@@ -282,6 +297,8 @@ def parse_cmd_args(self, _args=None, cwd=None, allow_unknown=False):
+ elif arg != 'options':
+ commands.append(arg)
+
++ if options.jobs < 1:
++ options.jobs = 1
+ for name in 'top out destdir prefix bindir libdir'.split():
+ # those paths are usually expanded from Context.launch_dir
+ if getattr(options, name, None):
+diff --git a/waflib/Runner.py b/waflib/Runner.py
+index 261084d27..350c86a22 100644
+--- a/waflib/Runner.py
++++ b/waflib/Runner.py
+@@ -37,6 +37,8 @@ def __len__(self):
+ return len(self.lst)
+ def __iter__(self):
+ return iter(self.lst)
++ def __str__(self):
++ return 'PriorityTasks: [%s]' % '\n '.join(str(x) for x in self.lst)
+ def clear(self):
+ self.lst = []
+ def append(self, task):
+@@ -69,7 +71,7 @@ def __init__(self, spawner, task):
+ """Task to execute"""
+ self.spawner = spawner
+ """Coordinator object"""
+- self.setDaemon(1)
++ self.daemon = True
+ self.start()
+ def run(self):
+ """
+@@ -96,7 +98,7 @@ def __init__(self, master):
+ """:py:class:`waflib.Runner.Parallel` producer instance"""
+ self.sem = Utils.threading.Semaphore(master.numjobs)
+ """Bounded semaphore that prevents spawning more than *n* concurrent consumers"""
+- self.setDaemon(1)
++ self.daemon = True
+ self.start()
+ def run(self):
+ """
+@@ -181,10 +183,12 @@ def __init__(self, bld, j=2):
+ The reverse dependency graph of dependencies obtained from Task.run_after
+ """
+
+- self.spawner = Spawner(self)
++ self.spawner = None
+ """
+ Coordinating daemon thread that spawns thread consumers
+ """
++ if self.numjobs > 1:
++ self.spawner = Spawner(self)
+
+ def get_next_task(self):
+ """
+@@ -254,6 +258,8 @@ def refill_task_list(self):
+ self.outstanding.append(x)
+ break
+ else:
++ if self.stop or self.error:
++ break
+ raise Errors.WafError('Broken revdeps detected on %r' % self.incomplete)
+ else:
+ tasks = next(self.biter)
+@@ -331,11 +337,16 @@ def try_unfreeze(x):
+
+ if hasattr(tsk, 'semaphore'):
+ sem = tsk.semaphore
+- sem.release(tsk)
+- while sem.waiting and not sem.is_locked():
+- # take a frozen task, make it ready to run
+- x = sem.waiting.pop()
+- self._add_task(x)
++ try:
++ sem.release(tsk)
++ except KeyError:
++ # TODO
++ pass
++ else:
++ while sem.waiting and not sem.is_locked():
++ # take a frozen task, make it ready to run
++ x = sem.waiting.pop()
++ self._add_task(x)
+
+ def get_out(self):
+ """
+diff --git a/waflib/Scripting.py b/waflib/Scripting.py
+index 749d4f2e6..a80cb3678 100644
+--- a/waflib/Scripting.py
++++ b/waflib/Scripting.py
+@@ -216,7 +216,10 @@ def parse_options():
+ ctx = Context.create_context('options')
+ ctx.execute()
+ if not Options.commands:
+- Options.commands.append(default_cmd)
++ if isinstance(default_cmd, list):
++ Options.commands.extend(default_cmd)
++ else:
++ Options.commands.append(default_cmd)
+ if Options.options.whelp:
+ ctx.parser.print_help()
+ sys.exit(0)
+@@ -280,7 +283,7 @@ def distclean_dir(dirname):
+ pass
+
+ try:
+- shutil.rmtree('c4che')
++ shutil.rmtree(Build.CACHE_DIR)
+ except OSError:
+ pass
+
+@@ -303,7 +306,7 @@ def remove_and_log(k, fun):
+
+ # remove a build folder, if any
+ cur = '.'
+- if ctx.options.no_lock_in_top:
++ if os.environ.get('NO_LOCK_IN_TOP') or ctx.options.no_lock_in_top:
+ cur = ctx.options.out
+
+ try:
+@@ -329,7 +332,12 @@ def remove_and_log(k, fun):
+ else:
+ remove_and_log(env.out_dir, shutil.rmtree)
+
+- for k in (env.out_dir, env.top_dir, env.run_dir):
++ env_dirs = [env.out_dir]
++ if not (os.environ.get('NO_LOCK_IN_TOP') or ctx.options.no_lock_in_top):
++ env_dirs.append(env.top_dir)
++ if not (os.environ.get('NO_LOCK_IN_RUN') or ctx.options.no_lock_in_run):
++ env_dirs.append(env.run_dir)
++ for k in env_dirs:
+ p = os.path.join(k, Options.lockfile)
+ remove_and_log(p, os.remove)
+
+@@ -380,7 +388,11 @@ def archive(self):
+
+ for x in files:
+ archive_name = self.get_base_name() + '/' + x.path_from(self.base_path)
+- zip.write(x.abspath(), archive_name, zipfile.ZIP_DEFLATED)
++ if os.environ.get('SOURCE_DATE_EPOCH'):
++ # TODO: parse that timestamp
++ zip.writestr(zipfile.ZipInfo(archive_name), x.read(), zipfile.ZIP_DEFLATED)
++ else:
++ zip.write(x.abspath(), archive_name, zipfile.ZIP_DEFLATED)
+ zip.close()
+ else:
+ self.fatal('Valid algo types are tar.bz2, tar.gz, tar.xz or zip')
+@@ -417,6 +429,8 @@ def add_tar_file(self, x, tar):
+ tinfo.gid = 0
+ tinfo.uname = 'root'
+ tinfo.gname = 'root'
++ if os.environ.get('SOURCE_DATE_EPOCH'):
++ tinfo.mtime = int(os.environ.get('SOURCE_DATE_EPOCH'))
+
+ if os.path.isfile(p):
+ with open(p, 'rb') as f:
+@@ -598,12 +612,15 @@ def execute(self):
+ cmd = env.config_cmd or 'configure'
+ if Configure.autoconfig == 'clobber':
+ tmp = Options.options.__dict__
++ launch_dir_tmp = Context.launch_dir
+ if env.options:
+ Options.options.__dict__ = env.options
++ Context.launch_dir = env.launch_dir
+ try:
+ run_command(cmd)
+ finally:
+ Options.options.__dict__ = tmp
++ Context.launch_dir = launch_dir_tmp
+ else:
+ run_command(cmd)
+ run_command(self.cmd)
+diff --git a/waflib/Task.py b/waflib/Task.py
+index 6aebc6074..cb49a7394 100644
+--- a/waflib/Task.py
++++ b/waflib/Task.py
+@@ -163,10 +163,10 @@ class Task(evil):
+ """File extensions that objects of this task class may create"""
+
+ before = []
+- """List of task class names to execute before instances of this class"""
++ """The instances of this class are executed before the instances of classes whose names are in this list"""
+
+ after = []
+- """List of task class names to execute after instances of this class"""
++ """The instances of this class are executed after the instances of classes whose names are in this list"""
+
+ hcode = Utils.SIG_NIL
+ """String representing an additional hash for the class representation"""
+@@ -306,25 +306,31 @@ def exec_command(self, cmd, **kw):
+ if hasattr(self, 'stderr'):
+ kw['stderr'] = self.stderr
+
+- # workaround for command line length limit:
+- # http://support.microsoft.com/kb/830473
+- if not isinstance(cmd, str) and (len(repr(cmd)) >= 8192 if Utils.is_win32 else len(cmd) > 200000):
+- cmd, args = self.split_argfile(cmd)
+- try:
+- (fd, tmp) = tempfile.mkstemp()
+- os.write(fd, '\r\n'.join(args).encode())
+- os.close(fd)
+- if Logs.verbose:
+- Logs.debug('argfile: @%r -> %r', tmp, args)
+- return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw)
+- finally:
++ if not isinstance(cmd, str):
++ if Utils.is_win32:
++ # win32 compares the resulting length http://support.microsoft.com/kb/830473
++ too_long = sum([len(arg) for arg in cmd]) + len(cmd) > 8192
++ else:
++ # non-win32 counts the amount of arguments (200k)
++ too_long = len(cmd) > 200000
++
++ if too_long and getattr(self, 'allow_argsfile', True):
++ # Shunt arguments to a temporary file if the command is too long.
++ cmd, args = self.split_argfile(cmd)
+ try:
+- os.remove(tmp)
+- except OSError:
+- # anti-virus and indexers can keep files open -_-
+- pass
+- else:
+- return self.generator.bld.exec_command(cmd, **kw)
++ (fd, tmp) = tempfile.mkstemp()
++ os.write(fd, '\r\n'.join(args).encode())
++ os.close(fd)
++ if Logs.verbose:
++ Logs.debug('argfile: @%r -> %r', tmp, args)
++ return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw)
++ finally:
++ try:
++ os.remove(tmp)
++ except OSError:
++ # anti-virus and indexers can keep files open -_-
++ pass
++ return self.generator.bld.exec_command(cmd, **kw)
+
+ def process(self):
+ """
+@@ -1044,7 +1050,7 @@ def funex(c):
+ exec(c, dc)
+ return dc['f']
+
+-re_cond = re.compile('(?P<var>\w+)|(?P<or>\|)|(?P<and>&)')
++re_cond = re.compile(r'(?P<var>\w+)|(?P<or>\|)|(?P<and>&)')
+ re_novar = re.compile(r'^(SRC|TGT)\W+.*?$')
+ reg_act = re.compile(r'(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})', re.M)
+ def compile_fun_shell(line):
+diff --git a/waflib/TaskGen.py b/waflib/TaskGen.py
+index a74e6431d..32468f03d 100644
+--- a/waflib/TaskGen.py
++++ b/waflib/TaskGen.py
+@@ -74,7 +74,7 @@ def __init__(self, *k, **kw):
+ else:
+ self.bld = kw['bld']
+ self.env = self.bld.env.derive()
+- self.path = self.bld.path # emulate chdir when reading scripts
++ self.path = kw.get('path', self.bld.path) # by default, emulate chdir when reading scripts
+
+ # Provide a unique index per folder
+ # This is part of a measure to prevent output file name collisions
+@@ -400,7 +400,7 @@ def feature(*k):
+ Decorator that registers a task generator method that will be executed when the
+ object attribute ``feature`` contains the corresponding key(s)::
+
+- from waflib.Task import feature
++ from waflib.TaskGen import feature
+ @feature('myfeature')
+ def myfunction(self):
+ print('that is my feature!')
+@@ -631,12 +631,8 @@ def chmod_fun(tsk):
+ cls.scan = self.scan
+ elif has_deps:
+ def scan(self):
+- nodes = []
+- for x in self.generator.to_list(getattr(self.generator, 'deps', None)):
+- node = self.generator.path.find_resource(x)
+- if not node:
+- self.generator.bld.fatal('Could not find %r (was it declared?)' % x)
+- nodes.append(node)
++ deps = getattr(self.generator, 'deps', None)
++ nodes = self.generator.to_nodes(deps)
+ return [nodes, []]
+ cls.scan = scan
+
+@@ -727,7 +723,7 @@ def sequence_order(self):
+ self.bld.prev = self
+
+
+-re_m4 = re.compile('@(\w+)@', re.M)
++re_m4 = re.compile(r'@(\w+)@', re.M)
+
+ class subst_pc(Task.Task):
+ """
+@@ -905,7 +901,7 @@ def build(bld):
+ # paranoid safety measure for the general case foo.in->foo.h with ambiguous dependencies
+ for xt in HEADER_EXTS:
+ if b.name.endswith(xt):
+- tsk.ext_in = tsk.ext_in + ['.h']
++ tsk.ext_out = tsk.ext_out + ['.h']
+ break
+
+ inst_to = getattr(self, 'install_path', None)
+diff --git a/waflib/Tools/c_aliases.py b/waflib/Tools/c_aliases.py
+index c9d53692e..928cfe29c 100644
+--- a/waflib/Tools/c_aliases.py
++++ b/waflib/Tools/c_aliases.py
+@@ -38,7 +38,7 @@ def sniff_features(**kw):
+ :return: the list of features for a task generator processing the source files
+ :rtype: list of string
+ """
+- exts = get_extensions(kw['source'])
++ exts = get_extensions(kw.get('source', []))
+ typ = kw['typ']
+ feats = []
+
+@@ -47,10 +47,12 @@ def sniff_features(**kw):
+ if x in exts:
+ feats.append('cxx')
+ break
+-
+ if 'c' in exts or 'vala' in exts or 'gs' in exts:
+ feats.append('c')
+
++ if 's' in exts or 'S' in exts:
++ feats.append('asm')
++
+ for x in 'f f90 F F90 for FOR'.split():
+ if x in exts:
+ feats.append('fc')
+@@ -66,11 +68,11 @@ def sniff_features(**kw):
+ if typ in ('program', 'shlib', 'stlib'):
+ will_link = False
+ for x in feats:
+- if x in ('cxx', 'd', 'fc', 'c'):
++ if x in ('cxx', 'd', 'fc', 'c', 'asm'):
+ feats.append(x + typ)
+ will_link = True
+ if not will_link and not kw.get('features', []):
+- raise Errors.WafError('Cannot link from %r, try passing eg: features="c cprogram"?' % kw)
++ raise Errors.WafError('Unable to determine how to link %r, try adding eg: features="c cshlib"?' % kw)
+ return feats
+
+ def set_features(kw, typ):
+diff --git a/waflib/Tools/c_config.py b/waflib/Tools/c_config.py
+index d2b3c0d8f..f5ab19bf6 100644
+--- a/waflib/Tools/c_config.py
++++ b/waflib/Tools/c_config.py
+@@ -68,6 +68,8 @@
+ '__s390__' : 's390',
+ '__sh__' : 'sh',
+ '__xtensa__' : 'xtensa',
++'__e2k__' : 'e2k',
++'__riscv' : 'riscv',
+ }
+
+ @conf
+@@ -86,6 +88,10 @@ def configure(conf):
+ :type uselib_store: string
+ :param env: config set or conf.env by default
+ :type env: :py:class:`waflib.ConfigSet.ConfigSet`
++ :param force_static: force usage of static libraries
++ :type force_static: bool default False
++ :param posix: usage of POSIX mode for shlex lexical analiysis library
++ :type posix: bool default True
+ """
+
+ assert(isinstance(line, str))
+@@ -103,6 +109,8 @@ def configure(conf):
+ lex.commenters = ''
+ lst = list(lex)
+
++ so_re = re.compile(r"\.so(?:\.[0-9]+)*$")
++
+ # append_unique is not always possible
+ # for example, apple flags may require both -arch i386 and -arch ppc
+ uselib = uselib_store
+@@ -144,7 +152,7 @@ def appu(var, val):
+ elif x.startswith('-std='):
+ prefix = 'CXXFLAGS' if '++' in x else 'CFLAGS'
+ app(prefix, x)
+- elif x.startswith('+') or x in ('-pthread', '-fPIC', '-fpic', '-fPIE', '-fpie'):
++ elif x.startswith('+') or x in ('-pthread', '-fPIC', '-fpic', '-fPIE', '-fpie', '-flto', '-fno-lto'):
+ app('CFLAGS', x)
+ app('CXXFLAGS', x)
+ app('LINKFLAGS', x)
+@@ -180,7 +188,7 @@ def appu(var, val):
+ app('CFLAGS', tmp)
+ app('CXXFLAGS', tmp)
+ app('LINKFLAGS', tmp)
+- elif x.endswith(('.a', '.so', '.dylib', '.lib')):
++ elif x.endswith(('.a', '.dylib', '.lib')) or so_re.search(x):
+ appu('LINKFLAGS', x) # not cool, #762
+ else:
+ self.to_log('Unhandled flag %r' % x)
+@@ -246,13 +254,15 @@ def exec_cfg(self, kw):
+ * if modversion is given, then return the module version
+ * else, execute the *-config* program with the *args* and *variables* given, and set the flags on the *conf.env.FLAGS_name* variable
+
++ :param path: the **-config program to use**
++ :type path: list of string
+ :param atleast_pkgconfig_version: minimum pkg-config version to use (disable other tests)
+ :type atleast_pkgconfig_version: string
+ :param package: package name, for example *gtk+-2.0*
+ :type package: string
+- :param uselib_store: if the test is successful, define HAVE\_*name*. It is also used to define *conf.env.FLAGS_name* variables.
++ :param uselib_store: if the test is successful, define HAVE\\_*name*. It is also used to define *conf.env.FLAGS_name* variables.
+ :type uselib_store: string
+- :param modversion: if provided, return the version of the given module and define *name*\_VERSION
++ :param modversion: if provided, return the version of the given module and define *name*\\_VERSION
+ :type modversion: string
+ :param args: arguments to give to *package* when retrieving flags
+ :type args: list of string
+@@ -260,6 +270,12 @@ def exec_cfg(self, kw):
+ :type variables: list of string
+ :param define_variable: additional variables to define (also in conf.env.PKG_CONFIG_DEFINES)
+ :type define_variable: dict(string: string)
++ :param pkg_config_path: paths where pkg-config should search for .pc config files (overrides env.PKG_CONFIG_PATH if exists)
++ :type pkg_config_path: string, list of directories separated by colon
++ :param force_static: force usage of static libraries
++ :type force_static: bool default False
++ :param posix: usage of POSIX mode for shlex lexical analiysis library
++ :type posix: bool default True
+ """
+
+ path = Utils.to_list(kw['path'])
+@@ -334,6 +350,7 @@ def check_cfg(self, *k, **kw):
+ """
+ Checks for configuration flags using a **-config**-like program (pkg-config, sdl-config, etc).
+ This wraps internal calls to :py:func:`waflib.Tools.c_config.validate_cfg` and :py:func:`waflib.Tools.c_config.exec_cfg`
++ so check exec_cfg parameters descriptions for more details on kw passed
+
+ A few examples::
+
+@@ -659,20 +676,21 @@ class test_exec(Task.Task):
+ """
+ color = 'PINK'
+ def run(self):
++ cmd = [self.inputs[0].abspath()] + getattr(self.generator, 'test_args', [])
+ if getattr(self.generator, 'rpath', None):
+ if getattr(self.generator, 'define_ret', False):
+- self.generator.bld.retval = self.generator.bld.cmd_and_log([self.inputs[0].abspath()])
++ self.generator.bld.retval = self.generator.bld.cmd_and_log(cmd)
+ else:
+- self.generator.bld.retval = self.generator.bld.exec_command([self.inputs[0].abspath()])
++ self.generator.bld.retval = self.generator.bld.exec_command(cmd)
+ else:
+ env = self.env.env or {}
+ env.update(dict(os.environ))
+ for var in ('LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'PATH'):
+ env[var] = self.inputs[0].parent.abspath() + os.path.pathsep + env.get(var, '')
+ if getattr(self.generator, 'define_ret', False):
+- self.generator.bld.retval = self.generator.bld.cmd_and_log([self.inputs[0].abspath()], env=env)
++ self.generator.bld.retval = self.generator.bld.cmd_and_log(cmd, env=env)
+ else:
+- self.generator.bld.retval = self.generator.bld.exec_command([self.inputs[0].abspath()], env=env)
++ self.generator.bld.retval = self.generator.bld.exec_command(cmd, env=env)
+
+ @feature('test_exec')
+ @after_method('apply_link')
+@@ -1266,10 +1284,11 @@ def to_log(self, *k, **kw):
+ tasks = []
+
+ id_to_task = {}
+- for dct in k:
++ for counter, dct in enumerate(k):
+ x = Task.classes['cfgtask'](bld=bld, env=None)
+ tasks.append(x)
+ x.args = dct
++ x.args['multicheck_counter'] = counter
+ x.bld = bld
+ x.conf = self
+ x.args = dct
+diff --git a/waflib/Tools/c_preproc.py b/waflib/Tools/c_preproc.py
+index 7e04b4a7c..68e5f5aea 100644
+--- a/waflib/Tools/c_preproc.py
++++ b/waflib/Tools/c_preproc.py
+@@ -75,13 +75,13 @@ class PreprocError(Errors.WafError):
+ re.IGNORECASE | re.MULTILINE)
+ """Match #include lines"""
+
+-re_mac = re.compile("^[a-zA-Z_]\w*")
++re_mac = re.compile(r"^[a-zA-Z_]\w*")
+ """Match macro definitions"""
+
+ re_fun = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*[(]')
+ """Match macro functions"""
+
+-re_pragma_once = re.compile('^\s*once\s*', re.IGNORECASE)
++re_pragma_once = re.compile(r'^\s*once\s*', re.IGNORECASE)
+ """Match #pragma once statements"""
+
+ re_nl = re.compile('\\\\\r*\n', re.MULTILINE)
+@@ -660,7 +660,7 @@ def extract_macro(txt):
+ # empty define, assign an empty token
+ return (v, [[], [('T','')]])
+
+-re_include = re.compile('^\s*(<(?:.*)>|"(?:.*)")')
++re_include = re.compile(r'^\s*(<(?:.*)>|"(?:.*)")')
+ def extract_include(txt, defs):
+ """
+ Process a line in the form::
+diff --git a/waflib/Tools/c_tests.py b/waflib/Tools/c_tests.py
+index f858df576..bdd186c6b 100644
+--- a/waflib/Tools/c_tests.py
++++ b/waflib/Tools/c_tests.py
+@@ -180,9 +180,15 @@ def check_large_file(self, **kw):
+ ########################################################################################
+
+ ENDIAN_FRAGMENT = '''
++#ifdef _MSC_VER
++#define testshlib_EXPORT __declspec(dllexport)
++#else
++#define testshlib_EXPORT
++#endif
++
+ short int ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+ short int ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+-int use_ascii (int i) {
++int testshlib_EXPORT use_ascii (int i) {
+ return ascii_mm[i] + ascii_ii[i];
+ }
+ short int ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+@@ -208,12 +214,12 @@ def run(self):
+ return -1
+
+ @feature('grep_for_endianness')
+-@after_method('process_source')
++@after_method('apply_link')
+ def grep_for_endianness_fun(self):
+ """
+ Used by the endianness configuration test
+ """
+- self.create_task('grep_for_endianness', self.compiled_tasks[0].outputs[0])
++ self.create_task('grep_for_endianness', self.link_task.outputs[0])
+
+ @conf
+ def check_endianness(self):
+@@ -223,7 +229,9 @@ def check_endianness(self):
+ tmp = []
+ def check_msg(self):
+ return tmp[0]
+- self.check(fragment=ENDIAN_FRAGMENT, features='c grep_for_endianness',
+- msg='Checking for endianness', define='ENDIANNESS', tmp=tmp, okmsg=check_msg)
++
++ self.check(fragment=ENDIAN_FRAGMENT, features='c cshlib grep_for_endianness',
++ msg='Checking for endianness', define='ENDIANNESS', tmp=tmp,
++ okmsg=check_msg, confcache=None)
+ return tmp[0]
+
+diff --git a/waflib/Tools/ccroot.py b/waflib/Tools/ccroot.py
+index 484846f5f..533992903 100644
+--- a/waflib/Tools/ccroot.py
++++ b/waflib/Tools/ccroot.py
+@@ -111,7 +111,7 @@ def apply_incpaths(self):
+ tg = bld(features='includes', includes='.')
+
+ The folders only need to be relative to the current directory, the equivalent build directory is
+- added automatically (for headers created in the build directory). This enable using a build directory
++ added automatically (for headers created in the build directory). This enables using a build directory
+ or not (``top == out``).
+
+ This method will add a list of nodes read by :py:func:`waflib.Tools.ccroot.to_incnodes` in ``tg.env.INCPATHS``,
+@@ -128,6 +128,7 @@ class link_task(Task.Task):
+ Base class for all link tasks. A task generator is supposed to have at most one link task bound in the attribute *link_task*. See :py:func:`waflib.Tools.ccroot.apply_link`.
+
+ .. inheritance-diagram:: waflib.Tools.ccroot.stlink_task waflib.Tools.c.cprogram waflib.Tools.c.cshlib waflib.Tools.cxx.cxxstlib waflib.Tools.cxx.cxxprogram waflib.Tools.cxx.cxxshlib waflib.Tools.d.dprogram waflib.Tools.d.dshlib waflib.Tools.d.dstlib waflib.Tools.ccroot.fake_shlib waflib.Tools.ccroot.fake_stlib waflib.Tools.asm.asmprogram waflib.Tools.asm.asmshlib waflib.Tools.asm.asmstlib
++ :top-classes: waflib.Tools.ccroot.link_task
+ """
+ color = 'YELLOW'
+
+@@ -238,6 +239,17 @@ def wrap(self):
+ setattr(cls, 'run', wrap)
+ rm_tgt(stlink_task)
+
++@feature('skip_stlib_link_deps')
++@before_method('process_use')
++def apply_skip_stlib_link_deps(self):
++ """
++ This enables an optimization in the :py:func:wafilb.Tools.ccroot.processes_use: method that skips dependency and
++ link flag optimizations for targets that generate static libraries (via the :py:class:Tools.ccroot.stlink_task task).
++ The actual behavior is implemented in :py:func:wafilb.Tools.ccroot.processes_use: method so this feature only tells waf
++ to enable the new behavior.
++ """
++ self.env.SKIP_STLIB_LINK_DEPS = True
++
+ @feature('c', 'cxx', 'd', 'fc', 'asm')
+ @after_method('process_source')
+ def apply_link(self):
+@@ -386,7 +398,11 @@ def build(bld):
+ y = self.bld.get_tgen_by_name(x)
+ var = y.tmp_use_var
+ if var and link_task:
+- if var == 'LIB' or y.tmp_use_stlib or x in names:
++ if self.env.SKIP_STLIB_LINK_DEPS and isinstance(link_task, stlink_task):
++ # If the skip_stlib_link_deps feature is enabled then we should
++ # avoid adding lib deps to the stlink_task instance.
++ pass
++ elif var == 'LIB' or y.tmp_use_stlib or x in names:
+ self.env.append_value(var, [y.target[y.target.rfind(os.sep) + 1:]])
+ self.link_task.dep_nodes.extend(y.link_task.outputs)
+ tmp_path = y.link_task.outputs[0].parent.path_from(self.get_cwd())
+diff --git a/waflib/Tools/compiler_c.py b/waflib/Tools/compiler_c.py
+index 2dba3f827..e033ce6c5 100644
+--- a/waflib/Tools/compiler_c.py
++++ b/waflib/Tools/compiler_c.py
+@@ -36,18 +36,19 @@ def build(bld):
+ from waflib.Logs import debug
+
+ c_compiler = {
+-'win32': ['msvc', 'gcc', 'clang'],
+-'cygwin': ['gcc'],
+-'darwin': ['clang', 'gcc'],
+-'aix': ['xlc', 'gcc', 'clang'],
+-'linux': ['gcc', 'clang', 'icc'],
+-'sunos': ['suncc', 'gcc'],
+-'irix': ['gcc', 'irixcc'],
+-'hpux': ['gcc'],
+-'osf1V': ['gcc'],
+-'gnu': ['gcc', 'clang'],
+-'java': ['gcc', 'msvc', 'clang', 'icc'],
+-'default':['clang', 'gcc'],
++'win32': ['msvc', 'gcc', 'clang'],
++'cygwin': ['gcc', 'clang'],
++'darwin': ['clang', 'gcc'],
++'aix': ['xlc', 'gcc', 'clang'],
++'linux': ['gcc', 'clang', 'icc'],
++'sunos': ['suncc', 'gcc'],
++'irix': ['gcc', 'irixcc'],
++'hpux': ['gcc'],
++'osf1V': ['gcc'],
++'gnu': ['gcc', 'clang'],
++'java': ['gcc', 'msvc', 'clang', 'icc'],
++'gnukfreebsd': ['gcc', 'clang'],
++'default': ['clang', 'gcc'],
+ }
+ """
+ Dict mapping platform names to Waf tools finding specific C compilers::
+diff --git a/waflib/Tools/compiler_cxx.py b/waflib/Tools/compiler_cxx.py
+index 1af65a226..42658c584 100644
+--- a/waflib/Tools/compiler_cxx.py
++++ b/waflib/Tools/compiler_cxx.py
+@@ -37,18 +37,19 @@ def build(bld):
+ from waflib.Logs import debug
+
+ cxx_compiler = {
+-'win32': ['msvc', 'g++', 'clang++'],
+-'cygwin': ['g++'],
+-'darwin': ['clang++', 'g++'],
+-'aix': ['xlc++', 'g++', 'clang++'],
+-'linux': ['g++', 'clang++', 'icpc'],
+-'sunos': ['sunc++', 'g++'],
+-'irix': ['g++'],
+-'hpux': ['g++'],
+-'osf1V': ['g++'],
+-'gnu': ['g++', 'clang++'],
+-'java': ['g++', 'msvc', 'clang++', 'icpc'],
+-'default': ['clang++', 'g++']
++'win32': ['msvc', 'g++', 'clang++'],
++'cygwin': ['g++', 'clang++'],
++'darwin': ['clang++', 'g++'],
++'aix': ['xlc++', 'g++', 'clang++'],
++'linux': ['g++', 'clang++', 'icpc'],
++'sunos': ['sunc++', 'g++'],
++'irix': ['g++'],
++'hpux': ['g++'],
++'osf1V': ['g++'],
++'gnu': ['g++', 'clang++'],
++'java': ['g++', 'msvc', 'clang++', 'icpc'],
++'gnukfreebsd': ['g++', 'clang++'],
++'default': ['clang++', 'g++']
+ }
+ """
+ Dict mapping the platform names to Waf tools finding specific C++ compilers::
+diff --git a/waflib/Tools/irixcc.py b/waflib/Tools/irixcc.py
+index c3ae1ac91..0335c13cb 100644
+--- a/waflib/Tools/irixcc.py
++++ b/waflib/Tools/irixcc.py
+@@ -13,22 +13,11 @@
+ @conf
+ def find_irixcc(conf):
+ v = conf.env
+- cc = None
+- if v.CC:
+- cc = v.CC
+- elif 'CC' in conf.environ:
+- cc = conf.environ['CC']
+- if not cc:
+- cc = conf.find_program('cc', var='CC')
+- if not cc:
+- conf.fatal('irixcc was not found')
+-
++ cc = conf.find_program('cc', var='CC')
+ try:
+ conf.cmd_and_log(cc + ['-version'])
+ except Errors.WafError:
+ conf.fatal('%r -version could not be executed' % cc)
+-
+- v.CC = cc
+ v.CC_NAME = 'irix'
+
+ @conf
+@@ -57,7 +46,6 @@ def irixcc_common_flags(conf):
+
+ def configure(conf):
+ conf.find_irixcc()
+- conf.find_cpp()
+ conf.find_ar()
+ conf.irixcc_common_flags()
+ conf.cc_load_tools()
+diff --git a/waflib/Tools/msvc.py b/waflib/Tools/msvc.py
+index 17b347d45..d60f67026 100644
+--- a/waflib/Tools/msvc.py
++++ b/waflib/Tools/msvc.py
+@@ -99,10 +99,31 @@ def build(bld):
+ """List of icl platforms"""
+
+ def options(opt):
+- opt.add_option('--msvc_version', type='string', help = 'msvc version, eg: "msvc 10.0,msvc 9.0"', default='')
++ default_ver = ''
++ vsver = os.getenv('VSCMD_VER')
++ if vsver:
++ m = re.match(r'(^\d+\.\d+).*', vsver)
++ if m:
++ default_ver = 'msvc %s' % m.group(1)
++ opt.add_option('--msvc_version', type='string', help = 'msvc version, eg: "msvc 10.0,msvc 9.0"', default=default_ver)
+ opt.add_option('--msvc_targets', type='string', help = 'msvc targets, eg: "x64,arm"', default='')
+ opt.add_option('--no-msvc-lazy', action='store_false', help = 'lazily check msvc target environments', default=True, dest='msvc_lazy')
+
++class MSVCVersion(object):
++ def __init__(self, ver):
++ m = re.search(r'^(.*)\s+(\d+[.]\d+)', ver)
++ if m:
++ self.name = m.group(1)
++ self.number = float(m.group(2))
++ else:
++ self.name = ver
++ self.number = 0.
++
++ def __lt__(self, other):
++ if self.number == other.number:
++ return self.name < other.name
++ return self.number < other.number
++
+ @conf
+ def setup_msvc(conf, versiondict):
+ """
+@@ -119,7 +140,7 @@ def setup_msvc(conf, versiondict):
+ platforms=Utils.to_list(conf.env.MSVC_TARGETS) or [i for i,j in all_msvc_platforms+all_icl_platforms+all_wince_platforms]
+ desired_versions = getattr(Options.options, 'msvc_version', '').split(',')
+ if desired_versions == ['']:
+- desired_versions = conf.env.MSVC_VERSIONS or list(reversed(sorted(versiondict.keys())))
++ desired_versions = conf.env.MSVC_VERSIONS or list(sorted(versiondict.keys(), key=MSVCVersion, reverse=True))
+
+ # Override lazy detection by evaluating after the fact.
+ lazy_detect = getattr(Options.options, 'msvc_lazy', True)
+@@ -187,7 +208,7 @@ def get_msvc_version(conf, compiler, version, target, vcvars):
+ echo INCLUDE=%%INCLUDE%%
+ echo LIB=%%LIB%%;%%LIBPATH%%
+ """ % (vcvars,target))
+- sout = conf.cmd_and_log(['cmd.exe', '/E:on', '/V:on', '/C', batfile.abspath()])
++ sout = conf.cmd_and_log(['cmd.exe', '/E:on', '/V:on', '/C', batfile.abspath()], stdin=getattr(Utils.subprocess, 'DEVNULL', None))
+ lines = sout.splitlines()
+
+ if not lines[0]:
+@@ -281,7 +302,7 @@ def gather_wince_supported_platforms():
+
+ def gather_msvc_detected_versions():
+ #Detected MSVC versions!
+- version_pattern = re.compile('^(\d\d?\.\d\d?)(Exp)?$')
++ version_pattern = re.compile(r'^(\d\d?\.\d\d?)(Exp)?$')
+ detected_versions = []
+ for vcver,vcvar in (('VCExpress','Exp'), ('VisualStudio','')):
+ prefix = 'SOFTWARE\\Wow6432node\\Microsoft\\' + vcver
+@@ -367,7 +388,7 @@ def gather_wsdk_versions(conf, versions):
+ :param versions: list to modify
+ :type versions: list
+ """
+- version_pattern = re.compile('^v..?.?\...?.?')
++ version_pattern = re.compile(r'^v..?.?\...?.?')
+ try:
+ all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Microsoft\\Microsoft SDKs\\Windows')
+ except OSError:
+@@ -525,7 +546,7 @@ def gather_icl_versions(conf, versions):
+ :param versions: list to modify
+ :type versions: list
+ """
+- version_pattern = re.compile('^...?.?\....?.?')
++ version_pattern = re.compile(r'^...?.?\....?.?')
+ try:
+ all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Intel\\Compilers\\C++')
+ except OSError:
+@@ -579,7 +600,7 @@ def gather_intel_composer_versions(conf, versions):
+ :param versions: list to modify
+ :type versions: list
+ """
+- version_pattern = re.compile('^...?.?\...?.?.?')
++ version_pattern = re.compile(r'^...?.?\...?.?.?')
+ try:
+ all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Intel\\Suites')
+ except OSError:
+@@ -683,7 +704,7 @@ def find_lt_names_msvc(self, libname, is_static=False):
+ if not is_static and ltdict.get('library_names', ''):
+ dllnames=ltdict['library_names'].split()
+ dll=dllnames[0].lower()
+- dll=re.sub('\.dll$', '', dll)
++ dll=re.sub(r'\.dll$', '', dll)
+ return (lt_libdir, dll, False)
+ elif ltdict.get('old_library', ''):
+ olib=ltdict['old_library']
+@@ -700,7 +721,7 @@ def find_lt_names_msvc(self, libname, is_static=False):
+ @conf
+ def libname_msvc(self, libname, is_static=False):
+ lib = libname.lower()
+- lib = re.sub('\.lib$','',lib)
++ lib = re.sub(r'\.lib$','',lib)
+
+ if lib in g_msvc_systemlibs:
+ return lib
+@@ -747,11 +768,11 @@ def libname_msvc(self, libname, is_static=False):
+ for libn in libnames:
+ if os.path.exists(os.path.join(path, libn)):
+ Logs.debug('msvc: lib found: %s', os.path.join(path,libn))
+- return re.sub('\.lib$', '',libn)
++ return re.sub(r'\.lib$', '',libn)
+
+ #if no lib can be found, just return the libname as msvc expects it
+ self.fatal('The library %r could not be found' % libname)
+- return re.sub('\.lib$', '', libname)
++ return re.sub(r'\.lib$', '', libname)
+
+ @conf
+ def check_lib_msvc(self, libname, is_static=False, uselib_store=None):
+@@ -969,7 +990,7 @@ def build(bld):
+ if not is_static:
+ for f in self.env.LINKFLAGS:
+ d = f.lower()
+- if d[1:] == 'debug':
++ if d[1:] in ('debug', 'debug:full', 'debug:fastlink'):
+ pdbnode = self.link_task.outputs[0].change_ext('.pdb')
+ self.link_task.outputs.append(pdbnode)
+
+diff --git a/waflib/Tools/waf_unit_test.py b/waflib/Tools/waf_unit_test.py
+index 74d6c0561..8cff89bde 100644
+--- a/waflib/Tools/waf_unit_test.py
++++ b/waflib/Tools/waf_unit_test.py
+@@ -97,6 +97,7 @@ def make_interpreted_test(self):
+ if isinstance(v, str):
+ v = v.split(os.pathsep)
+ self.ut_env[k] = os.pathsep.join(p + v)
++ self.env.append_value('UT_DEPS', ['%r%r' % (key, self.ut_env[key]) for key in self.ut_env])
+
+ @feature('test')
+ @after_method('apply_link', 'process_use')
+@@ -108,7 +109,8 @@ def make_test(self):
+ tsk = self.create_task('utest', self.link_task.outputs)
+ if getattr(self, 'ut_str', None):
+ self.ut_run, lst = Task.compile_fun(self.ut_str, shell=getattr(self, 'ut_shell', False))
+- tsk.vars = lst + tsk.vars
++ tsk.vars = tsk.vars + lst
++ self.env.append_value('UT_DEPS', self.ut_str)
+
+ self.handle_ut_cwd('ut_cwd')
+
+@@ -139,6 +141,10 @@ def add_path(var):
+ if not hasattr(self, 'ut_cmd'):
+ self.ut_cmd = getattr(Options.options, 'testcmd', False)
+
++ self.env.append_value('UT_DEPS', str(self.ut_cmd))
++ self.env.append_value('UT_DEPS', self.ut_paths)
++ self.env.append_value('UT_DEPS', ['%r%r' % (key, self.ut_env[key]) for key in self.ut_env])
++
+ @taskgen_method
+ def add_test_results(self, tup):
+ """Override and return tup[1] to interrupt the build immediately if a test does not run"""
+@@ -159,7 +165,7 @@ class utest(Task.Task):
+ """
+ color = 'PINK'
+ after = ['vnum', 'inst']
+- vars = []
++ vars = ['UT_DEPS']
+
+ def runnable_status(self):
+ """
+@@ -200,7 +206,7 @@ def run(self):
+ self.ut_exec = getattr(self.generator, 'ut_exec', [self.inputs[0].abspath()])
+ ut_cmd = getattr(self.generator, 'ut_cmd', False)
+ if ut_cmd:
+- self.ut_exec = shlex.split(ut_cmd % ' '.join(self.ut_exec))
++ self.ut_exec = shlex.split(ut_cmd % Utils.shell_escape(self.ut_exec))
+
+ return self.exec_command(self.ut_exec)
+
+@@ -214,7 +220,7 @@ def exec_command(self, cmd, **kw):
+ 'cmd': cmd
+ }
+ script_file = self.inputs[0].abspath() + '_run.py'
+- Utils.writef(script_file, script_code)
++ Utils.writef(script_file, script_code, encoding='utf-8')
+ os.chmod(script_file, Utils.O755)
+ if Logs.verbose > 1:
+ Logs.info('Test debug file written as %r' % script_file)
+diff --git a/waflib/Utils.py b/waflib/Utils.py
+index a0cc2a09d..ea0f7a9db 100644
+--- a/waflib/Utils.py
++++ b/waflib/Utils.py
+@@ -11,7 +11,7 @@
+
+ from __future__ import with_statement
+
+-import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time
++import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time, shlex
+
+ try:
+ import cPickle
+@@ -49,10 +49,16 @@ class TimeoutExpired(Exception):
+ from hashlib import md5
+ except ImportError:
+ try:
+- from md5 import md5
++ from hashlib import sha1 as md5
+ except ImportError:
+- # never fail to enable fixes from another module
++ # never fail to enable potential fixes from another module
+ pass
++else:
++ try:
++ md5().digest()
++ except ValueError:
++ # Fips? #2213
++ from hashlib import sha1 as md5
+
+ try:
+ import threading
+@@ -202,7 +208,7 @@ def __next__(self):
+
+ next = __next__
+
+-is_win32 = os.sep == '\\' or sys.platform == 'win32' # msys2
++is_win32 = os.sep == '\\' or sys.platform == 'win32' or os.name == 'nt' # msys2
+ """
+ Whether this system is a Windows series
+ """
+@@ -446,6 +452,8 @@ def console_encoding():
+ pass
+ else:
+ if codepage:
++ if 65001 == codepage and sys.version_info < (3, 3):
++ return 'utf-8'
+ return 'cp%d' % codepage
+ return sys.stdout.encoding or ('cp1252' if is_win32 else 'latin-1')
+
+@@ -484,7 +492,9 @@ def split_path_msys(path):
+ if sys.platform == 'cygwin':
+ split_path = split_path_cygwin
+ elif is_win32:
+- if os.environ.get('MSYSTEM'):
++ # Consider this an MSYSTEM environment if $MSYSTEM is set and python
++ # reports is executable from a unix like path on a windows host.
++ if os.environ.get('MSYSTEM') and sys.executable.startswith('/'):
+ split_path = split_path_msys
+ else:
+ split_path = split_path_win32
+@@ -569,10 +579,13 @@ def quote_define_name(s):
+ fu = fu.upper()
+ return fu
+
+-re_sh = re.compile('\\s|\'|"')
+-"""
+-Regexp used for shell_escape below
+-"""
++# shlex.quote didn't exist until python 3.3. Prior to that it was a non-documented
++# function in pipes.
++try:
++ shell_quote = shlex.quote
++except AttributeError:
++ import pipes
++ shell_quote = pipes.quote
+
+ def shell_escape(cmd):
+ """
+@@ -581,7 +594,7 @@ def shell_escape(cmd):
+ """
+ if isinstance(cmd, str):
+ return cmd
+- return ' '.join(repr(x) if re_sh.search(x) else x for x in cmd)
++ return ' '.join(shell_quote(x) for x in cmd)
+
+ def h_list(lst):
+ """
+@@ -596,6 +609,12 @@ def h_list(lst):
+ """
+ return md5(repr(lst).encode()).digest()
+
++if sys.hexversion < 0x3000000:
++ def h_list_python2(lst):
++ return md5(repr(lst)).digest()
++ h_list_python2.__doc__ = h_list.__doc__
++ h_list = h_list_python2
++
+ def h_fun(fun):
+ """
+ Hash functions
+@@ -615,7 +634,7 @@ def h_fun(fun):
+ #
+ # The sorting result outcome will be consistent because:
+ # 1. tuples are compared in order of their elements
+- # 2. optional argument names are unique
++ # 2. optional argument namess are unique
+ code.extend(sorted(fun.keywords.items()))
+ code.append(h_fun(fun.func))
+ fun.code = h_list(code)
+@@ -730,7 +749,7 @@ def unversioned_sys_platform():
+ if s == 'cli' and os.name == 'nt':
+ # ironpython is only on windows as far as we know
+ return 'win32'
+- return re.split('\d+$', s)[0]
++ return re.split(r'\d+$', s)[0]
+
+ def nada(*k, **kw):
+ """
+@@ -851,6 +870,19 @@ def lib64():
+ return '64'
+ return ''
+
++def loose_version(ver_str):
++ # private for the time being!
++ # see #2402
++ lst = re.split(r'([.]|\\d+|[a-zA-Z])', ver_str)
++ ver = []
++ for i, val in enumerate(lst):
++ try:
++ ver.append(int(val))
++ except ValueError:
++ if val != '.':
++ ver.append(val)
++ return ver
++
+ def sane_path(p):
+ # private function for the time being!
+ return os.path.abspath(os.path.expanduser(p))
+@@ -871,13 +903,13 @@ def get_process():
+ except IndexError:
+ filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'processor.py'
+ cmd = [sys.executable, '-c', readf(filepath)]
+- return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0)
++ return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0, close_fds=not is_win32)
+
+ def run_prefork_process(cmd, kwargs, cargs):
+ """
+ Delegates process execution to a pre-forked process instance.
+ """
+- if not 'env' in kwargs:
++ if not kwargs.get('env'):
+ kwargs['env'] = dict(os.environ)
+ try:
+ obj = base64.b64encode(cPickle.dumps([cmd, kwargs, cargs]))
+diff --git a/waflib/ansiterm.py b/waflib/ansiterm.py
+index 0d20c6374..027f0ad68 100644
+--- a/waflib/ansiterm.py
++++ b/waflib/ansiterm.py
+@@ -264,7 +264,7 @@ def hide_cursor(self,param):
+ 'u': pop_cursor,
+ }
+ # Match either the escape sequence or text not containing escape sequence
+- ansi_tokens = re.compile('(?:\x1b\[([0-9?;]*)([a-zA-Z])|([^\x1b]+))')
++ ansi_tokens = re.compile(r'(?:\x1b\[([0-9?;]*)([a-zA-Z])|([^\x1b]+))')
+ def write(self, text):
+ try:
+ wlock.acquire()
+diff --git a/waflib/extras/clang_cross.py b/waflib/extras/clang_cross.py
+new file mode 100644
+index 000000000..1b51e2886
+--- /dev/null
++++ b/waflib/extras/clang_cross.py
+@@ -0,0 +1,92 @@
++#!/usr/bin/env python
++# encoding: utf-8
++# Krzysztof Kosiński 2014
++# DragoonX6 2018
++
++"""
++Detect the Clang C compiler
++This version is an attempt at supporting the -target and -sysroot flag of Clang.
++"""
++
++from waflib.Tools import ccroot, ar, gcc
++from waflib.Configure import conf
++import waflib.Context
++import waflib.extras.clang_cross_common
++
++def options(opt):
++ """
++ Target triplet for clang::
++ $ waf configure --clang-target-triple=x86_64-pc-linux-gnu
++ """
++ cc_compiler_opts = opt.add_option_group('Configuration options')
++ cc_compiler_opts.add_option('--clang-target-triple', default=None,
++ help='Target triple for clang',
++ dest='clang_target_triple')
++ cc_compiler_opts.add_option('--clang-sysroot', default=None,
++ help='Sysroot for clang',
++ dest='clang_sysroot')
++
++@conf
++def find_clang(conf):
++ """
++ Finds the program clang and executes it to ensure it really is clang
++ """
++
++ import os
++
++ cc = conf.find_program('clang', var='CC')
++
++ if conf.options.clang_target_triple != None:
++ conf.env.append_value('CC', ['-target', conf.options.clang_target_triple])
++
++ if conf.options.clang_sysroot != None:
++ sysroot = str()
++
++ if os.path.isabs(conf.options.clang_sysroot):
++ sysroot = conf.options.clang_sysroot
++ else:
++ sysroot = os.path.normpath(os.path.join(os.getcwd(), conf.options.clang_sysroot))
++
++ conf.env.append_value('CC', ['--sysroot', sysroot])
++
++ conf.get_cc_version(cc, clang=True)
++ conf.env.CC_NAME = 'clang'
++
++@conf
++def clang_modifier_x86_64_w64_mingw32(conf):
++ conf.gcc_modifier_win32()
++
++@conf
++def clang_modifier_i386_w64_mingw32(conf):
++ conf.gcc_modifier_win32()
++
++@conf
++def clang_modifier_x86_64_windows_msvc(conf):
++ conf.clang_modifier_msvc()
++
++ # Allow the user to override any flags if they so desire.
++ clang_modifier_user_func = getattr(conf, 'clang_modifier_x86_64_windows_msvc_user', None)
++ if clang_modifier_user_func:
++ clang_modifier_user_func()
++
++@conf
++def clang_modifier_i386_windows_msvc(conf):
++ conf.clang_modifier_msvc()
++
++ # Allow the user to override any flags if they so desire.
++ clang_modifier_user_func = getattr(conf, 'clang_modifier_i386_windows_msvc_user', None)
++ if clang_modifier_user_func:
++ clang_modifier_user_func()
++
++def configure(conf):
++ conf.find_clang()
++ conf.find_program(['llvm-ar', 'ar'], var='AR')
++ conf.find_ar()
++ conf.gcc_common_flags()
++ # Allow the user to provide flags for the target platform.
++ conf.gcc_modifier_platform()
++ # And allow more fine grained control based on the compiler's triplet.
++ conf.clang_modifier_target_triple()
++ conf.cc_load_tools()
++ conf.cc_add_flags()
++ conf.link_add_flags()
+diff --git a/waflib/extras/clang_cross_common.py b/waflib/extras/clang_cross_common.py
+new file mode 100644
+index 000000000..b76a07006
+--- /dev/null
++++ b/waflib/extras/clang_cross_common.py
+@@ -0,0 +1,113 @@
++#!/usr/bin/env python
++# encoding: utf-8
++# DragoonX6 2018
++
++"""
++Common routines for cross_clang.py and cross_clangxx.py
++"""
++
++from waflib.Configure import conf
++import waflib.Context
++
++def normalize_target_triple(target_triple):
++ target_triple = target_triple[:-1]
++ normalized_triple = target_triple.replace('--', '-unknown-')
++
++ if normalized_triple.startswith('-'):
++ normalized_triple = 'unknown' + normalized_triple
++
++ if normalized_triple.endswith('-'):
++ normalized_triple += 'unknown'
++
++ # Normalize MinGW builds to *arch*-w64-mingw32
++ if normalized_triple.endswith('windows-gnu'):
++ normalized_triple = normalized_triple[:normalized_triple.index('-')] + '-w64-mingw32'
++
++ # Strip the vendor when doing msvc builds, since it's unused anyway.
++ if normalized_triple.endswith('windows-msvc'):
++ normalized_triple = normalized_triple[:normalized_triple.index('-')] + '-windows-msvc'
++
++ return normalized_triple.replace('-', '_')
++
++@conf
++def clang_modifier_msvc(conf):
++ import os
++
++ """
++ Really basic setup to use clang in msvc mode.
++ We actually don't really want to do a lot, even though clang is msvc compatible
++ in this mode, that doesn't mean we're actually using msvc.
++ It's probably the best to leave it to the user, we can assume msvc mode if the user
++ uses the clang-cl frontend, but this module only concerns itself with the gcc-like frontend.
++ """
++ v = conf.env
++ v.cprogram_PATTERN = '%s.exe'
++
++ v.cshlib_PATTERN = '%s.dll'
++ v.implib_PATTERN = '%s.lib'
++ v.IMPLIB_ST = '-Wl,-IMPLIB:%s'
++ v.SHLIB_MARKER = []
++
++ v.CFLAGS_cshlib = []
++ v.LINKFLAGS_cshlib = ['-Wl,-DLL']
++ v.cstlib_PATTERN = '%s.lib'
++ v.STLIB_MARKER = []
++
++ del(v.AR)
++ conf.find_program(['llvm-lib', 'lib'], var='AR')
++ v.ARFLAGS = ['-nologo']
++ v.AR_TGT_F = ['-out:']
++
++ # Default to the linker supplied with llvm instead of link.exe or ld
++ v.LINK_CC = v.CC + ['-fuse-ld=lld', '-nostdlib']
++ v.CCLNK_TGT_F = ['-o']
++ v.def_PATTERN = '-Wl,-def:%s'
++
++ v.LINKFLAGS = []
++
++ v.LIB_ST = '-l%s'
++ v.LIBPATH_ST = '-Wl,-LIBPATH:%s'
++ v.STLIB_ST = '-l%s'
++ v.STLIBPATH_ST = '-Wl,-LIBPATH:%s'
++
++ CFLAGS_CRT_COMMON = [
++ '-Xclang', '--dependent-lib=oldnames',
++ '-Xclang', '-fno-rtti-data',
++ '-D_MT'
++ ]
++
++ v.CFLAGS_CRT_MULTITHREADED = CFLAGS_CRT_COMMON + [
++ '-Xclang', '-flto-visibility-public-std',
++ '-Xclang', '--dependent-lib=libcmt',
++ ]
++ v.CXXFLAGS_CRT_MULTITHREADED = v.CFLAGS_CRT_MULTITHREADED
++
++ v.CFLAGS_CRT_MULTITHREADED_DBG = CFLAGS_CRT_COMMON + [
++ '-D_DEBUG',
++ '-Xclang', '-flto-visibility-public-std',
++ '-Xclang', '--dependent-lib=libcmtd',
++ ]
++ v.CXXFLAGS_CRT_MULTITHREADED_DBG = v.CFLAGS_CRT_MULTITHREADED_DBG
++
++ v.CFLAGS_CRT_MULTITHREADED_DLL = CFLAGS_CRT_COMMON + [
++ '-D_DLL',
++ '-Xclang', '--dependent-lib=msvcrt'
++ ]
++ v.CXXFLAGS_CRT_MULTITHREADED_DLL = v.CFLAGS_CRT_MULTITHREADED_DLL
++
++ v.CFLAGS_CRT_MULTITHREADED_DLL_DBG = CFLAGS_CRT_COMMON + [
++ '-D_DLL',
++ '-D_DEBUG',
++ '-Xclang', '--dependent-lib=msvcrtd',
++ ]
++ v.CXXFLAGS_CRT_MULTITHREADED_DLL_DBG = v.CFLAGS_CRT_MULTITHREADED_DLL_DBG
++
++@conf
++def clang_modifier_target_triple(conf, cpp=False):
++ compiler = conf.env.CXX if cpp else conf.env.CC
++ output = conf.cmd_and_log(compiler + ['-dumpmachine'], output=waflib.Context.STDOUT)
++
++ modifier = ('clangxx' if cpp else 'clang') + '_modifier_'
++ clang_modifier_func = getattr(conf, modifier + normalize_target_triple(output), None)
++ if clang_modifier_func:
++ clang_modifier_func()
+diff --git a/waflib/extras/clangxx_cross.py b/waflib/extras/clangxx_cross.py
+new file mode 100644
+index 000000000..0ad38ad46
+--- /dev/null
++++ b/waflib/extras/clangxx_cross.py
+@@ -0,0 +1,106 @@
++#!/usr/bin/env python
++# encoding: utf-8
++# Thomas Nagy 2009-2018 (ita)
++# DragoonX6 2018
++
++"""
++Detect the Clang++ C++ compiler
++This version is an attempt at supporting the -target and -sysroot flag of Clang++.
++"""
++
++from waflib.Tools import ccroot, ar, gxx
++from waflib.Configure import conf
++import waflib.extras.clang_cross_common
++
++def options(opt):
++ """
++ Target triplet for clang++::
++ $ waf configure --clangxx-target-triple=x86_64-pc-linux-gnu
++ """
++ cxx_compiler_opts = opt.add_option_group('Configuration options')
++ cxx_compiler_opts.add_option('--clangxx-target-triple', default=None,
++ help='Target triple for clang++',
++ dest='clangxx_target_triple')
++ cxx_compiler_opts.add_option('--clangxx-sysroot', default=None,
++ help='Sysroot for clang++',
++ dest='clangxx_sysroot')
++
++@conf
++def find_clangxx(conf):
++ """
++ Finds the program clang++, and executes it to ensure it really is clang++
++ """
++
++ import os
++
++ cxx = conf.find_program('clang++', var='CXX')
++
++ if conf.options.clangxx_target_triple != None:
++ conf.env.append_value('CXX', ['-target', conf.options.clangxx_target_triple])
++
++ if conf.options.clangxx_sysroot != None:
++ sysroot = str()
++
++ if os.path.isabs(conf.options.clangxx_sysroot):
++ sysroot = conf.options.clangxx_sysroot
++ else:
++ sysroot = os.path.normpath(os.path.join(os.getcwd(), conf.options.clangxx_sysroot))
++
++ conf.env.append_value('CXX', ['--sysroot', sysroot])
++
++ conf.get_cc_version(cxx, clang=True)
++ conf.env.CXX_NAME = 'clang'
++
++@conf
++def clangxx_modifier_x86_64_w64_mingw32(conf):
++ conf.gcc_modifier_win32()
++
++@conf
++def clangxx_modifier_i386_w64_mingw32(conf):
++ conf.gcc_modifier_win32()
++
++@conf
++def clangxx_modifier_msvc(conf):
++ v = conf.env
++ v.cxxprogram_PATTERN = v.cprogram_PATTERN
++ v.cxxshlib_PATTERN = v.cshlib_PATTERN
++
++ v.CXXFLAGS_cxxshlib = []
++ v.LINKFLAGS_cxxshlib = v.LINKFLAGS_cshlib
++ v.cxxstlib_PATTERN = v.cstlib_PATTERN
++
++ v.LINK_CXX = v.CXX + ['-fuse-ld=lld', '-nostdlib']
++ v.CXXLNK_TGT_F = v.CCLNK_TGT_F
++
++@conf
++def clangxx_modifier_x86_64_windows_msvc(conf):
++ conf.clang_modifier_msvc()
++ conf.clangxx_modifier_msvc()
++
++ # Allow the user to override any flags if they so desire.
++ clang_modifier_user_func = getattr(conf, 'clangxx_modifier_x86_64_windows_msvc_user', None)
++ if clang_modifier_user_func:
++ clang_modifier_user_func()
++
++@conf
++def clangxx_modifier_i386_windows_msvc(conf):
++ conf.clang_modifier_msvc()
++ conf.clangxx_modifier_msvc()
++
++ # Allow the user to override any flags if they so desire.
++ clang_modifier_user_func = getattr(conf, 'clangxx_modifier_i386_windows_msvc_user', None)
++ if clang_modifier_user_func:
++ clang_modifier_user_func()
++
++def configure(conf):
++ conf.find_clangxx()
++ conf.find_program(['llvm-ar', 'ar'], var='AR')
++ conf.find_ar()
++ conf.gxx_common_flags()
++ # Allow the user to provide flags for the target platform.
++ conf.gxx_modifier_platform()
++ # And allow more fine grained control based on the compiler's triplet.
++ conf.clang_modifier_target_triple(cpp=True)
++ conf.cxx_load_tools()
++ conf.cxx_add_flags()
++ conf.link_add_flags()
+diff --git a/waflib/extras/classic_runner.py b/waflib/extras/classic_runner.py
+new file mode 100644
+index 000000000..b08c794e8
+--- /dev/null
++++ b/waflib/extras/classic_runner.py
+@@ -0,0 +1,68 @@
++#!/usr/bin/env python
++# encoding: utf-8
++# Thomas Nagy, 2021 (ita)
++
++from waflib import Utils, Runner
++
++"""
++Re-enable the classic threading system from waf 1.x
++
++def configure(conf):
++ conf.load('classic_runner')
++"""
++
++class TaskConsumer(Utils.threading.Thread):
++ """
++ Task consumers belong to a pool of workers
++
++ They wait for tasks in the queue and then use ``task.process(...)``
++ """
++ def __init__(self, spawner):
++ Utils.threading.Thread.__init__(self)
++ """
++ Obtain :py:class:`waflib.Task.TaskBase` instances from this queue.
++ """
++ self.spawner = spawner
++ self.daemon = True
++ self.start()
++
++ def run(self):
++ """
++ Loop over the tasks to execute
++ """
++ try:
++ self.loop()
++ except Exception:
++ pass
++
++ def loop(self):
++ """
++ Obtain tasks from :py:attr:`waflib.Runner.TaskConsumer.ready` and call
++ :py:meth:`waflib.Task.TaskBase.process`. If the object is a function, execute it.
++ """
++ master = self.spawner.master
++ while 1:
++ if not master.stop:
++ try:
++ tsk = master.ready.get()
++ if tsk:
++ tsk.log_display(tsk.generator.bld)
++ master.process_task(tsk)
++ else:
++ break
++ finally:
++ master.out.put(tsk)
++
++class Spawner(object):
++ """
++ Daemon thread that consumes tasks from :py:class:`waflib.Runner.Parallel` producer and
++ spawns a consuming thread :py:class:`waflib.Runner.Consumer` for each
++ :py:class:`waflib.Task.Task` instance.
++ """
++ def __init__(self, master):
++ self.master = master
++ """:py:class:`waflib.Runner.Parallel` producer instance"""
++
++ self.pool = [TaskConsumer(self) for i in range(master.numjobs)]
++
++Runner.Spawner = Spawner
+diff --git a/waflib/extras/color_msvc.py b/waflib/extras/color_msvc.py
+new file mode 100644
+index 000000000..60bacb7b2
+--- /dev/null
++++ b/waflib/extras/color_msvc.py
+@@ -0,0 +1,59 @@
++#!/usr/bin/env python
++# encoding: utf-8
++
++# Replaces the default formatter by one which understands MSVC output and colorizes it.
++# Modified from color_gcc.py
++
++__author__ = __maintainer__ = "Alibek Omarov <a1ba.omarov@gmail.com>"
++__copyright__ = "Alibek Omarov, 2019"
++
++import sys
++from waflib import Logs
++
++class ColorMSVCFormatter(Logs.formatter):
++ def __init__(self, colors):
++ self.colors = colors
++ Logs.formatter.__init__(self)
++
++ def parseMessage(self, line, color):
++ # Split messaage from 'disk:filepath: type: message'
++ arr = line.split(':', 3)
++ if len(arr) < 4:
++ return line
++
++ colored = self.colors.BOLD + arr[0] + ':' + arr[1] + ':' + self.colors.NORMAL
++ colored += color + arr[2] + ':' + self.colors.NORMAL
++ colored += arr[3]
++ return colored
++
++ def format(self, rec):
++ frame = sys._getframe()
++ while frame:
++ func = frame.f_code.co_name
++ if func == 'exec_command':
++ cmd = frame.f_locals.get('cmd')
++ if isinstance(cmd, list):
++ # Fix file case, it may be CL.EXE or cl.exe
++ argv0 = cmd[0].lower()
++ if 'cl.exe' in argv0:
++ lines = []
++ # This will not work with "localized" versions
++ # of MSVC
++ for line in rec.msg.splitlines():
++ if ': warning ' in line:
++ lines.append(self.parseMessage(line, self.colors.YELLOW))
++ elif ': error ' in line:
++ lines.append(self.parseMessage(line, self.colors.RED))
++ elif ': fatal error ' in line:
++ lines.append(self.parseMessage(line, self.colors.RED + self.colors.BOLD))
++ elif ': note: ' in line:
++ lines.append(self.parseMessage(line, self.colors.CYAN))
++ else:
++ lines.append(line)
++ rec.msg = "\n".join(lines)
++ frame = frame.f_back
++ return Logs.formatter.format(self, rec)
++
++def options(opt):
++ Logs.log.handlers[0].setFormatter(ColorMSVCFormatter(Logs.colors))
++
+diff --git a/waflib/extras/fc_fujitsu.py b/waflib/extras/fc_fujitsu.py
+new file mode 100644
+index 000000000..cae676c20
+--- /dev/null
++++ b/waflib/extras/fc_fujitsu.py
+@@ -0,0 +1,52 @@
++#! /usr/bin/env python
++# encoding: utf-8
++# Detection of the Fujitsu Fortran compiler for ARM64FX
++
++import re
++from waflib.Tools import fc,fc_config,fc_scan
++from waflib.Configure import conf
++from waflib.Tools.compiler_fc import fc_compiler
++fc_compiler['linux'].append('fc_fujitsu')
++
++@conf
++def find_fujitsu(conf):
++ fc=conf.find_program(['frtpx'],var='FC')
++ conf.get_fujitsu_version(fc)
++ conf.env.FC_NAME='FUJITSU'
++ conf.env.FC_MOD_CAPITALIZATION='lower'
++
++@conf
++def fujitsu_flags(conf):
++ v=conf.env
++ v['_FCMODOUTFLAGS']=[]
++ v['FCFLAGS_DEBUG']=[]
++ v['FCFLAGS_fcshlib']=[]
++ v['LINKFLAGS_fcshlib']=[]
++ v['FCSTLIB_MARKER']=''
++ v['FCSHLIB_MARKER']=''
++
++@conf
++def get_fujitsu_version(conf,fc):
++ version_re=re.compile(r"frtpx\s*\(FRT\)\s*(?P<major>\d+)\.(?P<minor>\d+)\.",re.I).search
++ cmd=fc+['--version']
++ out,err=fc_config.getoutput(conf,cmd,stdin=False)
++ if out:
++ match=version_re(out)
++ else:
++ match=version_re(err)
++ if not match:
++ return(False)
++ conf.fatal('Could not determine the Fujitsu FRT Fortran compiler version.')
++ else:
++ k=match.groupdict()
++ conf.env['FC_VERSION']=(k['major'],k['minor'])
++
++def configure(conf):
++ conf.find_fujitsu()
++ conf.find_program('ar',var='AR')
++ conf.add_os_flags('ARFLAGS')
++ if not conf.env.ARFLAGS:
++ conf.env.ARFLAGS=['rcs']
++ conf.fc_flags()
++ conf.fc_add_flags()
++ conf.fujitsu_flags()
+diff --git a/waflib/extras/fc_nfort.py b/waflib/extras/fc_nfort.py
+new file mode 100644
+index 000000000..c25886b8e
+--- /dev/null
++++ b/waflib/extras/fc_nfort.py
+@@ -0,0 +1,52 @@
++#! /usr/bin/env python
++# encoding: utf-8
++# Detection of the NEC Fortran compiler for Aurora Tsubasa
++
++import re
++from waflib.Tools import fc,fc_config,fc_scan
++from waflib.Configure import conf
++from waflib.Tools.compiler_fc import fc_compiler
++fc_compiler['linux'].append('fc_nfort')
++
++@conf
++def find_nfort(conf):
++ fc=conf.find_program(['nfort'],var='FC')
++ conf.get_nfort_version(fc)
++ conf.env.FC_NAME='NFORT'
++ conf.env.FC_MOD_CAPITALIZATION='lower'
++
++@conf
++def nfort_flags(conf):
++ v=conf.env
++ v['_FCMODOUTFLAGS']=[]
++ v['FCFLAGS_DEBUG']=[]
++ v['FCFLAGS_fcshlib']=[]
++ v['LINKFLAGS_fcshlib']=[]
++ v['FCSTLIB_MARKER']=''
++ v['FCSHLIB_MARKER']=''
++
++@conf
++def get_nfort_version(conf,fc):
++ version_re=re.compile(r"nfort\s*\(NFORT\)\s*(?P<major>\d+)\.(?P<minor>\d+)\.",re.I).search
++ cmd=fc+['--version']
++ out,err=fc_config.getoutput(conf,cmd,stdin=False)
++ if out:
++ match=version_re(out)
++ else:
++ match=version_re(err)
++ if not match:
++ return(False)
++ conf.fatal('Could not determine the NEC NFORT Fortran compiler version.')
++ else:
++ k=match.groupdict()
++ conf.env['FC_VERSION']=(k['major'],k['minor'])
++
++def configure(conf):
++ conf.find_nfort()
++ conf.find_program('nar',var='AR')
++ conf.add_os_flags('ARFLAGS')
++ if not conf.env.ARFLAGS:
++ conf.env.ARFLAGS=['rcs']
++ conf.fc_flags()
++ conf.fc_add_flags()
++ conf.nfort_flags()
+diff --git a/waflib/extras/genpybind.py b/waflib/extras/genpybind.py
+new file mode 100644
+index 000000000..ac206ee8a
+--- /dev/null
++++ b/waflib/extras/genpybind.py
+@@ -0,0 +1,194 @@
++import os
++import pipes
++import subprocess
++import sys
++
++from waflib import Logs, Task, Context
++from waflib.Tools.c_preproc import scan as scan_impl
++# ^-- Note: waflib.extras.gccdeps.scan does not work for us,
++# due to its current implementation:
++# The -MD flag is injected into the {C,CXX}FLAGS environment variable and
++# dependencies are read out in a separate step after compiling by reading
++# the .d file saved alongside the object file.
++# As the genpybind task refers to a header file that is never compiled itself,
++# gccdeps will not be able to extract the list of dependencies.
++
++from waflib.TaskGen import feature, before_method
++
++
++def join_args(args):
++ return " ".join(pipes.quote(arg) for arg in args)
++
++
++def configure(cfg):
++ cfg.load("compiler_cxx")
++ cfg.load("python")
++ cfg.check_python_version(minver=(2, 7))
++ if not cfg.env.LLVM_CONFIG:
++ cfg.find_program("llvm-config", var="LLVM_CONFIG")
++ if not cfg.env.GENPYBIND:
++ cfg.find_program("genpybind", var="GENPYBIND")
++
++ # find clang reasource dir for builtin headers
++ cfg.env.GENPYBIND_RESOURCE_DIR = os.path.join(
++ cfg.cmd_and_log(cfg.env.LLVM_CONFIG + ["--libdir"]).strip(),
++ "clang",
++ cfg.cmd_and_log(cfg.env.LLVM_CONFIG + ["--version"]).strip())
++ if os.path.exists(cfg.env.GENPYBIND_RESOURCE_DIR):
++ cfg.msg("Checking clang resource dir", cfg.env.GENPYBIND_RESOURCE_DIR)
++ else:
++ cfg.fatal("Clang resource dir not found")
++
++
++@feature("genpybind")
++@before_method("process_source")
++def generate_genpybind_source(self):
++ """
++ Run genpybind on the headers provided in `source` and compile/link the
++ generated code instead. This works by generating the code on the fly and
++ swapping the source node before `process_source` is run.
++ """
++ # name of module defaults to name of target
++ module = getattr(self, "module", self.target)
++
++ # create temporary source file in build directory to hold generated code
++ out = "genpybind-%s.%d.cpp" % (module, self.idx)
++ out = self.path.get_bld().find_or_declare(out)
++
++ task = self.create_task("genpybind", self.to_nodes(self.source), out)
++ # used to detect whether CFLAGS or CXXFLAGS should be passed to genpybind
++ task.features = self.features
++ task.module = module
++ # can be used to select definitions to include in the current module
++ # (when header files are shared by more than one module)
++ task.genpybind_tags = self.to_list(getattr(self, "genpybind_tags", []))
++ # additional include directories
++ task.includes = self.to_list(getattr(self, "includes", []))
++ task.genpybind = self.env.GENPYBIND
++
++ # Tell waf to compile/link the generated code instead of the headers
++ # originally passed-in via the `source` parameter. (see `process_source`)
++ self.source = [out]
++
++
++class genpybind(Task.Task): # pylint: disable=invalid-name
++ """
++ Runs genpybind on headers provided as input to this task.
++ Generated code will be written to the first (and only) output node.
++ """
++ quiet = True
++ color = "PINK"
++ scan = scan_impl
++
++ @staticmethod
++ def keyword():
++ return "Analyzing"
++
++ def run(self):
++ if not self.inputs:
++ return
++
++ args = self.find_genpybind() + self._arguments(
++ resource_dir=self.env.GENPYBIND_RESOURCE_DIR)
++
++ output = self.run_genpybind(args)
++
++ # For debugging / log output
++ pasteable_command = join_args(args)
++
++ # write generated code to file in build directory
++ # (will be compiled during process_source stage)
++ (output_node,) = self.outputs
++ output_node.write("// {}\n{}\n".format(
++ pasteable_command.replace("\n", "\n// "), output))
++
++ def find_genpybind(self):
++ return self.genpybind
++
++ def run_genpybind(self, args):
++ bld = self.generator.bld
++
++ kwargs = dict(cwd=bld.variant_dir)
++ if hasattr(bld, "log_command"):
++ bld.log_command(args, kwargs)
++ else:
++ Logs.debug("runner: {!r}".format(args))
++ proc = subprocess.Popen(
++ args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
++ stdout, stderr = proc.communicate()
++
++ if not isinstance(stdout, str):
++ stdout = stdout.decode(sys.stdout.encoding, errors="replace")
++ if not isinstance(stderr, str):
++ stderr = stderr.decode(sys.stderr.encoding, errors="replace")
++
++ if proc.returncode != 0:
++ bld.fatal(
++ "genpybind returned {code} during the following call:"
++ "\n{command}\n\n{stdout}\n\n{stderr}".format(
++ code=proc.returncode,
++ command=join_args(args),
++ stdout=stdout,
++ stderr=stderr,
++ ))
++
++ if stderr.strip():
++ Logs.debug("non-fatal warnings during genpybind run:\n{}".format(stderr))
++
++ return stdout
++
++ def _include_paths(self):
++ return self.generator.to_incnodes(self.includes + self.env.INCLUDES)
++
++ def _inputs_as_relative_includes(self):
++ include_paths = self._include_paths()
++ relative_includes = []
++ for node in self.inputs:
++ for inc in include_paths:
++ if node.is_child_of(inc):
++ relative_includes.append(node.path_from(inc))
++ break
++ else:
++ self.generator.bld.fatal("could not resolve {}".format(node))
++ return relative_includes
++
++ def _arguments(self, genpybind_parse=None, resource_dir=None):
++ args = []
++ relative_includes = self._inputs_as_relative_includes()
++ is_cxx = "cxx" in self.features
++
++ # options for genpybind
++ args.extend(["--genpybind-module", self.module])
++ if self.genpybind_tags:
++ args.extend(["--genpybind-tag"] + self.genpybind_tags)
++ if relative_includes:
++ args.extend(["--genpybind-include"] + relative_includes)
++ if genpybind_parse:
++ args.extend(["--genpybind-parse", genpybind_parse])
++
++ args.append("--")
++
++ # headers to be processed by genpybind
++ args.extend(node.abspath() for node in self.inputs)
++
++ args.append("--")
++
++ # options for clang/genpybind-parse
++ args.append("-D__GENPYBIND__")
++ args.append("-xc++" if is_cxx else "-xc")
++ has_std_argument = False
++ for flag in self.env["CXXFLAGS" if is_cxx else "CFLAGS"]:
++ flag = flag.replace("-std=gnu", "-std=c")
++ if flag.startswith("-std=c"):
++ has_std_argument = True
++ args.append(flag)
++ if not has_std_argument:
++ args.append("-std=c++14")
++ args.extend("-I{}".format(n.abspath()) for n in self._include_paths())
++ args.extend("-D{}".format(p) for p in self.env.DEFINES)
++
++ # point to clang resource dir, if specified
++ if resource_dir:
++ args.append("-resource-dir={}".format(resource_dir))
++
++ return args
+diff --git a/waflib/extras/haxe.py b/waflib/extras/haxe.py
+new file mode 100644
+index 000000000..4ff374579
+--- /dev/null
++++ b/waflib/extras/haxe.py
+@@ -0,0 +1,154 @@
++import re
++
++from waflib import Utils, Task, Errors, Logs
++from waflib.Configure import conf
++from waflib.TaskGen import extension, taskgen_method
++
++HAXE_COMPILERS = {
++ 'JS': {'tgt': '--js', 'ext_out': ['.js']},
++ 'LUA': {'tgt': '--lua', 'ext_out': ['.lua']},
++ 'SWF': {'tgt': '--swf', 'ext_out': ['.swf']},
++ 'NEKO': {'tgt': '--neko', 'ext_out': ['.n']},
++ 'PHP': {'tgt': '--php', 'ext_out': ['.php']},
++ 'CPP': {'tgt': '--cpp', 'ext_out': ['.h', '.cpp']},
++ 'CPPIA': {'tgt': '--cppia', 'ext_out': ['.cppia']},
++ 'CS': {'tgt': '--cs', 'ext_out': ['.cs']},
++ 'JAVA': {'tgt': '--java', 'ext_out': ['.java']},
++ 'JVM': {'tgt': '--jvm', 'ext_out': ['.jar']},
++ 'PYTHON': {'tgt': '--python', 'ext_out': ['.py']},
++ 'HL': {'tgt': '--hl', 'ext_out': ['.hl']},
++ 'HLC': {'tgt': '--hl', 'ext_out': ['.h', '.c']},
++}
++
++@conf
++def check_haxe_pkg(self, **kw):
++ self.find_program('haxelib')
++ libs = kw.get('libs')
++ if not libs or not (type(libs) == str or (type(libs) == list and all(isinstance(s, str) for s in libs))):
++ self.fatal('Specify correct libs value in ensure call')
++ return
++ fetch = kw.get('fetch')
++ if not fetch is None and not type(fetch) == bool:
++ self.fatal('Specify correct fetch value in ensure call')
++
++ libs = [libs] if type(libs) == str else libs
++ halt = False
++ for lib in libs:
++ try:
++ self.start_msg('Checking for library %s' % lib)
++ output = self.cmd_and_log(self.env.HAXELIB + ['list', lib])
++ except Errors.WafError:
++ self.end_msg(False)
++ self.fatal('Can\'t run haxelib list, ensuring halted')
++ return
++
++ if lib in output:
++ self.end_msg(lib in output)
++ else:
++ if not fetch:
++ self.end_msg(False)
++ halt = True
++ continue
++ try:
++ status = self.exec_command(self.env.HAXELIB + ['install', lib])
++ if status:
++ self.end_msg(False)
++ self.fatal('Can\'t get %s with haxelib, ensuring halted' % lib)
++ return
++ else:
++ self.end_msg('downloaded', color='YELLOW')
++ except Errors.WafError:
++ self.end_msg(False)
++ self.fatal('Can\'t run haxelib install, ensuring halted')
++ return
++ postfix = kw.get('uselib_store') or lib.upper()
++ self.env.append_unique('LIB_' + postfix, lib)
++
++ if halt:
++ self.fatal('Can\'t find libraries in haxelib list, ensuring halted')
++ return
++
++class haxe(Task.Task):
++ vars = ['HAXE_VERSION', 'HAXE_FLAGS']
++ ext_in = ['.hx']
++
++ def run(self):
++ cmd = self.env.HAXE + self.env.HAXE_FLAGS_DEFAULT + self.env.HAXE_FLAGS
++ return self.exec_command(cmd)
++
++for COMP in HAXE_COMPILERS:
++ # create runners for each compile target
++ type("haxe_" + COMP, (haxe,), {'ext_out': HAXE_COMPILERS[COMP]['ext_out']})
++
++@taskgen_method
++def init_haxe(self):
++ errmsg = '%s not found, specify correct value'
++ try:
++ compiler = HAXE_COMPILERS[self.compiler]
++ comp_tgt = compiler['tgt']
++ comp_mod = '/main.c' if self.compiler == 'HLC' else ''
++ except (AttributeError, KeyError):
++ self.bld.fatal(errmsg % 'COMPILER' + ': ' + ', '.join(HAXE_COMPILERS.keys()))
++ return
++
++ self.env.append_value(
++ 'HAXE_FLAGS',
++ [comp_tgt, self.path.get_bld().make_node(self.target + comp_mod).abspath()])
++ if hasattr(self, 'use'):
++ if not (type(self.use) == str or type(self.use) == list):
++ self.bld.fatal(errmsg % 'USE')
++ return
++ self.use = [self.use] if type(self.use) == str else self.use
++
++ for dep in self.use:
++ if self.env['LIB_' + dep]:
++ for lib in self.env['LIB_' + dep]:
++ self.env.append_value('HAXE_FLAGS', ['-lib', lib])
++
++ if hasattr(self, 'res'):
++ if not type(self.res) == str:
++ self.bld.fatal(errmsg % 'RES')
++ return
++ self.env.append_value('HAXE_FLAGS', ['-D', 'resourcesPath=%s' % self.res])
++
++@extension('.hx')
++def haxe_hook(self, node):
++ if len(self.source) > 1:
++ self.bld.fatal('Use separate task generators for multiple files')
++ return
++
++ src = node
++ tgt = self.path.get_bld().find_or_declare(self.target)
++
++ self.init_haxe()
++ self.create_task('haxe_' + self.compiler, src, tgt)
++
++@conf
++def check_haxe(self, mini=None, maxi=None):
++ self.start_msg('Checking for haxe version')
++ try:
++ curr = re.search(
++ r'(\d+.?)+',
++ self.cmd_and_log(self.env.HAXE + ['-version'])).group()
++ except Errors.WafError:
++ self.end_msg(False)
++ self.fatal('Can\'t get haxe version')
++ return
++
++ if mini and Utils.num2ver(curr) < Utils.num2ver(mini):
++ self.end_msg('wrong', color='RED')
++ self.fatal('%s is too old, need >= %s' % (curr, mini))
++ return
++ if maxi and Utils.num2ver(curr) > Utils.num2ver(maxi):
++ self.end_msg('wrong', color='RED')
++ self.fatal('%s is too new, need <= %s' % (curr, maxi))
++ return
++ self.end_msg(curr, color='GREEN')
++ self.env.HAXE_VERSION = curr
++
++def configure(self):
++ self.env.append_value(
++ 'HAXE_FLAGS_DEFAULT',
++ ['-D', 'no-compilation', '-cp', self.path.abspath()])
++ Logs.warn('Default flags: %s' % ' '.join(self.env.HAXE_FLAGS_DEFAULT))
++ self.find_program('haxe')
+diff --git a/waflib/extras/msvc_pdb.py b/waflib/extras/msvc_pdb.py
+new file mode 100644
+index 000000000..077656b4f
+--- /dev/null
++++ b/waflib/extras/msvc_pdb.py
+@@ -0,0 +1,46 @@
++#!/usr/bin/env python
++# encoding: utf-8
++# Rafaël Kooi 2019
++
++from waflib import TaskGen
++
++@TaskGen.feature('c', 'cxx', 'fc')
++@TaskGen.after_method('propagate_uselib_vars')
++def add_pdb_per_object(self):
++ """For msvc/fortran, specify a unique compile pdb per object, to work
++ around LNK4099. Flags are updated with a unique /Fd flag based on the
++ task output name. This is separate from the link pdb.
++ """
++ if not hasattr(self, 'compiled_tasks'):
++ return
++
++ link_task = getattr(self, 'link_task', None)
++
++ for task in self.compiled_tasks:
++ if task.inputs and task.inputs[0].name.lower().endswith('.rc'):
++ continue
++
++ add_pdb = False
++ for flagname in ('CFLAGS', 'CXXFLAGS', 'FCFLAGS'):
++ # several languages may be used at once
++ for flag in task.env[flagname]:
++ if flag[1:].lower() == 'zi':
++ add_pdb = True
++ break
++
++ if add_pdb:
++ node = task.outputs[0].change_ext('.pdb')
++ pdb_flag = '/Fd:' + node.abspath()
++
++ for flagname in ('CFLAGS', 'CXXFLAGS', 'FCFLAGS'):
++ buf = [pdb_flag]
++ for flag in task.env[flagname]:
++ if flag[1:3] == 'Fd' or flag[1:].lower() == 'fs' or flag[1:].lower() == 'mp':
++ continue
++ buf.append(flag)
++ task.env[flagname] = buf
++
++ if link_task and not node in link_task.dep_nodes:
++ link_task.dep_nodes.append(node)
++ if not node in task.outputs:
++ task.outputs.append(node)
+diff --git a/waflib/extras/sphinx.py b/waflib/extras/sphinx.py
+new file mode 100644
+index 000000000..08f3cfd8a
+--- /dev/null
++++ b/waflib/extras/sphinx.py
+@@ -0,0 +1,120 @@
++"""Support for Sphinx documentation
++
++This is a wrapper for sphinx-build program. Please note that sphinx-build supports only
++one output format at a time, but the tool can create multiple tasks to handle more.
++The output formats can be passed via the sphinx_output_format, which is an array of
++strings. For backwards compatibility if only one output is needed, it can be passed
++as a single string.
++The default output format is html.
++
++Specific formats can be installed in different directories by specifying the
++install_path_<FORMAT> attribute. If not defined, the standard install_path
++will be used instead.
++
++Example wscript:
++
++def configure(cnf):
++ conf.load('sphinx')
++
++def build(bld):
++ bld(
++ features='sphinx',
++ sphinx_source='sources', # path to source directory
++ sphinx_options='-a -v', # sphinx-build program additional options
++ sphinx_output_format=['html', 'man'], # output format of sphinx documentation
++ install_path_man='${DOCDIR}/man' # put man pages in a specific directory
++ )
++
++"""
++
++from waflib.Node import Node
++from waflib import Utils
++from waflib import Task
++from waflib.TaskGen import feature, after_method
++
++
++def configure(cnf):
++ """Check if sphinx-build program is available and loads gnu_dirs tool."""
++ cnf.find_program('sphinx-build', var='SPHINX_BUILD', mandatory=False)
++ cnf.load('gnu_dirs')
++
++
++@feature('sphinx')
++def build_sphinx(self):
++ """Builds sphinx sources.
++ """
++ if not self.env.SPHINX_BUILD:
++ self.bld.fatal('Program SPHINX_BUILD not defined.')
++ if not getattr(self, 'sphinx_source', None):
++ self.bld.fatal('Attribute sphinx_source not defined.')
++ if not isinstance(self.sphinx_source, Node):
++ self.sphinx_source = self.path.find_node(self.sphinx_source)
++ if not self.sphinx_source:
++ self.bld.fatal('Can\'t find sphinx_source: %r' % self.sphinx_source)
++
++ # In the taskgen we have the complete list of formats
++ Utils.def_attrs(self, sphinx_output_format='html')
++ self.sphinx_output_format = Utils.to_list(self.sphinx_output_format)
++
++ self.env.SPHINX_OPTIONS = getattr(self, 'sphinx_options', [])
++
++ for source_file in self.sphinx_source.ant_glob('**/*'):
++ self.bld.add_manual_dependency(self.sphinx_source, source_file)
++
++ for cfmt in self.sphinx_output_format:
++ sphinx_build_task = self.create_task('SphinxBuildingTask')
++ sphinx_build_task.set_inputs(self.sphinx_source)
++ # In task we keep the specific format this task is generating
++ sphinx_build_task.env.SPHINX_OUTPUT_FORMAT = cfmt
++
++ # the sphinx-build results are in <build + output_format> directory
++ sphinx_build_task.sphinx_output_directory = self.path.get_bld().make_node(cfmt)
++ sphinx_build_task.set_outputs(sphinx_build_task.sphinx_output_directory)
++ sphinx_build_task.sphinx_output_directory.mkdir()
++
++ Utils.def_attrs(sphinx_build_task, install_path=getattr(self, 'install_path_' + cfmt, getattr(self, 'install_path', get_install_path(sphinx_build_task))))
++
++
++def get_install_path(object):
++ if object.env.SPHINX_OUTPUT_FORMAT == 'man':
++ return object.env.MANDIR
++ elif object.env.SPHINX_OUTPUT_FORMAT == 'info':
++ return object.env.INFODIR
++ else:
++ return object.env.DOCDIR
++
++
++class SphinxBuildingTask(Task.Task):
++ color = 'BOLD'
++ run_str = '${SPHINX_BUILD} -M ${SPHINX_OUTPUT_FORMAT} ${SRC} ${TGT} -d ${TGT[0].bld_dir()}/doctrees-${SPHINX_OUTPUT_FORMAT} ${SPHINX_OPTIONS}'
++
++ def keyword(self):
++ return 'Compiling (%s)' % self.env.SPHINX_OUTPUT_FORMAT
++
++ def runnable_status(self):
++
++ for x in self.run_after:
++ if not x.hasrun:
++ return Task.ASK_LATER
++
++ self.signature()
++ ret = Task.Task.runnable_status(self)
++ if ret == Task.SKIP_ME:
++ # in case the files were removed
++ self.add_install()
++ return ret
++
++
++ def post_run(self):
++ self.add_install()
++ return Task.Task.post_run(self)
++
++
++ def add_install(self):
++ nodes = self.sphinx_output_directory.ant_glob('**/*', quiet=True)
++ self.outputs += nodes
++ self.generator.add_install_files(install_to=self.install_path,
++ install_from=nodes,
++ postpone=False,
++ cwd=self.sphinx_output_directory.make_node(self.env.SPHINX_OUTPUT_FORMAT),
++ relative_trick=True)
+diff --git a/waflib/extras/wafcache.py b/waflib/extras/wafcache.py
+new file mode 100644
+index 000000000..30ac3ef51
+--- /dev/null
++++ b/waflib/extras/wafcache.py
+@@ -0,0 +1,648 @@
++#! /usr/bin/env python
++# encoding: utf-8
++# Thomas Nagy, 2019 (ita)
++
++"""
++Filesystem-based cache system to share and re-use build artifacts
++
++Cache access operations (copy to and from) are delegated to
++independent pre-forked worker subprocesses.
++
++The following environment variables may be set:
++* WAFCACHE: several possibilities:
++ - File cache:
++ absolute path of the waf cache (~/.cache/wafcache_user,
++ where `user` represents the currently logged-in user)
++ - URL to a cache server, for example:
++ export WAFCACHE=http://localhost:8080/files/
++ in that case, GET/POST requests are made to urls of the form
++ http://localhost:8080/files/000000000/0 (cache management is delegated to the server)
++ - GCS, S3 or MINIO bucket
++ gs://my-bucket/ (uses gsutil command line tool or WAFCACHE_CMD)
++ s3://my-bucket/ (uses aws command line tool or WAFCACHE_CMD)
++ minio://my-bucket/ (uses mc command line tool or WAFCACHE_CMD)
++* WAFCACHE_CMD: bucket upload/download command, for example:
++ WAFCACHE_CMD="gsutil cp %{SRC} %{TGT}"
++ Note that the WAFCACHE bucket value is used for the source or destination
++ depending on the operation (upload or download). For example, with:
++ WAFCACHE="gs://mybucket/"
++ the following commands may be run:
++ gsutil cp build/myprogram gs://mybucket/aa/aaaaa/1
++ gsutil cp gs://mybucket/bb/bbbbb/2 build/somefile
++* WAFCACHE_NO_PUSH: if set, disables pushing to the cache
++* WAFCACHE_VERBOSITY: if set, displays more detailed cache operations
++* WAFCACHE_STATS: if set, displays cache usage statistics on exit
++
++File cache specific options:
++ Files are copied using hard links by default; if the cache is located
++ onto another partition, the system switches to file copies instead.
++* WAFCACHE_TRIM_MAX_FOLDER: maximum amount of tasks to cache (1M)
++* WAFCACHE_EVICT_MAX_BYTES: maximum amount of cache size in bytes (10GB)
++* WAFCACHE_EVICT_INTERVAL_MINUTES: minimum time interval to try
++ and trim the cache (3 minutes)
++
++Upload specific options:
++* WAFCACHE_ASYNC_WORKERS: define a number of workers to upload results asynchronously
++ this may improve build performance with many/long file uploads
++ the default is unset (synchronous uploads)
++* WAFCACHE_ASYNC_NOWAIT: do not wait for uploads to complete (default: False)
++ this requires asynchonous uploads to have an effect
++
++Usage::
++
++ def build(bld):
++ bld.load('wafcache')
++ ...
++
++To troubleshoot::
++
++ waf clean build --zone=wafcache
++"""
++
++import atexit, base64, errno, fcntl, getpass, os, re, shutil, sys, time, threading, traceback, urllib3, shlex
++try:
++ import subprocess32 as subprocess
++except ImportError:
++ import subprocess
++
++base_cache = os.path.expanduser('~/.cache/')
++if not os.path.isdir(base_cache):
++ base_cache = '/tmp/'
++default_wafcache_dir = os.path.join(base_cache, 'wafcache_' + getpass.getuser())
++
++CACHE_DIR = os.environ.get('WAFCACHE', default_wafcache_dir)
++WAFCACHE_CMD = os.environ.get('WAFCACHE_CMD')
++TRIM_MAX_FOLDERS = int(os.environ.get('WAFCACHE_TRIM_MAX_FOLDER', 1000000))
++EVICT_INTERVAL_MINUTES = int(os.environ.get('WAFCACHE_EVICT_INTERVAL_MINUTES', 3))
++EVICT_MAX_BYTES = int(os.environ.get('WAFCACHE_EVICT_MAX_BYTES', 10**10))
++WAFCACHE_NO_PUSH = 1 if os.environ.get('WAFCACHE_NO_PUSH') else 0
++WAFCACHE_VERBOSITY = 1 if os.environ.get('WAFCACHE_VERBOSITY') else 0
++WAFCACHE_STATS = 1 if os.environ.get('WAFCACHE_STATS') else 0
++WAFCACHE_ASYNC_WORKERS = os.environ.get('WAFCACHE_ASYNC_WORKERS')
++WAFCACHE_ASYNC_NOWAIT = os.environ.get('WAFCACHE_ASYNC_NOWAIT')
++OK = "ok"
++
++re_waf_cmd = re.compile('(?P<src>%{SRC})|(?P<tgt>%{TGT})')
++
++try:
++ import cPickle
++except ImportError:
++ import pickle as cPickle
++
++if __name__ != '__main__':
++ from waflib import Task, Logs, Utils, Build
++
++def can_retrieve_cache(self):
++ """
++ New method for waf Task classes
++ """
++ if not self.outputs:
++ return False
++
++ self.cached = False
++
++ sig = self.signature()
++ ssig = Utils.to_hex(self.uid() + sig)
++
++ if WAFCACHE_STATS:
++ self.generator.bld.cache_reqs += 1
++
++ files_to = [node.abspath() for node in self.outputs]
++ proc = get_process()
++ err = cache_command(proc, ssig, [], files_to)
++ process_pool.append(proc)
++ if err.startswith(OK):
++ if WAFCACHE_VERBOSITY:
++ Logs.pprint('CYAN', ' Fetched %r from cache' % files_to)
++ else:
++ Logs.debug('wafcache: fetched %r from cache', files_to)
++ if WAFCACHE_STATS:
++ self.generator.bld.cache_hits += 1
++ else:
++ if WAFCACHE_VERBOSITY:
++ Logs.pprint('YELLOW', ' No cache entry %s' % files_to)
++ else:
++ Logs.debug('wafcache: No cache entry %s: %s', files_to, err)
++ return False
++
++ self.cached = True
++ return True
++
++def put_files_cache(self):
++ """
++ New method for waf Task classes
++ """
++ if WAFCACHE_NO_PUSH or getattr(self, 'cached', None) or not self.outputs:
++ return
++
++ files_from = []
++ for node in self.outputs:
++ path = node.abspath()
++ if not os.path.isfile(path):
++ return
++ files_from.append(path)
++
++ bld = self.generator.bld
++ old_sig = self.signature()
++
++ for node in self.inputs:
++ try:
++ del node.ctx.cache_sig[node]
++ except KeyError:
++ pass
++
++ delattr(self, 'cache_sig')
++ sig = self.signature()
++
++ def _async_put_files_cache(bld, ssig, files_from):
++ proc = get_process()
++ if WAFCACHE_ASYNC_WORKERS:
++ with bld.wafcache_lock:
++ if bld.wafcache_stop:
++ process_pool.append(proc)
++ return
++ bld.wafcache_procs.add(proc)
++
++ err = cache_command(proc, ssig, files_from, [])
++ process_pool.append(proc)
++ if err.startswith(OK):
++ if WAFCACHE_VERBOSITY:
++ Logs.pprint('CYAN', ' Successfully uploaded %s to cache' % files_from)
++ else:
++ Logs.debug('wafcache: Successfully uploaded %r to cache', files_from)
++ if WAFCACHE_STATS:
++ bld.cache_puts += 1
++ else:
++ if WAFCACHE_VERBOSITY:
++ Logs.pprint('RED', ' Error caching step results %s: %s' % (files_from, err))
++ else:
++ Logs.debug('wafcache: Error caching results %s: %s', files_from, err)
++
++ if old_sig == sig:
++ ssig = Utils.to_hex(self.uid() + sig)
++ if WAFCACHE_ASYNC_WORKERS:
++ fut = bld.wafcache_executor.submit(_async_put_files_cache, bld, ssig, files_from)
++ bld.wafcache_uploads.append(fut)
++ else:
++ _async_put_files_cache(bld, ssig, files_from)
++ else:
++ Logs.debug('wafcache: skipped %r upload due to late input modifications %r', self.outputs, self.inputs)
++
++ bld.task_sigs[self.uid()] = self.cache_sig
++
++def hash_env_vars(self, env, vars_lst):
++ """
++ Reimplement BuildContext.hash_env_vars so that the resulting hash does not depend on local paths
++ """
++ if not env.table:
++ env = env.parent
++ if not env:
++ return Utils.SIG_NIL
++
++ idx = str(id(env)) + str(vars_lst)
++ try:
++ cache = self.cache_env
++ except AttributeError:
++ cache = self.cache_env = {}
++ else:
++ try:
++ return self.cache_env[idx]
++ except KeyError:
++ pass
++
++ v = str([env[a] for a in vars_lst])
++ v = v.replace(self.srcnode.abspath().__repr__()[:-1], '')
++ m = Utils.md5()
++ m.update(v.encode())
++ ret = m.digest()
++
++ Logs.debug('envhash: %r %r', ret, v)
++
++ cache[idx] = ret
++
++ return ret
++
++def uid(self):
++ """
++ Reimplement Task.uid() so that the signature does not depend on local paths
++ """
++ try:
++ return self.uid_
++ except AttributeError:
++ m = Utils.md5()
++ src = self.generator.bld.srcnode
++ up = m.update
++ up(self.__class__.__name__.encode())
++ for x in self.inputs + self.outputs:
++ up(x.path_from(src).encode())
++ self.uid_ = m.digest()
++ return self.uid_
++
++
++def make_cached(cls):
++ """
++ Enable the waf cache for a given task class
++ """
++ if getattr(cls, 'nocache', None) or getattr(cls, 'has_cache', False):
++ return
++
++ full_name = "%s.%s" % (cls.__module__, cls.__name__)
++ if full_name in ('waflib.Tools.ccroot.vnum', 'waflib.Build.inst'):
++ return
++
++ m1 = getattr(cls, 'run', None)
++ def run(self):
++ if getattr(self, 'nocache', False):
++ return m1(self)
++ if self.can_retrieve_cache():
++ return 0
++ return m1(self)
++ cls.run = run
++
++ m2 = getattr(cls, 'post_run', None)
++ def post_run(self):
++ if getattr(self, 'nocache', False):
++ return m2(self)
++ ret = m2(self)
++ self.put_files_cache()
++ return ret
++ cls.post_run = post_run
++ cls.has_cache = True
++
++process_pool = []
++def get_process():
++ """
++ Returns a worker process that can process waf cache commands
++ The worker process is assumed to be returned to the process pool when unused
++ """
++ try:
++ return process_pool.pop()
++ except IndexError:
++ filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'wafcache.py'
++ cmd = [sys.executable, '-c', Utils.readf(filepath)]
++ return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0)
++
++def atexit_pool():
++ for proc in process_pool:
++ proc.kill()
++atexit.register(atexit_pool)
++
++def build(bld):
++ """
++ Called during the build process to enable file caching
++ """
++
++ if WAFCACHE_ASYNC_WORKERS:
++ try:
++ num_workers = int(WAFCACHE_ASYNC_WORKERS)
++ except ValueError:
++ Logs.warn('Invalid WAFCACHE_ASYNC_WORKERS specified: %r' % WAFCACHE_ASYNC_WORKERS)
++ else:
++ from concurrent.futures import ThreadPoolExecutor
++ bld.wafcache_executor = ThreadPoolExecutor(max_workers=num_workers)
++ bld.wafcache_uploads = []
++ bld.wafcache_procs = set([])
++ bld.wafcache_stop = False
++ bld.wafcache_lock = threading.Lock()
++
++ def finalize_upload_async(bld):
++ if WAFCACHE_ASYNC_NOWAIT:
++ with bld.wafcache_lock:
++ bld.wafcache_stop = True
++
++ for fut in reversed(bld.wafcache_uploads):
++ fut.cancel()
++
++ for proc in bld.wafcache_procs:
++ proc.kill()
++
++ bld.wafcache_procs.clear()
++ else:
++ Logs.pprint('CYAN', '... waiting for wafcache uploads to complete (%s uploads)' % len(bld.wafcache_uploads))
++ bld.wafcache_executor.shutdown(wait=True)
++ bld.add_post_fun(finalize_upload_async)
++
++ if WAFCACHE_STATS:
++ # Init counter for statistics and hook to print results at the end
++ bld.cache_reqs = bld.cache_hits = bld.cache_puts = 0
++
++ def printstats(bld):
++ hit_ratio = 0
++ if bld.cache_reqs > 0:
++ hit_ratio = (bld.cache_hits / bld.cache_reqs) * 100
++ Logs.pprint('CYAN', ' wafcache stats: %s requests, %s hits (ratio: %.2f%%), %s writes' %
++ (bld.cache_reqs, bld.cache_hits, hit_ratio, bld.cache_puts) )
++ bld.add_post_fun(printstats)
++
++ if process_pool:
++ # already called once
++ return
++
++ # pre-allocation
++ processes = [get_process() for x in range(bld.jobs)]
++ process_pool.extend(processes)
++
++ Task.Task.can_retrieve_cache = can_retrieve_cache
++ Task.Task.put_files_cache = put_files_cache
++ Task.Task.uid = uid
++ Build.BuildContext.hash_env_vars = hash_env_vars
++ for x in reversed(list(Task.classes.values())):
++ make_cached(x)
++
++def cache_command(proc, sig, files_from, files_to):
++ """
++ Create a command for cache worker processes, returns a pickled
++ base64-encoded tuple containing the task signature, a list of files to
++ cache and a list of files files to get from cache (one of the lists
++ is assumed to be empty)
++ """
++ obj = base64.b64encode(cPickle.dumps([sig, files_from, files_to]))
++ proc.stdin.write(obj)
++ proc.stdin.write('\n'.encode())
++ proc.stdin.flush()
++ obj = proc.stdout.readline()
++ if not obj:
++ raise OSError('Preforked sub-process %r died' % proc.pid)
++ return cPickle.loads(base64.b64decode(obj))
++
++try:
++ copyfun = os.link
++except NameError:
++ copyfun = shutil.copy2
++
++def atomic_copy(orig, dest):
++ """
++ Copy files to the cache, the operation is atomic for a given file
++ """
++ global copyfun
++ tmp = dest + '.tmp'
++ up = os.path.dirname(dest)
++ try:
++ os.makedirs(up)
++ except OSError:
++ pass
++
++ try:
++ copyfun(orig, tmp)
++ except OSError as e:
++ if e.errno == errno.EXDEV:
++ copyfun = shutil.copy2
++ copyfun(orig, tmp)
++ else:
++ raise
++ os.rename(tmp, dest)
++
++def lru_trim():
++ """
++ the cache folders take the form:
++ `CACHE_DIR/0b/0b180f82246d726ece37c8ccd0fb1cde2650d7bfcf122ec1f169079a3bfc0ab9`
++ they are listed in order of last access, and then removed
++ until the amount of folders is within TRIM_MAX_FOLDERS and the total space
++ taken by files is less than EVICT_MAX_BYTES
++ """
++ lst = []
++ for up in os.listdir(CACHE_DIR):
++ if len(up) == 2:
++ sub = os.path.join(CACHE_DIR, up)
++ for hval in os.listdir(sub):
++ path = os.path.join(sub, hval)
++
++ size = 0
++ for fname in os.listdir(path):
++ try:
++ size += os.lstat(os.path.join(path, fname)).st_size
++ except OSError:
++ pass
++ lst.append((os.stat(path).st_mtime, size, path))
++
++ lst.sort(key=lambda x: x[0])
++ lst.reverse()
++
++ tot = sum(x[1] for x in lst)
++ while tot > EVICT_MAX_BYTES or len(lst) > TRIM_MAX_FOLDERS:
++ _, tmp_size, path = lst.pop()
++ tot -= tmp_size
++
++ tmp = path + '.remove'
++ try:
++ shutil.rmtree(tmp)
++ except OSError:
++ pass
++ try:
++ os.rename(path, tmp)
++ except OSError:
++ sys.stderr.write('Could not rename %r to %r\n' % (path, tmp))
++ else:
++ try:
++ shutil.rmtree(tmp)
++ except OSError:
++ sys.stderr.write('Could not remove %r\n' % tmp)
++ sys.stderr.write("Cache trimmed: %r bytes in %r folders left\n" % (tot, len(lst)))
++
++
++def lru_evict():
++ """
++ Reduce the cache size
++ """
++ lockfile = os.path.join(CACHE_DIR, 'all.lock')
++ try:
++ st = os.stat(lockfile)
++ except EnvironmentError as e:
++ if e.errno == errno.ENOENT:
++ with open(lockfile, 'w') as f:
++ f.write('')
++ return
++ else:
++ raise
++
++ if st.st_mtime < time.time() - EVICT_INTERVAL_MINUTES * 60:
++ # check every EVICT_INTERVAL_MINUTES minutes if the cache is too big
++ # OCLOEXEC is unnecessary because no processes are spawned
++ fd = os.open(lockfile, os.O_RDWR | os.O_CREAT, 0o755)
++ try:
++ try:
++ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
++ except EnvironmentError:
++ if WAFCACHE_VERBOSITY:
++ sys.stderr.write('wafcache: another cleaning process is running\n')
++ else:
++ # now dow the actual cleanup
++ lru_trim()
++ os.utime(lockfile, None)
++ finally:
++ os.close(fd)
++
++class netcache(object):
++ def __init__(self):
++ self.http = urllib3.PoolManager()
++
++ def url_of(self, sig, i):
++ return "%s/%s/%s" % (CACHE_DIR, sig, i)
++
++ def upload(self, file_path, sig, i):
++ url = self.url_of(sig, i)
++ with open(file_path, 'rb') as f:
++ file_data = f.read()
++ r = self.http.request('POST', url, timeout=60,
++ fields={ 'file': ('%s/%s' % (sig, i), file_data), })
++ if r.status >= 400:
++ raise OSError("Invalid status %r %r" % (url, r.status))
++
++ def download(self, file_path, sig, i):
++ url = self.url_of(sig, i)
++ with self.http.request('GET', url, preload_content=False, timeout=60) as inf:
++ if inf.status >= 400:
++ raise OSError("Invalid status %r %r" % (url, inf.status))
++ with open(file_path, 'wb') as out:
++ shutil.copyfileobj(inf, out)
++
++ def copy_to_cache(self, sig, files_from, files_to):
++ try:
++ for i, x in enumerate(files_from):
++ if not os.path.islink(x):
++ self.upload(x, sig, i)
++ except Exception:
++ return traceback.format_exc()
++ return OK
++
++ def copy_from_cache(self, sig, files_from, files_to):
++ try:
++ for i, x in enumerate(files_to):
++ self.download(x, sig, i)
++ except Exception:
++ return traceback.format_exc()
++ return OK
++
++class fcache(object):
++ def __init__(self):
++ if not os.path.exists(CACHE_DIR):
++ try:
++ os.makedirs(CACHE_DIR)
++ except OSError:
++ pass
++ if not os.path.exists(CACHE_DIR):
++ raise ValueError('Could not initialize the cache directory')
++
++ def copy_to_cache(self, sig, files_from, files_to):
++ """
++ Copy files to the cache, existing files are overwritten,
++ and the copy is atomic only for a given file, not for all files
++ that belong to a given task object
++ """
++ try:
++ for i, x in enumerate(files_from):
++ dest = os.path.join(CACHE_DIR, sig[:2], sig, str(i))
++ atomic_copy(x, dest)
++ except Exception:
++ return traceback.format_exc()
++ else:
++ # attempt trimming if caching was successful:
++ # we may have things to trim!
++ try:
++ lru_evict()
++ except Exception:
++ return traceback.format_exc()
++ return OK
++
++ def copy_from_cache(self, sig, files_from, files_to):
++ """
++ Copy files from the cache
++ """
++ try:
++ for i, x in enumerate(files_to):
++ orig = os.path.join(CACHE_DIR, sig[:2], sig, str(i))
++ atomic_copy(orig, x)
++
++ # success! update the cache time
++ os.utime(os.path.join(CACHE_DIR, sig[:2], sig), None)
++ except Exception:
++ return traceback.format_exc()
++ return OK
++
++class bucket_cache(object):
++ def bucket_copy(self, source, target):
++ if WAFCACHE_CMD:
++ def replacer(match):
++ if match.group('src'):
++ return source
++ elif match.group('tgt'):
++ return target
++ cmd = [re_waf_cmd.sub(replacer, x) for x in shlex.split(WAFCACHE_CMD)]
++ elif CACHE_DIR.startswith('s3://'):
++ cmd = ['aws', 's3', 'cp', source, target]
++ elif CACHE_DIR.startswith('gs://'):
++ cmd = ['gsutil', 'cp', source, target]
++ else:
++ cmd = ['mc', 'cp', source, target]
++
++ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
++ out, err = proc.communicate()
++ if proc.returncode:
++ raise OSError('Error copy %r to %r using: %r (exit %r):\n out:%s\n err:%s' % (
++ source, target, cmd, proc.returncode, out.decode(errors='replace'), err.decode(errors='replace')))
++
++ def copy_to_cache(self, sig, files_from, files_to):
++ try:
++ for i, x in enumerate(files_from):
++ dest = os.path.join(CACHE_DIR, sig[:2], sig, str(i))
++ self.bucket_copy(x, dest)
++ except Exception:
++ return traceback.format_exc()
++ return OK
++
++ def copy_from_cache(self, sig, files_from, files_to):
++ try:
++ for i, x in enumerate(files_to):
++ orig = os.path.join(CACHE_DIR, sig[:2], sig, str(i))
++ self.bucket_copy(orig, x)
++ except EnvironmentError:
++ return traceback.format_exc()
++ return OK
++
++def loop(service):
++ """
++ This function is run when this file is run as a standalone python script,
++ it assumes a parent process that will communicate the commands to it
++ as pickled-encoded tuples (one line per command)
++
++ The commands are to copy files to the cache or copy files from the
++ cache to a target destination
++ """
++ # one operation is performed at a single time by a single process
++ # therefore stdin never has more than one line
++ txt = sys.stdin.readline().strip()
++ if not txt:
++ # parent process probably ended
++ sys.exit(1)
++ ret = OK
++
++ [sig, files_from, files_to] = cPickle.loads(base64.b64decode(txt))
++ if files_from:
++ # TODO return early when pushing files upstream
++ ret = service.copy_to_cache(sig, files_from, files_to)
++ elif files_to:
++ # the build process waits for workers to (possibly) obtain files from the cache
++ ret = service.copy_from_cache(sig, files_from, files_to)
++ else:
++ ret = "Invalid command"
++
++ obj = base64.b64encode(cPickle.dumps(ret))
++ sys.stdout.write(obj.decode())
++ sys.stdout.write('\n')
++ sys.stdout.flush()
++
++if __name__ == '__main__':
++ if CACHE_DIR.startswith('s3://') or CACHE_DIR.startswith('gs://') or CACHE_DIR.startswith('minio://'):
++ if CACHE_DIR.startswith('minio://'):
++ CACHE_DIR = CACHE_DIR[8:] # minio doesn't need the protocol part, uses config aliases
++ service = bucket_cache()
++ elif CACHE_DIR.startswith('http'):
++ service = netcache()
++ else:
++ service = fcache()
++ while 1:
++ try:
++ loop(service)
++ except KeyboardInterrupt:
++ break
++
+diff --git a/waflib/extras/xcode6.py b/waflib/extras/xcode6.py
+index 91bbff181..c5b309120 100644
+--- a/waflib/extras/xcode6.py
++++ b/waflib/extras/xcode6.py
+@@ -99,7 +99,7 @@ def delete_invalid_values(dct):
+ ...
+ }
+ 'Release': {
+- 'ARCHS' x86_64'
++ 'ARCHS': x86_64'
+ ...
+ }
+ }
+@@ -163,12 +163,12 @@ def tostring(self, value):
+ result = result + "\t\t}"
+ return result
+ elif isinstance(value, str):
+- return "\"%s\"" % value
++ return '"%s"' % value.replace('"', '\\\\\\"')
+ elif isinstance(value, list):
+ result = "(\n"
+ for i in value:
+- result = result + "\t\t\t%s,\n" % self.tostring(i)
+- result = result + "\t\t)"
++ result = result + "\t\t\t\t%s,\n" % self.tostring(i)
++ result = result + "\t\t\t)"
+ return result
+ elif isinstance(value, XCodeNode):
+ return value._id
+@@ -565,13 +565,13 @@ def process_xcode(self):
+ # Override target specific build settings
+ bldsettings = {
+ 'HEADER_SEARCH_PATHS': ['$(inherited)'] + self.env['INCPATHS'],
+- 'LIBRARY_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.LIBPATH) + Utils.to_list(self.env.STLIBPATH) + Utils.to_list(self.env.LIBDIR) ,
++ 'LIBRARY_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.LIBPATH) + Utils.to_list(self.env.STLIBPATH) + Utils.to_list(self.env.LIBDIR),
+ 'FRAMEWORK_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.FRAMEWORKPATH),
+- 'OTHER_LDFLAGS': libs + ' ' + frameworks,
+- 'OTHER_LIBTOOLFLAGS': bld.env['LINKFLAGS'],
++ 'OTHER_LDFLAGS': libs + ' ' + frameworks + ' ' + ' '.join(bld.env['LINKFLAGS']),
+ 'OTHER_CPLUSPLUSFLAGS': Utils.to_list(self.env['CXXFLAGS']),
+ 'OTHER_CFLAGS': Utils.to_list(self.env['CFLAGS']),
+- 'INSTALL_PATH': []
++ 'INSTALL_PATH': [],
++ 'GCC_PREPROCESSOR_DEFINITIONS': self.env['DEFINES']
+ }
+
+ # Install path
+@@ -591,7 +591,7 @@ def process_xcode(self):
+
+ # The keys represents different build configuration, e.g. Debug, Release and so on..
+ # Insert our generated build settings to all configuration names
+- keys = set(settings.keys() + bld.env.PROJ_CONFIGURATION.keys())
++ keys = set(settings.keys()) | set(bld.env.PROJ_CONFIGURATION.keys())
+ for k in keys:
+ if k in settings:
+ settings[k].update(bldsettings)
+diff --git a/waflib/fixpy2.py b/waflib/fixpy2.py
+index 24176e066..c99bff4b9 100644
+--- a/waflib/fixpy2.py
++++ b/waflib/fixpy2.py
+@@ -56,7 +56,7 @@ def r1(code):
+ @subst('Runner.py')
+ def r4(code):
+ "generator syntax"
+- return code.replace('next(self.biter)', 'self.biter.next()')
++ return code.replace('next(self.biter)', 'self.biter.next()').replace('self.daemon = True', 'self.setDaemon(1)')
+
+ @subst('Context.py')
+ def r5(code):
+diff --git a/waflib/processor.py b/waflib/processor.py
+index 2eecf3bd9..eff2e69ad 100755
+--- a/waflib/processor.py
++++ b/waflib/processor.py
+@@ -27,6 +27,10 @@ def run():
+ [cmd, kwargs, cargs] = cPickle.loads(base64.b64decode(txt))
+ cargs = cargs or {}
+
++ if not 'close_fds' in kwargs:
++ # workers have no fds
++ kwargs['close_fds'] = False
++
+ ret = 1
+ out, err, ex, trace = (None, None, None, None)
+ try:
--
2.43.0
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply related [flat|nested] 2+ messages in thread* Re: [Buildroot] [PATCH 1/1] package/jack2: fix build with python 3.12
2024-02-11 22:02 [Buildroot] [PATCH 1/1] package/jack2: fix build with python 3.12 Fabrice Fontaine
@ 2024-07-12 13:01 ` Thomas Petazzoni via buildroot
0 siblings, 0 replies; 2+ messages in thread
From: Thomas Petazzoni via buildroot @ 2024-07-12 13:01 UTC (permalink / raw)
To: Fabrice Fontaine; +Cc: Wojciech M . Zabolotny, buildroot
On Sun, 11 Feb 2024 23:02:22 +0100
Fabrice Fontaine <fontaine.fabrice@gmail.com> wrote:
> Fix the following build failure with python 3.12 (which removed imp
> module):
>
> Traceback (most recent call last):
> File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/./waf", line 166, in <module>
> from waflib import Scripting
> File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/waflib/Scripting.py", line 10, in <module>
> from waflib import Utils, Configure, Logs, Options, ConfigSet, Context, Errors, Build, Node
> File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/waflib/Configure.py", line 16, in <module>
> from waflib import ConfigSet, Utils, Options, Logs, Context, Build, Errors
> File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/waflib/Options.py", line 14, in <module>
> from waflib import Logs, Utils, Context, Errors
> File "/home/autobuild/autobuild/instance-7/output-1/build/jack2-1.9.22/waflib/Context.py", line 9, in <module>
> import os, re, imp, sys
> ModuleNotFoundError: No module named 'imp'
>
> Fixes: 36e635d2d5c0166476858aa239ccbe78e8f2af14
> - http://autobuild.buildroot.org/results/1bfe34e10ffdab80647ac01863165e93bcc9b0d8
>
> Signed-off-by: Fabrice Fontaine <fontaine.fabrice@gmail.com>
> ---
> package/jack2/0001-Update-to-waf-2-0-26.patch | 3480 +++++++++++++++++
> 1 file changed, 3480 insertions(+)
> create mode 100644 package/jack2/0001-Update-to-waf-2-0-26.patch
Since the patch has been merged upstream, I changed the reference in
the Upstream: tag to point to the final commit in the upstream github
repo. Applied with this change. Thanks!
Thomas
--
Thomas Petazzoni, CTO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2024-07-12 13:02 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-02-11 22:02 [Buildroot] [PATCH 1/1] package/jack2: fix build with python 3.12 Fabrice Fontaine
2024-07-12 13:01 ` Thomas Petazzoni via buildroot
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.