From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 778C720C038 for ; Fri, 1 Aug 2025 19:46:38 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=198.137.202.133 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1754077607; cv=none; b=D4+9S4lRxBSJjTB5JT9u5nC7LlbeLYnLozT9ZCmkL7Lj8jlZvakEXJVZG3G3K5ITElOeCTFGTKqOvjNIFeBVkypf6DgEiFKMXPAaeeioAutCGaWMBL/ljQpDaExZdQI5sHDQQ3b37MJDOHDgd5dArZpXdUQzKNdeNIqNwb24oMY= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1754077607; c=relaxed/simple; bh=fR96Uoezh5dT4A0a+GQPYWgC6iAwlbR5a6PqrGahPAI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=SmaPCkE1I5lgIC6rahbOSl11APbbG7vdo/QV0f87M5ScJJI5QpkFKSBBkqOs/9NchTatfOjJOeIxUdShocoVClD/FmjLxuqNtHqFA5XZvRX6F3R0DOanDym47zC8PGfzoyXdR54UdchfIZ/kH4GiD3xCgh+Kfm6ROD/rwuxj8aA= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dmarc=fail (p=quarantine dis=none) header.from=kernel.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=NryVmrvE; arc=none smtp.client-ip=198.137.202.133 Authentication-Results: smtp.subspace.kernel.org; dmarc=fail (p=quarantine dis=none) header.from=kernel.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="NryVmrvE" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=bombadil.20210309; h=Sender:Content-Transfer-Encoding: Content-Type:MIME-Version:References:In-Reply-To:Message-ID:Date:Subject:Cc: To:From:Reply-To:Content-ID:Content-Description; bh=DYNUoQxxmBmy0qepwGW/C+4OdmuzDsVacj7KyN6U3rM=; b=NryVmrvEiFOABiXklZj2QOgNaG CSVfJ+mJbXkgY/4NItskL4tBSUfIQyFruGCwlZx0Ol9nK8oI9BInIQprcLUEoFyQM9GEjWePK2+4Z /y0Tnf45USCIdMdeX9Ls1g1EtIfke/ESAUjiG+H/wZpzSMc1k49NvGAWAKs03OGSgg09TGll1tRYa lsWlOYdkjpuLrhXC8CoTofuivKzfxGBJ1s5EK7L7x6SDs/Zg+gk7uolPHMKzjujCHmgYdG4XrJwU+ hBABIYV8Hcwno9179XwraHY3fiCVQ/T07mMbXFP1iafLujvzqANosdXFQRvr218rpOpCT9bWE86w7 3Ksnlgug==; Received: from mcgrof by bombadil.infradead.org with local (Exim 4.98.2 #2 (Red Hat Linux)) id 1uhvi9-00000006hre-10xH; Fri, 01 Aug 2025 19:46:37 +0000 From: Luis Chamberlain To: Chuck Lever , Daniel Gomez , kdevops@lists.linux.dev Cc: Luis Chamberlain , Daniel Gomez Subject: [PATCH v3 07/11] all: run black Date: Fri, 1 Aug 2025 12:46:31 -0700 Message-ID: <20250801194635.1598544-8-mcgrof@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250801194635.1598544-1-mcgrof@kernel.org> References: <20250801194635.1598544-1-mcgrof@kernel.org> Precedence: bulk X-Mailing-List: kdevops@lists.linux.dev List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sender: Luis Chamberlain Run black to fix tons of styling issues with tons of Python scripts. In order to help bots ensure they don't add odd styling we need a convention. Acked-by: Daniel Gomez Signed-off-by: Luis Chamberlain --- .../blktests/augment_expunge_list.py | 95 ++-- .../workflows/blktests/gen-expunge-args.py | 46 +- .../workflows/blktests/gen-results-dir.py | 39 +- .../blktests/get_new_expunge_files.py | 16 +- .../dynamic-kconfig/gen-dynamic-pci.py | 89 ++-- .../workflows/fstests/augment_expunge_list.py | 163 +++++-- .../workflows/fstests/bad_files_summary.py | 46 +- .../fstests/fstests-checktime-distribution.py | 46 +- .../workflows/fstests/gen_results_summary.py | 132 +++--- .../fstests/get_new_expunge_files.py | 24 +- playbooks/python/workflows/fstests/lib/git.py | 21 +- .../workflows/fstests/xunit_merge_all.py | 33 +- .../sysbench/sysbench-tps-compare.py | 84 +++- .../workflows/sysbench/sysbench-tps-plot.py | 36 +- .../sysbench/sysbench-tps-variance.py | 435 +++++++++++++----- .../gen_pcie_passthrough_guestfs_xml.py | 49 +- .../linux-mirror/python/gen-mirror-files.py | 131 +++--- .../linux-mirror/python/start-mirroring.py | 116 +++-- scripts/check_commit_format.py | 28 +- .../generation/check_for_atomic_calls.py | 71 +-- .../generation/check_for_sleepy_calls.py | 202 +++++--- scripts/detect_whitespace_issues.py | 38 +- scripts/fix_whitespace_issues.py | 44 +- scripts/generate_refs.py | 6 +- scripts/honey-badger.py | 103 +++-- scripts/spdxcheck.py | 201 ++++---- scripts/update_ssh_config_guestfs.py | 49 +- .../workflows/blktests/blktests_watchdog.py | 75 ++- scripts/workflows/cxl/gen_qemu_cxl.py | 235 +++++++--- scripts/workflows/fstests/fstests_watchdog.py | 99 ++-- scripts/workflows/generic/crash_report.py | 4 +- scripts/workflows/generic/crash_watchdog.py | 78 +++- scripts/workflows/lib/blktests.py | 47 +- scripts/workflows/lib/crash.py | 12 +- scripts/workflows/lib/fstests.py | 155 ++++--- scripts/workflows/lib/kssh.py | 178 ++++--- scripts/workflows/lib/systemd_remote.py | 101 ++-- .../workflows/pynfs/check_pynfs_results.py | 17 +- 38 files changed, 2250 insertions(+), 1094 deletions(-) diff --git a/playbooks/python/workflows/blktests/augment_expunge_list.py b/playbooks/python/workflows/blktests/augment_expunge_list.py index 4fe856a3339f..a7b8eb42c874 100755 --- a/playbooks/python/workflows/blktests/augment_expunge_list.py +++ b/playbooks/python/workflows/blktests/augment_expunge_list.py @@ -15,58 +15,81 @@ import configparser from itertools import chain oscheck_ansible_python_dir = os.path.dirname(os.path.abspath(__file__)) -oscheck_sort_expunge = oscheck_ansible_python_dir + "/../../../scripts/workflows/fstests/sort-expunges.sh" +oscheck_sort_expunge = ( + oscheck_ansible_python_dir + "/../../../scripts/workflows/fstests/sort-expunges.sh" +) top_dir = oscheck_ansible_python_dir + "/../../../../" -blktests_last_kernel = top_dir + 'workflows/blktests/results/last-kernel.txt' +blktests_last_kernel = top_dir + "workflows/blktests/results/last-kernel.txt" expunge_name = "failures.txt" + def append_line(output_file, test_failure_line): # We want to now add entries like block/xxx where xxx are digits output = open(output_file, "a+") output.write("%s\n" % test_failure_line) output.close() + def is_config_bool_true(config, name): - if name in config and config[name].strip('\"') == "y": + if name in config and config[name].strip('"') == "y": return True return False + def config_string(config, name): if name in config: - return config[name].strip('\"') + return config[name].strip('"') return None + def get_config(dotconfig): - config = configparser.ConfigParser(allow_no_value=True, strict=False, interpolation=None) + config = configparser.ConfigParser( + allow_no_value=True, strict=False, interpolation=None + ) with open(dotconfig) as lines: lines = chain(("[top]",), lines) config.read_file(lines) return config["top"] return None + def read_blktest_last_kernel(): if not os.path.isfile(blktests_last_kernel): return None - kfile = open(blktests_last_kernel, 'r') + kfile = open(blktests_last_kernel, "r") all_lines = kfile.readlines() kfile.close() for line in all_lines: return line.strip() return None + def main(): - parser = argparse.ArgumentParser(description='Augments expunge list for blktest') - parser.add_argument('results', metavar='', type=str, - help='directory with results file') - parser.add_argument('outputdir', metavar='', type=str, - help='The directory where to generate the expunge failure.txt ') - parser.add_argument('--verbose', const=True, default=False, action="store_const", - help='Print more verbose information') + parser = argparse.ArgumentParser(description="Augments expunge list for blktest") + parser.add_argument( + "results", + metavar="", + type=str, + help="directory with results file", + ) + parser.add_argument( + "outputdir", + metavar="", + type=str, + help="The directory where to generate the expunge failure.txt ", + ) + parser.add_argument( + "--verbose", + const=True, + default=False, + action="store_const", + help="Print more verbose information", + ) args = parser.parse_args() expunge_kernel_dir = "" - dotconfig = top_dir + '/.config' + dotconfig = top_dir + "/.config" config = get_config(dotconfig) if not config: sys.stdout.write("%s does not exist\n" % (dotconfig)) @@ -85,7 +108,7 @@ def main(): continue if not os.path.isfile(f): continue - if not f.endswith('.bad') and not f.endswith('.dmesg'): + if not f.endswith(".bad") and not f.endswith(".dmesg"): continue bad_files.append(f) @@ -97,13 +120,13 @@ def main(): # f may be results/last-run/nodev/meta/009.dmesg bad_file_list = f.split("/") bad_file_list_len = len(bad_file_list) - 1 - bad_file = bad_file_list[bad_file_list_len] - test_group = bad_file_list[bad_file_list_len-1] - bdev = bad_file_list[bad_file_list_len-2] + bad_file = bad_file_list[bad_file_list_len] + test_group = bad_file_list[bad_file_list_len - 1] + bdev = bad_file_list[bad_file_list_len - 2] if args.verbose: sys.stdout.write("%s\n" % bad_file_list) - sys.stdout.write("\tbad_file: %s\n" %bad_file) + sys.stdout.write("\tbad_file: %s\n" % bad_file) sys.stdout.write("\ttest_group: %s\n" % test_group) sys.stdout.write("\tkernel: %s\n" % kernel) @@ -126,11 +149,11 @@ def main(): sys.exit(1) # This is like for example block/xxx where xxx are digits - test_failure_line = test_group + '/' + bad_file_test_number + test_failure_line = test_group + "/" + bad_file_test_number # now to stuff this into expunge files such as: # expunges/sles/15.3/failures.txt - expunge_kernel_dir = args.outputdir + '/' + kernel + '/' + expunge_kernel_dir = args.outputdir + "/" + kernel + "/" output_dir = expunge_kernel_dir output_file = output_dir + expunge_name shortcut_kernel_dir = None @@ -145,19 +168,23 @@ def main(): sles_release_name = config_string(config, "CONFIG_KDEVOPS_HOSTS_PREFIX") sles_release_parts = sles_release_name.split("sp") if len(sles_release_parts) <= 1: - sys.stderr.write("Unexpected sles_release_name: %s\n" % sles_release_name) + sys.stderr.write( + "Unexpected sles_release_name: %s\n" % sles_release_name + ) sys.exit(1) - sles_point_release = sles_release_parts[0].split("sles")[1] + "." + sles_release_parts[1] + sles_point_release = ( + sles_release_parts[0].split("sles")[1] + "." + sles_release_parts[1] + ) # This becomes generic release directory, not specific to any # kernel. - shortcut_dir = args.outputdir + '/' + "sles/" + sles_point_release + '/' + shortcut_dir = args.outputdir + "/" + "sles/" + sles_point_release + "/" shortcut_kernel_dir = shortcut_dir shortcut_file = shortcut_dir + expunge_name else: ksplit = kernel.split(".") shortcut_kernel = ksplit[0] + "." + ksplit[1] + "." + ksplit[2] - shortcut_kernel_dir = args.outputdir + '/' + shortcut_kernel + '/' + shortcut_kernel_dir = args.outputdir + "/" + shortcut_kernel + "/" shortcut_dir = shortcut_kernel_dir shortcut_file = shortcut_dir + expunge_name @@ -170,10 +197,13 @@ def main(): os.makedirs(output_dir) if not os.path.isfile(output_file): - sys.stdout.write("====%s/%s new failure found file was empty\n" % (test_group, test_failure_line)) + sys.stdout.write( + "====%s/%s new failure found file was empty\n" + % (test_group, test_failure_line) + ) append_line(output_file, test_failure_line) else: - existing_file = open(output_file, 'r') + existing_file = open(output_file, "r") all_lines = existing_file.readlines() existing_file.close() found = False @@ -182,13 +212,18 @@ def main(): found = True break if not found: - sys.stdout.write("%s %s new failure found\n" % (test_group, test_failure_line)) + sys.stdout.write( + "%s %s new failure found\n" % (test_group, test_failure_line) + ) append_line(output_file, test_failure_line) if expunge_kernel_dir != "": sys.stdout.write("Sorting %s ...\n" % (expunge_kernel_dir)) - sys.stdout.write("Running %s %s...\n" % (oscheck_sort_expunge, expunge_kernel_dir)) + sys.stdout.write( + "Running %s %s...\n" % (oscheck_sort_expunge, expunge_kernel_dir) + ) subprocess.call([oscheck_sort_expunge, expunge_kernel_dir]) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/playbooks/python/workflows/blktests/gen-expunge-args.py b/playbooks/python/workflows/blktests/gen-expunge-args.py index e73713f826dd..4d7f6d4c61d7 100755 --- a/playbooks/python/workflows/blktests/gen-expunge-args.py +++ b/playbooks/python/workflows/blktests/gen-expunge-args.py @@ -14,16 +14,37 @@ import os import sys import subprocess + def main(): - parser = argparse.ArgumentParser(description='Generates expunge arguments to run blktests check based on results directory') - parser.add_argument('--test-group', metavar='', type=str, - help='group of tests to focus on otherwise all groups are considered') - parser.add_argument('results', metavar='', type=str, - help='directory with blktests results') - parser.add_argument('--gen-exclude-args', const=True, default=False, action="store_const", - help='Generate exclude arguments so to be passed to blktests check') - parser.add_argument('--verbose', const=True, default=False, action="store_const", - help='Print more verbose information') + parser = argparse.ArgumentParser( + description="Generates expunge arguments to run blktests check based on results directory" + ) + parser.add_argument( + "--test-group", + metavar="", + type=str, + help="group of tests to focus on otherwise all groups are considered", + ) + parser.add_argument( + "results", + metavar="", + type=str, + help="directory with blktests results", + ) + parser.add_argument( + "--gen-exclude-args", + const=True, + default=False, + action="store_const", + help="Generate exclude arguments so to be passed to blktests check", + ) + parser.add_argument( + "--verbose", + const=True, + default=False, + action="store_const", + help="Print more verbose information", + ) args = parser.parse_args() bad_files = [] @@ -34,7 +55,7 @@ def main(): continue if not os.path.isfile(f): continue - if f.endswith('.bad') or f.endswith('.dmesg'): + if f.endswith(".bad") or f.endswith(".dmesg"): bad_files.append(f) continue exclude_args = "" @@ -58,12 +79,13 @@ def main(): if args.test_group and args.test_group != group: continue if args.gen_exclude_args: - exclude_args += (" -x %s/%s" % (group, fail)) + exclude_args += " -x %s/%s" % (group, fail) else: sys.stdout.write("%s/%s\n" % (group, fail)) if args.gen_exclude_args: sys.stdout.write("%s\n" % (exclude_args)) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/playbooks/python/workflows/blktests/gen-results-dir.py b/playbooks/python/workflows/blktests/gen-results-dir.py index cb3e76aabce6..e18504bcffd0 100755 --- a/playbooks/python/workflows/blktests/gen-results-dir.py +++ b/playbooks/python/workflows/blktests/gen-results-dir.py @@ -18,7 +18,8 @@ oscheck_ansible_python_dir = os.path.dirname(os.path.abspath(__file__)) top_dir = oscheck_ansible_python_dir + "/../../../../" results_dir = top_dir + "workflows/blktests/results/" last_run_dir = results_dir + "last-run/" -blktests_last_kernel = top_dir + 'workflows/blktests/results/last-kernel.txt' +blktests_last_kernel = top_dir + "workflows/blktests/results/last-kernel.txt" + def clean_empty_dir(target_results): for i in range(1, 3): @@ -31,12 +32,23 @@ def clean_empty_dir(target_results): else: clean_empty_dir(f) + def main(): - parser = argparse.ArgumentParser(description='Get list of expunge files not yet committed in git') - parser.add_argument('--clean-dir-only', metavar='', type=str, default='none', - help='Do not perform an evaluation, just clean empty directories on the specified directory') - parser.add_argument('--copy-all', action='store_true', - help='Copy all test results without filtering') + parser = argparse.ArgumentParser( + description="Get list of expunge files not yet committed in git" + ) + parser.add_argument( + "--clean-dir-only", + metavar="", + type=str, + default="none", + help="Do not perform an evaluation, just clean empty directories on the specified directory", + ) + parser.add_argument( + "--copy-all", + action="store_true", + help="Copy all test results without filtering", + ) args = parser.parse_args() if not os.path.isfile(blktests_last_kernel): @@ -44,7 +56,7 @@ def main(): sys.exit(1) kernel = None - f = open(blktests_last_kernel, 'r') + f = open(blktests_last_kernel, "r") for line in f: kernel = line.strip() if not line: @@ -56,9 +68,11 @@ def main(): clean_empty_dir(args.clean_dir_only) sys.exit(0) - target_results = results_dir + kernel + '/' + target_results = results_dir + kernel + "/" if not os.path.isdir(last_run_dir): - sys.stdout.write("Ignoring last-run directory %s as it is empty ...\n" % (last_run_dir)) + sys.stdout.write( + "Ignoring last-run directory %s as it is empty ...\n" % (last_run_dir) + ) sys.exit(0) sys.stdout.write("Copying %s to %s ...\n" % (last_run_dir, target_results)) copytree(last_run_dir, target_results, dirs_exist_ok=True) @@ -89,8 +103,8 @@ def main(): test_name = test_name_file_list[0] test_dir = os.path.dirname(f) - name_lookup_base = test_dir + '/' + test_name + '*' - name_lookup = test_dir + '/' + test_name + '.*' + name_lookup_base = test_dir + "/" + test_name + "*" + name_lookup = test_dir + "/" + test_name + ".*" listing = glob.glob(name_lookup) bad_ext_found = False if len(listing) > 0: @@ -102,5 +116,6 @@ def main(): os.unlink(r) clean_empty_dir(target_results) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/playbooks/python/workflows/blktests/get_new_expunge_files.py b/playbooks/python/workflows/blktests/get_new_expunge_files.py index 45f0e1a3c3d8..f1f1241bfc7b 100755 --- a/playbooks/python/workflows/blktests/get_new_expunge_files.py +++ b/playbooks/python/workflows/blktests/get_new_expunge_files.py @@ -14,10 +14,17 @@ import sys import subprocess from lib import git + def main(): - parser = argparse.ArgumentParser(description='Get list of expunge files not yet committed in git') - parser.add_argument('expunge_dir', metavar='', type=str, - help='directory with expunge files') + parser = argparse.ArgumentParser( + description="Get list of expunge files not yet committed in git" + ) + parser.add_argument( + "expunge_dir", + metavar="", + type=str, + help="directory with expunge files", + ) args = parser.parse_args() block_expunge_dir = args.expunge_dir @@ -36,5 +43,6 @@ def main(): short_file = f.split("../")[1] sys.stdout.write("%s\n" % (short_file)) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/playbooks/python/workflows/dynamic-kconfig/gen-dynamic-pci.py b/playbooks/python/workflows/dynamic-kconfig/gen-dynamic-pci.py index ed984ae1d778..107bb25b40a5 100755 --- a/playbooks/python/workflows/dynamic-kconfig/gen-dynamic-pci.py +++ b/playbooks/python/workflows/dynamic-kconfig/gen-dynamic-pci.py @@ -15,27 +15,31 @@ sys_bus_prefix = "/sys/bus/pci/devices/" debug = 0 + def get_first_dir(path): if len(os.listdir(path)) > 0: return os.listdir(path)[0] return None + def get_sysname(sys_path, entry): sys_entry_path = sys_path + entry if not os.path.isfile(sys_entry_path): return None - entry_fd = open(sys_entry_path, 'r') + entry_fd = open(sys_entry_path, "r") line = entry_fd.readlines()[0] line = line.strip() entry_fd.close() return line + # kconfig does not like some characters def strip_kconfig_name(name): - fixed_name = name.replace("\"", "") + fixed_name = name.replace('"', "") fixed_name = fixed_name.replace("'", "") return fixed_name + def get_special_device_nvme(pci_id, IOMMUGroup): pci_id_name = strip_kconfig_name(pci_id) sys_path = sys_bus_prefix + pci_id + "/nvme/" @@ -51,7 +55,13 @@ def get_special_device_nvme(pci_id, IOMMUGroup): fw = get_sysname(block_sys_path, "firmware_rev") if not fw: return None - return "%s IOMMU group %s - /dev/%s - %s with FW %s" % (pci_id_name, IOMMUGroup, block_device_name, model, fw) + return "%s IOMMU group %s - /dev/%s - %s with FW %s" % ( + pci_id_name, + IOMMUGroup, + block_device_name, + model, + fw, + ) def get_kconfig_device_name(pci_id, sdevice, IOMMUGroup): @@ -63,26 +73,31 @@ def get_kconfig_device_name(pci_id, sdevice, IOMMUGroup): return strip_kconfig_name(default_name) return strip_kconfig_name(special_name) + def add_pcie_kconfig_string(prefix, val, name): config_name = prefix + "_" + name.upper() sys.stdout.write("config %s\n" % (config_name)) sys.stdout.write("\tstring\n") - sys.stdout.write("\tdefault \"%s\"\n" % (strip_kconfig_name(str(val)))) + sys.stdout.write('\tdefault "%s"\n' % (strip_kconfig_name(str(val)))) sys.stdout.write("\n") + def add_pcie_kconfig_name(config_name, sdevice): sys.stdout.write("config %s\n" % (config_name)) - sys.stdout.write("\tbool \"%s\"\n" % (sdevice)) + sys.stdout.write('\tbool "%s"\n' % (sdevice)) sys.stdout.write("\tdefault n\n") sys.stdout.write("\thelp\n") sys.stdout.write("\t Enabling this will PCI-E passthrough this device onto the\n") sys.stdout.write("\t target guest.\n") sys.stdout.write("\n") + def add_pcie_kconfig_target(config_name, sdevice): sys.stdout.write("config %s_TARGET_GUEST\n" % (config_name)) - sys.stdout.write("\tstring \"Taret guest to offload %s\"\n" % (strip_kconfig_name(sdevice))) - sys.stdout.write("\tdefault \"\"\n") + sys.stdout.write( + '\tstring "Taret guest to offload %s"\n' % (strip_kconfig_name(sdevice)) + ) + sys.stdout.write('\tdefault ""\n') sys.stdout.write("\tdepends on %s\n" % config_name) sys.stdout.write("\tdepends on KDEVOPS_LIBVIRT_PCIE_PASSTHROUGH_TYPE_EACH\n") sys.stdout.write("\thelp\n") @@ -90,7 +105,10 @@ def add_pcie_kconfig_target(config_name, sdevice): sys.stdout.write("\t target guest.\n") sys.stdout.write("\n") -def add_pcie_kconfig_entry(pci_id, sdevice, domain, bus, slot, function, IOMMUGroup, config_id): + +def add_pcie_kconfig_entry( + pci_id, sdevice, domain, bus, slot, function, IOMMUGroup, config_id +): prefix = passthrough_prefix + "_%04d" % config_id name = get_kconfig_device_name(pci_id, sdevice, IOMMUGroup) add_pcie_kconfig_name(prefix, name) @@ -104,22 +122,23 @@ def add_pcie_kconfig_entry(pci_id, sdevice, domain, bus, slot, function, IOMMUGr add_pcie_kconfig_string(prefix, slot, "slot") add_pcie_kconfig_string(prefix, function, "function") + def add_new_device(slot, sdevice, IOMMUGroup, possible_id): # Example expeced format 0000:2d:00.0 - m = re.match(r"^(?P\w+):" - "(?P\w+):" - "(?P\w+)\." - "(?P\w+)$", slot) + m = re.match( + r"^(?P\w+):" "(?P\w+):" "(?P\w+)\." "(?P\w+)$", + slot, + ) if not m: return possible_id possible_id += 1 slot_dict = m.groupdict() - domain = "0x" + slot_dict['DOMAIN'] - bus = "0x" + slot_dict['BUS'] - mslot = "0x" + slot_dict['MSLOT'] - function = "0x" + slot_dict['FUNCTION'] + domain = "0x" + slot_dict["DOMAIN"] + bus = "0x" + slot_dict["BUS"] + mslot = "0x" + slot_dict["MSLOT"] + function = "0x" + slot_dict["FUNCTION"] if debug: sys.stdout.write("\tslot: %s\n" % (slot)) @@ -130,17 +149,26 @@ def add_new_device(slot, sdevice, IOMMUGroup, possible_id): sys.stdout.write("\tIOMMUGroup: %s\n" % (IOMMUGroup)) if possible_id == 1: - sys.stdout.write("# Automatically generated PCI-E passthrough Kconfig by kdevops\n\n") + sys.stdout.write( + "# Automatically generated PCI-E passthrough Kconfig by kdevops\n\n" + ) - add_pcie_kconfig_entry(slot, sdevice, domain, bus, mslot, function, IOMMUGroup, possible_id) + add_pcie_kconfig_entry( + slot, sdevice, domain, bus, mslot, function, IOMMUGroup, possible_id + ) return possible_id + def main(): num_candidate_devices = 0 - parser = argparse.ArgumentParser(description='Creates a Kconfig file lspci output') - parser.add_argument('input', metavar='', type=str, - help='input file wth lspci -Dvmmm output') + parser = argparse.ArgumentParser(description="Creates a Kconfig file lspci output") + parser.add_argument( + "input", + metavar="", + type=str, + help="input file wth lspci -Dvmmm output", + ) args = parser.parse_args() lspci_output = args.input @@ -149,7 +177,7 @@ def main(): sys.stdout.write("input file did not exist: %s\n" % (lspci_output)) sys.exit(1) - lspci = open(lspci_output, 'r') + lspci = open(lspci_output, "r") all_lines = lspci.readlines() lspci.close() @@ -159,17 +187,18 @@ def main(): for line in all_lines: line = line.strip() - m = re.match(r"^(?P\w+):" - "(?P.*)$", line) + m = re.match(r"^(?P\w+):" "(?P.*)$", line) if not m: continue eval_line = m.groupdict() - tag = eval_line['TAG'] - data = eval_line['STRING'] + tag = eval_line["TAG"] + data = eval_line["STRING"] data = data.strip() if tag == "Slot": if sdevice: - num_candidate_devices = add_new_device(slot, sdevice, IOMMUGroup, num_candidate_devices) + num_candidate_devices = add_new_device( + slot, sdevice, IOMMUGroup, num_candidate_devices + ) slot = data sdevice = None IOMMUGroup = None @@ -180,11 +209,13 @@ def main(): # Handle the last device if sdevice and slot: - num_candidate_devices = add_new_device(slot, sdevice, IOMMUGroup, num_candidate_devices) + num_candidate_devices = add_new_device( + slot, sdevice, IOMMUGroup, num_candidate_devices + ) add_pcie_kconfig_string(passthrough_prefix, num_candidate_devices, "NUM_DEVICES") os.unlink(lspci_output) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/playbooks/python/workflows/fstests/augment_expunge_list.py b/playbooks/python/workflows/fstests/augment_expunge_list.py index 9265cd8bc169..7f31401e0b88 100755 --- a/playbooks/python/workflows/fstests/augment_expunge_list.py +++ b/playbooks/python/workflows/fstests/augment_expunge_list.py @@ -16,44 +16,69 @@ import configparser from itertools import chain oscheck_ansible_python_dir = os.path.dirname(os.path.abspath(__file__)) -oscheck_sort_expunge = oscheck_ansible_python_dir + "/../../../scripts/workflows/fstests/sort-expunges.sh" +oscheck_sort_expunge = ( + oscheck_ansible_python_dir + "/../../../scripts/workflows/fstests/sort-expunges.sh" +) top_dir = oscheck_ansible_python_dir + "/../../../../" + def append_line(output_file, test_failure_line): # We want to now add entries like generic/xxx where xxx are digits output = open(output_file, "a+") output.write("%s\n" % test_failure_line) output.close() + def is_config_bool_true(config, name): - if name in config and config[name].strip('\"') == "y": + if name in config and config[name].strip('"') == "y": return True return False + def get_config(dotconfig): - config = configparser.ConfigParser(allow_no_value=True, strict=False, interpolation=None) + config = configparser.ConfigParser( + allow_no_value=True, strict=False, interpolation=None + ) with open(dotconfig) as lines: lines = chain(("[top]",), lines) config.read_file(lines) return config["top"] return None + def main(): - parser = argparse.ArgumentParser(description='Augments expunge list for oscheck') - parser.add_argument('filesystem', metavar='', type=str, - help='filesystem which was tested') - parser.add_argument('results', metavar='', type=str, - help='directory with results file') - parser.add_argument('outputdir', metavar='', type=str, - help='The directory where to generate the expunge lists to') - parser.add_argument('--verbose', const=True, default=False, action="store_const", - help='Print more verbose information') + parser = argparse.ArgumentParser(description="Augments expunge list for oscheck") + parser.add_argument( + "filesystem", + metavar="", + type=str, + help="filesystem which was tested", + ) + parser.add_argument( + "results", + metavar="", + type=str, + help="directory with results file", + ) + parser.add_argument( + "outputdir", + metavar="", + type=str, + help="The directory where to generate the expunge lists to", + ) + parser.add_argument( + "--verbose", + const=True, + default=False, + action="store_const", + help="Print more verbose information", + ) args = parser.parse_args() expunge_kernel_dir = "" -# all_files = os.listdir(args.results) - dotconfig = top_dir + '/.config' + # all_files = os.listdir(args.results) + dotconfig = top_dir + "/.config" config = get_config(dotconfig) if not config: sys.stdout.write("%s does not exist\n" % (dotconfig)) @@ -67,7 +92,7 @@ def main(): continue if not os.path.isfile(f): continue - if f.endswith('.bad') or f.endswith('.dmesg'): + if f.endswith(".bad") or f.endswith(".dmesg"): bad_files.append(f) for f in bad_files: if args.verbose: @@ -78,15 +103,15 @@ def main(): # where xxx are digits bad_file_list = f.split("/") bad_file_list_len = len(bad_file_list) - 1 - bad_file = bad_file_list[bad_file_list_len] - test_group = bad_file_list[bad_file_list_len-1] - section = bad_file_list[bad_file_list_len-2] - kernel = bad_file_list[bad_file_list_len-3] - hostname = bad_file_list[bad_file_list_len-4] + bad_file = bad_file_list[bad_file_list_len] + test_group = bad_file_list[bad_file_list_len - 1] + section = bad_file_list[bad_file_list_len - 2] + kernel = bad_file_list[bad_file_list_len - 3] + hostname = bad_file_list[bad_file_list_len - 4] if args.verbose: sys.stdout.write("%s\n" % bad_file_list) - sys.stdout.write("\tbad_file: %s\n" %bad_file) + sys.stdout.write("\tbad_file: %s\n" % bad_file) sys.stdout.write("\ttest_group: %s\n" % test_group) sys.stdout.write("\tsection: %s\n" % section) sys.stdout.write("\thostname: %s\n" % hostname) @@ -94,20 +119,22 @@ def main(): bad_file_parts = bad_file.split(".") bad_file_test_number = bad_file_parts[0] # This is like for example generic/xxx where xxx are digits - test_failure_line = test_group + '/' + bad_file_test_number + test_failure_line = test_group + "/" + bad_file_test_number # now to stuff this into expunge files such as: # path/4.19.17/xfs/unassigned/xfs_nocrc.txt - expunge_kernel_dir = args.outputdir + '/' + kernel + '/' + args.filesystem + '/' - output_dir = expunge_kernel_dir + 'unassigned/' - output_file = output_dir + section + '.txt' + expunge_kernel_dir = args.outputdir + "/" + kernel + "/" + args.filesystem + "/" + output_dir = expunge_kernel_dir + "unassigned/" + output_file = output_dir + section + ".txt" base_kernel = kernel if base_kernel.endswith("+"): base_kernel = kernel.replace("+", "") - base_expunge_kernel_dir = args.outputdir + '/' + base_kernel + '/' + args.filesystem + '/' - base_output_dir = base_expunge_kernel_dir + 'unassigned/' - base_output_file = base_output_dir + section + '.txt' + base_expunge_kernel_dir = ( + args.outputdir + "/" + base_kernel + "/" + args.filesystem + "/" + ) + base_output_dir = base_expunge_kernel_dir + "unassigned/" + base_output_file = base_output_dir + section + ".txt" shortcut_kernel_dir = None shortcut_dir = None @@ -124,24 +151,39 @@ def main(): sles_release_name = sles_release_parts[0] sles_release_parts = sles_release_name.split("sp") if len(sles_release_parts) <= 1: - sys.stderr.write("Unexpected sles_release_name: %s\n" % sles_release_name) + sys.stderr.write( + "Unexpected sles_release_name: %s\n" % sles_release_name + ) sys.exit(1) sles_point_release = sles_release_parts[0] + "." + sles_release_parts[1] # This becomes generic release directory, not specific to any # kernel. - shortcut_kernel_dir = args.outputdir + '/' + "sles/" + sles_point_release + '/' + args.filesystem + '/' + shortcut_kernel_dir = ( + args.outputdir + + "/" + + "sles/" + + sles_point_release + + "/" + + args.filesystem + + "/" + ) - shortcut_dir = shortcut_kernel_dir + 'unassigned/' - shortcut_file = shortcut_dir + section + '.txt' + shortcut_dir = shortcut_kernel_dir + "unassigned/" + shortcut_file = shortcut_dir + section + ".txt" else: ksplit = kernel.split(".") shortcut_kernel = ksplit[0] + "." + ksplit[1] + "." + ksplit[2] - shortcut_kernel_dir = args.outputdir + '/' + shortcut_kernel + '/' + args.filesystem + '/' - shortcut_dir = shortcut_kernel_dir + 'unassigned/' - shortcut_file = shortcut_dir + section + '.txt' + shortcut_kernel_dir = ( + args.outputdir + "/" + shortcut_kernel + "/" + args.filesystem + "/" + ) + shortcut_dir = shortcut_kernel_dir + "unassigned/" + shortcut_file = shortcut_dir + section + ".txt" elif is_config_bool_true(config, "CONFIG_LIBVIRT_OPENSUSE"): - if is_config_bool_true(config, "CONFIG_WORKFLOW_KOTD_ENABLE") and "leap" in hostname: + if ( + is_config_bool_true(config, "CONFIG_WORKFLOW_KOTD_ENABLE") + and "leap" in hostname + ): leap_host_parts = hostname.split("leap") if len(leap_host_parts) <= 1: sys.stderr.write("Invalid hostname: %s\n" % hostname) @@ -150,22 +192,34 @@ def main(): leap_release_name = leap_release_parts[0] leap_release_parts = leap_release_name.split("sp") if len(leap_release_parts) <= 1: - sys.stderr.write("Unexpected sles_release_name: %s\n" % leap_release_name) + sys.stderr.write( + "Unexpected sles_release_name: %s\n" % leap_release_name + ) sys.exit(1) leap_point_release = leap_release_parts[0] + "." + leap_release_parts[1] # This becomes generic release directory, not specific to any # kernel. - shortcut_kernel_dir = args.outputdir + '/' + "opensuse-leap/" + leap_point_release + '/' + args.filesystem + '/' + shortcut_kernel_dir = ( + args.outputdir + + "/" + + "opensuse-leap/" + + leap_point_release + + "/" + + args.filesystem + + "/" + ) - shortcut_dir = shortcut_kernel_dir + 'unassigned/' - shortcut_file = shortcut_dir + section + '.txt' + shortcut_dir = shortcut_kernel_dir + "unassigned/" + shortcut_file = shortcut_dir + section + ".txt" else: ksplit = kernel.split(".") shortcut_kernel = ksplit[0] + "." + ksplit[1] + "." + ksplit[2] - shortcut_kernel_dir = args.outputdir + '/' + shortcut_kernel + '/' + args.filesystem + '/' - shortcut_dir = shortcut_kernel_dir + 'unassigned/' - shortcut_file = shortcut_dir + section + '.txt' + shortcut_kernel_dir = ( + args.outputdir + "/" + shortcut_kernel + "/" + args.filesystem + "/" + ) + shortcut_dir = shortcut_kernel_dir + "unassigned/" + shortcut_file = shortcut_dir + section + ".txt" if not os.path.isdir(output_dir): if shortcut_dir and os.path.isdir(shortcut_dir): @@ -173,7 +227,10 @@ def main(): output_file = shortcut_file expunge_kernel_dir = shortcut_kernel_dir elif base_kernel != kernel and os.path.isdir(base_output_dir): - sys.stdout.write("<== expunges for %s not found but found base kernel %s expunge directory ==>\n" % (kernel, base_kernel)) + sys.stdout.write( + "<== expunges for %s not found but found base kernel %s expunge directory ==>\n" + % (kernel, base_kernel) + ) expunge_kernel_dir = base_expunge_kernel_dir output_dir = base_output_dir output_file = base_output_file @@ -182,10 +239,13 @@ def main(): os.makedirs(output_dir) if not os.path.isfile(output_file): - sys.stdout.write("====%s %s new failure found file was empty\n" % (section, test_failure_line)) + sys.stdout.write( + "====%s %s new failure found file was empty\n" + % (section, test_failure_line) + ) append_line(output_file, test_failure_line) else: - existing_file = open(output_file, 'r') + existing_file = open(output_file, "r") all_lines = existing_file.readlines() existing_file.close() found = False @@ -194,13 +254,18 @@ def main(): found = True break if not found: - sys.stdout.write("%s %s new failure found\n" % (section, test_failure_line)) + sys.stdout.write( + "%s %s new failure found\n" % (section, test_failure_line) + ) append_line(output_file, test_failure_line) if expunge_kernel_dir != "": sys.stdout.write("Sorting %s ...\n" % (expunge_kernel_dir)) - sys.stdout.write("Running %s %s...\n" % (oscheck_sort_expunge, expunge_kernel_dir)) + sys.stdout.write( + "Running %s %s...\n" % (oscheck_sort_expunge, expunge_kernel_dir) + ) subprocess.call([oscheck_sort_expunge, expunge_kernel_dir]) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/playbooks/python/workflows/fstests/bad_files_summary.py b/playbooks/python/workflows/fstests/bad_files_summary.py index 164409b52466..d0cf4058d596 100755 --- a/playbooks/python/workflows/fstests/bad_files_summary.py +++ b/playbooks/python/workflows/fstests/bad_files_summary.py @@ -11,6 +11,7 @@ import argparse import os import sys + def parse_results_ascii(sections, results, kernel, filesystem): sys.stdout.write("%s on %s\n" % (filesystem, kernel)) for section in sections: @@ -18,6 +19,7 @@ def parse_results_ascii(sections, results, kernel, filesystem): for test in results[section]: sys.stdout.write("\t%s\n" % test) + def parse_results_html(sections, results, kernel, filesystem): sys.stdout.write("%s on %s" % (filesystem, kernel)) sys.stdout.write("

%s on %s

\n" % (filesystem, kernel)) @@ -33,15 +35,28 @@ def parse_results_html(sections, results, kernel, filesystem): sys.stdout.write("\n") sys.stdout.write("") + def main(): - parser = argparse.ArgumentParser(description='generate html file from results') - parser.add_argument('filesystem', metavar='', type=str, - help='filesystem which was tested') - parser.add_argument('results', metavar='', type=str, - help='directory with results file') - parser.add_argument('--format', metavar='', type=str, - help='Output format: ascii html, the default is ascii', - default='txt') + parser = argparse.ArgumentParser(description="generate html file from results") + parser.add_argument( + "filesystem", + metavar="", + type=str, + help="filesystem which was tested", + ) + parser.add_argument( + "results", + metavar="", + type=str, + help="directory with results file", + ) + parser.add_argument( + "--format", + metavar="", + type=str, + help="Output format: ascii html, the default is ascii", + default="txt", + ) args = parser.parse_args() results = dict() sections = list() @@ -51,27 +66,27 @@ def main(): for root, dirs, all_files in os.walk(args.results): for fname in all_files: f = os.path.join(root, fname) - #sys.stdout.write("%s\n" % f) + # sys.stdout.write("%s\n" % f) if os.path.isdir(f): continue if not os.path.isfile(f): continue - if not f.endswith('.bad'): + if not f.endswith(".bad"): continue # f may be results/oscheck-xfs/4.19.0-4-amd64/xfs/generic/091.out.bad bad_file_list = f.split("/") bad_file_list_len = len(bad_file_list) - 1 bad_file = bad_file_list[bad_file_list_len] - test_type = bad_file_list[bad_file_list_len-1] - section = bad_file_list[bad_file_list_len-2] - kernel = bad_file_list[bad_file_list_len-3] + test_type = bad_file_list[bad_file_list_len - 1] + section = bad_file_list[bad_file_list_len - 2] + kernel = bad_file_list[bad_file_list_len - 3] bad_file_parts = bad_file.split(".") bad_file_part_len = len(bad_file_parts) - 1 bad_file_test_number = bad_file_parts[bad_file_part_len - 2] # This is like for example generic/091 - test_failure_line = test_type + '/' + bad_file_test_number + test_failure_line = test_type + "/" + bad_file_test_number test_section = results.get(section) if not test_section: @@ -86,5 +101,6 @@ def main(): else: parse_results_ascii(sections, results, kernel, args.filesystem) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/playbooks/python/workflows/fstests/fstests-checktime-distribution.py b/playbooks/python/workflows/fstests/fstests-checktime-distribution.py index b84e0a290110..75025fa2a541 100755 --- a/playbooks/python/workflows/fstests/fstests-checktime-distribution.py +++ b/playbooks/python/workflows/fstests/fstests-checktime-distribution.py @@ -16,12 +16,21 @@ import subprocess import collections oscheck_ansible_python_dir = os.path.dirname(os.path.abspath(__file__)) -oscheck_sort_expunge = oscheck_ansible_python_dir + "/../../../scripts/workflows/fstests/sort-expunges.sh" +oscheck_sort_expunge = ( + oscheck_ansible_python_dir + "/../../../scripts/workflows/fstests/sort-expunges.sh" +) + def main(): - parser = argparse.ArgumentParser(description='Creates check.time.distribution files for all found check.time files') - parser.add_argument('results', metavar='', type=str, - help='directory with check.time files') + parser = argparse.ArgumentParser( + description="Creates check.time.distribution files for all found check.time files" + ) + parser.add_argument( + "results", + metavar="", + type=str, + help="directory with check.time files", + ) args = parser.parse_args() expunge_kernel_dir = "" @@ -31,22 +40,22 @@ def main(): for root, dirs, all_files in os.walk(args.results): for fname in all_files: f = os.path.join(root, fname) - #sys.stdout.write("%s\n" % f) + # sys.stdout.write("%s\n" % f) if os.path.isdir(f): continue if not os.path.isfile(f): continue - if not f.endswith('check.time'): + if not f.endswith("check.time"): continue # f may be results/oscheck-xfs/4.19.0-4-amd64/check.time - time_distribution = f + '.distribution' + time_distribution = f + ".distribution" if os.path.isfile(time_distribution): os.unlink(time_distribution) - checktime = open(f, 'r') - distribution = open(time_distribution, 'w') + checktime = open(f, "r") + distribution = open(time_distribution, "w") sys.stdout.write("checktime: %s\n" % f) @@ -57,17 +66,17 @@ def main(): num_tests = 0 for line in all_lines: line = line.strip() - m = re.match(r"^(?P\w+)/" - "(?P\d+)\s+" - "(?P