From 75a555daa874424e28624e0d27c96a1d9ae9eb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Thu, 27 Sep 2018 00:00:00 +0000 Subject: [PATCH] common: print cd pdw on paths print only modified variables on PATH use common.run_cmd everywhere to get full bash bash commands readme: recommend private/ instead of the cryptic p/ --- README.adoc | 10 +++++----- build-buildroot | 8 ++++---- build-crosstool-ng | 2 +- build-gem5 | 16 ++++++++-------- build-qemu | 7 ++++--- common.py | 25 +++++++++++++++++++------ 6 files changed, 41 insertions(+), 27 deletions(-) diff --git a/README.adoc b/README.adoc index 4f591ea..c771bab 100644 --- a/README.adoc +++ b/README.adoc @@ -10679,18 +10679,18 @@ Next, when you want to build with this repository, use the `--gem5-src` argument .... cd linux-kernel-module-cheat ./build-gem5 \ - --gem5-build-id p/master \ + --gem5-build-id private/master \ --gem5-src "$gem5_internal" \ - --gem5-worktree p/master \ + --gem5-worktree private/master \ ; ./run-gem5 - --gem5-build-id p/master \ + --gem5-build-id private/master \ --gem5-src "$gem5_internal" \ - --gem5-worktree p/master \ + --gem5-worktree private/master \ ; .... -`p` stands for Private, and is not mandatory, but it is a sane default that will remind you at all times that you are dealing with private code instead of public. +`private` is not mandatory, but it is a sane default that will remind you at all times that you are dealing with private code instead of public. This setup only creates gitignored worktrees of the private repository inside this repository, which is pretty safe, while still allowing fully use our infrastructure as usual. diff --git a/build-buildroot b/build-buildroot index 7cbce9e..24a8b88 100755 --- a/build-buildroot +++ b/build-buildroot @@ -156,7 +156,7 @@ def main(args, extra_args=None): if os.path.isdir(package_dir_abs): br2_external_dirs.append(path_relative_to_buildroot(package_dir_abs)) br2_external_str = ':'.join(br2_external_dirs) - subprocess.check_call( + assert common.run_cmd( [ 'make', 'O={}'.format(common.buildroot_build_dir), @@ -164,7 +164,7 @@ def main(args, extra_args=None): defconfig, ], cwd=common.buildroot_src_dir, - ) + ) == 0 buildroot_configs = args.buildroot_config buildroot_configs.extend([ 'BR2_JLEVEL={}'.format(nproc), @@ -254,14 +254,14 @@ def main(args, extra_args=None): buildroot_kernel_config_fragment_str = 'BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{}"'.format(' '.join(kernel_config_fragments)) buildroot_configs.append(buildroot_kernel_config_fragment_str) common.write_configs(common.buildroot_config_file, buildroot_configs, buildroot_config_fragments) - subprocess.check_call( + assert common.run_cmd( [ 'make', 'O={}'.format(common.buildroot_build_dir), 'olddefconfig', ], cwd=common.buildroot_src_dir, - ) + ) == 0 # Manage Linux kernel and QEMU variants. def symlink_buildroot_variant(custom_dir, variant_dir): diff --git a/build-crosstool-ng b/build-crosstool-ng index f9ac297..d074727 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -70,7 +70,7 @@ def main(args, extra_args=None): ], out_file=os.path.join(common.crosstool_ng_build_dir, 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], - extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']}, + extra_paths=[common.ccache_dir], ) == 0 if __name__ == '__main__': diff --git a/build-gem5 b/build-gem5 index 28246ec..a55feca 100755 --- a/build-gem5 +++ b/build-gem5 @@ -30,20 +30,20 @@ else: if not os.path.exists(os.path.join(common.gem5_src_dir, '.git')): if common.gem5_src_dir == common.gem5_default_src_dir: raise Exception('gem5 submodule not checked out') - subprocess.check_call([ + assert common.run_cmd([ 'git', '-C', common.gem5_default_src_dir, 'worktree', 'add', '-b', os.path.join('wt', args.gem5_build_id), common.gem5_src_dir - ]) + ]) == 0 if args.arch == 'x86_64': dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img') with open(dummy_img_path, 'wb') as dummy_img_file: zeroes = b'\x00' * (2 ** 16) for i in range(2 ** 10): dummy_img_file.write(zeroes) - subprocess.check_call(['mkswap', dummy_img_path]) + assert common.run_cmd(['mkswap', dummy_img_path]) == 0 with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'): # This file must always be present, despite --kernel overriding that default and selecting the kernel. # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present. @@ -54,7 +54,7 @@ else: # dtb dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt') - subprocess.check_call(['make', '-C', dt_src_dir]) + assert common.run_cmd(['make', '-C', dt_src_dir]) == 0 os.makedirs(dt_build_dir, exist_ok=True) for dt in glob.glob(os.path.join(dt_src_dir, '*.dtb')): shutil.copy2(dt, dt_build_dir) @@ -62,14 +62,14 @@ else: # Bootloader 32. bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader') # TODO use the buildroot cross compiler here, and remove the dependencies from configure. - subprocess.check_call(['make', '-C', bootloader32_dir]) + assert common.run_cmd(['make', '-C', bootloader32_dir]) == 0 # bootloader shutil.copy2(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) # Bootloader 64. bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') # TODO cross_compile is ignored because the make does not use CC... - subprocess.check_call(['make', '-C', bootloader64_dir]) + assert common.run_cmd(['make', '-C', bootloader64_dir]) == 0 shutil.copy2(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) assert common.run_cmd( [ @@ -81,11 +81,11 @@ else: ] + args.extra_scons_args, cwd=common.gem5_src_dir, - extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']}, + extra_paths=[common.ccache_dir], ) == 0 term_src_dir = os.path.join(common.gem5_src_dir, 'util/term') m5term_build = os.path.join(term_src_dir, 'm5term') - subprocess.check_call(['make', '-C', term_src_dir]) + assert common.run_cmd(['make', '-C', term_src_dir]) == 0 if os.path.exists(common.gem5_m5term): # Otherwise shutil.copy2 would fail with "Text file busy" if you # tried to rebuild while running m5term: diff --git a/build-qemu b/build-qemu index 8ec6564..c573e41 100755 --- a/build-qemu +++ b/build-qemu @@ -21,7 +21,7 @@ if args.clean: else: start_time = time.time() os.makedirs(common.qemu_build_dir, exist_ok=True) - subprocess.check_call( + assert common.run_cmd( [ os.path.join(common.qemu_src_dir, 'configure'), '--enable-debug', @@ -31,8 +31,9 @@ else: '--with-sdlabi=2.0', ] + args.extra_config_args, + extra_paths=[common.ccache_dir], cwd=common.qemu_build_dir - ) + ) == 0 assert common.run_cmd( [ 'make', @@ -40,7 +41,7 @@ else: '-j', str(multiprocessing.cpu_count()), ], cwd=common.qemu_build_dir, - extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']}, + extra_paths=[common.ccache_dir], ) == 0 end_time = time.time() common.print_time(end_time - start_time) diff --git a/common.py b/common.py index b54ed80..7765a04 100644 --- a/common.py +++ b/common.py @@ -265,7 +265,7 @@ def mkdir(): os.makedirs(this.qemu_run_dir, exist_ok=True) os.makedirs(this.p9_dir, exist_ok=True) -def print_cmd(cmd, cmd_file=None, extra_env=None): +def print_cmd(cmd, cwd, cmd_file=None, extra_env=None, extra_paths=None): ''' Format a command given as a list of strings so that it can be viewed nicely and executed by bash directly and print it to stdout. @@ -275,12 +275,15 @@ def print_cmd(cmd, cmd_file=None, extra_env=None): ''' newline_separator = ' \\\n' out = [] + out.append('cd {} &&{}'.format(shlex.quote(cwd), newline_separator)) + if extra_paths is not None: + out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)) + newline_separator) for key in extra_env: - out.extend(['{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])), newline_separator]) + out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])) + newline_separator) for arg in cmd: - out.extend([shlex.quote(arg), newline_separator]) - out = ''.join(out) - print(out) + out.append(shlex.quote(arg) + newline_separator) + out = ' '.join(out) + ';\n' + print('+ ' + out, end='') if cmd_file is not None: with open(cmd_file, 'w') as f: f.write('#!/usr/bin/env bash\n') @@ -343,6 +346,7 @@ def run_cmd( show_stdout=True, show_cmd=True, extra_env=None, + extra_paths=None, delete_env=None, **kwargs ): @@ -380,13 +384,22 @@ def run_cmd( extra_env = {} if delete_env is None: delete_env = [] + if 'cwd' in kwargs: + cwd = kwargs['cwd'] + else: + cwd = os.getcwd() env = os.environ.copy() env.update(extra_env) + if extra_paths is not None: + path = ':'.join(extra_paths) + if 'PATH' in os.environ: + path += ':' + os.environ['PATH'] + env['PATH'] = path for key in delete_env: if key in env: del env[key] if show_cmd: - print_cmd(cmd, cmd_file, extra_env=extra_env) + print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) # Otherwise Ctrl + C gives: # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).