diff --git a/build-baremetal b/build-baremetal index 74d148a..b0c0126 100755 --- a/build-baremetal +++ b/build-baremetal @@ -36,7 +36,7 @@ class BaremetalComponent(common.Component): uart_address = 0x09000000 os.makedirs(build_dir, exist_ok=True) os.makedirs(common.baremetal_build_lib_dir, exist_ok=True) - assert common.run_cmd( + common.run_cmd( [gcc] + cflags + [ @@ -44,8 +44,8 @@ class BaremetalComponent(common.Component): '-o', bootloader_obj, os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)), ] - ) == 0 - assert common.run_cmd( + ) + common.run_cmd( [gcc] + cflags + [ @@ -55,7 +55,7 @@ class BaremetalComponent(common.Component): '-o', common_obj, os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext), ] - ) == 0 + ) self._build_dir( '', gcc=gcc, @@ -126,7 +126,7 @@ Build the baremetal examples with crosstool-NG. if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext): in_name = os.path.splitext(in_basename)[0] main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext)) - assert common.run_cmd( + common.run_cmd( [gcc] + cflags + [ @@ -134,8 +134,8 @@ Build the baremetal examples with crosstool-NG. '-o', main_obj, os.path.join(common.baremetal_src_dir, in_path), ] - ) == 0 - assert common.run_cmd( + ) + common.run_cmd( [gcc] + cflags + [ @@ -148,7 +148,7 @@ Build the baremetal examples with crosstool-NG. common_obj, main_obj, ] - ) == 0 + ) if __name__ == '__main__': BaremetalComponent().build() diff --git a/build-buildroot b/build-buildroot index 8f459fb..0592760 100755 --- a/build-buildroot +++ b/build-buildroot @@ -133,7 +133,7 @@ usually extra Buildroot targets. if os.path.isdir(package_dir_abs): br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs)) br2_external_str = ':'.join(br2_external_dirs) - assert common.run_cmd( + common.run_cmd( [ 'make', 'O={}'.format(common.buildroot_build_dir), @@ -141,7 +141,7 @@ usually extra Buildroot targets. defconfig, ], cwd=common.buildroot_src_dir, - ) == 0 + ) buildroot_configs = args.buildroot_config buildroot_configs.extend([ 'BR2_JLEVEL={}'.format(nproc), @@ -241,20 +241,20 @@ usually extra Buildroot targets. buildroot_kernel_config_fragment_str = 'BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{}"'.format(' '.join(kernel_config_fragments)) buildroot_configs.append(buildroot_kernel_config_fragment_str) common.write_configs(common.buildroot_config_file, buildroot_configs, buildroot_config_fragments) - assert common.run_cmd( + common.run_cmd( [ 'make', 'O={}'.format(common.buildroot_build_dir), 'olddefconfig', ], cwd=common.buildroot_src_dir, - ) == 0 + ) # Do the actual build. common.make_build_dirs() if not args.no_all: extra_make_args.append('all') - assert common.run_cmd( + common.run_cmd( [ 'make', 'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_src_dir), @@ -267,7 +267,7 @@ usually extra Buildroot targets. out_file=os.path.join(common.buildroot_build_dir, 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], cwd=common.buildroot_src_dir, - ) == 0 + ) # Create the qcow2 from ext2. # Skip if qemu is not present, because gem5 does not need the qcow2. diff --git a/build-crosstool-ng b/build-crosstool-ng index 75602a7..07ae78c 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -17,22 +17,22 @@ class CrosstoolNgComponent(common.Component): # Bootstrap out-ot-tree WONTFIX. I've tried. # https://github.com/crosstool-ng/crosstool-ng/issues/1021 os.chdir(common.crosstool_ng_src_dir) - assert common.run_cmd( + common.run_cmd( [os.path.join(common.crosstool_ng_src_dir, 'bootstrap')], - ) == 0 + ) os.chdir(common.crosstool_ng_util_dir) - assert common.run_cmd( + common.run_cmd( [ os.path.join(common.crosstool_ng_src_dir, 'configure'), '--enable-local', ], - ) == 0 - assert common.run_cmd( + ) + common.run_cmd( [ 'make', '-j', str(nproc), ], - ) == 0 + ) # Build the toolchain. common.cp( @@ -47,14 +47,14 @@ class CrosstoolNgComponent(common.Component): 'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir), ] ) - assert common.run_cmd( + common.run_cmd( [ common.crosstool_ng_executable, 'defconfig', ], - ) == 0 + ) os.unlink(defconfig_dest) - assert common.run_cmd( + common.run_cmd( [ common.crosstool_ng_executable, 'build', @@ -63,7 +63,7 @@ class CrosstoolNgComponent(common.Component): out_file=os.path.join(build_dir, 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], extra_paths=[common.ccache_dir], - ) == 0 + ) def get_argparse_args(self): return { diff --git a/build-docker b/build-docker index 9ad7444..4e4063f 100755 --- a/build-docker +++ b/build-docker @@ -31,12 +31,12 @@ if not args.clean: '--format', '{{.Names}}', ]).decode() if container_name in containers.split(): - assert common.run_cmd([ + common.run_cmd([ 'docker', 'rm', container_name, - ]) == 0 - assert common.run_cmd([ + ]) + common.run_cmd([ 'docker', 'create', '--name', container_name, @@ -49,27 +49,27 @@ if not args.clean: '-v', '{}:{}'.format(common.root_dir, target_dir), 'ubuntu:18.04', 'bash', - ]) == 0 - assert common.run_cmd([ + ]) + common.run_cmd([ 'docker', 'export', '-o', common.docker_tar_file, container_name, - ]) == 0 + ]) tar = tarfile.open(common.docker_tar_file) tar.extractall(common.docker_tar_dir) tar.close() # sudo not required in theory # https://askubuntu.com/questions/1046828/how-to-run-libguestfs-tools-tools-such-as-virt-make-fs-without-sudo - assert common.run_cmd([ + common.run_cmd([ 'virt-make-fs', '--format', 'raw', '--size', '+1G', '--type', 'ext2', common.docker_tar_dir, common.docker_rootfs_raw_file, - ]) == 0 + ]) common.raw_to_qcow2(prebuilt=True) end_time = time.time() common.print_time(end_time - start_time) diff --git a/build-gem5 b/build-gem5 index cae8134..2097ac5 100755 --- a/build-gem5 +++ b/build-gem5 @@ -27,13 +27,13 @@ class Gem5Component(common.Component): if not os.path.exists(os.path.join(common.gem5_src_dir, '.git')): if common.gem5_src_dir == common.gem5_default_src_dir: raise Exception('gem5 submodule not checked out') - assert common.run_cmd([ + common.run_cmd([ 'git', '-C', common.gem5_default_src_dir, 'worktree', 'add', '-b', os.path.join('wt', args.gem5_build_id), common.gem5_src_dir - ]) == 0 + ]) if args.verbose: verbose = ['--verbose'] else: @@ -44,7 +44,7 @@ class Gem5Component(common.Component): zeroes = b'\x00' * (2 ** 16) for i in range(2 ** 10): dummy_img_file.write(zeroes) - assert common.run_cmd(['mkswap', dummy_img_path]) == 0 + common.run_cmd(['mkswap', dummy_img_path]) with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'): # This file must always be present, despite --kernel overriding that default and selecting the kernel. # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present. @@ -55,7 +55,7 @@ class Gem5Component(common.Component): # dtb dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt') - assert common.run_cmd(['make', '-C', dt_src_dir]) == 0 + common.run_cmd(['make', '-C', dt_src_dir]) os.makedirs(dt_build_dir, exist_ok=True) for dt in glob.glob(os.path.join(dt_src_dir, '*.dtb')): common.cp(dt, dt_build_dir) @@ -63,16 +63,16 @@ class Gem5Component(common.Component): # Bootloader 32. bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader') # TODO use the buildroot cross compiler here, and remove the dependencies from configure. - assert common.run_cmd(['make', '-C', bootloader32_dir]) == 0 + common.run_cmd(['make', '-C', bootloader32_dir]) # bootloader common.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) # Bootloader 64. bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') # TODO cross_compile is ignored because the make does not use CC... - assert common.run_cmd(['make', '-C', bootloader64_dir]) == 0 + common.run_cmd(['make', '-C', bootloader64_dir]) common.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) - assert common.run_cmd( + common.run_cmd( ( [ 'scons', @@ -85,10 +85,10 @@ class Gem5Component(common.Component): ), cwd=common.gem5_src_dir, extra_paths=[common.ccache_dir], - ) == 0 + ) term_src_dir = os.path.join(common.gem5_src_dir, 'util/term') m5term_build = os.path.join(term_src_dir, 'm5term') - assert common.run_cmd(['make', '-C', term_src_dir]) == 0 + common.run_cmd(['make', '-C', term_src_dir]) if os.path.exists(common.gem5_m5term): # Otherwise common.cp would fail with "Text file busy" if you # tried to rebuild while running m5term: diff --git a/build-linux b/build-linux index c2269ca..d4934ab 100755 --- a/build-linux +++ b/build-linux @@ -43,7 +43,7 @@ class LinuxComponent(common.Component): verbose = ['V=1'] else: verbose = [] - assert common.run_cmd( + common.run_cmd( [ os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), '-m', @@ -52,8 +52,8 @@ class LinuxComponent(common.Component): os.path.join(common.linux_config_dir, 'min'), os.path.join(common.linux_config_dir, 'default'), ], - ) == 0 - assert common.run_cmd( + ) + common.run_cmd( ( [ 'make', @@ -65,8 +65,8 @@ class LinuxComponent(common.Component): ] ), **common_args, - ) == 0 - assert common.run_cmd( + ) + common.run_cmd( ( [ 'make', @@ -77,7 +77,7 @@ class LinuxComponent(common.Component): args.extra_make_args ), **common_args, - ) == 0 + ) def get_argparse_args(self): return { diff --git a/build-m5 b/build-m5 index c15ebd5..ce886a3 100755 --- a/build-m5 +++ b/build-m5 @@ -28,19 +28,19 @@ class M5Component(common.Component): def do_build(self, args): os.makedirs(common.gem5_m5_build_dir, exist_ok=True) - assert common.run_cmd( + common.run_cmd( self.get_make_cmd(args), cwd=common.gem5_m5_src_dir, - ) == 0 + ) print(common.out_rootfs_overlay_bin_dir) os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True) common.cp(os.path.join(common.gem5_m5_src_dir, 'm5'), common.out_rootfs_overlay_bin_dir) def clean(self, args): - assert common.run_cmd( + common.run_cmd( self.get_make_cmd(args) + ['clean'], cwd=common.gem5_m5_src_dir, - ) == 0 + ) if __name__ == '__main__': M5Component().build() diff --git a/build-modules b/build-modules index b3e14f0..2d83ca2 100755 --- a/build-modules +++ b/build-modules @@ -79,7 +79,7 @@ class ModulesComponent(common.Component): else: linux_dir = common.linux_build_dir build_subdir = os.path.join(build_dir, common.kernel_modules_subdir) - assert common.run_cmd( + common.run_cmd( ( [ 'make', @@ -94,7 +94,7 @@ class ModulesComponent(common.Component): verbose ), cwd=os.path.join(build_subdir), - ) == 0 + ) common.copy_dir_if_update_non_recursive( srcdir=build_subdir, destdir=common.out_rootfs_overlay_dir, diff --git a/build-qemu b/build-qemu index f8cfbe2..2caef78 100755 --- a/build-qemu +++ b/build-qemu @@ -22,7 +22,7 @@ class QemuComponent(common.Component): verbose = ['V=1'] else: verbose = [] - assert common.run_cmd( + common.run_cmd( [ os.path.join(common.qemu_src_dir, 'configure'), '--enable-debug', @@ -34,8 +34,8 @@ class QemuComponent(common.Component): args.extra_config_args, extra_paths=[common.ccache_dir], cwd=build_dir - ) == 0 - assert common.run_cmd( + ) + common.run_cmd( ( [ 'make', @@ -46,7 +46,7 @@ class QemuComponent(common.Component): ), cwd=build_dir, extra_paths=[common.ccache_dir], - ) == 0 + ) def get_build_dir(self, args): return common.qemu_build_dir diff --git a/build-userland b/build-userland index 44a0396..872bc95 100755 --- a/build-userland +++ b/build-userland @@ -38,7 +38,7 @@ has the OpenBLAS libraries and headers installed. allowed_toolchains = ['buildroot'] cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) cxx = common.get_toolchain_tool('g++', allowed_toolchains=allowed_toolchains) - assert common.run_cmd( + common.run_cmd( ( [ 'make', @@ -54,7 +54,7 @@ has the OpenBLAS libraries and headers installed. ), cwd=common.userland_src_dir, extra_paths=[common.ccache_dir], - ) == 0 + ) common.copy_dir_if_update_non_recursive( srcdir=build_dir, destdir=common.out_rootfs_overlay_dir, diff --git a/common.py b/common.py index b465690..d2c2926 100644 --- a/common.py +++ b/common.py @@ -415,13 +415,10 @@ def make_run_dirs(): os.makedirs(this_module.p9_dir, exist_ok=True) os.makedirs(this_module.qemu_run_dir, exist_ok=True) -def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): +def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None): ''' Format a command given as a list of strings so that it can be viewed nicely and executed by bash directly and print it to stdout. - - Optionally save the command to cmd_file file, and add extra_env - environment variables to the command generated. ''' newline_separator = ' \\\n' out = [] @@ -435,12 +432,21 @@ def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])) + newline_separator) for arg in cmd: out.append(shlex.quote(arg) + newline_separator) - out = ' '.join(out) + ';\n' - print(this_module.command_prefix + out, end='') + return ' '.join(out) + ';\n' + +def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): + ''' + Print cmd_to_string to stdout. + + Optionally save the command to cmd_file file, and add extra_env + environment variables to the command generated. + ''' + cmd_string = cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None) + print(this_module.command_prefix + cmd_string, end='') if cmd_file is not None: with open(cmd_file, 'w') as f: f.write('#!/usr/bin/env bash\n') - f.write(out) + f.write(cmd_string) st = os.stat(cmd_file) os.chmod(cmd_file, st.st_mode | stat.S_IXUSR) @@ -466,7 +472,7 @@ def raw_to_qcow2(prebuilt=False, reverse=False): tmp = infile infile = outfile outfile = tmp - assert this_module.run_cmd([ + this_module.run_cmd([ qemu_img_executable, # Prevent qemu-img from generating trace files like QEMU. Disgusting. '-T', 'pr_manager_run,file=/dev/null', @@ -475,7 +481,7 @@ def raw_to_qcow2(prebuilt=False, reverse=False): '-O', outfmt, infile, outfile, - ]) == 0 + ]) def raise_no_x86(arch): if (arch == 'x86_64'): @@ -508,6 +514,7 @@ def run_cmd( extra_paths=None, delete_env=None, dry_run=False, + raise_on_failure=True, **kwargs ): ''' @@ -598,7 +605,10 @@ def run_cmd( break signal.signal(signal.SIGINT, sigint_old) #signal.signal(signal.SIGPIPE, sigpipe_old) - return proc.returncode + returncode = proc.returncode + if returncode != 0 and raise_on_failure: + raise Exception('Command exited with status: {}'.format(returncode)) + return returncode else: return 0 diff --git a/run b/run index b463de9..6c7381a 100755 --- a/run +++ b/run @@ -118,7 +118,7 @@ def main(args, extra_args=None): # This is to run gem5 from a prebuilt download. if (not args.baremetal is None) or (not os.path.exists(common.linux_image)): raise_image_not_found() - assert common.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) == 0 + common.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True) with open(common.gem5_readfile, 'w') as readfile: readfile.write(args.gem5_readfile) @@ -313,10 +313,7 @@ def main(args, extra_args=None): out_file = None else: out_file = common.termout_file - returncode = common.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) - if returncode != 0: - common.log_error('simulator exited with status != 0') - return returncode + common.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) # Check if guest panicked. if args.gem5: # We have to do some parsing here because gem5 exits with status 0 even when panic happens.