From 33af5648996a91b77333a7635a5e64e0bf1773af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sat, 8 Dec 2018 00:00:00 +0000 Subject: [PATCH] bak --- build | 16 ++++++++-------- build-baremetal | 8 ++++---- build-buildroot | 6 +++--- build-crosstool-ng | 12 ++++++------ build-docker | 8 ++++---- build-linux | 30 ++++++++++++------------------ build-m5 | 6 +++--- build-modules | 2 +- build-qemu | 4 ++-- build-userland | 2 +- common.py | 10 ++++++++-- gem5-shell | 2 +- qemu-trace2txt | 2 +- run | 14 ++------------ run-docker | 14 +++++++------- run-gdb | 2 +- run-toolchain | 2 +- 17 files changed, 65 insertions(+), 75 deletions(-) diff --git a/build b/build index 050f3ff..400a8d0 100755 --- a/build +++ b/build @@ -53,7 +53,7 @@ def run_cmd(cmd, arch): cmd_abs.extend(['--arch', arch]) if args.extra_args: cmd_abs.append(args.extra_args) - common.run_cmd(cmd_abs, dry_run=args.dry_run) + self.sh.run_cmd(cmd_abs, dry_run=args.dry_run) buildroot_component = Component( lambda arch: run_cmd(['build-buildroot'], arch), @@ -410,29 +410,29 @@ if args.download_dependencies: y = ['-y'] else: y = [] - common.run_cmd( + self.sh.run_cmd( sudo + ['apt-get', 'update', common.Newline] ) if apt_get_pkgs: - common.run_cmd( + self.sh.run_cmd( sudo + ['apt-get', 'install'] + y + [common.Newline] + common.add_newlines(sorted(apt_get_pkgs)) ) if apt_build_deps: - common.run_cmd( + self.sh.run_cmd( sudo + ['apt-get', 'build-dep'] + y + [common.Newline] + common.add_newlines(sorted(apt_build_deps)) ) if python2_pkgs: - common.run_cmd( + self.sh.run_cmd( ['python', '-m', 'pip', 'install', '--user', common.Newline] + common.add_newlines(sorted(python2_pkgs)) ) if python3_pkgs: # Not with pip executable directly: # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 - common.run_cmd( + self.sh.run_cmd( ['python3', '-m', 'pip', 'install', '--user', common.Newline] + common.add_newlines(sorted(python3_pkgs)) ) @@ -448,7 +448,7 @@ if args.download_dependencies: # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone # # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 - common.run_cmd( + self.sh.run_cmd( git_cmd_common + ['--', common.Newline] + common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules)]) ) @@ -472,7 +472,7 @@ if args.download_dependencies: # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702 # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac # - common.run_cmd( + self.sh.run_cmd( git_cmd_common + ['--depth', '1', '--', common.Newline] + common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules_shallow)]) ) diff --git a/build-baremetal b/build-baremetal index 16594bb..75c76c7 100755 --- a/build-baremetal +++ b/build-baremetal @@ -46,7 +46,7 @@ class BaremetalComponent(common.Component): os.makedirs(common.baremetal_build_lib_dir, exist_ok=True) src = os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)) if common.need_rebuild([src], bootloader_obj): - common.run_cmd( + self.sh.run_cmd( [gcc, common.Newline] + cflags + [ @@ -60,7 +60,7 @@ class BaremetalComponent(common.Component): (syscalls_src, syscalls_obj), ]: if common.need_rebuild([src], obj): - common.run_cmd( + self.sh.run_cmd( [gcc, common.Newline] + cflags + [ @@ -150,7 +150,7 @@ Build the baremetal examples with crosstool-NG. main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext)) src = os.path.join(common.baremetal_src_dir, in_path) if common.need_rebuild([src], main_obj): - common.run_cmd( + self.sh.run_cmd( [gcc, common.Newline] + cflags + [ @@ -163,7 +163,7 @@ Build the baremetal examples with crosstool-NG. out = os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext) link_script = os.path.join(common.baremetal_src_dir, 'link.ld') if common.need_rebuild(objs + [link_script], out): - common.run_cmd( + self.sh.run_cmd( [gcc, common.Newline] + cflags + [ diff --git a/build-buildroot b/build-buildroot index 2685460..6b2ea87 100755 --- a/build-buildroot +++ b/build-buildroot @@ -85,7 +85,7 @@ usually extra Buildroot targets. if os.path.isdir(package_dir_abs): br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs)) br2_external_str = ':'.join(br2_external_dirs) - common.run_cmd( + self.sh.run_cmd( [ 'make', common.Newline, 'O={}'.format(common.buildroot_build_dir), common.Newline, @@ -132,7 +132,7 @@ usually extra Buildroot targets. # TODO Can't get rid of these for now with nice fragments on Buildroot: # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config self.sh.write_configs(common.buildroot_config_file, configs, config_fragments) - common.run_cmd( + self.sh.run_cmd( [ 'make', common.Newline, 'O={}'.format(common.buildroot_build_dir), common.Newline, @@ -143,7 +143,7 @@ usually extra Buildroot targets. common.make_build_dirs() if not args.no_all: extra_make_args.extend(['all', common.Newline]) - common.run_cmd( + self.sh.run_cmd( [ 'make', common.Newline, 'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_source_dir), common.Newline, diff --git a/build-crosstool-ng b/build-crosstool-ng index 3386ad6..57799a0 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -15,17 +15,17 @@ class CrosstoolNgComponent(common.Component): # Bootstrap out-ot-tree WONTFIX. I've tried. # https://github.com/crosstool-ng/crosstool-ng/issues/1021 os.chdir(common.crosstool_ng_src_dir) - common.run_cmd( + self.sh.run_cmd( [os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), common.Newline], ) os.chdir(common.crosstool_ng_util_dir) - common.run_cmd( + self.sh.run_cmd( [ os.path.join(common.crosstool_ng_src_dir, 'configure'), common.Newline, '--enable-local', common.Newline, ], ) - common.run_cmd( + self.sh.run_cmd( [ 'make', common.Newline, '-j', str(args.nproc), common.Newline, @@ -33,7 +33,7 @@ class CrosstoolNgComponent(common.Component): ) # Build the toolchain. - common.cp( + self.sh.cp( os.path.join(common.root_dir, 'crosstool_ng_config', args.arch), defconfig_dest ) @@ -45,14 +45,14 @@ class CrosstoolNgComponent(common.Component): 'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir), ] ) - common.run_cmd( + self.sh.run_cmd( [ common.crosstool_ng_executable, common.Newline, 'defconfig', common.Newline, ], ) os.unlink(defconfig_dest) - common.run_cmd( + self.sh.run_cmd( [ common.crosstool_ng_executable, common.Newline, 'build', common.Newline, diff --git a/build-docker b/build-docker index 1882b56..7fab055 100755 --- a/build-docker +++ b/build-docker @@ -29,12 +29,12 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- '--format', '{{.Names}}', ]).decode() if container_name in containers.split(): - common.run_cmd([ + self.sh.run_cmd([ 'docker', 'rm', container_name, ]) - common.run_cmd([ + self.sh.run_cmd([ 'docker', 'create', '--name', container_name, @@ -48,7 +48,7 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- 'ubuntu:18.04', 'bash', ]) - common.run_cmd([ + self.sh.run_cmd([ 'docker', 'export', '-o', @@ -60,7 +60,7 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- tar.close() # sudo not required in theory # https://askubuntu.com/questions/1046828/how-to-run-libguestfs-tools-tools-such-as-virt-make-fs-without-sudo - common.run_cmd([ + self.sh.run_cmd([ 'virt-make-fs', '--format', 'raw', '--size', '+1G', diff --git a/build-linux b/build-linux index b173adf..90a8a85 100755 --- a/build-linux +++ b/build-linux @@ -6,8 +6,8 @@ import shutil import common class LinuxComponent(common.Component): - def add_parser_arguments(self, parser): - parser.add_argument( + def __init__(self, parser): + self.add_argument( '--config', default=[], action='append', help='''\ Add a single kernel config configs to the current build. Sample value: @@ -15,14 +15,14 @@ Add a single kernel config configs to the current build. Sample value: configs. Takes precedence over any config files. ''' ) - parser.add_argument( + self.add_argument( '--config-fragment', default=[], action='append', help='''\ Also use the given kernel configuration fragment file. Pass multiple times to use multiple fragment files. ''' ) - parser.add_argument( + self.add_argument( '--custom-config-file', help='''\ Ignore all default kernel configurations and use this file instead. @@ -30,26 +30,20 @@ Still uses options explicitly passed with `--config` and `--config-fragment` on top of it. ''' ) - parser.add_argument( + self.add_argument( '--config-only', default=False, action='store_true', help='''\ Configure the kernel, but don't build it. ''' ) - parser.add_argument( - '--initramfs', default=False, action='store_true', - ) - parser.add_argument( - '--initrd', default=False, action='store_true', - ) - parser.add_argument( + self.add_argument( 'extra_make_args', default=[], metavar='extra-make-args', nargs='*' ) - def do_main(self, **kwargs): + def build(self, **kwargs): build_dir = self.get_build_dir(**kwargs) if args.initrd or args.initramfs: raise Exception('just trolling, --initrd and --initramfs are broken for now') @@ -93,11 +87,11 @@ Configure the kernel, but don't build it. cli_config_str = '\n'.join(args.config) common.write_string_to_file(cli_config_fragment_path, cli_config_str) config_fragments.append(cli_config_fragment_path) - common.cp( + self.sh.cp( base_config_file, os.path.join(build_dir, '.config'), ) - common.run_cmd( + self.sh.run_cmd( [ os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), common.Newline, '-m', common.Newline, @@ -106,7 +100,7 @@ Configure the kernel, but don't build it. ] + common.add_newlines(config_fragments) ) - common.run_cmd( + self.sh.run_cmd( ( common_make_args + ['olddefconfig', common.Newline] @@ -114,14 +108,14 @@ Configure the kernel, but don't build it. **common_args ) if not args.config_only: - common.run_cmd( + self.sh.run_cmd( ( common_make_args + common.add_newlines(args.extra_make_args) ), **common_args ) - common.run_cmd( + self.sh.run_cmd( ( common_make_args + [ diff --git a/build-m5 b/build-m5 index 37c4c48..c7f022a 100755 --- a/build-m5 +++ b/build-m5 @@ -27,15 +27,15 @@ class M5Component(common.Component): # We must clean first or else the build outputs of one arch can conflict with the other. # I should stop being lazy and go actually patch gem5 to support out of tree m5 build... self.clean(args) - common.run_cmd( + self.sh.run_cmd( self.get_make_cmd(args), cwd=common.gem5_m5_source_dir, ) os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True) - common.cp(os.path.join(common.gem5_m5_source_dir, 'm5'), common.out_rootfs_overlay_bin_dir) + self.sh.cp(os.path.join(common.gem5_m5_source_dir, 'm5'), common.out_rootfs_overlay_bin_dir) def clean(self, args): - common.run_cmd( + self.sh.run_cmd( self.get_make_cmd(args) + ['clean', common.Newline], cwd=common.gem5_m5_source_dir, ) diff --git a/build-modules b/build-modules index 73432e5..5fe1a08 100755 --- a/build-modules +++ b/build-modules @@ -85,7 +85,7 @@ Use the host packaged cross toolchain. linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build') else: linux_dir = common.linux_build_dir - common.run_cmd( + self.sh.run_cmd( ( [ 'make', common.Newline, diff --git a/build-qemu b/build-qemu index 5299d07..f867052 100755 --- a/build-qemu +++ b/build-qemu @@ -30,7 +30,7 @@ class QemuComponent(common.Component): target_list = '{}-linux-user'.format(args.arch) else: target_list = '{}-softmmu'.format(args.arch) - common.run_cmd( + self.sh.run_cmd( [ os.path.join(common.qemu_src_dir, 'configure'), common.Newline, '--enable-debug', common.Newline, @@ -43,7 +43,7 @@ class QemuComponent(common.Component): extra_paths=[common.ccache_dir], cwd=build_dir ) - common.run_cmd( + self.sh.run_cmd( ( [ 'make', common.Newline, diff --git a/build-userland b/build-userland index 3a7e527..be39c51 100755 --- a/build-userland +++ b/build-userland @@ -53,7 +53,7 @@ has the OpenBLAS libraries and headers installed. allowed_toolchains = ['buildroot'] cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) cxx = common.get_toolchain_tool('g++', allowed_toolchains=allowed_toolchains) - common.run_cmd( + self.sh.run_cmd( ( [ 'make', common.Newline, diff --git a/common.py b/common.py index 47df412..195ed56 100644 --- a/common.py +++ b/common.py @@ -162,6 +162,12 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree Linux build ID. Allows you to keep multiple separate Linux builds. ''' ) + parser.add_argument( + '--initramfs', default=False, action='store_true', + ) + parser.add_argument( + '--initrd', default=False, action='store_true', + ) # Baremetal. self.add_argument( @@ -271,7 +277,7 @@ to allow overriding configs from the CLI. common.print_time(end_time - start_time) def run_cmd(self, *args, **kwargs): - common.run_cmd(*args, **kwargs) + self.sh.run_cmd(*args, **kwargs) def timed_main(self, **kwargs): raise NotImplementedError() @@ -480,7 +486,7 @@ def raw_to_qcow2(prebuilt=False, reverse=False): tmp = infile infile = outfile outfile = tmp - common.run_cmd( + self.sh.run_cmd( [ qemu_img_executable, common.Newline, ] + diff --git a/gem5-shell b/gem5-shell index f669b57..699181e 100755 --- a/gem5-shell +++ b/gem5-shell @@ -9,7 +9,7 @@ parser = common.get_argparse( argparse_args={'description':'Connect a terminal to a running gem5 instance'} ) args = common.setup(parser) -sys.exit(common.run_cmd([ +sys.exit(self.sh.run_cmd([ common.gem5_m5term, common.Newline, 'localhost', common.Newline, str(common.gem5_telnet_port), common.Newline, diff --git a/qemu-trace2txt b/qemu-trace2txt index fa4ab1f..89d634c 100755 --- a/qemu-trace2txt +++ b/qemu-trace2txt @@ -7,7 +7,7 @@ import sys import common def main(): - return common.run_cmd( + return self.sh.run_cmd( [ os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), common.Newline, os.path.join(common.qemu_build_dir, 'trace-events-all'), common.Newline, diff --git a/run b/run index 967ade4..1654279 100755 --- a/run +++ b/run @@ -21,8 +21,6 @@ defaults = { 'gem5_readfile': '', 'gem5_restore': None, 'graphic': False, - 'initramfs': False, - 'initrd': False, 'kernel_cli': None, 'kernel_cli_after_dash': None, 'eval_after': None, @@ -142,7 +140,7 @@ def main(args, extra_args=None): # This is to run gem5 from a prebuilt download. if (not common.baremetal is None) or (not os.path.exists(common.linux_image)): raise_image_not_found() - common.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) + self.sh.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True) common.write_string_to_file(common.gem5_readfile, args.gem5_readfile) memory = '{}B'.format(args.memory) @@ -403,7 +401,7 @@ def main(args, extra_args=None): out_file = None else: out_file = common.termout_file - common.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) + self.sh.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) # Check if guest panicked. if common.emulator == 'gem5': # We have to do some parsing here because gem5 exits with status 0 even when panic happens. @@ -507,14 +505,6 @@ gem.op5 --debug-flags=Exec fs.py --cpu-type=HPI --caches '--gem5-readfile', default=defaults['gem5_readfile'], help='Set the contents of m5 readfile to this string.' ) - init_group.add_argument( - '-I', '--initramfs', default=defaults['initramfs'], action='store_true', - help='Use initramfs instead of a root filesystem' - ) - init_group.add_argument( - '-i', '--initrd', default=defaults['initrd'], action='store_true', - help='Use initrd instead of a root filesystem' - ) kvm_group.add_argument( '-K', '--kvm', default=defaults['kvm'], action='store_true', help='Use KVM. Only works if guest arch == host arch' diff --git a/run-docker b/run-docker index 56e471b..bfc3cba 100755 --- a/run-docker +++ b/run-docker @@ -11,10 +11,10 @@ image_name = common.repo_short_id target_dir = '/root/{}'.format(common.repo_short_id) docker = ['sudo', 'docker'] def create(args): - common.run_cmd(docker + ['build', '-t', image_name, '.', common.Newline]) + self.sh.run_cmd(docker + ['build', '-t', image_name, '.', common.Newline]) # --privileged for KVM: # https://stackoverflow.com/questions/48422001/launching-qemu-kvm-from-inside-docker-container - common.run_cmd( + self.sh.run_cmd( docker + [ 'create', common.Newline, @@ -31,23 +31,23 @@ def create(args): ) def destroy(args): stop(args) - common.run_cmd(docker + ['rm', container_name, common.Newline]) - common.run_cmd(docker + ['rmi', image_name, common.Newline]) + self.sh.run_cmd(docker + ['rm', container_name, common.Newline]) + self.sh.run_cmd(docker + ['rmi', image_name, common.Newline]) def sh(args): start(args) if args: sh_args = args else: sh_args = ['bash'] - common.run_cmd( + self.sh.run_cmd( docker + ['exec', '-i', '-t', container_name] + sh_args + [common.Newline], ) def start(args): - common.run_cmd(docker + ['start', container_name, common.Newline]) + self.sh.run_cmd(docker + ['start', container_name, common.Newline]) def stop(args): - common.run_cmd(docker + ['stop', container_name, common.Newline]) + self.sh.run_cmd(docker + ['stop', container_name, common.Newline]) cmd_action_map = { 'create': lambda args: create(args), 'DESTROY': lambda args: destroy(args), diff --git a/run-gdb b/run-gdb index 55fc02f..e0a80b6 100755 --- a/run-gdb +++ b/run-gdb @@ -188,7 +188,7 @@ def main(args, extra_args=None): # I would rather have cwd be out_rootfs_overlay_dir, # but then lx-symbols cannot fine the vmlinux and fails with: # vmlinux: No such file or directory. - return common.run_cmd( + return self.sh.run_cmd( cmd, cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'), cwd=common.linux_build_dir diff --git a/run-toolchain b/run-toolchain index 041e1b5..1ac8724 100755 --- a/run-toolchain +++ b/run-toolchain @@ -43,7 +43,7 @@ tool= common.get_toolchain_tool(args.tool) if args.dry: print(tool) else: - sys.exit(common.run_cmd( + sys.exit(self.sh.run_cmd( [tool, common.Newline] + common.add_newlines(args.extra_args), cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'),