common: print cd pdw on paths

print only modified variables on PATH

use common.run_cmd everywhere to get full bash bash commands

readme: recommend private/ instead of the cryptic p/

README.adoc

@@ -10679,18 +10679,18 @@ Next, when you want to build with this repository, use the `--gem5-src` argument
 ....
 cd linux-kernel-module-cheat
 ./build-gem5 \
-  --gem5-build-id p/master \
+  --gem5-build-id private/master \
   --gem5-src "$gem5_internal" \
-  --gem5-worktree p/master \
+  --gem5-worktree private/master \
 ;
 ./run-gem5
-  --gem5-build-id p/master \
+  --gem5-build-id private/master \
   --gem5-src "$gem5_internal" \
-  --gem5-worktree p/master \
+  --gem5-worktree private/master \
 ;
 ....
 
-`p` stands for Private, and is not mandatory, but it is a sane default that will remind you at all times that you are dealing with private code instead of public.
+`private` is not mandatory, but it is a sane default that will remind you at all times that you are dealing with private code instead of public.
 
 This setup only creates gitignored worktrees of the private repository inside this repository, which is pretty safe, while still allowing fully use our infrastructure as usual.
 

build-buildroot

@@ -156,7 +156,7 @@ def main(args, extra_args=None):
                 if os.path.isdir(package_dir_abs):
                     br2_external_dirs.append(path_relative_to_buildroot(package_dir_abs))
             br2_external_str = ':'.join(br2_external_dirs)
-            subprocess.check_call(
+            assert common.run_cmd(
                 [
                     'make',
                     'O={}'.format(common.buildroot_build_dir),
@@ -164,7 +164,7 @@ def main(args, extra_args=None):
                     defconfig,
                 ],
                 cwd=common.buildroot_src_dir,
-            )
+            ) == 0
             buildroot_configs = args.buildroot_config
             buildroot_configs.extend([
                 'BR2_JLEVEL={}'.format(nproc),
@@ -254,14 +254,14 @@ def main(args, extra_args=None):
                 buildroot_kernel_config_fragment_str = 'BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{}"'.format(' '.join(kernel_config_fragments))
                 buildroot_configs.append(buildroot_kernel_config_fragment_str)
                 common.write_configs(common.buildroot_config_file, buildroot_configs, buildroot_config_fragments)
-                subprocess.check_call(
+                assert common.run_cmd(
                     [
                         'make',
                         'O={}'.format(common.buildroot_build_dir),
                         'olddefconfig',
                     ],
                     cwd=common.buildroot_src_dir,
-                )
+                ) == 0
 
         # Manage Linux kernel and QEMU variants.
         def symlink_buildroot_variant(custom_dir, variant_dir):

build-crosstool-ng

@@ -70,7 +70,7 @@ def main(args, extra_args=None):
         ],
         out_file=os.path.join(common.crosstool_ng_build_dir, 'lkmc.log'),
         delete_env=['LD_LIBRARY_PATH'],
-        extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']},
+        extra_paths=[common.ccache_dir],
     ) == 0
 
 if __name__ == '__main__':

build-gem5

@@ -30,20 +30,20 @@ else:
     if  not os.path.exists(os.path.join(common.gem5_src_dir, '.git')):
         if common.gem5_src_dir == common.gem5_default_src_dir:
             raise Exception('gem5 submodule not checked out')
-        subprocess.check_call([
+        assert common.run_cmd([
             'git',
             '-C', common.gem5_default_src_dir,
             'worktree', 'add',
             '-b', os.path.join('wt', args.gem5_build_id),
             common.gem5_src_dir
-        ])
+        ]) == 0
     if args.arch == 'x86_64':
         dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img')
         with open(dummy_img_path, 'wb') as dummy_img_file:
             zeroes = b'\x00' * (2 ** 16)
             for i in range(2 ** 10):
                 dummy_img_file.write(zeroes)
-        subprocess.check_call(['mkswap', dummy_img_path])
+        assert common.run_cmd(['mkswap', dummy_img_path]) == 0
         with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'):
             # This file must always be present, despite --kernel overriding that default and selecting the kernel.
             # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present.
@@ -54,7 +54,7 @@ else:
         # dtb
         dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt')
         dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt')
-        subprocess.check_call(['make', '-C', dt_src_dir])
+        assert common.run_cmd(['make', '-C', dt_src_dir]) == 0
         os.makedirs(dt_build_dir, exist_ok=True)
         for dt in glob.glob(os.path.join(dt_src_dir, '*.dtb')):
             shutil.copy2(dt, dt_build_dir)
@@ -62,14 +62,14 @@ else:
         # Bootloader 32.
         bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader')
         # TODO use the buildroot cross compiler here, and remove the dependencies from configure.
-        subprocess.check_call(['make', '-C', bootloader32_dir])
+        assert common.run_cmd(['make', '-C', bootloader32_dir]) == 0
         # bootloader
         shutil.copy2(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir)
 
         # Bootloader 64.
         bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader')
         # TODO cross_compile is ignored because the make does not use CC...
-        subprocess.check_call(['make', '-C', bootloader64_dir])
+        assert common.run_cmd(['make', '-C', bootloader64_dir]) == 0
         shutil.copy2(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir)
     assert common.run_cmd(
         [
@@ -81,11 +81,11 @@ else:
         ] +
         args.extra_scons_args,
         cwd=common.gem5_src_dir,
-        extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']},
+        extra_paths=[common.ccache_dir],
     ) == 0
     term_src_dir = os.path.join(common.gem5_src_dir, 'util/term')
     m5term_build = os.path.join(term_src_dir, 'm5term')
-    subprocess.check_call(['make', '-C', term_src_dir])
+    assert common.run_cmd(['make', '-C', term_src_dir]) == 0
     if os.path.exists(common.gem5_m5term):
         # Otherwise shutil.copy2 would fail with "Text file busy" if you
         # tried to rebuild while running m5term:

build-qemu

@@ -21,7 +21,7 @@ if args.clean:
 else:
     start_time = time.time()
     os.makedirs(common.qemu_build_dir, exist_ok=True)
-    subprocess.check_call(
+    assert common.run_cmd(
         [
             os.path.join(common.qemu_src_dir, 'configure'),
             '--enable-debug',
@@ -31,8 +31,9 @@ else:
             '--with-sdlabi=2.0',
         ] +
         args.extra_config_args,
+        extra_paths=[common.ccache_dir],
         cwd=common.qemu_build_dir
-    )
+    ) == 0
     assert common.run_cmd(
         [
             'make',
@@ -40,7 +41,7 @@ else:
             '-j', str(multiprocessing.cpu_count()),
         ],
         cwd=common.qemu_build_dir,
-        extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']},
+        extra_paths=[common.ccache_dir],
     ) == 0
     end_time = time.time()
     common.print_time(end_time - start_time)

common.py

@@ -265,7 +265,7 @@ def mkdir():
     os.makedirs(this.qemu_run_dir, exist_ok=True)
     os.makedirs(this.p9_dir, exist_ok=True)
 
-def print_cmd(cmd, cmd_file=None, extra_env=None):
+def print_cmd(cmd, cwd, cmd_file=None, extra_env=None, extra_paths=None):
     '''
     Format a command given as a list of strings so that it can
     be viewed nicely and executed by bash directly and print it to stdout.
@@ -275,12 +275,15 @@ def print_cmd(cmd, cmd_file=None, extra_env=None):
     '''
     newline_separator = ' \\\n'
     out = []
+    out.append('cd {} &&{}'.format(shlex.quote(cwd), newline_separator))
+    if extra_paths is not None:
+        out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)) + newline_separator)
     for key in extra_env:
-        out.extend(['{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])), newline_separator])
+        out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])) + newline_separator)
     for arg in cmd:
-        out.extend([shlex.quote(arg), newline_separator])
+        out.append(shlex.quote(arg) + newline_separator)
-    out = ''.join(out)
+    out = '  '.join(out) + ';\n'
-    print(out)
+    print('+ ' + out, end='')
     if cmd_file is not None:
         with open(cmd_file, 'w') as f:
             f.write('#!/usr/bin/env bash\n')
@@ -343,6 +346,7 @@ def run_cmd(
         show_stdout=True,
         show_cmd=True,
         extra_env=None,
+        extra_paths=None,
         delete_env=None,
         **kwargs
     ):
@@ -380,13 +384,22 @@ def run_cmd(
         extra_env = {}
     if delete_env is None:
         delete_env = []
+    if 'cwd' in kwargs:
+        cwd = kwargs['cwd']
+    else:
+        cwd = os.getcwd()
     env = os.environ.copy()
     env.update(extra_env)
+    if extra_paths is not None:
+        path = ':'.join(extra_paths)
+        if 'PATH' in os.environ:
+            path += ':' + os.environ['PATH']
+        env['PATH'] = path
     for key in delete_env:
         if key in env:
             del env[key]
     if show_cmd:
-        print_cmd(cmd, cmd_file, extra_env=extra_env)
+        print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths)
 
     # Otherwise Ctrl + C gives:
     # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).