shell_helpers: create a check_stdout

./build-doc --dry-run was failing if asciidoctor is not installed

Also catch BrokenPipeError on ./build --dry-run all | less if you quit less quickly.

build

@@ -7,6 +7,7 @@ import cli_function
 import collections
 import common
 import copy
+import math
 import subprocess
 import shell_helpers
 from shell_helpers import LF
@@ -513,7 +514,6 @@ Which components to build. Default: qemu-buildroot
                     ['python3', '-m', 'pip', 'install', '--user', LF] +
                     self.sh.add_newlines(sorted(python3_pkgs))
                 )
-            git_version_tuple = tuple(int(x) for x in subprocess.check_output(['git', '--version']).decode().split(' ')[-1].split('.'))
             git_cmd_common = [
                 'git', LF,
                 'submodule', LF,
@@ -521,13 +521,20 @@ Which components to build. Default: qemu-buildroot
                 '--init', LF,
                 '--recursive', LF,
             ]
-            if git_version_tuple >= (2, 9, 0):
-                # https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
-                git_cmd_common.extend(['--jobs', str(len(os.sched_getaffinity(0))), LF])
-            if git_version_tuple >= (2, 10, 0):
-                # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
-                # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
-                git_cmd_common.extend(['--progress', LF])
+            if self.env['dry_run']:
+                git_version_tuple = (math.inf, math.inf, math.inf)
+            else:
+                git_version_tuple = tuple(
+                    int(x) for x in self.sh.check_output(['git', '--version']) \
+                    .split(' ')[-1].split('.')
+                )
+                if git_version_tuple >= (2, 9, 0):
+                    # https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
+                    git_cmd_common.extend(['--jobs', str(len(os.sched_getaffinity(0))), LF])
+                if git_version_tuple >= (2, 10, 0):
+                    # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
+                    # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
+                    git_cmd_common.extend(['--progress', LF])
             def submodule_ids_to_cmd(submodules):
                 return self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
             if submodules:

build-doc

@@ -35,10 +35,10 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#build-the-documentatio
 
         # Check that all local files linked from README exist.
         external_link_re = re.compile('^https?://')
-        for link in subprocess.check_output([
+        for link in self.sh.check_output([
             os.path.join(asciidoctor_dir, 'extract-link-targets'),
             self.env['readme']
-        ]).decode().splitlines():
+        ]).splitlines():
             if not external_link_re.match(link):
                 if not os.path.lexists(link):
                     self.log_error('broken link: ' + link)
@@ -48,17 +48,18 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#build-the-documentatio
         header_ids = set()
         grep_line_location_re = re.compile('^(.*?:\d+):')
         grep_line_hash_re = re.compile('^([a-z0-9_-]+)')
-        for header_id in subprocess.check_output([
+        for header_id in self.sh.check_output([
             os.path.join(asciidoctor_dir, 'extract-header-ids'),
             self.env['readme']
-        ]).decode().splitlines():
+        ]).splitlines():
             header_ids.add(header_id)
-        for grep_line in subprocess.check_output([
+        for grep_line in self.sh.check_output([
             'git',
             'grep',
             '--fixed-strings',
-            self.env['github_repo_id_url'] + '#'
-        ]).decode().splitlines():
+            self.env['github_repo_id_url'] + '#',
+            LF
+        ]).splitlines():
             url_index = grep_line.index(self.env['github_repo_id_url'])
             hash_start_index = url_index + len(self.env['github_repo_id_url'])
             if len(grep_line) > hash_start_index:

build-docker

@@ -22,7 +22,7 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheatTODO#ubuntu-g
         container_name = 'lkmc-guest'
         target_dir = os.path.join('/root', 'linux-kernel-module-cheat')
         os.makedirs(build_dir, exist_ok=True)
-        containers = subprocess.check_output([
+        containers = self.sh.check_output([
                 'docker',
                 'ps',
                 '-a',

common.py

@@ -1066,7 +1066,7 @@ lunch aosp_{}-eng
             )
 
     def get_elf_entry(self, elf_file_path):
-        readelf_header = subprocess.check_output([
+        readelf_header = self.sh.check_output([
             self.get_toolchain_tool('readelf'),
             '-h',
             elf_file_path
@@ -1592,7 +1592,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-debug-build
                                     if 'cc_flags' in package:
                                         cc_flags.extend(package['cc_flags'])
                                     else:
-                                        pkg_config_output = subprocess.check_output([
+                                        pkg_config_output = self.sh.check_output([
                                             self.env['pkg_config'],
                                             '--cflags',
                                             package_key

release-upload

@@ -2,7 +2,6 @@
 
 import json
 import os
-import subprocess
 import sys
 import urllib.error
 
@@ -19,7 +18,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#release-upload
 
     def timed_main(self):
         # https://stackoverflow.com/questions/3404936/show-which-git-tag-you-are-on
-        tag = subprocess.check_output([
+        tag = self.sh.check_output([
             'git',
             'describe',
             '--exact-match',

shell_helpers.py

@@ -2,6 +2,7 @@
 
 import base64
 import distutils.file_util
+import io
 import itertools
 import os
 import shlex
@@ -54,8 +55,12 @@ class ShellHelpers:
         The initial use case was test-gdb which must create a thread for GDB to run the program in parallel.
         '''
         with cls._print_lock:
-            sys.stdout.write(string + '\n')
-            sys.stdout.flush()
+            try:
+                print(string, flush=True)
+            except BrokenPipeError:
+                # https://stackoverflow.com/questions/26692284/how-to-prevent-brokenpipeerror-when-doing-a-flush-in-python
+                # https://stackoverflow.com/questions/16314321/suppressing-printout-of-exception-ignored-message-in-python-3
+                pass
 
     def add_newlines(self, cmd):
         out = []
@@ -72,6 +77,17 @@ class ShellHelpers:
     def base64_decode(self, string):
         return base64.b64decode(string.encode()).decode()
 
+    def check_output(self, *args, **kwargs):
+        out_str = []
+        self.run_cmd(
+            *args,
+            out_str=out_str,
+            show_stdout=False,
+            show_cmd=False,
+            **kwargs
+        )
+        return out_str[0]
+
     def chmod(self, path, add_rm_abs='+', mode_delta=stat.S_IXUSR):
         '''
         TODO extend further, shell print equivalent.
@@ -245,6 +261,8 @@ class ShellHelpers:
         extra_paths=None,
         delete_env=None,
         raise_on_failure=True,
+        *,
+        out_str=None,
         **kwargs
     ):
         '''
@@ -261,6 +279,9 @@ class ShellHelpers:
         :param out_file: if not None, write the stdout and stderr of the command the file
         :type out_file: str
 
+        :param out_str: if not None, append the stdout and stderr string to this list
+        :type out_str: Union(List,None)
+
         :param show_stdout: wether to show stdout and stderr on the terminal or not
         :type show_stdout: bool
 
@@ -270,7 +291,7 @@ class ShellHelpers:
         :return: exit status of the command
         :rtype: int
         '''
-        if out_file is None:
+        if out_file is None and out_str is None:
             if show_stdout:
                 stdout = None
                 stderr = None
@@ -299,14 +320,21 @@ class ShellHelpers:
             if key in env:
                 del env[key]
         if show_cmd:
-            self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths)
+            self.print_cmd(
+                cmd,
+                cwd=cwd,
+                cmd_file=cmd_file,
+                extra_env=extra_env,
+                extra_paths=extra_paths
+            )
 
         # Otherwise, if called from a non-main thread:
         # ValueError: signal only works in main thread
         if threading.current_thread() == threading.main_thread():
             # Otherwise Ctrl + C gives:
             # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
-            # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
+            # - kills Python, and that then kills GDB:
+            #   https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
             sigint_old = signal.getsignal(signal.SIGINT)
             signal.signal(signal.SIGINT, signal.SIG_IGN)
 
@@ -320,23 +348,39 @@ class ShellHelpers:
         cmd = self.strip_newlines(cmd)
         if not self.dry_run:
             # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
-            with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
-                if out_file is not None:
-                    os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True)
-                    with open(out_file, 'bw') as logfile:
-                        while True:
-                            byte = proc.stdout.read(1)
-                            if byte:
-                                if show_stdout:
-                                    sys.stdout.buffer.write(byte)
-                                    try:
-                                        sys.stdout.flush()
-                                    except BlockingIOError:
-                                        # TODO understand. Why, Python, why.
-                                        pass
+            with subprocess.Popen(
+                cmd,
+                stdout=stdout,
+                stderr=stderr,
+                env=env,
+                **kwargs
+            ) as proc:
+                if out_file is not None or out_str is not None:
+                    if out_file is not None:
+                        os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True)
+                    if out_file is not None:
+                        logfile = open(out_file, 'bw')
+                    logfile_str = []
+                    while True:
+                        byte = proc.stdout.read(1)
+                        if byte:
+                            if show_stdout:
+                                sys.stdout.buffer.write(byte)
+                                try:
+                                    sys.stdout.flush()
+                                except BlockingIOError:
+                                    # TODO understand. Why, Python, why.
+                                    pass
+                            if out_file is not None:
                                 logfile.write(byte)
-                            else:
-                                break
+                            if out_str is not None:
+                                logfile_str.append(byte)
+                        else:
+                            break
+                    if out_file is not None:
+                        logfile.close()
+                    if out_str is not None:
+                        out_str.append((b''.join(logfile_str)).decode())
             if threading.current_thread() == threading.main_thread():
                 signal.signal(signal.SIGINT, sigint_old)
                 #signal.signal(signal.SIGPIPE, sigpipe_old)
@@ -347,6 +391,8 @@ class ShellHelpers:
                 raise e
             return returncode
         else:
+            if not out_str is None:
+                out_str.append('')
             return 0
 
     def shlex_split(self, string):