From 45f2d630cbb8b98bfdc94a42fad24ae620e9b90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Fri, 21 Jun 2019 00:00:00 +0000 Subject: [PATCH] shell_helpers: create a check_stdout ./build-doc --dry-run was failing if asciidoctor is not installed Also catch BrokenPipeError on ./build --dry-run all | less if you quit less quickly. --- build | 23 ++++++++----- build-doc | 15 +++++---- build-docker | 2 +- common.py | 4 +-- release-upload | 3 +- shell_helpers.py | 88 ++++++++++++++++++++++++++++++++++++------------ 6 files changed, 94 insertions(+), 41 deletions(-) diff --git a/build b/build index a7bff54..bb87e3d 100755 --- a/build +++ b/build @@ -7,6 +7,7 @@ import cli_function import collections import common import copy +import math import subprocess import shell_helpers from shell_helpers import LF @@ -513,7 +514,6 @@ Which components to build. Default: qemu-buildroot ['python3', '-m', 'pip', 'install', '--user', LF] + self.sh.add_newlines(sorted(python3_pkgs)) ) - git_version_tuple = tuple(int(x) for x in subprocess.check_output(['git', '--version']).decode().split(' ')[-1].split('.')) git_cmd_common = [ 'git', LF, 'submodule', LF, @@ -521,13 +521,20 @@ Which components to build. Default: qemu-buildroot '--init', LF, '--recursive', LF, ] - if git_version_tuple >= (2, 9, 0): - # https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 - git_cmd_common.extend(['--jobs', str(len(os.sched_getaffinity(0))), LF]) - if git_version_tuple >= (2, 10, 0): - # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching - # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone - git_cmd_common.extend(['--progress', LF]) + if self.env['dry_run']: + git_version_tuple = (math.inf, math.inf, math.inf) + else: + git_version_tuple = tuple( + int(x) for x in self.sh.check_output(['git', '--version']) \ + .split(' ')[-1].split('.') + ) + if git_version_tuple >= (2, 9, 0): + # https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 + git_cmd_common.extend(['--jobs', str(len(os.sched_getaffinity(0))), LF]) + if git_version_tuple >= (2, 10, 0): + # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching + # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone + git_cmd_common.extend(['--progress', LF]) def submodule_ids_to_cmd(submodules): return self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)]) if submodules: diff --git a/build-doc b/build-doc index 2560436..751ff70 100755 --- a/build-doc +++ b/build-doc @@ -35,10 +35,10 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#build-the-documentatio # Check that all local files linked from README exist. external_link_re = re.compile('^https?://') - for link in subprocess.check_output([ + for link in self.sh.check_output([ os.path.join(asciidoctor_dir, 'extract-link-targets'), self.env['readme'] - ]).decode().splitlines(): + ]).splitlines(): if not external_link_re.match(link): if not os.path.lexists(link): self.log_error('broken link: ' + link) @@ -48,17 +48,18 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#build-the-documentatio header_ids = set() grep_line_location_re = re.compile('^(.*?:\d+):') grep_line_hash_re = re.compile('^([a-z0-9_-]+)') - for header_id in subprocess.check_output([ + for header_id in self.sh.check_output([ os.path.join(asciidoctor_dir, 'extract-header-ids'), self.env['readme'] - ]).decode().splitlines(): + ]).splitlines(): header_ids.add(header_id) - for grep_line in subprocess.check_output([ + for grep_line in self.sh.check_output([ 'git', 'grep', '--fixed-strings', - self.env['github_repo_id_url'] + '#' - ]).decode().splitlines(): + self.env['github_repo_id_url'] + '#', + LF + ]).splitlines(): url_index = grep_line.index(self.env['github_repo_id_url']) hash_start_index = url_index + len(self.env['github_repo_id_url']) if len(grep_line) > hash_start_index: diff --git a/build-docker b/build-docker index 4b06200..b68979d 100755 --- a/build-docker +++ b/build-docker @@ -22,7 +22,7 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheatTODO#ubuntu-g container_name = 'lkmc-guest' target_dir = os.path.join('/root', 'linux-kernel-module-cheat') os.makedirs(build_dir, exist_ok=True) - containers = subprocess.check_output([ + containers = self.sh.check_output([ 'docker', 'ps', '-a', diff --git a/common.py b/common.py index 20b754c..7d6685b 100644 --- a/common.py +++ b/common.py @@ -1066,7 +1066,7 @@ lunch aosp_{}-eng ) def get_elf_entry(self, elf_file_path): - readelf_header = subprocess.check_output([ + readelf_header = self.sh.check_output([ self.get_toolchain_tool('readelf'), '-h', elf_file_path @@ -1592,7 +1592,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-debug-build if 'cc_flags' in package: cc_flags.extend(package['cc_flags']) else: - pkg_config_output = subprocess.check_output([ + pkg_config_output = self.sh.check_output([ self.env['pkg_config'], '--cflags', package_key diff --git a/release-upload b/release-upload index 4b014d3..30a9843 100755 --- a/release-upload +++ b/release-upload @@ -2,7 +2,6 @@ import json import os -import subprocess import sys import urllib.error @@ -19,7 +18,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#release-upload def timed_main(self): # https://stackoverflow.com/questions/3404936/show-which-git-tag-you-are-on - tag = subprocess.check_output([ + tag = self.sh.check_output([ 'git', 'describe', '--exact-match', diff --git a/shell_helpers.py b/shell_helpers.py index 9d2bd99..01f15cf 100644 --- a/shell_helpers.py +++ b/shell_helpers.py @@ -2,6 +2,7 @@ import base64 import distutils.file_util +import io import itertools import os import shlex @@ -54,8 +55,12 @@ class ShellHelpers: The initial use case was test-gdb which must create a thread for GDB to run the program in parallel. ''' with cls._print_lock: - sys.stdout.write(string + '\n') - sys.stdout.flush() + try: + print(string, flush=True) + except BrokenPipeError: + # https://stackoverflow.com/questions/26692284/how-to-prevent-brokenpipeerror-when-doing-a-flush-in-python + # https://stackoverflow.com/questions/16314321/suppressing-printout-of-exception-ignored-message-in-python-3 + pass def add_newlines(self, cmd): out = [] @@ -72,6 +77,17 @@ class ShellHelpers: def base64_decode(self, string): return base64.b64decode(string.encode()).decode() + def check_output(self, *args, **kwargs): + out_str = [] + self.run_cmd( + *args, + out_str=out_str, + show_stdout=False, + show_cmd=False, + **kwargs + ) + return out_str[0] + def chmod(self, path, add_rm_abs='+', mode_delta=stat.S_IXUSR): ''' TODO extend further, shell print equivalent. @@ -245,6 +261,8 @@ class ShellHelpers: extra_paths=None, delete_env=None, raise_on_failure=True, + *, + out_str=None, **kwargs ): ''' @@ -261,6 +279,9 @@ class ShellHelpers: :param out_file: if not None, write the stdout and stderr of the command the file :type out_file: str + :param out_str: if not None, append the stdout and stderr string to this list + :type out_str: Union(List,None) + :param show_stdout: wether to show stdout and stderr on the terminal or not :type show_stdout: bool @@ -270,7 +291,7 @@ class ShellHelpers: :return: exit status of the command :rtype: int ''' - if out_file is None: + if out_file is None and out_str is None: if show_stdout: stdout = None stderr = None @@ -299,14 +320,21 @@ class ShellHelpers: if key in env: del env[key] if show_cmd: - self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) + self.print_cmd( + cmd, + cwd=cwd, + cmd_file=cmd_file, + extra_env=extra_env, + extra_paths=extra_paths + ) # Otherwise, if called from a non-main thread: # ValueError: signal only works in main thread if threading.current_thread() == threading.main_thread(): # Otherwise Ctrl + C gives: # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). - # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec + # - kills Python, and that then kills GDB: + # https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec sigint_old = signal.getsignal(signal.SIGINT) signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -320,23 +348,39 @@ class ShellHelpers: cmd = self.strip_newlines(cmd) if not self.dry_run: # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 - with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: - if out_file is not None: - os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True) - with open(out_file, 'bw') as logfile: - while True: - byte = proc.stdout.read(1) - if byte: - if show_stdout: - sys.stdout.buffer.write(byte) - try: - sys.stdout.flush() - except BlockingIOError: - # TODO understand. Why, Python, why. - pass + with subprocess.Popen( + cmd, + stdout=stdout, + stderr=stderr, + env=env, + **kwargs + ) as proc: + if out_file is not None or out_str is not None: + if out_file is not None: + os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True) + if out_file is not None: + logfile = open(out_file, 'bw') + logfile_str = [] + while True: + byte = proc.stdout.read(1) + if byte: + if show_stdout: + sys.stdout.buffer.write(byte) + try: + sys.stdout.flush() + except BlockingIOError: + # TODO understand. Why, Python, why. + pass + if out_file is not None: logfile.write(byte) - else: - break + if out_str is not None: + logfile_str.append(byte) + else: + break + if out_file is not None: + logfile.close() + if out_str is not None: + out_str.append((b''.join(logfile_str)).decode()) if threading.current_thread() == threading.main_thread(): signal.signal(signal.SIGINT, sigint_old) #signal.signal(signal.SIGPIPE, sigpipe_old) @@ -347,6 +391,8 @@ class ShellHelpers: raise e return returncode else: + if not out_str is None: + out_str.append('') return 0 def shlex_split(self, string):