3rd-party/linux-kernel-module-cheat
1
0
Fork 0
mirror of https://github.com/cirosantilli/linux-kernel-module-cheat.git synced 2026-01-23 02:05:57 +01:00
Code Issues Releases Wiki Activity

shell_helpers: create a check_stdout

Browse Source 
./build-doc --dry-run was failing if asciidoctor is not installed

Also catch BrokenPipeError on ./build --dry-run all | less if you quit less quickly.
This commit is contained in:
Ciro Santilli 六四事件 法轮功
2019-06-21 00:00:00 +00:00
parent 6a9299599e
commit 45f2d630cb
6 changed files with 94 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
build
View File

@@ -7,6 +7,7 @@ import cli_function
import collections import collections
import common import common
import copy import copy
import math
import subprocess import subprocess
import shell_helpers import shell_helpers
from shell_helpers import LF from shell_helpers import LF
@@ -513,7 +514,6 @@ Which components to build. Default: qemu-buildroot
['python3', '-m', 'pip', 'install', '--user', LF] + ['python3', '-m', 'pip', 'install', '--user', LF] +
self.sh.add_newlines(sorted(python3_pkgs)) self.sh.add_newlines(sorted(python3_pkgs))
) )
git_version_tuple = tuple(int(x) for x in subprocess.check_output(['git', '--version']).decode().split(' ')[-1].split('.'))
git_cmd_common = [ git_cmd_common = [
'git', LF, 'git', LF,
'submodule', LF, 'submodule', LF,
@@ -521,13 +521,20 @@ Which components to build. Default: qemu-buildroot
'--init', LF, '--init', LF,
'--recursive', LF, '--recursive', LF,
] ]
if git_version_tuple >= (2, 9, 0): if self.env['dry_run']:
# https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 git_version_tuple = (math.inf, math.inf, math.inf)
git_cmd_common.extend(['--jobs', str(len(os.sched_getaffinity(0))), LF]) else:
if git_version_tuple >= (2, 10, 0): git_version_tuple = tuple(
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching int(x) for x in self.sh.check_output(['git', '--version']) \
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone .split(' ')[-1].split('.')
git_cmd_common.extend(['--progress', LF]) )
if git_version_tuple >= (2, 9, 0):
# https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
git_cmd_common.extend(['--jobs', str(len(os.sched_getaffinity(0))), LF])
if git_version_tuple >= (2, 10, 0):
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
git_cmd_common.extend(['--progress', LF])
def submodule_ids_to_cmd(submodules): def submodule_ids_to_cmd(submodules):
return self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)]) return self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
if submodules: if submodules:

15
build-doc
View File

@@ -35,10 +35,10 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#build-the-documentatio
# Check that all local files linked from README exist. # Check that all local files linked from README exist.
external_link_re = re.compile('^https?://') external_link_re = re.compile('^https?://')
for link in subprocess.check_output([ for link in self.sh.check_output([
os.path.join(asciidoctor_dir, 'extract-link-targets'), os.path.join(asciidoctor_dir, 'extract-link-targets'),
self.env['readme'] self.env['readme']
]).decode().splitlines(): ]).splitlines():
if not external_link_re.match(link): if not external_link_re.match(link):
if not os.path.lexists(link): if not os.path.lexists(link):
self.log_error('broken link: ' + link) self.log_error('broken link: ' + link)
@@ -48,17 +48,18 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#build-the-documentatio
header_ids = set() header_ids = set()
grep_line_location_re = re.compile('^(.*?:\d+):') grep_line_location_re = re.compile('^(.*?:\d+):')
grep_line_hash_re = re.compile('^([a-z0-9_-]+)') grep_line_hash_re = re.compile('^([a-z0-9_-]+)')
for header_id in subprocess.check_output([ for header_id in self.sh.check_output([
os.path.join(asciidoctor_dir, 'extract-header-ids'), os.path.join(asciidoctor_dir, 'extract-header-ids'),
self.env['readme'] self.env['readme']
]).decode().splitlines(): ]).splitlines():
header_ids.add(header_id) header_ids.add(header_id)
for grep_line in subprocess.check_output([ for grep_line in self.sh.check_output([
'git', 'git',
'grep', 'grep',
'--fixed-strings', '--fixed-strings',
self.env['github_repo_id_url'] + '#' self.env['github_repo_id_url'] + '#',
]).decode().splitlines(): LF
]).splitlines():
url_index = grep_line.index(self.env['github_repo_id_url']) url_index = grep_line.index(self.env['github_repo_id_url'])
hash_start_index = url_index + len(self.env['github_repo_id_url']) hash_start_index = url_index + len(self.env['github_repo_id_url'])
if len(grep_line) > hash_start_index: if len(grep_line) > hash_start_index:

2
build-docker
View File

@@ -22,7 +22,7 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheatTODO#ubuntu-g
container_name = 'lkmc-guest' container_name = 'lkmc-guest'
target_dir = os.path.join('/root', 'linux-kernel-module-cheat') target_dir = os.path.join('/root', 'linux-kernel-module-cheat')
os.makedirs(build_dir, exist_ok=True) os.makedirs(build_dir, exist_ok=True)
containers = subprocess.check_output([ containers = self.sh.check_output([
'docker', 'docker',
'ps', 'ps',
'-a', '-a',

4
common.py
View File

@@ -1066,7 +1066,7 @@ lunch aosp_{}-eng
) )
def get_elf_entry(self, elf_file_path): def get_elf_entry(self, elf_file_path):
readelf_header = subprocess.check_output([ readelf_header = self.sh.check_output([
self.get_toolchain_tool('readelf'), self.get_toolchain_tool('readelf'),
'-h', '-h',
elf_file_path elf_file_path
@@ -1592,7 +1592,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-debug-build
if 'cc_flags' in package: if 'cc_flags' in package:
cc_flags.extend(package['cc_flags']) cc_flags.extend(package['cc_flags'])
else: else:
pkg_config_output = subprocess.check_output([ pkg_config_output = self.sh.check_output([
self.env['pkg_config'], self.env['pkg_config'],
'--cflags', '--cflags',
package_key package_key

3
release-upload
View File

@@ -2,7 +2,6 @@
import json import json
import os import os
import subprocess
import sys import sys
import urllib.error import urllib.error
@@ -19,7 +18,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#release-upload
def timed_main(self): def timed_main(self):
# https://stackoverflow.com/questions/3404936/show-which-git-tag-you-are-on # https://stackoverflow.com/questions/3404936/show-which-git-tag-you-are-on
tag = subprocess.check_output([ tag = self.sh.check_output([
'git', 'git',
'describe', 'describe',
'--exact-match', '--exact-match',

88
shell_helpers.py
View File

@@ -2,6 +2,7 @@
import base64 import base64
import distutils.file_util import distutils.file_util
import io
import itertools import itertools
import os import os
import shlex import shlex
@@ -54,8 +55,12 @@ class ShellHelpers:
The initial use case was test-gdb which must create a thread for GDB to run the program in parallel. The initial use case was test-gdb which must create a thread for GDB to run the program in parallel.
''' '''
with cls._print_lock: with cls._print_lock:
sys.stdout.write(string + '\n') try:
sys.stdout.flush() print(string, flush=True)
except BrokenPipeError:
# https://stackoverflow.com/questions/26692284/how-to-prevent-brokenpipeerror-when-doing-a-flush-in-python
# https://stackoverflow.com/questions/16314321/suppressing-printout-of-exception-ignored-message-in-python-3
pass
def add_newlines(self, cmd): def add_newlines(self, cmd):
out = [] out = []
@@ -72,6 +77,17 @@ class ShellHelpers:
def base64_decode(self, string): def base64_decode(self, string):
return base64.b64decode(string.encode()).decode() return base64.b64decode(string.encode()).decode()
def check_output(self, *args, **kwargs):
out_str = []
self.run_cmd(
*args,
out_str=out_str,
show_stdout=False,
show_cmd=False,
**kwargs
)
return out_str[0]
def chmod(self, path, add_rm_abs='+', mode_delta=stat.S_IXUSR): def chmod(self, path, add_rm_abs='+', mode_delta=stat.S_IXUSR):
''' '''
TODO extend further, shell print equivalent. TODO extend further, shell print equivalent.
@@ -245,6 +261,8 @@ class ShellHelpers:
extra_paths=None, extra_paths=None,
delete_env=None, delete_env=None,
raise_on_failure=True, raise_on_failure=True,
*,
out_str=None,
**kwargs **kwargs
): ):
''' '''
@@ -261,6 +279,9 @@ class ShellHelpers:
:param out_file: if not None, write the stdout and stderr of the command the file :param out_file: if not None, write the stdout and stderr of the command the file
:type out_file: str :type out_file: str
:param out_str: if not None, append the stdout and stderr string to this list
:type out_str: Union(List,None)
:param show_stdout: wether to show stdout and stderr on the terminal or not :param show_stdout: wether to show stdout and stderr on the terminal or not
:type show_stdout: bool :type show_stdout: bool
@@ -270,7 +291,7 @@ class ShellHelpers:
:return: exit status of the command :return: exit status of the command
:rtype: int :rtype: int
''' '''
if out_file is None: if out_file is None and out_str is None:
if show_stdout: if show_stdout:
stdout = None stdout = None
stderr = None stderr = None
@@ -299,14 +320,21 @@ class ShellHelpers:
if key in env: if key in env:
del env[key] del env[key]
if show_cmd: if show_cmd:
self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) self.print_cmd(
cmd,
cwd=cwd,
cmd_file=cmd_file,
extra_env=extra_env,
extra_paths=extra_paths
)
# Otherwise, if called from a non-main thread: # Otherwise, if called from a non-main thread:
# ValueError: signal only works in main thread # ValueError: signal only works in main thread
if threading.current_thread() == threading.main_thread(): if threading.current_thread() == threading.main_thread():
# Otherwise Ctrl + C gives: # Otherwise Ctrl + C gives:
# - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
# - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec # - kills Python, and that then kills GDB:
# https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
sigint_old = signal.getsignal(signal.SIGINT) sigint_old = signal.getsignal(signal.SIGINT)
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
@@ -320,23 +348,39 @@ class ShellHelpers:
cmd = self.strip_newlines(cmd) cmd = self.strip_newlines(cmd)
if not self.dry_run: if not self.dry_run:
# https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: with subprocess.Popen(
if out_file is not None: cmd,
os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True) stdout=stdout,
with open(out_file, 'bw') as logfile: stderr=stderr,
while True: env=env,
byte = proc.stdout.read(1) **kwargs
if byte: ) as proc:
if show_stdout: if out_file is not None or out_str is not None:
sys.stdout.buffer.write(byte) if out_file is not None:
try: os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True)
sys.stdout.flush() if out_file is not None:
except BlockingIOError: logfile = open(out_file, 'bw')
# TODO understand. Why, Python, why. logfile_str = []
pass while True:
byte = proc.stdout.read(1)
if byte:
if show_stdout:
sys.stdout.buffer.write(byte)
try:
sys.stdout.flush()
except BlockingIOError:
# TODO understand. Why, Python, why.
pass
if out_file is not None:
logfile.write(byte) logfile.write(byte)
else: if out_str is not None:
break logfile_str.append(byte)
else:
break
if out_file is not None:
logfile.close()
if out_str is not None:
out_str.append((b''.join(logfile_str)).decode())
if threading.current_thread() == threading.main_thread(): if threading.current_thread() == threading.main_thread():
signal.signal(signal.SIGINT, sigint_old) signal.signal(signal.SIGINT, sigint_old)
#signal.signal(signal.SIGPIPE, sigpipe_old) #signal.signal(signal.SIGPIPE, sigpipe_old)
@@ -347,6 +391,8 @@ class ShellHelpers:
raise e raise e
return returncode return returncode
else: else:
if not out_str is None:
out_str.append('')
return 0 return 0
def shlex_split(self, string): def shlex_split(self, string):