3rd-party/linux-kernel-module-cheat
1
0
Fork 0
mirror of https://github.com/cirosantilli/linux-kernel-module-cheat.git synced 2026-01-29 04:54:27 +01:00
Code Issues Releases Wiki Activity

start porting build

Browse Source
This commit is contained in:
Ciro Santilli 六四事件 法轮功
2019-01-22 00:00:00 +00:00
parent b72f75b531
commit 0a33292326
5 changed files with 428 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

801
build
View File

@@ -6,10 +6,12 @@ import platform
import re import re
import os import os
import cli_function
import common import common
import shell_helpers
from shell_helpers import LF from shell_helpers import LF
class Component: class _Component:
''' '''
Yes, we are re-inventing a crappy dependency resolution system. Yes, we are re-inventing a crappy dependency resolution system.
I can't believe it. I can't believe it.
@@ -40,6 +42,7 @@ class Component:
self.submodules_shallow = submodules_shallow or set() self.submodules_shallow = submodules_shallow or set()
self.python2_pkgs = python2_pkgs or set() self.python2_pkgs = python2_pkgs or set()
self.python3_pkgs = python3_pkgs or set() self.python3_pkgs = python3_pkgs or set()
def build(self, arch): def build(self, arch):
if ( if (
(self.build_callback is not None) and (self.build_callback is not None) and
@@ -47,208 +50,11 @@ class Component:
): ):
self.build_callback(arch) self.build_callback(arch)
def run_cmd(cmd, arch): class Main(cli_function.CliFunction):
global args def __init__(self):
cmd_abs = cmd.copy() super().__init__(
cmd_abs[0] = os.path.join(kwargs['root_dir'], cmd[0]) config_file=common.consts['config_file'],
cmd_abs.extend(['--arch', arch]) description='''\
if kwargs['extra_args']:
cmd_abs.append(kwargs['extra_args'])
self.sh.run_cmd(cmd_abs, dry_run=kwargs['dry_run'])
buildroot_component = Component(
lambda arch: run_cmd(['build-buildroot'], arch),
submodules = {'buildroot'},
# https://buildroot.org/downloads/manual/manual.html#requirement
apt_get_pkgs={
'bash',
'bc',
'binutils',
'build-essential',
'bzip2',
'cpio',
'g++',
'gcc',
'graphviz',
'gzip',
'make',
'patch',
'perl',
'python-matplotlib',
'python3',
'rsync',
'sed',
'tar',
'unzip',
},
)
name_to_component_map = {
# Leaves without dependencies.
'baremetal-qemu': Component(
lambda arch: run_cmd(['build-baremetal', '--emulator', 'qemu'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'baremetal-gem5': Component(
lambda arch: run_cmd(['build-baremetal', '--gem5'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'baremetal-gem5-pbx': Component(
lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'buildroot': buildroot_component,
'buildroot-gcc': buildroot_component,
'copy-overlay': Component(
lambda arch: run_cmd(['copy-overlay'], arch),
),
'crosstool-ng': Component(
lambda arch: run_cmd(['build-crosstool-ng'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'],
# http://crosstool-ng.github.io/docs/os-setup/
apt_get_pkgs={
'bison',
'docbook2x',
'flex',
'gawk',
'gcc',
'gperf',
'help2man',
'libncurses5-dev',
'libtool-bin',
'make',
'python-dev',
'texinfo',
},
submodules={'crosstool-ng'},
),
'gem5': Component(
lambda arch: run_cmd(['build-gem5'], arch),
# TODO test it out on Docker and answer that question properly:
# https://askubuntu.com/questions/350475/how-can-i-install-gem5
apt_get_pkgs={
'device-tree-compiler',
'diod',
'libgoogle-perftools-dev',
'm4',
'protobuf-compiler',
'python-dev',
'python-pip',
# For prebuilt qcow2 unpack.
'qemu-utils',
'scons',
'zlib1g-dev',
},
python2_pkgs={
# Generate graphs of config.ini under m5out.
'pydot',
},
submodules={'gem5'},
),
'gem5-debug': Component(
lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch),
),
'gem5-fast': Component(
lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch),
),
'linux': Component(
lambda arch: run_cmd(['build-linux'], arch),
submodules_shallow={'linux'},
apt_get_pkgs={
'bison',
'flex',
# Without this started failing in kernel 4.15 with:
# Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
'libelf-dev',
},
),
'modules': Component(
lambda arch: run_cmd(['build-modules'], arch),
),
'm5': Component(
lambda arch: run_cmd(['build-m5'], arch),
submodules={'gem5'},
),
'qemu': Component(
lambda arch: run_cmd(['build-qemu'], arch),
apt_build_deps={'qemu'},
apt_get_pkgs={'libsdl2-dev'},
submodules={'qemu'},
),
'qemu-user': Component(
lambda arch: run_cmd(['build-qemu', '--userland'], arch),
apt_build_deps = {'qemu'},
apt_get_pkgs={'libsdl2-dev'},
submodules = {'qemu'},
),
'parsec-benchmark': Component(
submodules = {'parsec-benchmark'},
),
'userland': Component(
lambda arch: run_cmd(['build-userland'], arch),
),
# Dependency only nodes.
'all': Component(dependencies=[
'all-linux',
'all-baremetal',
]),
'all-baremetal': Component(dependencies=[
'qemu-baremetal',
'gem5-baremetal',
'baremetal-gem5-pbx',
],
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'all-linux': Component(dependencies=[
'qemu-gem5-buildroot',
'gem5-debug',
'gem5-fast',
'qemu-user',
]),
'baremetal': Component(dependencies=[
'baremetal-gem5',
'baremetal-qemu',
]),
'gem5-buildroot': Component(dependencies=[
'buildroot-gcc',
'linux',
'm5',
'overlay',
'gem5',
]),
'gem5-baremetal': Component(dependencies=[
'gem5',
'crosstool-ng',
'baremetal-gem5',
]),
'overlay': Component(dependencies=[
'copy-overlay',
'modules',
'userland',
'buildroot',
]),
'qemu-baremetal': Component(dependencies=[
'qemu',
'crosstool-ng',
'baremetal-qemu',
]),
'qemu-buildroot': Component(dependencies=[
'qemu',
'buildroot-gcc',
'overlay',
'linux',
]),
'qemu-gem5-buildroot': Component(dependencies=[
'qemu',
'gem5-buildroot',
]),
'release': Component(dependencies=[
'qemu-buildroot',
]),
}
parser = argparse.ArgumentParser(
description= '''\
Shallow helper to build everything, or a subset of everything conveniently. Shallow helper to build everything, or a subset of everything conveniently.
Our build-* scripts don't build any dependencies to make iterative Our build-* scripts don't build any dependencies to make iterative
@@ -281,204 +87,425 @@ If `--arch` is given, build just for the given archs:
.... ....
This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`. This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`.
'''
)
buildroot_component = _Component(
lambda arch: self._run_cmd(['build-buildroot'], arch),
submodules = {'buildroot'},
# https://buildroot.org/downloads/manual/manual.html#requirement
apt_get_pkgs={
'bash',
'bc',
'binutils',
'build-essential',
'bzip2',
'cpio',
'g++',
'gcc',
'graphviz',
'gzip',
'make',
'patch',
'perl',
'python-matplotlib',
'python3',
'rsync',
'sed',
'tar',
'unzip',
},
)
Clean all Linux kernel builds: self.name_to_component_map = {
# Leaves without dependencies.
'baremetal-qemu': _Component(
lambda arch: self._run_cmd(['build-baremetal', '--emulator', 'qemu'], arch),
supported_archs=common.consts['crosstool_ng_supported_archs'],
),
'baremetal-gem5': _Component(
lambda arch: self._run_cmd(['build-baremetal', '--gem5'], arch),
supported_archs=common.consts['crosstool_ng_supported_archs'],
),
'baremetal-gem5-pbx': _Component(
lambda arch: self._run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
supported_archs=common.consts['crosstool_ng_supported_archs'],
),
'buildroot': buildroot_component,
'buildroot-gcc': buildroot_component,
'copy-overlay': _Component(
lambda arch: self._run_cmd(['copy-overlay'], arch),
),
'crosstool-ng': _Component(
lambda arch: self._run_cmd(['build-crosstool-ng'], arch),
supported_archs=common.consts['crosstool_ng_supported_archs'],
# http://crosstool-ng.github.io/docs/os-setup/
apt_get_pkgs={
'bison',
'docbook2x',
'flex',
'gawk',
'gcc',
'gperf',
'help2man',
'libncurses5-dev',
'libtool-bin',
'make',
'python-dev',
'texinfo',
},
submodules={'crosstool-ng'},
),
'gem5': _Component(
lambda arch: self._run_cmd(['build-gem5'], arch),
# TODO test it out on Docker and answer that question properly:
# https://askubuntu.com/questions/350475/how-can-i-install-gem5
apt_get_pkgs={
'device-tree-compiler',
'diod',
'libgoogle-perftools-dev',
'm4',
'protobuf-compiler',
'python-dev',
'python-pip',
# For prebuilt qcow2 unpack.
'qemu-utils',
'scons',
'zlib1g-dev',
},
python2_pkgs={
# Generate graphs of config.ini under m5out.
'pydot',
},
submodules={'gem5'},
),
'gem5-debug': _Component(
lambda arch: self._run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch),
),
'gem5-fast': _Component(
lambda arch: self._run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch),
),
'linux': _Component(
lambda arch: self._run_cmd(['build-linux'], arch),
submodules_shallow={'linux'},
apt_get_pkgs={
'bison',
'flex',
# Without this started failing in kernel 4.15 with:
# Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
'libelf-dev',
},
),
'modules': _Component(
lambda arch: self._run_cmd(['build-modules'], arch),
),
'm5': _Component(
lambda arch: self._run_cmd(['build-m5'], arch),
submodules={'gem5'},
),
'qemu': _Component(
lambda arch: self._run_cmd(['build-qemu'], arch),
apt_build_deps={'qemu'},
apt_get_pkgs={'libsdl2-dev'},
submodules={'qemu'},
),
'qemu-user': _Component(
lambda arch: self._run_cmd(['build-qemu', '--userland'], arch),
apt_build_deps = {'qemu'},
apt_get_pkgs={'libsdl2-dev'},
submodules = {'qemu'},
),
'parsec-benchmark': _Component(
submodules = {'parsec-benchmark'},
),
'userland': _Component(
lambda arch: self._run_cmd(['build-userland'], arch),
),
.... # Dependency only nodes.
./build --all-archs --extra-args=--clean buildroot 'all': _Component(dependencies=[
.... 'all-linux',
''', 'all-baremetal',
formatter_class=argparse.RawTextHelpFormatter, ]),
) 'all-baremetal': _Component(dependencies=[
parser.add_argument('--all', default=False, action='store_true', help='''\ 'qemu-baremetal',
'gem5-baremetal',
'baremetal-gem5-pbx',
],
supported_archs=common.consts['crosstool_ng_supported_archs'],
),
'all-linux': _Component(dependencies=[
'qemu-gem5-buildroot',
'gem5-debug',
'gem5-fast',
'qemu-user',
]),
'baremetal': _Component(dependencies=[
'baremetal-gem5',
'baremetal-qemu',
]),
'gem5-buildroot': _Component(dependencies=[
'buildroot-gcc',
'linux',
'm5',
'overlay',
'gem5',
]),
'gem5-baremetal': _Component(dependencies=[
'gem5',
'crosstool-ng',
'baremetal-gem5',
]),
'overlay': _Component(dependencies=[
'copy-overlay',
'modules',
'userland',
'buildroot',
]),
'qemu-baremetal': _Component(dependencies=[
'qemu',
'crosstool-ng',
'baremetal-qemu',
]),
'qemu-buildroot': _Component(dependencies=[
'qemu',
'buildroot-gcc',
'overlay',
'linux',
]),
'qemu-gem5-buildroot': _Component(dependencies=[
'qemu',
'gem5-buildroot',
]),
'release': _Component(dependencies=[
'qemu-buildroot',
]),
}
self.add_argument(
'--all',
default=False,
help='''\
Build absolutely everything for all archs. Build absolutely everything for all archs.
''') '''
group = parser.add_mutually_exclusive_group(required=False) )
group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\ self.add_argument(
'-A',
'--all-archs',
default=False,
help='''\
Build the selected components for all archs. Build the selected components for all archs.
''') ''')
group.add_argument('-a', '--arch', choices=kwargs['arch_choices'], default=[], action='append', help='''\ self.add_argument(
'-a',
'--arch',
choices=common.consts['arch_choices'],
default=[],
action='append',
help='''\
Build the selected components for this arch. Select multiple archs by Build the selected components for this arch. Select multiple archs by
passing this option multiple times. Default: [{}] passing this option multiple times. Default: [{}]
'''.format(kwargs['default_arch'])) '''.format(common.consts['default_arch']))
parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\ self.add_argument(
'-D',
'--download-dependencies',
default=False,
help='''\
Also download all dependencies required for a given build: Ubuntu packages, Also download all dependencies required for a given build: Ubuntu packages,
Python packages and git submodules. Python packages and git submodules.
''') '''
parser.add_argument('--extra-args', default='', help='''\ )
self.add_argument(
'--dry-run',
default=False,
)
self.add_argument(
'--travis',
default=False,
help='''\
Extra args to pass to all scripts. Extra args to pass to all scripts.
''' '''
) )
parser.add_argument('--travis', default=False, action='store_true', help='''\ self.add_argument(
Extra args to pass to all scripts. 'components',
''' choices=list(self.name_to_component_map.keys()) + [[]],
) default=[],
parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\ nargs='*',
help='''\
Which components to build. Default: qemu-buildroot Which components to build. Default: qemu-buildroot
'''.format(kwargs['default_arch'])) '''
self.add_dry_run_argument(parser) )
args = parser.parse_args()
self.setup_dry_run_arguments(args)
# Decide archs. def _run_cmd(self, python_file, **kwargs):
if kwargs['arch'] == []: python_file = os.path.join(common.consts['root_dir'], python_file)
if kwargs['all'] or kwargs['all_archs']: run = common.import_path(python_file).Main()
archs = kwargs['all_archs'].copy() run(**kwargs)
else: self.sh.run_cmd(cmd_abs)
archs = set([kwargs['default_arch']])
else:
archs = set()
for arch in kwargs['arch']:
if arch in kwargs['arch_short_to_long_dict']:
arch = kwargs['arch_short_to_long_dict'][arch]
archs.add(arch)
# Decide components. def main(self, **kwargs):
components = kwargs['components'] self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run'])
if kwargs['all']:
components = ['all']
elif components == []:
components = ['qemu-buildroot']
selected_components = []
selected_component_name_set = set()
for component_name in components:
todo = [component_name]
while todo:
current_name = todo.pop(0)
if current_name not in selected_component_name_set:
selected_component_name_set.add(current_name)
component = name_to_component_map[current_name]
selected_components.append(component)
todo.extend(component.dependencies)
if kwargs['download_dependencies']: # Decide archs.
apt_get_pkgs = { if kwargs['arch'] == []:
# Core requirements for this repo. if kwargs['all'] or kwargs['all_archs']:
'git', archs = kwargs['all_archs'].copy()
'moreutils', # ts else:
'python3-pip', archs = set([common.consts['default_arch']])
'tmux', else:
'vinagre', archs = set()
'wget', for arch in kwargs['arch']:
} if arch in common.consts['arch_short_to_long_dict']:
# E.e. on an ARM host, the package gcc-arm-linux-gnueabihf arch = common.consts['arch_short_to_long_dict'][arch]
# is called just gcc. archs.add(arch)
processor = platform.processor()
if processor != 'arm': # Decide components.
apt_get_pkgs.update({ components = kwargs['components']
'gcc-arm-linux-gnueabihf', if kwargs['all']:
'g++-arm-linux-gnueabihf', components = ['all']
}) elif components == []:
if processor != 'aarch64': components = ['qemu-buildroot']
apt_get_pkgs.update({ selected_components = []
'gcc-aarch64-linux-gnu', selected_component_name_set = set()
'g++-aarch64-linux-gnu', for component_name in components:
}) todo = [component_name]
apt_build_deps = set() while todo:
submodules = set() current_name = todo.pop(0)
submodules_shallow = set() if current_name not in selected_component_name_set:
python2_pkgs = set() selected_component_name_set.add(current_name)
python3_pkgs = { component = self.name_to_component_map[current_name]
'pexpect==4.6.0', selected_components.append(component)
} todo.extend(component.dependencies)
for component in selected_components:
apt_get_pkgs.update(component.apt_get_pkgs) if kwargs['download_dependencies']:
apt_build_deps.update(component.apt_build_deps) apt_get_pkgs = {
submodules.update(component.submodules) # Core requirements for this repo.
submodules_shallow.update(component.submodules_shallow) 'git',
python2_pkgs.update(component.python2_pkgs) 'moreutils', # ts
python3_pkgs.update(component.python3_pkgs) 'python3-pip',
if apt_get_pkgs or apt_build_deps: 'tmux',
if kwargs['travis']: 'vinagre',
interacive_pkgs = { 'wget',
'libsdl2-dev',
} }
apt_get_pkgs.difference_update(interacive_pkgs) # E.g. on an ARM host, the package gcc-arm-linux-gnueabihf
if kwargs['in_docker']: # is called just gcc.
sudo = [] processor = platform.processor()
# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai if processor != 'arm':
os.environ['DEBIAN_FRONTEND'] = 'noninteractive' apt_get_pkgs.update({
# https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list 'gcc-arm-linux-gnueabihf',
sources_path = os.path.join('/etc', 'apt', 'sources.list') 'g++-arm-linux-gnueabihf',
with open(sources_path, 'r') as f: })
sources_txt = f.read() if processor != 'aarch64':
sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE) apt_get_pkgs.update({
with open(sources_path, 'w') as f: 'gcc-aarch64-linux-gnu',
f.write(sources_txt) 'g++-aarch64-linux-gnu',
else: })
sudo = ['sudo'] apt_build_deps = set()
if kwargs['in_docker'] or kwargs['travis']: submodules = set()
y = ['-y'] submodules_shallow = set()
else: python2_pkgs = set()
y = [] python3_pkgs = {
self.sh.run_cmd( 'pexpect==4.6.0',
sudo + ['apt-get', 'update', LF] }
) for component in selected_components:
if apt_get_pkgs: apt_get_pkgs.update(component.apt_get_pkgs)
self.sh.run_cmd( apt_build_deps.update(component.apt_build_deps)
sudo + ['apt-get', 'install'] + y + [LF] + submodules.update(component.submodules)
self.sh.add_newlines(sorted(apt_get_pkgs)) submodules_shallow.update(component.submodules_shallow)
) python2_pkgs.update(component.python2_pkgs)
if apt_build_deps: python3_pkgs.update(component.python3_pkgs)
self.sh.run_cmd( if apt_get_pkgs or apt_build_deps:
sudo + if kwargs['travis']:
['apt-get', 'build-dep'] + y + [LF] + interacive_pkgs = {
self.sh.add_newlines(sorted(apt_build_deps)) 'libsdl2-dev',
) }
if python2_pkgs: apt_get_pkgs.difference_update(interacive_pkgs)
self.sh.run_cmd( if common.consts['in_docker']:
['python', '-m', 'pip', 'install', '--user', LF] + sudo = []
self.sh.add_newlines(sorted(python2_pkgs)) # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
) os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
if python3_pkgs: # https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
# Not with pip executable directly: sources_path = os.path.join('/etc', 'apt', 'sources.list')
# https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 with open(sources_path, 'r') as f:
self.sh.run_cmd( sources_txt = f.read()
['python3', '-m', 'pip', 'install', '--user', LF] + sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
self.sh.add_newlines(sorted(python3_pkgs)) with open(sources_path, 'w') as f:
) f.write(sources_txt)
git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive'] else:
if submodules: sudo = ['sudo']
# == Other nice git options for when distros move to newer Git if common.consts['in_docker'] or kwargs['travis']:
# y = ['-y']
# Currently not on Ubuntu 16.04: else:
# y = []
# `--progress`: added on Git 2.10: self.sh.run_cmd(
# sudo + ['apt-get', 'update', LF]
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching )
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone if apt_get_pkgs:
# self.sh.run_cmd(
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 sudo + ['apt-get', 'install'] + y + [LF] +
self.sh.run_cmd( self.sh.add_newlines(sorted(apt_get_pkgs))
git_cmd_common + ['--', LF] + )
self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules)]) if apt_build_deps:
) self.sh.run_cmd(
if submodules_shallow: sudo +
# == Shallow cloning. ['apt-get', 'build-dep'] + y + [LF] +
# self.sh.add_newlines(sorted(apt_build_deps))
# TODO Ideally we should shallow clone --depth 1 all of them. )
# if python2_pkgs:
# However, most git servers out there are crap or craply configured self.sh.run_cmd(
# and don't allow shallow cloning except for branches. ['python', '-m', 'pip', 'install', '--user', LF] +
# self.sh.add_newlines(sorted(python2_pkgs))
# So for now, let's shallow clone only the Linux kernel, which has by far )
# the largest .git repo history, and full clone the others. if python3_pkgs:
# # Not with pip executable directly:
# Then we will maintain a GitHub Linux kernel mirror / fork that always has a # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
# lkmc branch, and point to it, so that it will always succeed. self.sh.run_cmd(
# ['python3', '-m', 'pip', 'install', '--user', LF] +
# See also: self.sh.add_newlines(sorted(python3_pkgs))
# )
# * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
# * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702 if submodules:
# * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac # == Other nice git options for when distros move to newer Git
# #
self.sh.run_cmd( # Currently not on Ubuntu 16.04:
git_cmd_common + ['--depth', '1', '--', LF] + #
self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules_shallow)]) # `--progress`: added on Git 2.10:
) #
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
#
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
self.sh.run_cmd(
git_cmd_common + ['--', LF] +
self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
)
if submodules_shallow:
# == Shallow cloning.
#
# TODO Ideally we should shallow clone --depth 1 all of them.
#
# However, most git servers out there are crap or craply configured
# and don't allow shallow cloning except for branches.
#
# So for now, let's shallow clone only the Linux kernel, which has by far
# the largest .git repo history, and full clone the others.
#
# Then we will maintain a GitHub Linux kernel mirror / fork that always has a
# lkmc branch, and point to it, so that it will always succeed.
#
# See also:
#
# * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
# * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
# * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
#
self.sh.run_cmd(
git_cmd_common + ['--depth', '1', '--', LF] +
self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules_shallow)])
)
# Do the build. # Do the build.
for arch in archs: for arch in archs:
for component in selected_components: for component in selected_components:
component.build(arch) component.build(arch)
if __name__ == '__main__':
Main().cli()

8
cli_function.py
View File

@@ -151,7 +151,7 @@ class CliFunction:
config_file = args_with_defaults['config_file'] config_file = args_with_defaults['config_file']
else: else:
config_file = self._config_file config_file = self._config_file
if os.path.exists(config_file): if config_file is not None and os.path.exists(config_file):
config_configs = {} config_configs = {}
config = imp.load_source('config', config_file) config = imp.load_source('config', config_file)
config.set_args(config_configs) config.set_args(config_configs)
@@ -168,7 +168,8 @@ class CliFunction:
args_with_defaults[key] = argument.default args_with_defaults[key] = argument.default
else: else:
raise Exception('Value not given for mandatory argument: ' + key) raise Exception('Value not given for mandatory argument: ' + key)
del args_with_defaults['config_file'] if 'config_file' in args_with_defaults:
del args_with_defaults['config_file']
return args_with_defaults return args_with_defaults
def add_argument( def add_argument(
@@ -360,6 +361,9 @@ amazing function!
# Force a boolean value set on the config to be False on CLI. # Force a boolean value set on the config to be False on CLI.
assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False
# Pick another config file.
assert one_cli_function.cli(['--config-file', 'cli_function_test_config_2.py', '1'])['bool_cli'] is False
# get_cli # get_cli
assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)] assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)]
assert one_cli_function.get_cli(pos_mandatory=1, asdf='B', qwer='R') == [('--asdf', 'B'), ('--bool-cli',), ('--qwer', 'R'), ('1',)] assert one_cli_function.get_cli(pos_mandatory=1, asdf='B', qwer='R') == [('--asdf', 'B'), ('--bool-cli',), ('--qwer', 'R'), ('1',)]

5
cli_function_test_config_2.py Normal file
View File

@@ -0,0 +1,5 @@
def set_args(args):
'''
:type args: Dict[str, Any]
'''
args['bool_cli'] = False

2
common.py
View File

@@ -577,7 +577,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
def base64_encode(string): def base64_encode(string):
return base64.b64encode(string.encode()).decode() return base64.b64encode(string.encode()).decode()
def gem_list_checkpoint_dirs(self): def gem5_list_checkpoint_dirs(self):
''' '''
List checkpoint directory, oldest first. List checkpoint directory, oldest first.
''' '''

4
run
View File

@@ -351,7 +351,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
if self.env['gem5_script'] == 'fs': if self.env['gem5_script'] == 'fs':
# TODO port # TODO port
if self.env['gem5_restore'] is not None: if self.env['gem5_restore'] is not None:
cpt_dirs = self.gem_list_checkpoint_dirs() cpt_dirs = self.gem5_list_checkpoint_dirs()
cpt_dir = cpt_dirs[-self.env['gem5_restore']] cpt_dir = cpt_dirs[-self.env['gem5_restore']]
extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)])
cmd.extend([ cmd.extend([
@@ -401,7 +401,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
else: else:
cpu_type = 'atomic' cpu_type = 'atomic'
if self.env['gem5_restore'] is not None: if self.env['gem5_restore'] is not None:
cpt_dir = self.gem_list_checkpoint_dirs()[-self.env['gem5_restore']] cpt_dir = self.gem5_list_checkpoint_dirs()[-self.env['gem5_restore']]
extra_emulator_args.extend(['--restore-from', os.path.join(self.env['m5out_dir'], cpt_dir)]) extra_emulator_args.extend(['--restore-from', os.path.join(self.env['m5out_dir'], cpt_dir)])
cmd.extend([ cmd.extend([
os.path.join(self.env['gem5_source_dir'], 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF, os.path.join(self.env['gem5_source_dir'], 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF,