3rd-party/linux-kernel-module-cheat
1
0
Fork 0
mirror of https://github.com/cirosantilli/linux-kernel-module-cheat.git synced 2026-01-25 19:21:35 +01:00
Code Issues Releases Wiki Activity

start porting build

Browse Source
This commit is contained in:
Ciro Santilli 六四事件 法轮功
2019-01-22 00:00:00 +00:00
parent b72f75b531
commit 0a33292326
5 changed files with 428 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

801
build
View File

@@ -6,10 +6,12 @@ import platform
import re
import os
import cli_function
import common
import shell_helpers
from shell_helpers import LF
class Component:
class _Component:
'''
Yes, we are re-inventing a crappy dependency resolution system.
I can't believe it.
@@ -40,6 +42,7 @@ class Component:
self.submodules_shallow = submodules_shallow or set()
self.python2_pkgs = python2_pkgs or set()
self.python3_pkgs = python3_pkgs or set()
def build(self, arch):
if (
(self.build_callback is not None) and
@@ -47,208 +50,11 @@ class Component:
):
self.build_callback(arch)
def run_cmd(cmd, arch):
global args
cmd_abs = cmd.copy()
cmd_abs[0] = os.path.join(kwargs['root_dir'], cmd[0])
cmd_abs.extend(['--arch', arch])
if kwargs['extra_args']:
cmd_abs.append(kwargs['extra_args'])
self.sh.run_cmd(cmd_abs, dry_run=kwargs['dry_run'])
buildroot_component = Component(
lambda arch: run_cmd(['build-buildroot'], arch),
submodules = {'buildroot'},
# https://buildroot.org/downloads/manual/manual.html#requirement
apt_get_pkgs={
'bash',
'bc',
'binutils',
'build-essential',
'bzip2',
'cpio',
'g++',
'gcc',
'graphviz',
'gzip',
'make',
'patch',
'perl',
'python-matplotlib',
'python3',
'rsync',
'sed',
'tar',
'unzip',
},
)
name_to_component_map = {
# Leaves without dependencies.
'baremetal-qemu': Component(
lambda arch: run_cmd(['build-baremetal', '--emulator', 'qemu'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'baremetal-gem5': Component(
lambda arch: run_cmd(['build-baremetal', '--gem5'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'baremetal-gem5-pbx': Component(
lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'buildroot': buildroot_component,
'buildroot-gcc': buildroot_component,
'copy-overlay': Component(
lambda arch: run_cmd(['copy-overlay'], arch),
),
'crosstool-ng': Component(
lambda arch: run_cmd(['build-crosstool-ng'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'],
# http://crosstool-ng.github.io/docs/os-setup/
apt_get_pkgs={
'bison',
'docbook2x',
'flex',
'gawk',
'gcc',
'gperf',
'help2man',
'libncurses5-dev',
'libtool-bin',
'make',
'python-dev',
'texinfo',
},
submodules={'crosstool-ng'},
),
'gem5': Component(
lambda arch: run_cmd(['build-gem5'], arch),
# TODO test it out on Docker and answer that question properly:
# https://askubuntu.com/questions/350475/how-can-i-install-gem5
apt_get_pkgs={
'device-tree-compiler',
'diod',
'libgoogle-perftools-dev',
'm4',
'protobuf-compiler',
'python-dev',
'python-pip',
# For prebuilt qcow2 unpack.
'qemu-utils',
'scons',
'zlib1g-dev',
},
python2_pkgs={
# Generate graphs of config.ini under m5out.
'pydot',
},
submodules={'gem5'},
),
'gem5-debug': Component(
lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch),
),
'gem5-fast': Component(
lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch),
),
'linux': Component(
lambda arch: run_cmd(['build-linux'], arch),
submodules_shallow={'linux'},
apt_get_pkgs={
'bison',
'flex',
# Without this started failing in kernel 4.15 with:
# Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
'libelf-dev',
},
),
'modules': Component(
lambda arch: run_cmd(['build-modules'], arch),
),
'm5': Component(
lambda arch: run_cmd(['build-m5'], arch),
submodules={'gem5'},
),
'qemu': Component(
lambda arch: run_cmd(['build-qemu'], arch),
apt_build_deps={'qemu'},
apt_get_pkgs={'libsdl2-dev'},
submodules={'qemu'},
),
'qemu-user': Component(
lambda arch: run_cmd(['build-qemu', '--userland'], arch),
apt_build_deps = {'qemu'},
apt_get_pkgs={'libsdl2-dev'},
submodules = {'qemu'},
),
'parsec-benchmark': Component(
submodules = {'parsec-benchmark'},
),
'userland': Component(
lambda arch: run_cmd(['build-userland'], arch),
),
# Dependency only nodes.
'all': Component(dependencies=[
'all-linux',
'all-baremetal',
]),
'all-baremetal': Component(dependencies=[
'qemu-baremetal',
'gem5-baremetal',
'baremetal-gem5-pbx',
],
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'all-linux': Component(dependencies=[
'qemu-gem5-buildroot',
'gem5-debug',
'gem5-fast',
'qemu-user',
]),
'baremetal': Component(dependencies=[
'baremetal-gem5',
'baremetal-qemu',
]),
'gem5-buildroot': Component(dependencies=[
'buildroot-gcc',
'linux',
'm5',
'overlay',
'gem5',
]),
'gem5-baremetal': Component(dependencies=[
'gem5',
'crosstool-ng',
'baremetal-gem5',
]),
'overlay': Component(dependencies=[
'copy-overlay',
'modules',
'userland',
'buildroot',
]),
'qemu-baremetal': Component(dependencies=[
'qemu',
'crosstool-ng',
'baremetal-qemu',
]),
'qemu-buildroot': Component(dependencies=[
'qemu',
'buildroot-gcc',
'overlay',
'linux',
]),
'qemu-gem5-buildroot': Component(dependencies=[
'qemu',
'gem5-buildroot',
]),
'release': Component(dependencies=[
'qemu-buildroot',
]),
}
parser = argparse.ArgumentParser(
description= '''\
class Main(cli_function.CliFunction):
def __init__(self):
super().__init__(
config_file=common.consts['config_file'],
description='''\
Shallow helper to build everything, or a subset of everything conveniently.
Our build-* scripts don't build any dependencies to make iterative
@@ -281,204 +87,425 @@ If `--arch` is given, build just for the given archs:
....
This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`.
'''
)
buildroot_component = _Component(
lambda arch: self._run_cmd(['build-buildroot'], arch),
submodules = {'buildroot'},
# https://buildroot.org/downloads/manual/manual.html#requirement
apt_get_pkgs={
'bash',
'bc',
'binutils',
'build-essential',
'bzip2',
'cpio',
'g++',
'gcc',
'graphviz',
'gzip',
'make',
'patch',
'perl',
'python-matplotlib',
'python3',
'rsync',
'sed',
'tar',
'unzip',
},
)
Clean all Linux kernel builds:
self.name_to_component_map = {
# Leaves without dependencies.
'baremetal-qemu': _Component(
lambda arch: self._run_cmd(['build-baremetal', '--emulator', 'qemu'], arch),
supported_archs=common.consts['crosstool_ng_supported_archs'],
),
'baremetal-gem5': _Component(
lambda arch: self._run_cmd(['build-baremetal', '--gem5'], arch),
supported_archs=common.consts['crosstool_ng_supported_archs'],
),
'baremetal-gem5-pbx': _Component(
lambda arch: self._run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
supported_archs=common.consts['crosstool_ng_supported_archs'],
),
'buildroot': buildroot_component,
'buildroot-gcc': buildroot_component,
'copy-overlay': _Component(
lambda arch: self._run_cmd(['copy-overlay'], arch),
),
'crosstool-ng': _Component(
lambda arch: self._run_cmd(['build-crosstool-ng'], arch),
supported_archs=common.consts['crosstool_ng_supported_archs'],
# http://crosstool-ng.github.io/docs/os-setup/
apt_get_pkgs={
'bison',
'docbook2x',
'flex',
'gawk',
'gcc',
'gperf',
'help2man',
'libncurses5-dev',
'libtool-bin',
'make',
'python-dev',
'texinfo',
},
submodules={'crosstool-ng'},
),
'gem5': _Component(
lambda arch: self._run_cmd(['build-gem5'], arch),
# TODO test it out on Docker and answer that question properly:
# https://askubuntu.com/questions/350475/how-can-i-install-gem5
apt_get_pkgs={
'device-tree-compiler',
'diod',
'libgoogle-perftools-dev',
'm4',
'protobuf-compiler',
'python-dev',
'python-pip',
# For prebuilt qcow2 unpack.
'qemu-utils',
'scons',
'zlib1g-dev',
},
python2_pkgs={
# Generate graphs of config.ini under m5out.
'pydot',
},
submodules={'gem5'},
),
'gem5-debug': _Component(
lambda arch: self._run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch),
),
'gem5-fast': _Component(
lambda arch: self._run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch),
),
'linux': _Component(
lambda arch: self._run_cmd(['build-linux'], arch),
submodules_shallow={'linux'},
apt_get_pkgs={
'bison',
'flex',
# Without this started failing in kernel 4.15 with:
# Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
'libelf-dev',
},
),
'modules': _Component(
lambda arch: self._run_cmd(['build-modules'], arch),
),
'm5': _Component(
lambda arch: self._run_cmd(['build-m5'], arch),
submodules={'gem5'},
),
'qemu': _Component(
lambda arch: self._run_cmd(['build-qemu'], arch),
apt_build_deps={'qemu'},
apt_get_pkgs={'libsdl2-dev'},
submodules={'qemu'},
),
'qemu-user': _Component(
lambda arch: self._run_cmd(['build-qemu', '--userland'], arch),
apt_build_deps = {'qemu'},
apt_get_pkgs={'libsdl2-dev'},
submodules = {'qemu'},
),
'parsec-benchmark': _Component(
submodules = {'parsec-benchmark'},
),
'userland': _Component(
lambda arch: self._run_cmd(['build-userland'], arch),
),
....
./build --all-archs --extra-args=--clean buildroot
....
''',
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument('--all', default=False, action='store_true', help='''\
# Dependency only nodes.
'all': _Component(dependencies=[
'all-linux',
'all-baremetal',
]),
'all-baremetal': _Component(dependencies=[
'qemu-baremetal',
'gem5-baremetal',
'baremetal-gem5-pbx',
],
supported_archs=common.consts['crosstool_ng_supported_archs'],
),
'all-linux': _Component(dependencies=[
'qemu-gem5-buildroot',
'gem5-debug',
'gem5-fast',
'qemu-user',
]),
'baremetal': _Component(dependencies=[
'baremetal-gem5',
'baremetal-qemu',
]),
'gem5-buildroot': _Component(dependencies=[
'buildroot-gcc',
'linux',
'm5',
'overlay',
'gem5',
]),
'gem5-baremetal': _Component(dependencies=[
'gem5',
'crosstool-ng',
'baremetal-gem5',
]),
'overlay': _Component(dependencies=[
'copy-overlay',
'modules',
'userland',
'buildroot',
]),
'qemu-baremetal': _Component(dependencies=[
'qemu',
'crosstool-ng',
'baremetal-qemu',
]),
'qemu-buildroot': _Component(dependencies=[
'qemu',
'buildroot-gcc',
'overlay',
'linux',
]),
'qemu-gem5-buildroot': _Component(dependencies=[
'qemu',
'gem5-buildroot',
]),
'release': _Component(dependencies=[
'qemu-buildroot',
]),
}
self.add_argument(
'--all',
default=False,
help='''\
Build absolutely everything for all archs.
''')
group = parser.add_mutually_exclusive_group(required=False)
group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\
'''
)
self.add_argument(
'-A',
'--all-archs',
default=False,
help='''\
Build the selected components for all archs.
''')
group.add_argument('-a', '--arch', choices=kwargs['arch_choices'], default=[], action='append', help='''\
self.add_argument(
'-a',
'--arch',
choices=common.consts['arch_choices'],
default=[],
action='append',
help='''\
Build the selected components for this arch. Select multiple archs by
passing this option multiple times. Default: [{}]
'''.format(kwargs['default_arch']))
parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\
'''.format(common.consts['default_arch']))
self.add_argument(
'-D',
'--download-dependencies',
default=False,
help='''\
Also download all dependencies required for a given build: Ubuntu packages,
Python packages and git submodules.
''')
parser.add_argument('--extra-args', default='', help='''\
'''
)
self.add_argument(
'--dry-run',
default=False,
)
self.add_argument(
'--travis',
default=False,
help='''\
Extra args to pass to all scripts.
'''
)
parser.add_argument('--travis', default=False, action='store_true', help='''\
Extra args to pass to all scripts.
'''
)
parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\
)
self.add_argument(
'components',
choices=list(self.name_to_component_map.keys()) + [[]],
default=[],
nargs='*',
help='''\
Which components to build. Default: qemu-buildroot
'''.format(kwargs['default_arch']))
self.add_dry_run_argument(parser)
args = parser.parse_args()
self.setup_dry_run_arguments(args)
'''
)
# Decide archs.
if kwargs['arch'] == []:
if kwargs['all'] or kwargs['all_archs']:
archs = kwargs['all_archs'].copy()
else:
archs = set([kwargs['default_arch']])
else:
archs = set()
for arch in kwargs['arch']:
if arch in kwargs['arch_short_to_long_dict']:
arch = kwargs['arch_short_to_long_dict'][arch]
archs.add(arch)
def _run_cmd(self, python_file, **kwargs):
python_file = os.path.join(common.consts['root_dir'], python_file)
run = common.import_path(python_file).Main()
run(**kwargs)
self.sh.run_cmd(cmd_abs)
# Decide components.
components = kwargs['components']
if kwargs['all']:
components = ['all']
elif components == []:
components = ['qemu-buildroot']
selected_components = []
selected_component_name_set = set()
for component_name in components:
todo = [component_name]
while todo:
current_name = todo.pop(0)
if current_name not in selected_component_name_set:
selected_component_name_set.add(current_name)
component = name_to_component_map[current_name]
selected_components.append(component)
todo.extend(component.dependencies)
def main(self, **kwargs):
self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run'])
if kwargs['download_dependencies']:
apt_get_pkgs = {
# Core requirements for this repo.
'git',
'moreutils', # ts
'python3-pip',
'tmux',
'vinagre',
'wget',
}
# E.e. on an ARM host, the package gcc-arm-linux-gnueabihf
# is called just gcc.
processor = platform.processor()
if processor != 'arm':
apt_get_pkgs.update({
'gcc-arm-linux-gnueabihf',
'g++-arm-linux-gnueabihf',
})
if processor != 'aarch64':
apt_get_pkgs.update({
'gcc-aarch64-linux-gnu',
'g++-aarch64-linux-gnu',
})
apt_build_deps = set()
submodules = set()
submodules_shallow = set()
python2_pkgs = set()
python3_pkgs = {
'pexpect==4.6.0',
}
for component in selected_components:
apt_get_pkgs.update(component.apt_get_pkgs)
apt_build_deps.update(component.apt_build_deps)
submodules.update(component.submodules)
submodules_shallow.update(component.submodules_shallow)
python2_pkgs.update(component.python2_pkgs)
python3_pkgs.update(component.python3_pkgs)
if apt_get_pkgs or apt_build_deps:
if kwargs['travis']:
interacive_pkgs = {
'libsdl2-dev',
# Decide archs.
if kwargs['arch'] == []:
if kwargs['all'] or kwargs['all_archs']:
archs = kwargs['all_archs'].copy()
else:
archs = set([common.consts['default_arch']])
else:
archs = set()
for arch in kwargs['arch']:
if arch in common.consts['arch_short_to_long_dict']:
arch = common.consts['arch_short_to_long_dict'][arch]
archs.add(arch)
# Decide components.
components = kwargs['components']
if kwargs['all']:
components = ['all']
elif components == []:
components = ['qemu-buildroot']
selected_components = []
selected_component_name_set = set()
for component_name in components:
todo = [component_name]
while todo:
current_name = todo.pop(0)
if current_name not in selected_component_name_set:
selected_component_name_set.add(current_name)
component = self.name_to_component_map[current_name]
selected_components.append(component)
todo.extend(component.dependencies)
if kwargs['download_dependencies']:
apt_get_pkgs = {
# Core requirements for this repo.
'git',
'moreutils', # ts
'python3-pip',
'tmux',
'vinagre',
'wget',
}
apt_get_pkgs.difference_update(interacive_pkgs)
if kwargs['in_docker']:
sudo = []
# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
# https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
sources_path = os.path.join('/etc', 'apt', 'sources.list')
with open(sources_path, 'r') as f:
sources_txt = f.read()
sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
with open(sources_path, 'w') as f:
f.write(sources_txt)
else:
sudo = ['sudo']
if kwargs['in_docker'] or kwargs['travis']:
y = ['-y']
else:
y = []
self.sh.run_cmd(
sudo + ['apt-get', 'update', LF]
)
if apt_get_pkgs:
self.sh.run_cmd(
sudo + ['apt-get', 'install'] + y + [LF] +
self.sh.add_newlines(sorted(apt_get_pkgs))
)
if apt_build_deps:
self.sh.run_cmd(
sudo +
['apt-get', 'build-dep'] + y + [LF] +
self.sh.add_newlines(sorted(apt_build_deps))
)
if python2_pkgs:
self.sh.run_cmd(
['python', '-m', 'pip', 'install', '--user', LF] +
self.sh.add_newlines(sorted(python2_pkgs))
)
if python3_pkgs:
# Not with pip executable directly:
# https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
self.sh.run_cmd(
['python3', '-m', 'pip', 'install', '--user', LF] +
self.sh.add_newlines(sorted(python3_pkgs))
)
git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
if submodules:
# == Other nice git options for when distros move to newer Git
#
# Currently not on Ubuntu 16.04:
#
# `--progress`: added on Git 2.10:
#
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
#
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
self.sh.run_cmd(
git_cmd_common + ['--', LF] +
self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules)])
)
if submodules_shallow:
# == Shallow cloning.
#
# TODO Ideally we should shallow clone --depth 1 all of them.
#
# However, most git servers out there are crap or craply configured
# and don't allow shallow cloning except for branches.
#
# So for now, let's shallow clone only the Linux kernel, which has by far
# the largest .git repo history, and full clone the others.
#
# Then we will maintain a GitHub Linux kernel mirror / fork that always has a
# lkmc branch, and point to it, so that it will always succeed.
#
# See also:
#
# * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
# * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
# * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
#
self.sh.run_cmd(
git_cmd_common + ['--depth', '1', '--', LF] +
self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules_shallow)])
)
# E.g. on an ARM host, the package gcc-arm-linux-gnueabihf
# is called just gcc.
processor = platform.processor()
if processor != 'arm':
apt_get_pkgs.update({
'gcc-arm-linux-gnueabihf',
'g++-arm-linux-gnueabihf',
})
if processor != 'aarch64':
apt_get_pkgs.update({
'gcc-aarch64-linux-gnu',
'g++-aarch64-linux-gnu',
})
apt_build_deps = set()
submodules = set()
submodules_shallow = set()
python2_pkgs = set()
python3_pkgs = {
'pexpect==4.6.0',
}
for component in selected_components:
apt_get_pkgs.update(component.apt_get_pkgs)
apt_build_deps.update(component.apt_build_deps)
submodules.update(component.submodules)
submodules_shallow.update(component.submodules_shallow)
python2_pkgs.update(component.python2_pkgs)
python3_pkgs.update(component.python3_pkgs)
if apt_get_pkgs or apt_build_deps:
if kwargs['travis']:
interacive_pkgs = {
'libsdl2-dev',
}
apt_get_pkgs.difference_update(interacive_pkgs)
if common.consts['in_docker']:
sudo = []
# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
# https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
sources_path = os.path.join('/etc', 'apt', 'sources.list')
with open(sources_path, 'r') as f:
sources_txt = f.read()
sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
with open(sources_path, 'w') as f:
f.write(sources_txt)
else:
sudo = ['sudo']
if common.consts['in_docker'] or kwargs['travis']:
y = ['-y']
else:
y = []
self.sh.run_cmd(
sudo + ['apt-get', 'update', LF]
)
if apt_get_pkgs:
self.sh.run_cmd(
sudo + ['apt-get', 'install'] + y + [LF] +
self.sh.add_newlines(sorted(apt_get_pkgs))
)
if apt_build_deps:
self.sh.run_cmd(
sudo +
['apt-get', 'build-dep'] + y + [LF] +
self.sh.add_newlines(sorted(apt_build_deps))
)
if python2_pkgs:
self.sh.run_cmd(
['python', '-m', 'pip', 'install', '--user', LF] +
self.sh.add_newlines(sorted(python2_pkgs))
)
if python3_pkgs:
# Not with pip executable directly:
# https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
self.sh.run_cmd(
['python3', '-m', 'pip', 'install', '--user', LF] +
self.sh.add_newlines(sorted(python3_pkgs))
)
git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
if submodules:
# == Other nice git options for when distros move to newer Git
#
# Currently not on Ubuntu 16.04:
#
# `--progress`: added on Git 2.10:
#
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
#
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
self.sh.run_cmd(
git_cmd_common + ['--', LF] +
self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
)
if submodules_shallow:
# == Shallow cloning.
#
# TODO Ideally we should shallow clone --depth 1 all of them.
#
# However, most git servers out there are crap or craply configured
# and don't allow shallow cloning except for branches.
#
# So for now, let's shallow clone only the Linux kernel, which has by far
# the largest .git repo history, and full clone the others.
#
# Then we will maintain a GitHub Linux kernel mirror / fork that always has a
# lkmc branch, and point to it, so that it will always succeed.
#
# See also:
#
# * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
# * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
# * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
#
self.sh.run_cmd(
git_cmd_common + ['--depth', '1', '--', LF] +
self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules_shallow)])
)
# Do the build.
for arch in archs:
for component in selected_components:
component.build(arch)
# Do the build.
for arch in archs:
for component in selected_components:
component.build(arch)
if __name__ == '__main__':
Main().cli()

8
cli_function.py
View File

@@ -151,7 +151,7 @@ class CliFunction:
config_file = args_with_defaults['config_file']
else:
config_file = self._config_file
if os.path.exists(config_file):
if config_file is not None and os.path.exists(config_file):
config_configs = {}
config = imp.load_source('config', config_file)
config.set_args(config_configs)
@@ -168,7 +168,8 @@ class CliFunction:
args_with_defaults[key] = argument.default
else:
raise Exception('Value not given for mandatory argument: ' + key)
del args_with_defaults['config_file']
if 'config_file' in args_with_defaults:
del args_with_defaults['config_file']
return args_with_defaults
def add_argument(
@@ -360,6 +361,9 @@ amazing function!
# Force a boolean value set on the config to be False on CLI.
assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False
# Pick another config file.
assert one_cli_function.cli(['--config-file', 'cli_function_test_config_2.py', '1'])['bool_cli'] is False
# get_cli
assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)]
assert one_cli_function.get_cli(pos_mandatory=1, asdf='B', qwer='R') == [('--asdf', 'B'), ('--bool-cli',), ('--qwer', 'R'), ('1',)]

5
cli_function_test_config_2.py Normal file
View File

@@ -0,0 +1,5 @@
def set_args(args):
'''
:type args: Dict[str, Any]
'''
args['bool_cli'] = False

2
common.py
View File

@@ -577,7 +577,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
def base64_encode(string):
return base64.b64encode(string.encode()).decode()
def gem_list_checkpoint_dirs(self):
def gem5_list_checkpoint_dirs(self):
'''
List checkpoint directory, oldest first.
'''

4
run
View File

@@ -351,7 +351,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
if self.env['gem5_script'] == 'fs':
# TODO port
if self.env['gem5_restore'] is not None:
cpt_dirs = self.gem_list_checkpoint_dirs()
cpt_dirs = self.gem5_list_checkpoint_dirs()
cpt_dir = cpt_dirs[-self.env['gem5_restore']]
extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)])
cmd.extend([
@@ -401,7 +401,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
else:
cpu_type = 'atomic'
if self.env['gem5_restore'] is not None:
cpt_dir = self.gem_list_checkpoint_dirs()[-self.env['gem5_restore']]
cpt_dir = self.gem5_list_checkpoint_dirs()[-self.env['gem5_restore']]
extra_emulator_args.extend(['--restore-from', os.path.join(self.env['m5out_dir'], cpt_dir)])
cmd.extend([
os.path.join(self.env['gem5_source_dir'], 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF,