mirror of
https://github.com/cirosantilli/linux-kernel-module-cheat.git
synced 2026-01-25 19:21:35 +01:00
495 lines
19 KiB
Python
Executable File
495 lines
19 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import platform
|
|
import re
|
|
import os
|
|
|
|
import cli_function
|
|
import collections
|
|
import common
|
|
import copy
|
|
import shell_helpers
|
|
from shell_helpers import LF
|
|
|
|
class _Component:
|
|
'''
|
|
Yes, we are re-inventing a crappy dependency resolution system,
|
|
reminescent of scons or apt or Buildroot. I can't believe it.
|
|
|
|
The hard part is that we have optional dependencies as well...
|
|
e.g. buildroot optionally depends on m5 to put m5 in the root filesystem,
|
|
and buildroot optionally depends on qemu to build the qcow2 version
|
|
of the image.
|
|
'''
|
|
def __init__(
|
|
self,
|
|
build_callback=None,
|
|
supported_archs=None,
|
|
dependencies=None,
|
|
apt_get_pkgs=None,
|
|
apt_build_deps=None,
|
|
submodules=None,
|
|
submodules_shallow=None,
|
|
python2_pkgs=None,
|
|
python3_pkgs=None,
|
|
):
|
|
self.build_callback = build_callback
|
|
self.supported_archs = supported_archs
|
|
self.dependencies = dependencies or set()
|
|
self.apt_get_pkgs = apt_get_pkgs or set()
|
|
self.apt_build_deps = apt_build_deps or set()
|
|
self.submodules = submodules or set()
|
|
self.submodules_shallow = submodules_shallow or set()
|
|
self.python2_pkgs = python2_pkgs or set()
|
|
self.python3_pkgs = python3_pkgs or set()
|
|
|
|
def build(self, arch):
|
|
if (
|
|
(self.build_callback is not None) and
|
|
(self.supported_archs is None or arch in self.supported_archs)
|
|
):
|
|
self.build_callback()
|
|
|
|
class Main(common.LkmcCliFunction):
|
|
def __init__(self):
|
|
super().__init__(
|
|
description='''\
|
|
Build a component and all its dependencies.
|
|
|
|
Our build-* scripts don't build any dependencies to make iterative
|
|
development fast and more predictable.
|
|
|
|
It is currently not possible to configure indivitual components from the command line
|
|
when you build with this script. TODO.
|
|
|
|
Without any args, build only what is necessary for:
|
|
https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup
|
|
|
|
....
|
|
./%(prog)s
|
|
....
|
|
|
|
This is equivalent to:
|
|
|
|
....
|
|
./%(prog)s --arch x86_64 qemu-buildroot
|
|
....
|
|
''',
|
|
)
|
|
buildroot_component = _Component(
|
|
self._build_file('build-buildroot'),
|
|
submodules = {'buildroot'},
|
|
# https://buildroot.org/downloads/manual/manual.html#requirement
|
|
apt_get_pkgs={
|
|
'bash',
|
|
'bc',
|
|
'binutils',
|
|
'build-essential',
|
|
'bzip2',
|
|
'cpio',
|
|
'g++',
|
|
'gcc',
|
|
'graphviz',
|
|
'gzip',
|
|
'make',
|
|
'patch',
|
|
'perl',
|
|
'python-matplotlib',
|
|
'python3',
|
|
'rsync',
|
|
'sed',
|
|
'tar',
|
|
'unzip',
|
|
},
|
|
)
|
|
buildroot_overlay_qemu_component = copy.copy(buildroot_component)
|
|
# We need to build QEMU before the final Buildroot to get qemu-img.
|
|
buildroot_overlay_qemu_component.dependencies = ['overlay', 'qemu']
|
|
buildroot_overlay_gem5_component = copy.copy(buildroot_component)
|
|
buildroot_overlay_gem5_component.dependencies = ['overlay-gem5']
|
|
gem5_deps = {
|
|
# TODO test it out on Docker and answer that question properly:
|
|
# https://askubuntu.com/questions/350475/how-can-i-install-gem5
|
|
'apt_get_pkgs': {
|
|
'device-tree-compiler',
|
|
'diod',
|
|
'libgoogle-perftools-dev',
|
|
'm4',
|
|
'protobuf-compiler',
|
|
'python-dev',
|
|
'python-pip',
|
|
# For prebuilt qcow2 unpack.
|
|
'qemu-utils',
|
|
'scons',
|
|
'zlib1g-dev',
|
|
},
|
|
'python2_pkgs': {
|
|
# Generate graphs of config.ini under m5out.
|
|
'pydot',
|
|
},
|
|
'submodules': {'gem5'},
|
|
}
|
|
|
|
self.name_to_component_map = {
|
|
'all': _Component(dependencies=[
|
|
'all-linux',
|
|
'all-baremetal',
|
|
]),
|
|
'all-baremetal': _Component(dependencies=[
|
|
'qemu-baremetal',
|
|
'gem5-baremetal',
|
|
'baremetal-gem5-pbx',
|
|
],
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
),
|
|
'all-linux': _Component(dependencies=[
|
|
'qemu-gem5-buildroot',
|
|
'gem5-debug',
|
|
'gem5-fast',
|
|
'qemu-user',
|
|
]),
|
|
'baremetal': _Component(dependencies=[
|
|
'baremetal-gem5',
|
|
'baremetal-qemu',
|
|
]),
|
|
'baremetal-qemu': _Component(
|
|
self._build_file('build-baremetal', emulators=['qemu']),
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
dependencies=['crosstool-ng'],
|
|
),
|
|
'baremetal-gem5': _Component(
|
|
self._build_file('build-baremetal', emulators=['gem5']),
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
dependencies=['crosstool-ng'],
|
|
),
|
|
'baremetal-gem5-pbx': _Component(
|
|
self._build_file('build-baremetal', emulators=['gem5'], machine='RealViewPBX'),
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
dependencies=['crosstool-ng'],
|
|
),
|
|
'buildroot': buildroot_component,
|
|
# We need those to avoid cirtulcar dependencies, since we need to run Buildroot
|
|
# twice: once to get the toolchain, and a second time to put the overlay into
|
|
# the root filesystem.
|
|
'buildroot-overlay-qemu': buildroot_overlay_qemu_component,
|
|
'buildroot-overlay-gem5': buildroot_overlay_gem5_component,
|
|
'copy-overlay': _Component(
|
|
self._build_file('copy-overlay'),
|
|
),
|
|
'crosstool-ng': _Component(
|
|
self._build_file('build-crosstool-ng'),
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
# http://crosstool-ng.github.io/docs/os-setup/
|
|
apt_get_pkgs={
|
|
'bison',
|
|
'docbook2x',
|
|
'flex',
|
|
'gawk',
|
|
'gcc',
|
|
'gperf',
|
|
'help2man',
|
|
'libncurses5-dev',
|
|
'libtool-bin',
|
|
'make',
|
|
'python-dev',
|
|
'texinfo',
|
|
},
|
|
submodules={'crosstool-ng'},
|
|
),
|
|
'gem5': _Component(
|
|
self._build_file('build-gem5'),
|
|
**gem5_deps
|
|
),
|
|
'gem5-baremetal': _Component(dependencies=[
|
|
'gem5',
|
|
'baremetal-gem5',
|
|
]),
|
|
'gem5-buildroot': _Component(dependencies=[
|
|
'buildroot-overlay-gem5',
|
|
'linux',
|
|
'gem5',
|
|
]),
|
|
'gem5-debug': _Component(
|
|
self._build_file('build-gem5', gem5_build_type='debug'),
|
|
**gem5_deps
|
|
),
|
|
'gem5-fast': _Component(
|
|
self._build_file('build-gem5', gem5_build_type='fast'),
|
|
**gem5_deps
|
|
),
|
|
'linux': _Component(
|
|
self._build_file('build-linux', gem5_build_type='fast'),
|
|
dependencies={'buildroot'},
|
|
submodules_shallow={'linux'},
|
|
apt_get_pkgs={
|
|
'bison',
|
|
'flex',
|
|
# Without this started failing in kernel 4.15 with:
|
|
# Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
|
|
'libelf-dev',
|
|
},
|
|
),
|
|
'modules': _Component(
|
|
self._build_file('build-modules'),
|
|
dependencies=['buildroot', 'linux'],
|
|
),
|
|
'm5': _Component(
|
|
self._build_file('build-m5'),
|
|
dependencies=['buildroot'],
|
|
submodules={'gem5'},
|
|
),
|
|
'overlay': _Component(dependencies=[
|
|
'copy-overlay',
|
|
'modules',
|
|
'userland',
|
|
]),
|
|
'overlay-gem5': _Component(dependencies=[
|
|
'm5',
|
|
'overlay',
|
|
]),
|
|
'parsec-benchmark': _Component(
|
|
submodules={'parsec-benchmark'},
|
|
dependencies=['buildroot'],
|
|
),
|
|
'qemu': _Component(
|
|
self._build_file('build-qemu'),
|
|
apt_build_deps={'qemu'},
|
|
apt_get_pkgs={'libsdl2-dev'},
|
|
submodules={'qemu'},
|
|
),
|
|
'qemu-baremetal': _Component(dependencies=[
|
|
'qemu',
|
|
'baremetal-qemu',
|
|
]),
|
|
'qemu-buildroot': _Component(dependencies=[
|
|
'buildroot-overlay-qemu',
|
|
'linux',
|
|
]),
|
|
'qemu-gem5-buildroot': _Component(dependencies=[
|
|
'qemu',
|
|
'gem5-buildroot',
|
|
]),
|
|
'qemu-user': _Component(
|
|
self._build_file('build-qemu', user_mode=True),
|
|
apt_build_deps = {'qemu'},
|
|
apt_get_pkgs={'libsdl2-dev'},
|
|
submodules={'qemu'},
|
|
),
|
|
'release': _Component(dependencies=[
|
|
'qemu-buildroot',
|
|
]),
|
|
'test-gdb': _Component(dependencies=[
|
|
'all-baremetal',
|
|
],
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
),
|
|
'user-mode-qemu': _Component(
|
|
dependencies=['qemu-user', 'userland'],
|
|
),
|
|
'userland': _Component(
|
|
self._build_file('build-userland'),
|
|
dependencies=['buildroot'],
|
|
),
|
|
}
|
|
self.component_to_name_map = {self.name_to_component_map[key]:key for key in self.name_to_component_map}
|
|
|
|
self.add_argument(
|
|
'--download-dependencies',
|
|
default=False,
|
|
help='''\
|
|
Also download all dependencies required for a given build: Ubuntu packages,
|
|
Python packages and git submodules.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'--print-components',
|
|
default=False,
|
|
help='''\
|
|
Print the components that would be built, including dependencies, but don't
|
|
build them, nor show the build commands.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'--travis',
|
|
default=False,
|
|
help='''\
|
|
Extra args to pass to all scripts.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'components',
|
|
choices=list(self.name_to_component_map.keys()) + [[]],
|
|
default=[],
|
|
nargs='*',
|
|
help='''\
|
|
Which components to build. Default: qemu-buildroot
|
|
'''
|
|
)
|
|
|
|
def _build_file(self, component_file, **extra_args):
|
|
'''
|
|
Build something based on a component file that defines a Main class.
|
|
'''
|
|
def f():
|
|
args = self.get_common_args()
|
|
args.update(extra_args)
|
|
args['print_time'] = False
|
|
self.import_path_main(component_file)(**args)
|
|
return f
|
|
|
|
def timed_main(self):
|
|
self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
|
|
|
|
# Decide components.
|
|
components = self.env['components']
|
|
if components == []:
|
|
components = ['qemu-buildroot']
|
|
selected_components = []
|
|
for component_name in components:
|
|
todo = [component_name]
|
|
while todo:
|
|
current_name = todo.pop(0)
|
|
component = self.name_to_component_map[current_name]
|
|
selected_components.insert(0, component)
|
|
todo.extend(component.dependencies)
|
|
# Remove duplicates, keep only the first one of each.
|
|
# https://stackoverflow.com/questions/7961363/removing-duplicates-in-lists/7961390#7961390
|
|
selected_components = collections.OrderedDict.fromkeys(selected_components)
|
|
|
|
if self.env['download_dependencies']:
|
|
apt_get_pkgs = {
|
|
# Core requirements for this repo.
|
|
'git',
|
|
'moreutils', # ts
|
|
'python3-pip',
|
|
'tmux',
|
|
'vinagre',
|
|
'wget',
|
|
}
|
|
# E.g. on an ARM host, the package gcc-arm-linux-gnueabihf
|
|
# is called just gcc.
|
|
processor = platform.processor()
|
|
if processor != 'arm':
|
|
apt_get_pkgs.update({
|
|
'gcc-arm-linux-gnueabihf',
|
|
'g++-arm-linux-gnueabihf',
|
|
})
|
|
if processor != 'aarch64':
|
|
apt_get_pkgs.update({
|
|
'gcc-aarch64-linux-gnu',
|
|
'g++-aarch64-linux-gnu',
|
|
})
|
|
apt_build_deps = set()
|
|
submodules = set()
|
|
submodules_shallow = set()
|
|
python2_pkgs = set()
|
|
python3_pkgs = {
|
|
'pexpect==4.6.0',
|
|
}
|
|
for component in selected_components:
|
|
apt_get_pkgs.update(component.apt_get_pkgs)
|
|
apt_build_deps.update(component.apt_build_deps)
|
|
submodules.update(component.submodules)
|
|
submodules_shallow.update(component.submodules_shallow)
|
|
python2_pkgs.update(component.python2_pkgs)
|
|
python3_pkgs.update(component.python3_pkgs)
|
|
if apt_get_pkgs or apt_build_deps:
|
|
if self.env['travis']:
|
|
interacive_pkgs = {
|
|
'libsdl2-dev',
|
|
}
|
|
apt_get_pkgs.difference_update(interacive_pkgs)
|
|
if common.consts['in_docker']:
|
|
sudo = []
|
|
# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
|
|
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
|
|
# https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
|
|
sources_path = os.path.join('/etc', 'apt', 'sources.list')
|
|
with open(sources_path, 'r') as f:
|
|
sources_txt = f.read()
|
|
sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
|
|
with open(sources_path, 'w') as f:
|
|
f.write(sources_txt)
|
|
else:
|
|
sudo = ['sudo']
|
|
if common.consts['in_docker'] or self.env['travis']:
|
|
y = ['-y']
|
|
else:
|
|
y = []
|
|
self.sh.run_cmd(
|
|
sudo + ['apt-get', 'update', LF]
|
|
)
|
|
if apt_get_pkgs:
|
|
self.sh.run_cmd(
|
|
sudo + ['apt-get', 'install'] + y + [LF] +
|
|
self.sh.add_newlines(sorted(apt_get_pkgs))
|
|
)
|
|
if apt_build_deps:
|
|
self.sh.run_cmd(
|
|
sudo +
|
|
['apt-get', 'build-dep'] + y + [LF] +
|
|
self.sh.add_newlines(sorted(apt_build_deps))
|
|
)
|
|
if python2_pkgs:
|
|
self.sh.run_cmd(
|
|
['python', '-m', 'pip', 'install', '--user', LF] +
|
|
self.sh.add_newlines(sorted(python2_pkgs))
|
|
)
|
|
if python3_pkgs:
|
|
# Not with pip executable directly:
|
|
# https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
|
|
self.sh.run_cmd(
|
|
['python3', '-m', 'pip', 'install', '--user', LF] +
|
|
self.sh.add_newlines(sorted(python3_pkgs))
|
|
)
|
|
git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
|
|
if submodules:
|
|
# == Other nice git options for when distros move to newer Git
|
|
#
|
|
# Currently not on Ubuntu 16.04:
|
|
#
|
|
# `--progress`: added on Git 2.10:
|
|
#
|
|
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
|
|
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
|
|
#
|
|
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
|
|
self.sh.run_cmd(
|
|
git_cmd_common + ['--', LF] +
|
|
self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
|
|
)
|
|
if submodules_shallow:
|
|
# == Shallow cloning.
|
|
#
|
|
# TODO Ideally we should shallow clone --depth 1 all of them.
|
|
#
|
|
# However, most git servers out there are crap or craply configured
|
|
# and don't allow shallow cloning except for branches.
|
|
#
|
|
# So for now, let's shallow clone only the Linux kernel, which has by far
|
|
# the largest .git repo history, and full clone the others.
|
|
#
|
|
# Then we will maintain a GitHub Linux kernel mirror / fork that always has a
|
|
# lkmc branch, and point to it, so that it will always succeed.
|
|
#
|
|
# See also:
|
|
#
|
|
# * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
|
|
# * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
|
|
# * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
|
|
#
|
|
self.sh.run_cmd(
|
|
git_cmd_common + ['--depth', '1', '--', LF] +
|
|
self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules_shallow)])
|
|
)
|
|
|
|
# Do the build.
|
|
for component in selected_components:
|
|
if self.env['print_components']:
|
|
print(self.component_to_name_map[component])
|
|
else:
|
|
component.build(self.env['arch'])
|
|
|
|
if __name__ == '__main__':
|
|
Main().cli()
|