mirror of
https://github.com/cirosantilli/linux-kernel-module-cheat.git
synced 2026-01-26 11:41:35 +01:00
d41f7d9d23
It is impossible to sanely keep things tracked like that. All common algorithms work on unordered graphs, and now we match that as well.
497 lines
18 KiB
Python
Executable File
497 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import platform
|
|
import re
|
|
import os
|
|
|
|
import cli_function
|
|
import collections
|
|
import common
|
|
import copy
|
|
import shell_helpers
|
|
from shell_helpers import LF
|
|
|
|
class _Component:
|
|
'''
|
|
Yes, we are re-inventing a crappy dependency resolution system,
|
|
reminescent of scons or apt or Buildroot. I can't believe it.
|
|
|
|
The hard part is that we have optional dependencies as well...
|
|
e.g. buildroot optionally depends on m5 to put m5 in the root filesystem,
|
|
and buildroot optionally depends on qemu to build the qcow2 version
|
|
of the image.
|
|
'''
|
|
def __init__(
|
|
self,
|
|
build_callback=None,
|
|
supported_archs=None,
|
|
dependencies=None,
|
|
apt_get_pkgs=None,
|
|
apt_build_deps=None,
|
|
submodules=None,
|
|
submodules_shallow=None,
|
|
python2_pkgs=None,
|
|
python3_pkgs=None,
|
|
):
|
|
self.build_callback = build_callback
|
|
self.supported_archs = supported_archs
|
|
self.dependencies = dependencies or set()
|
|
self.apt_get_pkgs = apt_get_pkgs or set()
|
|
self.apt_build_deps = apt_build_deps or set()
|
|
self.submodules = submodules or set()
|
|
self.submodules_shallow = submodules_shallow or set()
|
|
self.python2_pkgs = python2_pkgs or set()
|
|
self.python3_pkgs = python3_pkgs or set()
|
|
|
|
def build(self, arch):
|
|
if (
|
|
(self.build_callback is not None) and
|
|
(self.supported_archs is None or arch in self.supported_archs)
|
|
):
|
|
self.build_callback()
|
|
|
|
class Main(common.LkmcCliFunction):
|
|
def __init__(self):
|
|
super().__init__(
|
|
description='''\
|
|
Build a component and all its dependencies.
|
|
|
|
Our build-* scripts don't build any dependencies to make iterative
|
|
development fast and more predictable.
|
|
|
|
It is currently not possible to configure indivitual components from the command line
|
|
when you build with this script. TODO.
|
|
|
|
Without any args, build only what is necessary for:
|
|
https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup
|
|
|
|
....
|
|
./%(prog)s
|
|
....
|
|
|
|
This is equivalent to:
|
|
|
|
....
|
|
./%(prog)s --arch x86_64 qemu-buildroot
|
|
....
|
|
''',
|
|
)
|
|
buildroot_component = _Component(
|
|
self._build_file('build-buildroot'),
|
|
submodules = {'buildroot'},
|
|
# https://buildroot.org/downloads/manual/manual.html#requirement
|
|
apt_get_pkgs={
|
|
'bash',
|
|
'bc',
|
|
'binutils',
|
|
'build-essential',
|
|
'bzip2',
|
|
'cpio',
|
|
'g++',
|
|
'gcc',
|
|
'graphviz',
|
|
'gzip',
|
|
'make',
|
|
'patch',
|
|
'perl',
|
|
'python-matplotlib',
|
|
'python3',
|
|
'rsync',
|
|
'sed',
|
|
'tar',
|
|
'unzip',
|
|
},
|
|
)
|
|
buildroot_overlay_qemu_component = copy.copy(buildroot_component)
|
|
buildroot_overlay_qemu_component.dependencies = ['overlay', 'qemu']
|
|
buildroot_overlay_gem5_component = copy.copy(buildroot_component)
|
|
buildroot_overlay_gem5_component.dependencies = ['overlay-gem5']
|
|
gem5_deps = {
|
|
# TODO test it out on Docker and answer that question properly:
|
|
# https://askubuntu.com/questions/350475/how-can-i-install-gem5
|
|
'apt_get_pkgs': {
|
|
'device-tree-compiler',
|
|
'diod',
|
|
'libgoogle-perftools-dev',
|
|
'm4',
|
|
'protobuf-compiler',
|
|
'python-dev',
|
|
'python-pip',
|
|
# For prebuilt qcow2 unpack.
|
|
'qemu-utils',
|
|
'scons',
|
|
'zlib1g-dev',
|
|
},
|
|
'python2_pkgs': {
|
|
# Generate graphs of config.ini under m5out.
|
|
'pydot',
|
|
},
|
|
'submodules': {'gem5'},
|
|
}
|
|
|
|
self.name_to_component_map = {
|
|
'all': _Component(dependencies=[
|
|
'all-linux',
|
|
'all-baremetal',
|
|
]),
|
|
'all-baremetal': _Component(dependencies=[
|
|
'qemu-baremetal',
|
|
'gem5-baremetal',
|
|
'baremetal-gem5-pbx',
|
|
],
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
),
|
|
'all-linux': _Component(dependencies=[
|
|
'qemu-gem5-buildroot',
|
|
'gem5-debug',
|
|
'gem5-fast',
|
|
'qemu-user',
|
|
]),
|
|
'baremetal': _Component(dependencies=[
|
|
'baremetal-gem5',
|
|
'baremetal-qemu',
|
|
]),
|
|
'baremetal-qemu': _Component(
|
|
self._build_file('build-baremetal', emulators=['qemu']),
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
dependencies=['crosstool-ng'],
|
|
),
|
|
'baremetal-gem5': _Component(
|
|
self._build_file('build-baremetal', emulators=['gem5']),
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
dependencies=['crosstool-ng'],
|
|
),
|
|
'baremetal-gem5-pbx': _Component(
|
|
self._build_file('build-baremetal', emulators=['gem5'], machine='RealViewPBX'),
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
dependencies=['crosstool-ng'],
|
|
),
|
|
'buildroot': buildroot_component,
|
|
'buildroot-overlay-qemu': buildroot_overlay_qemu_component,
|
|
'buildroot-overlay-gem5': buildroot_overlay_gem5_component,
|
|
'copy-overlay': _Component(
|
|
self._build_file('copy-overlay'),
|
|
),
|
|
'crosstool-ng': _Component(
|
|
self._build_file('build-crosstool-ng'),
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
# http://crosstool-ng.github.io/docs/os-setup/
|
|
apt_get_pkgs={
|
|
'bison',
|
|
'docbook2x',
|
|
'flex',
|
|
'gawk',
|
|
'gcc',
|
|
'gperf',
|
|
'help2man',
|
|
'libncurses5-dev',
|
|
'libtool-bin',
|
|
'make',
|
|
'python-dev',
|
|
'texinfo',
|
|
},
|
|
submodules={'crosstool-ng'},
|
|
),
|
|
'gem5': _Component(
|
|
self._build_file('build-gem5'),
|
|
**gem5_deps
|
|
),
|
|
'gem5-baremetal': _Component(dependencies=[
|
|
'gem5',
|
|
'baremetal-gem5',
|
|
]),
|
|
'gem5-buildroot': _Component(dependencies=[
|
|
'buildroot-overlay-gem5',
|
|
'linux',
|
|
'gem5',
|
|
]),
|
|
'gem5-debug': _Component(
|
|
self._build_file('build-gem5', gem5_build_type='debug'),
|
|
**gem5_deps
|
|
),
|
|
'gem5-fast': _Component(
|
|
self._build_file('build-gem5', gem5_build_type='fast'),
|
|
**gem5_deps
|
|
),
|
|
'linux': _Component(
|
|
self._build_file('build-linux', gem5_build_type='fast'),
|
|
dependencies={'buildroot'},
|
|
submodules_shallow={'linux'},
|
|
apt_get_pkgs={
|
|
'bison',
|
|
'flex',
|
|
# Without this started failing in kernel 4.15 with:
|
|
# Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
|
|
'libelf-dev',
|
|
},
|
|
),
|
|
'modules': _Component(
|
|
self._build_file('build-modules'),
|
|
dependencies=['buildroot', 'linux'],
|
|
),
|
|
'm5': _Component(
|
|
self._build_file('build-m5'),
|
|
dependencies=['buildroot'],
|
|
submodules={'gem5'},
|
|
),
|
|
'overlay': _Component(dependencies=[
|
|
'copy-overlay',
|
|
'modules',
|
|
'userland',
|
|
]),
|
|
'overlay-gem5': _Component(dependencies=[
|
|
'm5',
|
|
'overlay',
|
|
]),
|
|
'parsec-benchmark': _Component(
|
|
submodules={'parsec-benchmark'},
|
|
dependencies=['buildroot'],
|
|
),
|
|
'qemu': _Component(
|
|
self._build_file('build-qemu'),
|
|
apt_build_deps={'qemu'},
|
|
apt_get_pkgs={'libsdl2-dev'},
|
|
submodules={'qemu'},
|
|
),
|
|
'qemu-baremetal': _Component(dependencies=[
|
|
'qemu',
|
|
'baremetal-qemu',
|
|
]),
|
|
'qemu-buildroot': _Component(dependencies=[
|
|
'buildroot-overlay-qemu',
|
|
'linux',
|
|
]),
|
|
'qemu-gem5-buildroot': _Component(dependencies=[
|
|
'qemu',
|
|
'gem5-buildroot',
|
|
]),
|
|
'qemu-user': _Component(
|
|
self._build_file('build-qemu', user_mode=True),
|
|
apt_build_deps = {'qemu'},
|
|
apt_get_pkgs={'libsdl2-dev'},
|
|
submodules={'qemu'},
|
|
),
|
|
'release': _Component(dependencies=[
|
|
'qemu-buildroot',
|
|
]),
|
|
'test-gdb': _Component(dependencies=[
|
|
'all-baremetal',
|
|
],
|
|
supported_archs=common.consts['crosstool_ng_supported_archs'],
|
|
),
|
|
'userland': _Component(
|
|
self._build_file('build-userland'),
|
|
dependencies=['buildroot'],
|
|
),
|
|
}
|
|
self.component_to_name_map = {self.name_to_component_map[key]:key for key in self.name_to_component_map}
|
|
|
|
self.add_argument(
|
|
'--all',
|
|
default=False,
|
|
help='''\
|
|
Build absolutely everything for all archs.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'--download-dependencies',
|
|
default=False,
|
|
help='''\
|
|
Also download all dependencies required for a given build: Ubuntu packages,
|
|
Python packages and git submodules.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'--print-components',
|
|
default=False,
|
|
help='''\
|
|
Print the components that would be built, including dependencies, but don't
|
|
build them, nor show the build commands.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'--travis',
|
|
default=False,
|
|
help='''\
|
|
Extra args to pass to all scripts.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'components',
|
|
choices=list(self.name_to_component_map.keys()) + [[]],
|
|
default=[],
|
|
nargs='*',
|
|
help='''\
|
|
Which components to build. Default: qemu-buildroot
|
|
'''
|
|
)
|
|
|
|
def _build_file(self, component_file, **extra_args):
|
|
'''
|
|
Build something based on a component file that defines a Main class.
|
|
'''
|
|
def f():
|
|
args = self.get_common_args()
|
|
args.update(extra_args)
|
|
args['print_time'] = False
|
|
self.import_path_main(component_file)(**args)
|
|
return f
|
|
|
|
def timed_main(self):
|
|
self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
|
|
|
|
# Decide components.
|
|
components = self.env['components']
|
|
if self.env['all']:
|
|
components = ['all']
|
|
elif components == []:
|
|
components = ['qemu-buildroot']
|
|
selected_components = []
|
|
for component_name in components:
|
|
todo = [component_name]
|
|
while todo:
|
|
current_name = todo.pop(0)
|
|
component = self.name_to_component_map[current_name]
|
|
selected_components.insert(0, component)
|
|
todo.extend(component.dependencies)
|
|
# Remove duplicates, keep only the first one of each.
|
|
# https://stackoverflow.com/questions/7961363/removing-duplicates-in-lists/7961390#7961390
|
|
collections.OrderedDict.fromkeys(selected_components)
|
|
|
|
if self.env['download_dependencies']:
|
|
apt_get_pkgs = {
|
|
# Core requirements for this repo.
|
|
'git',
|
|
'moreutils', # ts
|
|
'python3-pip',
|
|
'tmux',
|
|
'vinagre',
|
|
'wget',
|
|
}
|
|
# E.g. on an ARM host, the package gcc-arm-linux-gnueabihf
|
|
# is called just gcc.
|
|
processor = platform.processor()
|
|
if processor != 'arm':
|
|
apt_get_pkgs.update({
|
|
'gcc-arm-linux-gnueabihf',
|
|
'g++-arm-linux-gnueabihf',
|
|
})
|
|
if processor != 'aarch64':
|
|
apt_get_pkgs.update({
|
|
'gcc-aarch64-linux-gnu',
|
|
'g++-aarch64-linux-gnu',
|
|
})
|
|
apt_build_deps = set()
|
|
submodules = set()
|
|
submodules_shallow = set()
|
|
python2_pkgs = set()
|
|
python3_pkgs = {
|
|
'pexpect==4.6.0',
|
|
}
|
|
for component in selected_components:
|
|
apt_get_pkgs.update(component.apt_get_pkgs)
|
|
apt_build_deps.update(component.apt_build_deps)
|
|
submodules.update(component.submodules)
|
|
submodules_shallow.update(component.submodules_shallow)
|
|
python2_pkgs.update(component.python2_pkgs)
|
|
python3_pkgs.update(component.python3_pkgs)
|
|
if apt_get_pkgs or apt_build_deps:
|
|
if self.env['travis']:
|
|
interacive_pkgs = {
|
|
'libsdl2-dev',
|
|
}
|
|
apt_get_pkgs.difference_update(interacive_pkgs)
|
|
if common.consts['in_docker']:
|
|
sudo = []
|
|
# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
|
|
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
|
|
# https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
|
|
sources_path = os.path.join('/etc', 'apt', 'sources.list')
|
|
with open(sources_path, 'r') as f:
|
|
sources_txt = f.read()
|
|
sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
|
|
with open(sources_path, 'w') as f:
|
|
f.write(sources_txt)
|
|
else:
|
|
sudo = ['sudo']
|
|
if common.consts['in_docker'] or self.env['travis']:
|
|
y = ['-y']
|
|
else:
|
|
y = []
|
|
self.sh.run_cmd(
|
|
sudo + ['apt-get', 'update', LF]
|
|
)
|
|
if apt_get_pkgs:
|
|
self.sh.run_cmd(
|
|
sudo + ['apt-get', 'install'] + y + [LF] +
|
|
self.sh.add_newlines(sorted(apt_get_pkgs))
|
|
)
|
|
if apt_build_deps:
|
|
self.sh.run_cmd(
|
|
sudo +
|
|
['apt-get', 'build-dep'] + y + [LF] +
|
|
self.sh.add_newlines(sorted(apt_build_deps))
|
|
)
|
|
if python2_pkgs:
|
|
self.sh.run_cmd(
|
|
['python', '-m', 'pip', 'install', '--user', LF] +
|
|
self.sh.add_newlines(sorted(python2_pkgs))
|
|
)
|
|
if python3_pkgs:
|
|
# Not with pip executable directly:
|
|
# https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
|
|
self.sh.run_cmd(
|
|
['python3', '-m', 'pip', 'install', '--user', LF] +
|
|
self.sh.add_newlines(sorted(python3_pkgs))
|
|
)
|
|
git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
|
|
if submodules:
|
|
# == Other nice git options for when distros move to newer Git
|
|
#
|
|
# Currently not on Ubuntu 16.04:
|
|
#
|
|
# `--progress`: added on Git 2.10:
|
|
#
|
|
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
|
|
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
|
|
#
|
|
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
|
|
self.sh.run_cmd(
|
|
git_cmd_common + ['--', LF] +
|
|
self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
|
|
)
|
|
if submodules_shallow:
|
|
# == Shallow cloning.
|
|
#
|
|
# TODO Ideally we should shallow clone --depth 1 all of them.
|
|
#
|
|
# However, most git servers out there are crap or craply configured
|
|
# and don't allow shallow cloning except for branches.
|
|
#
|
|
# So for now, let's shallow clone only the Linux kernel, which has by far
|
|
# the largest .git repo history, and full clone the others.
|
|
#
|
|
# Then we will maintain a GitHub Linux kernel mirror / fork that always has a
|
|
# lkmc branch, and point to it, so that it will always succeed.
|
|
#
|
|
# See also:
|
|
#
|
|
# * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
|
|
# * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
|
|
# * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
|
|
#
|
|
self.sh.run_cmd(
|
|
git_cmd_common + ['--depth', '1', '--', LF] +
|
|
self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules_shallow)])
|
|
)
|
|
|
|
# Do the build.
|
|
for component in selected_components:
|
|
if self.env['print_components']:
|
|
print(self.component_to_name_map[component])
|
|
else:
|
|
component.build(self.env['arch'])
|
|
|
|
if __name__ == '__main__':
|
|
Main().cli()
|