From 34085fd96dd45eb4458e2ac12c31ab4a6e96ac5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Tue, 22 Jan 2019 00:00:00 +0000 Subject: [PATCH] improve the release procedure --- README.adoc | 163 +++++++++++++++++----------- baremetal/{interactive => }/hello.c | 0 build-buildroot | 3 +- build-test-boot | 4 + cli_function.py | 34 +++--- common.py | 129 ++++++++++++++-------- gem5-shell | 2 +- getvar | 2 +- qemu-trace2txt | 2 +- release | 31 ------ release-upload | 125 +++++++++++---------- release-zip | 41 ++++--- run | 2 +- test | 4 +- test-boot | 3 +- test-gdb | 6 +- 16 files changed, 310 insertions(+), 241 deletions(-) rename baremetal/{interactive => }/hello.c (100%) delete mode 100755 release diff --git a/README.adoc b/README.adoc index ad8d010..d114f3a 100644 --- a/README.adoc +++ b/README.adoc @@ -219,7 +219,11 @@ insmod /mnt/9p/out_rootfs_overlay/hello.ko and the new `pr_info` message should now show on the terminal at the end of the boot. -This works because we have a <<9p>> mount there setup by default, which makes a host directory available on the guest. +This works because we have a <<9p>> mount there setup by default, which mounts the host directory that contains the Build outputs on the guest: + +.... +ls "$(./getvar out_rootfs_overlay_dir)" +.... The fast method is slightly risky because your previously insmodded buggy kernel module attempt might have corrupted the kernel memory, which could affect future runs. @@ -233,7 +237,7 @@ The safe way, is to fist quit QEMU, rebuild the modules, put them in the root fi ./run --eval-after 'insmod /hello.ko' .... -`./build-buildroot` is required after `./build-modules` because it generates the root filesystem with the modules that we compiled at `./build-modules`. +`./build-buildroot` is required after `./build-modules` because it re-generates the root filesystem with the modules that we compiled at `./build-modules`. You can see that `./build` does that as well, by running: @@ -844,9 +848,9 @@ For more information on baremetal, see the section: <>. The following Much like <>, this is another fun setup that does not require Buildroot or the Linux kernel. -See: <> +Introduction at: <>. -TODO: test it out on a clean repo. +Getting started at: <>. [[gdb]] == GDB step debug @@ -3329,7 +3333,7 @@ This would have several advantages: ** no need to regenerate the root filesystem at all and reboot ** overcomes the `check_bin_arch` problem: <> * we could keep the base root filesystem very small, which implies: -** less host disk usage, no need to copy the entire `out_rootfs_overlay_dir` to the image again +** less host disk usage, no need to copy the entire `./getvar out_rootfs_overlay_dir` to the image again ** no need to worry about <> We can already make host files appear on the guest with <<9p>>, but they appear on a subdirectory instead of the root. @@ -10244,6 +10248,8 @@ Once you've built a package in to the image, there is no easy way to remove it. Documented at: link:https://github.com/buildroot/buildroot/blob/2017.08/docs/manual/rebuilding-packages.txt#L90[] +Also mentioned at: https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot + See this for a sample manual workaround: <>. === BR2_TARGET_ROOTFS_EXT2_SIZE @@ -11351,6 +11357,7 @@ but note that this does not include script specific options. You don't need to depend on GitHub: .... +sudo apt install asciidoctor ./build-doc xdg-open out/README.html .... @@ -11903,11 +11910,9 @@ This directory is copied into the target filesystem by: Source: link:copy-overlay[] -`copy-overlay` by itself, only places the files into our intermediate `./getenv out_rootfs_overlay_dir` directory. +Build Buildroot is required for the same reason as described at: <>. -This directory combines files from several sources, including for example link:build-userland[], which the final `./build-buildroot` puts into the root filesystem. - -Since the link:rootfs_overlay[] directory does not require compilation, unlike say link:userland[] we also make it <<9p>> available to the guest directly even without `copy-overlay` at: +However, since the link:rootfs_overlay[] directory does not require compilation, unlike say <>, we also make it <<9p>> available to the guest directly even without `./copy-overlay` at: .... ls /mnt/9p/rootfs_overlay @@ -11915,15 +11920,6 @@ ls /mnt/9p/rootfs_overlay This way you can just hack away the scripts and try them out immediately without any further operations. -To add those scripts to the Buildroot root filesystem, you will need to run: - -We could add that directory to `BR2_ROOTFS_OVERLAY` but we don't do this because this mechanism: - -* also works for non Buildroot root filesystesms -* places everything in one place for a nice 9P mount - -and maintaining `BR2_ROOTFS_OVERLAY` in addition to our mechanism would duplicate some logic. - === Test this repo ==== Automated tests @@ -11942,16 +11938,16 @@ Sources: * link:build-test[] * link:test[] + +The link:test[] script runs several different types of tests, which can also be run separately as explained at: + +* link:test-boot[] +* <> * <> * <> +* <> -This is not all tests, because there are too many possible variations and that would take forever. - -Instead, we currently select on magic arch, currently `aarch64`, and for that arch run more stuff than on others. - -We could in the future we will add an option to select the large arch, or do something smarter. - -This full testing takes too much time to be feasible for every patch, but it should be done for every release. +link:test[] does not all possible tests, because there are too many possible variations and that would take forever. The rationale is the same as for `./build all` and is explained in `./build --help`. See the sources of those test scripts to learn how to run more specialized tests. @@ -11970,12 +11966,6 @@ This command would run the test four times, using `x86_64` and `aarch64` with bo Without those flags, it defaults to just running the default arch and emulator once: `x86_64` and `qemu`. -Test that the Internet works: - -.... -./run --arch x86_64 --kernel-cli '- lkmc_eval="ifup -a;wget -S google.com;poweroff;"' -.... - ===== Test userland in full system Run all userland tests from inside full system simulation (i.e. not <>): @@ -12031,7 +12021,26 @@ To debug GDB problems on gem5, you might want to enable the following <> +* link:rootfs_overlay/test_fail.sh[], which is used by <> + +=== Non-automated tests + +==== Test GDB Linux kernel For the Linux kernel, do the following manual tests for now. @@ -12054,23 +12063,14 @@ Then proceed to do the following tests: * `/count.sh` and `break __x64_sys_write` * `insmod /timer.ko` and `break lkmc_timer_callback` -==== Magic failure string +==== Test the Internet -Since there is no standardized exit status concept that works across all emulators for full system, we just parse the terminal output for a magic failure string to check if tests failed. - -If a full system simulation outputs a line containing only exactly the magic string: +You should also test that the Internet works: .... -lkmc_test_fail +./run --arch x86_64 --kernel-cli '- lkmc_eval="ifup -a;wget -S google.com;poweroff;"' .... -to the terminal, then our run scripts detect that and exit with status `1`. - -This magic output string is notably used by: - -* the `common_assert_fail()` function, which is used by <> -* link:rootfs_overlay/test_fail.sh[], which is used by <> - === Bisection When updating the Linux kernel, QEMU and gem5, things sometimes break. @@ -12174,37 +12174,62 @@ This can be used to check the determinism of: === Release -Create a release: +==== Release procedure + +Ensure that the <> are passing on a clean build: .... +mv out out.bak +./build-test --size 3 && ./test --size 3 +.... + +The clean build is necessary as it generates clean images since <> + +Run all tests in <> just QEMU x86_64 and QEMU aarch64. + +TODO: not working currently, so skipped: Ensure that the <> look fine: + +.... +./bench-all -A +.... + +Create a release candidate and upload it: + +.... +git tag -a -m '' v3.0-rc1 +git push --follow-tags +./release-zip --all-archs +# export LKMC_GITHUB_TOKEN= +./release-upload +.... + +Do an out-of-box testing for the release candidate: + +.... +cd .. git clone https://github.com/cirosantilli/linux-kernel-module-cheat linux-kernel-module-cheat-release cd linux-kernel-module-cheat-release -# export LKMC_GITHUB_TOKEN= -./release .... -Source: link:release[] +Test <>, and then through all of the <> section in order. -This scripts does: +Once everything looks fine, publish the release with: -* configure -* build -* package with <> -* creates a tag of form `sha-` -* upload to GitHub with link:release-create-github[] - -Cloning a clean tree is ideal as it generates clean images since <> - -This should in particular enable to easily update <>. - -TODO also run tests and only release if they pass. +.... +git tag -a v3.0 +# Describe the release int the tag message. +git push --follow-tags +./release-zip --all-archs +# export LKMC_GITHUB_TOKEN= +./release-upload +.... ==== release-zip Create a zip containing all files required for <>: .... -./build release && ./release-zip +./build --all-archs release && ./release-zip --all-archs .... Source: link:release-zip[] @@ -12217,7 +12242,14 @@ echo "$(./getvar release_zip_file)" which you can then upload somewhere. -For example, you can create or update a GitHub release and upload automatically with: +==== release-upload + +After: + +* running <> +* creating and pushing a tag to GitHub + +you can upload the release to GitHub automatically with: .... # export LKMC_GITHUB_TOKEN= @@ -12226,9 +12258,14 @@ For example, you can create or update a GitHub release and upload automatically Source: link:release-upload[] +The HEAD of the local repository must be on top of a tag that has been pushed for this to work. + Create `LKMC_GITHUB_TOKEN` under: https://github.com/settings/tokens/new and save it to your `.bashrc`. -TODO: generalize that so that people can upload to their forks. +The implementation of this script is described at: + +* https://stackoverflow.com/questions/5207269/how-to-release-a-build-artifact-asset-on-github-with-a-script/52354732#52354732 +* https://stackoverflow.com/questions/38153418/can-someone-give-a-python-requests-example-of-uploading-a-release-asset-in-githu/52354681#52354681 === Design rationale diff --git a/baremetal/interactive/hello.c b/baremetal/hello.c similarity index 100% rename from baremetal/interactive/hello.c rename to baremetal/hello.c diff --git a/build-buildroot b/build-buildroot index cf3f4da..ce04966 100755 --- a/build-buildroot +++ b/build-buildroot @@ -15,7 +15,8 @@ class Main(common.BuildCliFunction): def __init__(self): super().__init__( description='''\ -Run Linux on an emulator +Build Buildroot. This includes, notably: the userland GCC cross-toolchain, +and the root filesystem. ''') self.add_argument( '--build-linux', default=False, diff --git a/build-test-boot b/build-test-boot index 59aae6f..2cc5d9a 100755 --- a/build-test-boot +++ b/build-test-boot @@ -1,4 +1,8 @@ #!/usr/bin/env bash +# We want to move this into ./build. The only reason we haven't is that +# what to build depends on --size, which ./build does not support right now. +# The best way to solve this is to move the dependency checking into the run +# scripts, which will take a while to refactor. set -eu test_size=1 while [ $# -gt 0 ]; do diff --git a/cli_function.py b/cli_function.py index 41b3f9b..25bc7d2 100755 --- a/cli_function.py +++ b/cli_function.py @@ -209,10 +209,12 @@ class CliFunction: argument = _Argument(*args, **kwargs) self._arguments[argument.key] = argument - def cli(self, cli_args=None): + def cli_noexit(self, cli_args=None): ''' Call the function from the CLI. Parse command line arguments to get all arguments. + + :return: the return of main ''' parser = argparse.ArgumentParser( description=self._description, @@ -233,12 +235,17 @@ class CliFunction: args = parser.parse_args(args=cli_args) return self._do_main(vars(args)) - def cli_exit(self, *args, **kwargs): + def cli(self, *args, **kwargs): ''' Same as cli, but also exit the program with status equal to the return value of main. main must return an integer for this to be used. + + None is considered 0. ''' - sys.exit(self.cli(*args, **kwargs)) + exit_status = self.cli_noexit(*args, **kwargs) + if exit_status is None: + exit_status = 0 + sys.exit(exit_status) def get_cli(self, **kwargs): ''' @@ -324,6 +331,7 @@ amazing function! self.add_argument('pos-mandatory', help='Help for pos-mandatory', type=int), self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int), self.add_argument('args-star', help='Help for args-star', nargs='*'), + def main(self, **kwargs): del kwargs['_args_given'] return kwargs @@ -348,7 +356,7 @@ amazing function! } # Default CLI call with programmatic CLI arguments. - out = one_cli_function.cli(['1']) + out = one_cli_function.cli_noexit(['1']) assert out == default # asdf @@ -367,7 +375,7 @@ amazing function! if '--bool-true': out = one_cli_function(pos_mandatory=1, bool_true=False) - cli_out = one_cli_function.cli(['--no-bool-true', '1']) + cli_out = one_cli_function.cli_noexit(['--no-bool-true', '1']) assert out == cli_out assert out['bool_true'] == False out['bool_true'] = default['bool_true'] @@ -375,7 +383,7 @@ amazing function! if '--bool-false': out = one_cli_function(pos_mandatory=1, bool_false=True) - cli_out = one_cli_function.cli(['--bool-false', '1']) + cli_out = one_cli_function.cli_noexit(['--bool-false', '1']) assert out == cli_out assert out['bool_false'] == True out['bool_false'] = default['bool_false'] @@ -394,7 +402,7 @@ amazing function! # --dest out = one_cli_function(pos_mandatory=1, custom_dest='a') - cli_out = one_cli_function.cli(['--dest', 'a', '1']) + cli_out = one_cli_function.cli_noexit(['--dest', 'a', '1']) assert out == cli_out assert out['custom_dest'] == 'a' out['custom_dest'] = default['custom_dest'] @@ -405,7 +413,7 @@ amazing function! assert out['pos_mandatory'] == 1 assert out['pos_optional'] == 2 assert out['args_star'] == ['3', '4'] - cli_out = one_cli_function.cli(['1', '2', '3', '4']) + cli_out = one_cli_function.cli_noexit(['1', '2', '3', '4']) assert out == cli_out out['pos_mandatory'] = default['pos_mandatory'] out['pos_optional'] = default['pos_optional'] @@ -414,21 +422,21 @@ amazing function! # Star out = one_cli_function(append=['1', '2'], pos_mandatory=1) - cli_out = one_cli_function.cli(['--append', '1', '--append', '2', '1']) + cli_out = one_cli_function.cli_noexit(['--append', '1', '--append', '2', '1']) assert out == cli_out assert out['append'] == ['1', '2'] out['append'] = default['append'] assert out == default # Force a boolean value set on the config to be False on CLI. - assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False + assert one_cli_function.cli_noexit(['--no-bool-cli', '1'])['bool_cli'] is False # Pick another config file. - assert one_cli_function.cli(['--config-file', 'cli_function_test_config_2.py', '1'])['bool_cli'] is False + assert one_cli_function.cli_noexit(['--config-file', 'cli_function_test_config_2.py', '1'])['bool_cli'] is False # Extra config file for '*'. - assert one_cli_function.cli(['--config-file', 'cli_function_test_config_2.py', '1', '2', '3', '4'])['args_star'] == ['3', '4'] - assert one_cli_function.cli(['--config-file', 'cli_function_test_config_2.py', '1', '2'])['args_star'] == ['asdf', 'qwer'] + assert one_cli_function.cli_noexit(['--config-file', 'cli_function_test_config_2.py', '1', '2', '3', '4'])['args_star'] == ['3', '4'] + assert one_cli_function.cli_noexit(['--config-file', 'cli_function_test_config_2.py', '1', '2'])['args_star'] == ['asdf', 'qwer'] # get_cli assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)] diff --git a/common.py b/common.py index 2b0b598..b37a9e1 100644 --- a/common.py +++ b/common.py @@ -180,7 +180,14 @@ Implied by --quiet. '-q', '--quiet', default=False, help='''\ Don't print anything to stdout, except if it is part of an interactive terminal. -TODO: implement fully, some stuff is escaping currently. +TODO: implement fully, some stuff is escaping it currently. +''' + ) + self.add_argument( + '--quit-on-failure', + default=True, + help='''\ +Stop running at the first failed test. ''' ) self.add_argument( @@ -756,8 +763,8 @@ Valid emulators: {} def get_toolchain_tool(self, tool, allowed_toolchains=None): return '{}-{}'.format(self.get_toolchain_prefix(tool, allowed_toolchains), tool) - @staticmethod def github_make_request( + self, authenticate=False, data=None, extra_headers=None, @@ -775,7 +782,7 @@ Valid emulators: {} if url_params is not None: path += '?' + urllib.parse.urlencode(url_params) request = urllib.request.Request( - 'https://' + subdomain + '.github.com/repos/' + github_repo_id + path, + 'https://' + subdomain + '.github.com/repos/' + self.env['github_repo_id'] + path, headers=headers, data=data, **extra_request_args @@ -816,7 +823,10 @@ Valid emulators: {} def main(self, *args, **kwargs): ''' - Time the main of the derived class. + Run timed_main across all selected archs and emulators. + + :return: if any of the timed_mains exits non-zero and non-null, + return that. Otherwise, return 0. ''' env = kwargs.copy() self.input_args = env.copy() @@ -830,38 +840,52 @@ Valid emulators: {} real_emulators = consts['all_long_emulators'] else: real_emulators = env['emulators'] - for emulator in real_emulators: - for arch in real_archs: - if arch in env['arch_short_to_long_dict']: - arch = env['arch_short_to_long_dict'][arch] - if self.is_arch_supported(arch): - if not env['dry_run']: - start_time = time.time() - env['arch'] = arch - env['archs'] = [arch] - env['_args_given']['archs'] = True - env['all_archs'] = False - env['emulator'] = emulator - env['emulators'] = [emulator] - env['_args_given']['emulators'] = True - env['all_emulators'] = False - self.env = env.copy() - self._init_env(self.env) - self.sh = shell_helpers.ShellHelpers( - dry_run=self.env['dry_run'], - quiet=self.env['quiet'], - ) - ret = self.timed_main() - if not env['dry_run']: - end_time = time.time() - self.ellapsed_seconds = end_time - start_time - self.print_time(self.ellapsed_seconds) - if ret is not None and ret != 0: - return ret - elif not real_all_archs: - raise Exception('Unsupported arch for this action: ' + arch) - self.teardown() - return 0 + return_value = 0 + class GetOutOfLoop(Exception): pass + try: + ret = self.setup() + if ret is not None and ret != 0: + return_value = ret + raise GetOutOfLoop() + for emulator in real_emulators: + for arch in real_archs: + if arch in env['arch_short_to_long_dict']: + arch = env['arch_short_to_long_dict'][arch] + if self.is_arch_supported(arch): + if not env['dry_run']: + start_time = time.time() + env['arch'] = arch + env['archs'] = [arch] + env['_args_given']['archs'] = True + env['all_archs'] = False + env['emulator'] = emulator + env['emulators'] = [emulator] + env['_args_given']['emulators'] = True + env['all_emulators'] = False + self.env = env.copy() + self._init_env(self.env) + self.sh = shell_helpers.ShellHelpers( + dry_run=self.env['dry_run'], + quiet=self.env['quiet'], + ) + ret = self.timed_main() + if not env['dry_run']: + end_time = time.time() + self.ellapsed_seconds = end_time - start_time + self.print_time(self.ellapsed_seconds) + if ret is not None and ret != 0: + return_value = ret + if self.env['quit_on_failure']: + raise GetOutOfLoop() + elif not real_all_archs: + raise Exception('Unsupported arch for this action: ' + arch) + + except GetOutOfLoop: + pass + ret = self.teardown() + if ret is not None and ret != 0: + return_value = ret + return return_value def make_build_dirs(self): os.makedirs(self.env['buildroot_build_build_dir'], exist_ok=True) @@ -972,17 +996,29 @@ Valid emulators: {} self.env['userland_build_ext'], ) - def teardown(self): + def setup(self): ''' - Gets run just once after looping over all archs and emulators. + Similar to timed_main, but gets run only once for all --arch and --emulator, + before timed_main. + + Different from __init__, since at this point env has already been calculated, + so variables that don't depend on --arch or --emulator can be used. ''' pass def timed_main(self): ''' Main action of the derived class. + + Gets run once for every --arch and every --emulator. ''' - raise NotImplementedError() + pass + + def teardown(self): + ''' + Similar to setup, but run after timed_main. + ''' + pass class BuildCliFunction(LkmcCliFunction): ''' @@ -1070,13 +1106,6 @@ class TestCliFunction(LkmcCliFunction): kwargs['defaults'] = defaults super().__init__(*args, **kwargs) self.tests = [] - self.add_argument( - '--fail-early', - default=True, - help='''\ -Stop running at the first failed test. -''' - ) def run_test(self, run_obj, run_args=None, test_id=None): ''' @@ -1109,7 +1138,7 @@ Stop running at the first failed test. test_result = TestResult.PASS else: test_result = TestResult.FAIL - if self.env['fail_early']: + if self.env['quit_on_failure']: self.log_error('Test failed') sys.exit(1) self.log_info('test_result {}'.format(test_result.name)) @@ -1121,6 +1150,9 @@ Stop running at the first failed test. self.tests.append(Test(test_id_string, test_result, ellapsed_seconds)) def teardown(self): + ''' + :return: 1 if any test failed, 0 otherwise + ''' self.log_info('Test result summary') passes = [] fails = [] @@ -1136,4 +1168,5 @@ Stop running at the first failed test. for test in fails: self.log_info(test) self.log_error('A test failed') - sys.exit(1) + return 1 + return 0 diff --git a/gem5-shell b/gem5-shell index cca743e..5841697 100755 --- a/gem5-shell +++ b/gem5-shell @@ -20,4 +20,4 @@ class Main(common.LkmcCliFunction): ]) if __name__ == '__main__': - Main().cli_exit() + Main().cli() diff --git a/getvar b/getvar index 5c24380..e9c7c28 100755 --- a/getvar +++ b/getvar @@ -5,7 +5,7 @@ import common class Main(common.LkmcCliFunction): def __init__(self): super().__init__( - defaults={ + defaults = { 'print_time': False, }, description='''\ diff --git a/qemu-trace2txt b/qemu-trace2txt index 1e48514..725c09a 100755 --- a/qemu-trace2txt +++ b/qemu-trace2txt @@ -26,4 +26,4 @@ Convert a QEMU `-trace exec_tb` to text form. ) if __name__ == '__main__': - Main().cli_exit() + Main().cli() diff --git a/release b/release deleted file mode 100755 index 2b6cc31..0000000 --- a/release +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 - -''' -https://upload.com/cirosantilli/linux-kernel-module-cheat#release -''' - -import imp -import os -import subprocess -import time - -import common -release_zip = imp.load_source('release_zip', os.path.join(kwargs['root_dir'], 'release-zip')) -release_upload = imp.load_source('release_upload', os.path.join(kwargs['root_dir'], 'release-upload')) - -start_time = time.time() -# TODO factor those out so we don't redo the same thing multiple times. -# subprocess.check_call([os.path.join(kwargs['root_dir'], 'test')]) -# subprocess.check_call([os.path.join(kwargs['root_dir'], 'bench-all', '-A', '-u']) - -# A clean release requires a full rebuild unless we hack it :-( -# We can't just use our current build as it contains packages we've -# installed in random experiments. And with EXT2: we can't easily -# know what the smallest root filesystem size is and use it either... -# https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot -subprocess.check_call([os.path.join(kwargs['root_dir'], 'build'), '--all-archs', '--download-dependencies', 'release']) -release_zip.main() -subprocess.check_call(['git', 'push']) -release_upload.main() -end_time = time.time() -self.print_time(end_time - start_time) diff --git a/release-upload b/release-upload index 7149bdb..4b014d3 100755 --- a/release-upload +++ b/release-upload @@ -1,79 +1,84 @@ #!/usr/bin/env python3 -''' -Usage: https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip - -Implementation: - -* https://stackoverflow.com/questions/5207269/how-to-release-a-build-artifact-asset-on-github-with-a-script/52354732#52354732 -* https://stackoverflow.com/questions/38153418/can-someone-give-a-python-requests-example-of-uploading-a-release-asset-in-githu/52354681#52354681 -''' - import json import os +import subprocess import sys - import urllib.error import common from shell_helpers import LF -def main(): - repo = kwargs['github_repo_id'] - tag = 'sha-{}'.format(kwargs['sha']) - upload_path = kwargs['release_zip_file'] - - # Check the release already exists. - try: - _json = self.github_make_request(path='/releases/tags/' + tag) - except urllib.error.HTTPError as e: - if e.code == 404: - release_exists = False - else: - raise e - else: - release_exists = True - release_id = _json['id'] - - # Create release if not yet created. - if not release_exists: - _json = self.github_make_request( - authenticate=True, - data=json.dumps({ - 'tag_name': tag, - 'name': tag, - 'prerelease': True, - }).encode(), - path='/releases' +class Main(common.LkmcCliFunction): + def __init__(self): + super().__init__( + description='''\ +https://github.com/cirosantilli/linux-kernel-module-cheat#release-upload +''', ) - release_id = _json['id'] - asset_name = os.path.split(upload_path)[1] + def timed_main(self): + # https://stackoverflow.com/questions/3404936/show-which-git-tag-you-are-on + tag = subprocess.check_output([ + 'git', + 'describe', + '--exact-match', + '--tags' + ]).decode().rstrip() + upload_path = self.env['release_zip_file'] - # Clear the prebuilts for a upload. - _json = self.github_make_request( - path=('/releases/' + str(release_id) + '/assets'), - ) - for asset in _json: - if asset['name'] == asset_name: + # Check the release already exists. + try: + _json = self.github_make_request(path='/releases/tags/' + tag) + except urllib.error.HTTPError as e: + if e.code == 404: + release_exists = False + else: + raise e + else: + release_exists = True + release_id = _json['id'] + + # Create release if not yet created. + if not release_exists: _json = self.github_make_request( authenticate=True, - path=('/releases/assets/' + str(asset['id'])), - method='DELETE', + data=json.dumps({ + 'tag_name': tag, + 'name': tag, + 'prerelease': True, + }).encode(), + path='/releases' ) - break + release_id = _json['id'] - # Upload the prebuilt. - with open(upload_path, 'br') as myfile: - content = myfile.read() - _json = self.github_make_request( - authenticate=True, - data=content, - extra_headers={'Content-Type': 'application/zip'}, - path=('/releases/' + str(release_id) + '/assets'), - subdomain='uploads', - url_params={'name': asset_name}, - ) + asset_name = os.path.split(upload_path)[1] + + # Clear the prebuilts for a upload. + _json = self.github_make_request( + path=('/releases/' + str(release_id) + '/assets'), + ) + for asset in _json: + if asset['name'] == asset_name: + _json = self.github_make_request( + authenticate=True, + path=('/releases/assets/' + str(asset['id'])), + method='DELETE', + ) + break + + # Upload the prebuilt. + self.log_info('Uploading the release, this may take several seconds / a few minutes.') + with open(upload_path, 'br') as myfile: + content = myfile.read() + _json = self.github_make_request( + authenticate=True, + data=content, + extra_headers={'Content-Type': 'application/zip'}, + path=('/releases/' + str(release_id) + '/assets'), + subdomain='uploads', + url_params={'name': asset_name}, + ) if __name__ == '__main__': - main() + Main().cli() diff --git a/release-zip b/release-zip index 40e5ef6..39687d5 100755 --- a/release-zip +++ b/release-zip @@ -1,26 +1,35 @@ #!/usr/bin/env python3 -''' -https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip -''' - import os -import subprocess import zipfile import common -from shell_helpers import LF -def main(): - os.makedirs(self.env['release_dir'], exist_ok=True) - if os.path.exists(self.env['release_zip_file']): +class Main(common.LkmcCliFunction): + def __init__(self): + super().__init__( + description='''\ +https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip +''', + defaults = { + 'print_time': False, + } + ) + self.qcow2s_linux_images = [] + + def timed_main(self): + self.qcow2s_linux_images.append((self.env['qcow2_file'], self.env['linux_image'])) + + def teardown(self): + os.makedirs(self.env['release_dir'], exist_ok=True) self.sh.rmrf(self.env['release_zip_file']) - zipf = zipfile.ZipFile(self.env['release_zip_file'], 'w', zipfile.ZIP_DEFLATED) - for arch in self.env['all_long_archs']: - self.setup(common.get_argparse(default_args={'arch': arch})) - zipf.write(self.env['qcow2_file'], arcname=os.path.relpath(self.env['qcow2_file'], self.env['root_dir'])) - zipf.write(self.env['linux_image'], arcname=os.path.relpath(self.env['linux_image'], self.env['root_dir'])) - zipf.close() + self.log_info('Creating zip: ' + self.env['release_zip_file']) + with zipfile.ZipFile(self.env['release_zip_file'], 'w', zipfile.ZIP_DEFLATED) as zipf: + for qcow2, linux_image in self.qcow2s_linux_images: + self.log_info('Adding file: ' + qcow2) + zipf.write(qcow2, arcname=os.path.relpath(qcow2, self.env['root_dir'])) + self.log_info('Adding file: ' + linux_image) + zipf.write(linux_image, arcname=os.path.relpath(linux_image, self.env['root_dir'])) if __name__ == '__main__': - main() + Main().cli() diff --git a/run b/run index 0371131..825fd35 100755 --- a/run +++ b/run @@ -641,4 +641,4 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: return exit_status if __name__ == '__main__': - Main().cli_exit() + Main().cli() diff --git a/test b/test index ee9fdda..67b3b2f 100755 --- a/test +++ b/test @@ -8,7 +8,7 @@ class Main(common.TestCliFunction): def __init__(self): super().__init__( description='''\ -Run all tests in one go. +https://github.com/cirosantilli/linux-kernel-module-cheat#automated-tests ''' ) self.add_argument( @@ -35,5 +35,5 @@ Size of the tests to run. Scale: self.run_test(self.import_path_main('test-gdb'), run_args, 'test-gdb') if __name__ == '__main__': - Main().cli_exit() + Main().cli() diff --git a/test-boot b/test-boot index 3bed86b..33cdf9d 100755 --- a/test-boot +++ b/test-boot @@ -89,5 +89,4 @@ See ./test --help for --size. # self._bench(gem5_script='biglittle') if __name__ == '__main__': - Main().cli_exit() - + Main().cli() diff --git a/test-gdb b/test-gdb index 36861f6..1590abf 100755 --- a/test-gdb +++ b/test-gdb @@ -7,7 +7,11 @@ import common class Main(common.TestCliFunction): def __init__(self): - super().__init__() + super().__init__( + description='''\ +https://github.com/cirosantilli/linux-kernel-module-cheat#test-gdb +''' + ) self.add_argument( 'tests', nargs='*',