diff --git a/README.adoc b/README.adoc index f09f09d..e4dd94b 100644 --- a/README.adoc +++ b/README.adoc @@ -293,7 +293,7 @@ If you really want to develop semiconductors, your only choice is to join an uni ==== About the QEMU Buildroot setup -link:https://en.wikipedia.org/wiki/Buildroot[Buildroot] is a set of `make` scripts that downloads from source and compiles compatible versions of: +link:https://en.wikipedia.org/wiki/Buildroot[Buildroot] is a set of Make scripts that download and compile from source compatible versions of: * GCC * Linux kernel @@ -553,7 +553,7 @@ Some times it works with the host QEMU: sudo apt-get install qemu-system-x86 git clone https://github.com/cirosantilli/linux-kernel-module-cheat cd linux-kernel-module-cheat -./download-latest-release +./release-download-latest unzip lkmc-*.zip ./run --prebuilt .... @@ -9404,22 +9404,7 @@ vim "$(./getvar --arch arm run_cmd_file)" ./"$(./getvar --arch arm run_cmd_file)" .... -Next, you will also want to give the relevant images to save them time. Zip the images with: - -.... -./build-all -G -./zip-img -.... - -Source: link:zip-img[] - -This generates a zip file: - -.... -out/lkmc-*.zip -.... - -which you can then upload somewhere, e.g. GitHub release assets as in https://github.com/cirosantilli/linux-kernel-module-cheat/releases/tag/test-replay-arm +Next, you will also want to give the relevant images to save them time, see: <>. Finally, do a clone of the relevant repository out of tree and reproduce the bug there, to be 100% sure that it is an actual upstream bug, and to provide developers with the cleanest possible commands. @@ -10338,7 +10323,7 @@ This can be used to check the determinism of: * <> * <> -==== Releases +==== Release This is not yet super stable, but one day maybe this script will automatically do a release: @@ -10348,8 +10333,46 @@ This is not yet super stable, but one day maybe this script will automatically d Source: link:release[]. +When ready, that script should: + +* build +* test +* package with <> +* upload to GitHub with link:release-create-github[] + This should in particular enable to easily update <>. +===== release-zip + +Create a zip containing all files required for <> + +.... +./build-all -G +./release-zip +.... + +Source: link:release-zip[] + +This generates a zip file: + +.... +echo "$(./getvar release_zip_file)" +.... + +which you can then upload somewhere. + +For example, you can create or update a GitHub release and upload automatically with: + +.... +git push +printf "$GITHUB_TOKEN" > "$(./getvar github_token_file)" +./release-upload +.... + +Source: link:release-upload[] + +TODO: generalize that so that people can upload to their forks. + === Fairy tale ____ diff --git a/common.py b/common.py index 61cff70..14bef60 100644 --- a/common.py +++ b/common.py @@ -6,6 +6,7 @@ import copy import datetime import glob import imp +import json import os import re import shlex @@ -15,10 +16,10 @@ import stat import subprocess import sys import time +import urllib +import urllib.request this = sys.modules[__name__] - -# Default paths. root_dir = os.path.dirname(os.path.abspath(__file__)) data_dir = os.path.join(root_dir, 'data') p9_dir = os.path.join(data_dir, '9p') @@ -34,8 +35,7 @@ extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux') qemu_src_dir = os.path.join(submodules_dir, 'qemu') parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark') ccache_dir = os.path.join('/usr', 'lib', 'ccache') - -# Other default variables. +github_token_file = os.path.join(data_dir, 'github-token') arch_map = { 'a': 'arm', 'A': 'aarch64', @@ -44,6 +44,9 @@ arch_map = { arches = [arch_map[k] for k in arch_map] gem5_cpt_prefix = '^cpt\.' sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip() +release_dir = os.path.join(this.out_dir, 'release') +release_zip_file = os.path.join(this.release_dir, 'lkmc-{}.zip'.format(this.sha)) +github_repo_id = 'cirosantilli/linux-kernel-module-cheat' config_file = os.path.join(data_dir, 'config') if os.path.exists(config_file): config = imp.load_source('config', config_file) @@ -189,6 +192,39 @@ def get_toolchain_tool(tool): global this return glob.glob(os.path.join(this.host_bin_dir, '*-buildroot-*-{}'.format(tool)))[0] +def github_make_request( + authenticate=False, + data=None, + extra_headers=None, + path='', + subdomain='api', + url_params=None, + **extra_request_args + ): + global this + if extra_headers is None: + extra_headers = {} + headers = {'Accept': 'application/vnd.github.v3+json'} + headers.update(extra_headers) + if authenticate: + with open(this.github_token_file, 'r') as f: + token = f.read().rstrip() + headers['Authorization'] = 'token ' + token + if url_params is not None: + path += '?' + urllib.parse.urlencode(url_params) + request = urllib.request.Request( + 'https://' + subdomain + '.github.com/repos/' + github_repo_id + path, + headers=headers, + data=data, + **extra_request_args + ) + response_body = urllib.request.urlopen(request).read().decode() + if response_body: + _json = json.loads(response_body) + else: + _json = {} + return _json + def log_error(msg): print('error: {}'.format(msg), file=sys.stderr) @@ -340,7 +376,7 @@ def run_cmd( #signal.signal(signal.SIGPIPE, sigpipe_old) return proc.returncode -def setup(parser, **extra_args): +def setup(parser): ''' Parse the command line arguments, and setup several variables based on them. Typically done after getting inputs from the command line arguments. diff --git a/release b/release index d013669..18de22d 100755 --- a/release +++ b/release @@ -1,31 +1,27 @@ #!/usr/bin/env python3 +''' +https://upload.com/cirosantilli/linux-kernel-module-cheat#release +''' + import imp import os import subprocess import common -zip_img = imp.load_source('zip_img', os.path.join(common.root_dir, 'zip-img')) +release_zip = imp.load_source('release_zip', os.path.join(common.root_dir, 'release-zip')) +release_upload = imp.load_source('release_upload', os.path.join(common.root_dir, 'release-upload')) + +# TODO factor those out so we don't redo the same thing multiple times. +# subprocess.check_call([os.path.join(common.root_dir, 'test')]) +# subprocess.check_call([os.path.join(common.root_dir, ''bench-all', '-A', '-u']) -subprocess.check_call([os.path.join(common.root_dir, 'test')]) # A clean release requires a full rebuild unless we hack it :-( # We can't just use our curent build as it contains packages we've # installed in random experiments. And with EXT2: we can't easily # know what the smallest root filesystem size is and use it either... # https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot subprocess.check_call([os.path.join(common.root_dir, 'build-all')]) -zip_img.main() -tag = 'sha-{}'.format(common.sha) -subprocess.check_call(['git', 'tag', '-f', tag]) -subprocess.check_call(['git', 'push', '--tags']) -# TODO -# - https://stackoverflow.com/questions/41022470/curl-request-to-add-file-to-github-release -# - https://stackoverflow.com/questions/38627115/upload-files-to-github-directory-using-github-api -# upload_basename = 'images-{}.zip'.format(common.sha) -#curl "https://api.github.com/repos/cirosantilli/linux-kernel-module-cheat/releases/tags/${tag}/assets?access_token=$(cat data/access_token)&tag_name=${upload_basename}" \ -# --header 'Content-Type: application/zip' \ -# --upload-file "${common_out_dir}/${upload_basename}" \ -# -H 'Accept: application/vnd.github.v3+json' \ -# -X POST \ -#; -subprocess.check_call(['./bench-all', '-A', '-u']) +release_zip.main() +subprocess.check_call(['git', 'push']) +release_upload.main() diff --git a/download-latest-release b/release-download-latest similarity index 51% rename from download-latest-release rename to release-download-latest index bfb55dc..d30ad8c 100755 --- a/download-latest-release +++ b/release-download-latest @@ -1,16 +1,16 @@ #!/usr/bin/env python3 ''' -Download the latest release from GitHub: +Usage: https://github.com/cirosantilli/linux-kernel-module-cheat#prebuilt + +Implementation: https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downloading-a-file-in-the-latest-release-of-a-repo/50540591#50540591 ''' -import json import urllib.request -_json = json.loads(urllib.request.urlopen(urllib.request.Request( - 'https://api.github.com/repos/cirosantilli/linux-kernel-module-cheat/releases', - headers={'Accept' : 'application/vnd.github.full+json"text/html'} -)).read()) +import common + +_json = common.github_make_request(path='/releases') asset = _json[0]['assets'][0] urllib.request.urlretrieve(asset['browser_download_url'], asset['name']) diff --git a/release-upload b/release-upload new file mode 100755 index 0000000..5dc6581 --- /dev/null +++ b/release-upload @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +''' +Usage: https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip + +Implementation: + +* https://stackoverflow.com/questions/5207269/how-to-release-a-build-artifact-asset-on-github-with-a-script/52354732#52354732 +* https://stackoverflow.com/questions/38153418/can-someone-give-a-python-requests-example-of-uploading-a-release-asset-in-githu/52354681#52354681 +''' + +import json +import os +import sys + +import urllib.error + +import common + +def main(): + repo = common.github_repo_id + tag = 'sha-{}'.format(common.sha) + upload_path = common.release_zip_file + + # Check the release already exists. + try: + _json = common.github_make_request(path='/releases/tags/' + tag) + except urllib.error.HTTPError as e: + if e.code == 404: + release_exists = False + else: + raise e + else: + release_exists = True + release_id = _json['id'] + + # Create release if not yet created. + if not release_exists: + _json = common.github_make_request( + authenticate=True, + data=json.dumps({ + 'tag_name': tag, + 'name': tag, + 'prerelease': True, + }).encode(), + path='/releases' + ) + release_id = _json['id'] + + asset_name = os.path.split(upload_path)[1] + + # Clear the prebuilts for a upload. + _json = common.github_make_request( + path=('/releases/' + str(release_id) + '/assets'), + ) + for asset in _json: + if asset['name'] == asset_name: + _json = common.github_make_request( + authenticate=True, + path=('/releases/assets/' + str(asset['id'])), + method='DELETE', + ) + break + + # Upload the prebuilt. + with open(upload_path, 'br') as myfile: + content = myfile.read() + _json = common.github_make_request( + authenticate=True, + data=content, + extra_headers={'Content-Type': 'application/zip'}, + path=('/releases/' + str(release_id) + '/assets'), + subdomain='uploads', + url_params={'name': asset_name}, + ) + +if __name__ == '__main__': + main() diff --git a/zip-img b/release-zip similarity index 60% rename from zip-img rename to release-zip index ea2a487..8fff075 100755 --- a/zip-img +++ b/release-zip @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +''' +https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip +''' + import os import subprocess import zipfile @@ -7,10 +11,10 @@ import zipfile import common def main(): - outfile = os.path.join(common.out_dir, 'lkmc-{}.zip'.format(common.sha)) - if os.path.exists(outfile): - os.unlink(outfile) - zipf = zipfile.ZipFile(outfile, 'w', zipfile.ZIP_DEFLATED) + os.makedirs(common.release_dir, exist_ok=True) + if os.path.exists(common.release_zip_file): + os.unlink(common.release_zip_file) + zipf = zipfile.ZipFile(common.release_zip_file, 'w', zipfile.ZIP_DEFLATED) for arch in common.arches: common.setup(common.get_argparse(default_args={'arch': arch})) zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir))