common: factor -j --nproc to all builds

build-baremetal

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import multiprocessing
 import os
 
 import common

build-buildroot

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import multiprocessing
 import os
 import pathlib
 import shutil
@@ -63,10 +62,6 @@ Pass multiple times to use multiple fragment files.
         parser.add_argument(
             '-i', '--initrd', default=self._defaults['initrd'], action='store_true',
         )
-        parser.add_argument(
-            '-j', '--nproc', default=self._defaults['nproc'], type=int,
-            help='Number of processors to use for the build. Default: all.'
-        )
         parser.add_argument(
             '-K', '--kernel-custom-config-file', default=self._defaults['kernel_custom_config_file'],
             help='''\
@@ -113,10 +108,6 @@ usually extra Buildroot targets.
             extra_make_args.append('linux-reconfigure')
         if args.gem5:
             extra_make_args.append('gem5-reconfigure')
-        if args.nproc is None:
-            nproc = multiprocessing.cpu_count()
-        else:
-            nproc = args.nproc
         if args.arch == 'x86_64':
             defconfig = 'qemu_x86_64_defconfig'
         elif args.arch == 'arm':
@@ -144,7 +135,7 @@ usually extra Buildroot targets.
             )
             buildroot_configs = args.buildroot_config
             buildroot_configs.extend([
-                'BR2_JLEVEL={}'.format(nproc),
+                'BR2_JLEVEL={}'.format(args.nproc),
                 'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir),
             ])
             common.write_configs(common.buildroot_config_file, buildroot_configs)
@@ -297,7 +288,6 @@ Run Linux on an emulator
         'kernel_custom_config_file': None,
         'kernel_modules': False,
         'no_all': False,
-        'nproc': None,
         'skip_configure': False,
         'extra_make_args': [],
     }

build-crosstool-ng

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import multiprocessing
 import os
 
 import common
@@ -9,7 +8,6 @@ class CrosstoolNgComponent(common.Component):
     def do_build(self, args):
         common.raise_no_x86(args.arch)
         build_dir = self.get_build_dir(args)
-        nproc = multiprocessing.cpu_count()
         defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig')
         os.makedirs(common.crosstool_ng_util_dir, exist_ok=True)
         os.makedirs(common.crosstool_ng_download_dir, exist_ok=True)
@@ -30,7 +28,7 @@ class CrosstoolNgComponent(common.Component):
         common.run_cmd(
             [
                 'make',
-                '-j', str(nproc),
+                '-j', str(args.nproc),
             ],
         )
 
@@ -58,7 +56,7 @@ class CrosstoolNgComponent(common.Component):
             [
                 common.crosstool_ng_executable,
                 'build',
-                'CT_JOBS={}'.format(str(nproc)),
+                'CT_JOBS={}'.format(str(args.nproc)),
             ],
             out_file=os.path.join(build_dir, 'lkmc.log'),
             delete_env=['LD_LIBRARY_PATH'],

build-gem5

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import glob
-import multiprocessing
 import os
 import pathlib
 import subprocess
@@ -76,7 +75,7 @@ class Gem5Component(common.Component):
             (
                 [
                     'scons',
-                    '-j', str(multiprocessing.cpu_count()),
+                    '-j', str(args.nproc),
                     '--ignore-style',
                     common.gem5_executable
                 ] +

build-linux

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import multiprocessing
 import os
 import shutil
 
@@ -57,7 +56,7 @@ class LinuxComponent(common.Component):
             (
                 [
                     'make',
-                    '-j', str(multiprocessing.cpu_count()),
+                    '-j', str(args.nproc),
                 ] +
                 common_make_args +
                 [
@@ -70,7 +69,7 @@ class LinuxComponent(common.Component):
             (
                 [
                     'make',
-                    '-j', str(multiprocessing.cpu_count()),
+                    '-j', str(args.nproc),
                 ] +
                 common_make_args +
                 verbose +

build-m5

@@ -1,10 +1,6 @@
 #!/usr/bin/env python3
 
-import multiprocessing
 import os
-import platform
-import subprocess
-import time
 
 import common
 
@@ -19,7 +15,7 @@ class M5Component(common.Component):
             arch = args.arch
         return [
             'make',
-            '-j', str(multiprocessing.cpu_count()),
+            '-j', str(args.nproc),
             '-f', 'Makefile.{}'.format(arch),
             'CC={}'.format(cc),
             'LD={}'.format(ld),

build-modules

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import distutils.dir_util
-import multiprocessing
 import os
 import platform
 import shutil
@@ -83,7 +82,7 @@ class ModulesComponent(common.Component):
             (
                 [
                     'make',
-                    '-j', str(multiprocessing.cpu_count()),
+                    '-j', str(args.nproc),
                     'ARCH={}'.format(common.linux_arch),
                     'CC={}'.format(cc),
                     'CROSS_COMPILE={}'.format(prefix),

build-qemu

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
 
-import multiprocessing
 import os
-import subprocess
 
 import common
 
@@ -39,7 +37,7 @@ class QemuComponent(common.Component):
             (
                 [
                     'make',
-                    '-j', str(multiprocessing.cpu_count()),
+                    '-j', str(args.nproc),
 
                 ] +
                 verbose

build-userland

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import multiprocessing
 import os
 import platform
 import shutil
@@ -42,7 +41,7 @@ has the OpenBLAS libraries and headers installed.
             (
                 [
                     'make',
-                    '-j', str(multiprocessing.cpu_count()),
+                    '-j', str(args.nproc),
                     'CC={}'.format(cc),
                     'CXX={}'.format(cxx),
                     'PKG_CONFIG={}'.format(common.buildroot_pkg_config),

common.py

@@ -9,6 +9,7 @@ import distutils.file_util
 import glob
 import imp
 import json
+import multiprocessing
 import os
 import re
 import shlex
@@ -136,6 +137,12 @@ def add_build_arguments(parser):
         help='Clean the build instead of building.',
         action='store_true',
     )
+    parser.add_argument(
+        '-j', '--nproc',
+        help='Number of processors to use for the build. Default: use all cores.',
+        type=int,
+        default=multiprocessing.cpu_count(),
+    )
 
 def add_dry_run_argument(parser):
     parser.add_argument('--dry-run', default=False, action='store_true', help='''\