diff --git a/build-buildroot b/build-buildroot
index f5137b9..c24db99 100755
--- a/build-buildroot
+++ b/build-buildroot
@@ -91,11 +91,11 @@ usually extra Buildroot targets.
             'BR2_JLEVEL={}'.format(args.nproc),
             'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir),
         ])
-        common.write_configs(common.buildroot_config_file, configs)
         if not args.build_linux:
             configs.extend([
                 '# BR2_LINUX_KERNEL is not set',
             ])
+        config_fragments = []
         if not args.baseline:
             configs.extend([
                 'BR2_GLOBAL_PATCH_DIR="{}"'.format(
@@ -120,15 +120,15 @@ usually extra Buildroot targets.
             config_fragments = [
                 os.path.join(common.root_dir, 'buildroot_config', 'default')
             ] + args.config_fragment
-            common.write_configs(common.buildroot_config_file, configs, config_fragments)
-            common.run_cmd(
-                [
-                    'make',
-                    'O={}'.format(common.buildroot_build_dir),
-                    'olddefconfig',
-                ],
-                cwd=common.buildroot_src_dir,
-            )
+        common.write_configs(common.buildroot_config_file, configs, config_fragments)
+        common.run_cmd(
+            [
+                'make',
+                'O={}'.format(common.buildroot_build_dir),
+                'olddefconfig',
+            ],
+            cwd=common.buildroot_src_dir,
+        )
         common.make_build_dirs()
         if not args.no_all:
             extra_make_args.append('all')