Updated method of testing generate_test_runner script to be a FUNCTIONAL test instead of a perfect code match. Backfilled most tests and added some coverage not previously present. Still need to test new command line args and a few cmock details.

test/rakefile

@@ -5,6 +5,7 @@
 # ==========================================
 
 UNITY_ROOT = File.expand_path(File.dirname(__FILE__)) + '/'
+$verbose = false
 
 require 'rake'
 require 'rake/clean'
@@ -58,3 +59,7 @@ end
 task :no_color do
   $colour_output = false
 end
+
+task :verbose do
+  $verbose = true
+end

test/rakefile_helper.rb

@@ -92,12 +92,12 @@ module RakefileHelpers
     end
   end
 
-  def build_compiler_fields
+  def build_compiler_fields(inject_defines)
     command  = tackit($cfg['compiler']['path'])
     if $cfg['compiler']['defines']['items'].nil?
       defines  = ''
     else
-      defines  = squash($cfg['compiler']['defines']['prefix'], $cfg['compiler']['defines']['items'] + ['UNITY_OUTPUT_CHAR=putcharSpy'])
+      defines  = squash($cfg['compiler']['defines']['prefix'], $cfg['compiler']['defines']['items'] + ['UNITY_OUTPUT_CHAR=putcharSpy'] + inject_defines)
     end
     options  = squash('', $cfg['compiler']['options'])
     includes = squash($cfg['compiler']['includes']['prefix'], $cfg['compiler']['includes']['items'])
@@ -106,7 +106,8 @@ module RakefileHelpers
   end
 
   def compile(file, defines=[])
-    compiler = build_compiler_fields
+    compiler = build_compiler_fields(defines)
+    defines  =
     cmd_str  = "#{compiler[:command]}#{compiler[:defines]}#{compiler[:options]}#{compiler[:includes]} #{file} " +
                "#{$cfg['compiler']['object_files']['prefix']}#{$cfg['compiler']['object_files']['destination']}"
     obj_file = "#{File.basename(file, C_EXTENSION)}#{$cfg['compiler']['object_files']['extension']}"
@@ -160,11 +161,11 @@ module RakefileHelpers
     return {:command => command, :pre_support => pre_support, :post_support => post_support}
   end
 
-  def execute(command_string, verbose=true)
-    report command_string
+  def execute(command_string, ok_to_fail=false)
+    report command_string if $verbose
     output = `#{command_string}`.chomp
-    report(output) if (verbose && !output.nil? && (output.length > 0))
-    if $?.exitstatus != 0
+    report(output) if ($verbose && !output.nil? && (output.length > 0))
+    if (($?.exitstatus != 0) && !ok_to_fail)
       raise "Command failed. (Returned #{$?.exitstatus})"
     end
     return output

test/targets/clang_file.yml

@@ -53,6 +53,7 @@ compiler:
     items:
       - 'src/'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
   defines:
     prefix: '-D'

test/targets/clang_strict.yml

@@ -53,6 +53,7 @@ compiler:
     items:
       - 'src/'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
   defines:
     prefix: '-D'

test/targets/gcc_32.yml

@@ -15,6 +15,7 @@ compiler:
     items:
       - 'src/'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
   defines:
     prefix: '-D'

test/targets/gcc_64.yml

@@ -15,6 +15,7 @@ compiler:
     items:
       - 'src/'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
   defines:
     prefix: '-D'

test/targets/gcc_auto_limits.yml

@@ -15,6 +15,7 @@ compiler:
     items:
       - 'src/'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
   defines:
     prefix: '-D'

test/targets/gcc_auto_sizeof.yml

@@ -15,6 +15,7 @@ compiler:
     items:
       - 'src/'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
   defines:
     prefix: '-D'

test/targets/gcc_auto_stdint.yml

@@ -28,6 +28,7 @@ compiler:
     items:
       - 'src/'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
   defines:
     prefix: '-D'

test/targets/gcc_manual_math.yml

@@ -15,6 +15,7 @@ compiler:
     items:
       - 'src/'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
   defines:
     prefix: '-D'

test/targets/iar_arm_v4.yml

@@ -32,6 +32,7 @@ compiler:
       - [*tools_root, 'arm\inc\']
       - 'src\'
       - '..\src\'
+      - 'testdata/'
       - *unit_tests_path
       - 'vendor\unity\src\'
   defines:

test/targets/iar_arm_v5.yml

@@ -31,6 +31,7 @@ compiler:
       - [*tools_root, 'arm\inc\']
       - 'src\'
       - '..\src\'
+      - 'testdata/'
       - *unit_tests_path
       - 'vendor\unity\src\'
       - 'iar\iar_v5\incIAR\'

test/targets/iar_arm_v5_3.yml

@@ -31,6 +31,7 @@ compiler:
       - [*tools_root, 'arm\inc\']
       - 'src\'
       - '..\src\'
+      - 'testdata/'
       - *unit_tests_path
       - 'vendor\unity\src\'
       - 'iar\iar_v5\incIAR\'

test/targets/iar_armcortex_LM3S9B92_v5_4.yml

@@ -35,6 +35,7 @@ compiler:
       - [*tools_root, 'arm\inc\']
       - 'src\'
       - '..\src\'
+      - 'testdata/'
       - *unit_tests_path
       - 'vendor\unity\src\'
       - 'iar\iar_v5\incIAR\'

test/targets/iar_cortexm3_v5.yml

@@ -33,6 +33,7 @@ compiler:
      - [*tools_root, 'arm\inc\']
      - 'src\'
       - '..\src\'
+      - 'testdata/'
      - *unit_tests_path
      - 'vendor\unity\src\'
      - 'iar\iar_v5\incIAR\'

test/targets/iar_msp430.yml

@@ -34,6 +34,7 @@ compiler:
       - [*core_lib, 'dlib']
       - 'src\'
       - '../src/'
+      - 'testdata/'
       - *unit_tests_path
       - 'vendor\unity\src'
   defines:

test/targets/iar_sh2a_v6.yml

@@ -34,6 +34,7 @@ compiler:
       - [*tools_root, 'sh\inc\c']
       - 'src\'
       - '..\src\'
+      - 'testdata/'
       - *unit_tests_path
       - 'vendor\unity\src\'
   defines:

test/testdata/CException.h

@@ -0,0 +1,6 @@
+#ifndef CEXCEPTION_H
+#define CEXCEPTION_H
+
+#define CEXCEPTION_BEING_USED 1
+
+#endif //CEXCEPTION_H

test/testdata/Defs.h

@@ -0,0 +1,8 @@
+#ifndef DEF_H
+#define DEF_H
+
+#define EXTERN_DECL
+
+extern int CounterSuiteSetup;
+
+#endif //DEF_H

test/testdata/cmock.h

@@ -0,0 +1,14 @@
+#ifndef CMOCK_H
+#define CMOCK_H
+
+int CMockMemFreeFinalCounter = 0;
+int mockMock_Init_Counter = 0;
+int mockMock_Verify_Counter = 0;
+int mockMock_Destroy_Counter = 0;
+
+void CMock_Guts_MemFreeFinal(void) { CMockMemFreeFinalCounter++; }
+void mockMock_Init(void)           { mockMock_Init_Counter++;    }
+void mockMock_Verify(void)         { mockMock_Verify_Counter++;  }
+void mockMock_Destroy(void)        { mockMock_Destroy_Counter++; }
+
+#endif //CMOCK_H

test/testdata/mockMock.h

@@ -0,0 +1,13 @@
+#ifndef MOCK_MOCK_H
+#define MOCK_MOCK_H
+
+extern int mockMock_Init_Counter;
+extern int mockMock_Verify_Counter;
+extern int mockMock_Destroy_Counter;
+extern int CMockMemFreeFinalCounter;
+
+void mockMock_Init(void);
+void mockMock_Verify(void);
+void mockMock_Destroy(void);
+
+#endif //MOCK_MOCK_H

test/testdata/mocksample.c

@@ -1,51 +0,0 @@
-// This is just a sample test file to be used to test the generator script
-#ifndef TEST_SAMPLE_H
-#define TEST_SAMPLE_H
-
-#include <setjmp.h>
-#include "unity.h"
-#include "funky.h"
-#include "Mockstanky.h"
-
-void setUp(void)
-{
-  CustomSetupStuff();
-}
-
-void tearDown(void)
-{
-  CustomTeardownStuff
-}
-
-//Yup, nice comment
-void test_TheFirstThingToTest(void)
-{
-    TEST_ASSERT(1);
-
-    TEST_ASSERT_TRUE(1);
-}
-
-/*
-void test_ShouldBeIgnored(void)
-{
-    DoesStuff();
-}
-*/
-
-//void test_ShouldAlsoNotBeTested(void)
-//{
-//    Call_An_Expect();
-//    
-//    CallAFunction();
-//    test_CallAFunctionThatLooksLikeATest();
-//}
-
-void test_TheSecondThingToTest(void)
-{
-    Call_An_Expect();
-    
-    CallAFunction();
-    test_CallAFunctionThatLooksLikeATest();
-}
-
-#endif //TEST_SAMPLE_H

test/testdata/sample.yml

@@ -1,9 +0,0 @@
-:unity:
-  :includes:
-    - two.h
-    - three.h
-    - <four.h>
-  :plugins:
-    - :cexception
-  :suite_setup: |
-    a_yaml_setup();

test/testdata/testRunnerGenerator.c

@@ -0,0 +1,183 @@
+/* This Test File Is Used To Verify Many Combinations Of Using the Generate Test Runner Script */
+
+#include <stdio.h>
+#include "unity.h"
+#include "Defs.h"
+
+#ifdef USE_CEXCEPTION
+#include "CException.h"
+#endif
+
+/* Notes about prefixes:
+   test     - normal default prefix. these are "always run" tests for this procedure
+   spec     - normal default prefix. required to run default setup/teardown calls.
+   should   - normal default prefix.
+   qwiktest - custom prefix for when tests skip all setup/teardown calls.
+   custtest - custom prefix for when tests use custom setup/teardown calls.
+   paratest - custom prefix for when we want to verify parameterized tests.
+   extest   - custom prefix only used during cexception
+   suitetest- custom prefix for when we want to use custom suite setup/teardown
+*/
+
+/* Support for Meta Test Rig */
+#define TEST_CASE(a)
+void putcharSpy(int c) { (void)putchar(c);} // include passthrough for linking tests
+
+/* Global Variables Used During These Tests */
+int CounterSetup = 0;
+int CounterTeardown = 0;
+int CounterSuiteSetup = 0;
+
+void setUp(void)
+{
+    CounterSetup = 1;
+}
+
+void tearDown(void)
+{
+    CounterTeardown = 1;
+}
+
+void custom_setup(void)
+{
+    CounterSetup = 2;
+}
+
+void custom_teardown(void)
+{
+    CounterTeardown = 2;
+}
+
+/*
+void test_OldSchoolCommentsShouldBeIgnored(void)
+{
+    TEST_ASSERT_FAIL("Old-School Comments Should Be Ignored");
+}
+*/
+
+void test_ThisTestAlwaysPasses(void)
+{
+    TEST_PASS();
+}
+
+void test_ThisTestAlwaysFails(void)
+{
+    TEST_FAIL_MESSAGE("This Test Should Fail");
+}
+
+void test_ThisTestAlwaysIgnored(void)
+{
+    TEST_IGNORE_MESSAGE("This Test Should Be Ignored");
+}
+
+void qwiktest_ThisTestPassesWhenNoSetupRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(0, CounterSetup, "Setup Was Unexpectedly Run");
+}
+
+void qwiktest_ThisTestPassesWhenNoTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(0, CounterTeardown, "Teardown Was Unexpectedly Run");
+}
+
+void spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(0, CounterSuiteSetup, "Suite Setup Was Unexpectedly Run");
+}
+
+void spec_ThisTestPassesWhenNormalSetupRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(1, CounterSetup, "Normal Setup Wasn't Run");
+}
+
+void spec_ThisTestPassesWhenNormalTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(1, CounterTeardown, "Normal Teardown Wasn't Run");
+}
+
+void custtest_ThisTestPassesWhenCustomSetupRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(2, CounterSetup, "Custom Setup Wasn't Run");
+}
+
+void custtest_ThisTestPassesWhenCustomTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(2, CounterTeardown, "Custom Teardown Wasn't Run");
+}
+
+//void test_NewStyleCommentsShouldBeIgnored(void)
+//{
+//    TEST_ASSERT_FAIL("New Style Comments Should Be Ignored");
+//}
+
+void test_NotBeConfusedByLongComplicatedStrings(void)
+{
+    const char* crazyString = "GET / HTTP/1.1\r\nHost: 127.0.0.1:8081\r\nConnection: keep-alive\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36\r\nPostman-Token: 768c7149-c3fb-f704-71a2-63918d9195b2\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-GB,en-US;q=0.8,en;q=0.6\r\n\r\n";
+
+    TEST_ASSERT_EQUAL_STRING_MESSAGE(crazyString, crazyString, "These Strings Are The Same");
+}
+
+void test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings(void)
+{
+    TEST_ASSERT_TRUE_MESSAGE(1, "1 Should be True");
+}
+
+void test_StillNotBeConfusedByLongComplicatedStrings(void)
+{
+    const char* crazyString = "GET / HTTP/1.1\r\nHost: 127.0.0.1:8081\r\nConnection: keep-alive\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36\r\nPostman-Token: 768c7149-c3fb-f704-71a2-63918d9195b2\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-GB,en-US;q=0.8,en;q=0.6\r\n\r\n";
+
+    TEST_ASSERT_EQUAL_STRING_MESSAGE(crazyString, crazyString, "These Strings Are Still The Same");
+}
+
+void should_RunTestsStartingWithShouldByDefault(void)
+{
+    TEST_ASSERT_TRUE_MESSAGE(1, "1 Should be True");
+}
+
+TEST_CASE(25)
+TEST_CASE(125)
+TEST_CASE(5)
+void paratest_ShouldHandleParameterizedTests(int Num)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(0, (Num % 5), "All The Values Are Divisible By 5");
+}
+
+TEST_CASE(7)
+void paratest_ShouldHandleParameterizedTests2(int Num)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(7, Num, "The Only Call To This Passes");
+}
+
+void paratest_ShouldHandleNonParameterizedTestsWhenParameterizationValid(void)
+{
+    TEST_PASS();
+}
+
+TEST_CASE(17)
+void paratest_ShouldHandleParameterizedTestsThatFail(int Num)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(3, Num, "This call should fail");
+}
+
+#ifdef USE_CEXCEPTION
+void extest_ShouldHandleCExceptionInTest(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(1, CEXCEPTION_BEING_USED, "Should be pulling in CException");
+}
+#endif
+
+#ifdef USE_ANOTHER_MAIN
+int custom_main(void);
+
+int main(void)
+{
+    return custom_main();
+}
+#endif
+
+void suitetest_ThisTestPassesWhenCustomSuiteSetupAndTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(1, CounterSuiteSetup, "Suite Setup Should Have Run");
+}
+
+

test/testdata/testRunnerGeneratorWithMocks.c

@@ -0,0 +1,192 @@
+/* This Test File Is Used To Verify Many Combinations Of Using the Generate Test Runner Script */
+
+#include <stdio.h>
+#include "unity.h"
+#include "Defs.h"
+#include "mockMock.h"
+
+#ifdef USE_CEXCEPTION
+#include "CException.h"
+#endif
+
+/* Notes about prefixes:
+   test     - normal default prefix. these are "always run" tests for this procedure
+   spec     - normal default prefix. required to run default setup/teardown calls.
+   should   - normal default prefix.
+   qwiktest - custom prefix for when tests skip all setup/teardown calls.
+   custtest - custom prefix for when tests use custom setup/teardown calls.
+   paratest - custom prefix for when we want to verify parameterized tests.
+   extest   - custom prefix only used during cexception
+   suitetest- custom prefix for when we want to use custom suite setup/teardown
+*/
+
+/* Support for Meta Test Rig */
+#define TEST_CASE(a)
+void putcharSpy(int c) { (void)putchar(c);} // include passthrough for linking tests
+
+/* Global Variables Used During These Tests */
+int CounterSetup = 0;
+int CounterTeardown = 0;
+int CounterSuiteSetup = 0;
+
+void setUp(void)
+{
+    CounterSetup = 1;
+}
+
+void tearDown(void)
+{
+    CounterTeardown = 1;
+}
+
+void custom_setup(void)
+{
+    CounterSetup = 2;
+}
+
+void custom_teardown(void)
+{
+    CounterTeardown = 2;
+}
+
+/*
+void test_OldSchoolCommentsShouldBeIgnored(void)
+{
+    TEST_ASSERT_FAIL("Old-School Comments Should Be Ignored");
+}
+*/
+
+void test_ThisTestAlwaysPasses(void)
+{
+    TEST_PASS();
+}
+
+void test_ThisTestAlwaysFails(void)
+{
+    TEST_FAIL_MESSAGE("This Test Should Fail");
+}
+
+void test_ThisTestAlwaysIgnored(void)
+{
+    TEST_IGNORE_MESSAGE("This Test Should Be Ignored");
+}
+
+void qwiktest_ThisTestPassesWhenNoSetupRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(0, CounterSetup, "Setup Was Unexpectedly Run");
+}
+
+void qwiktest_ThisTestPassesWhenNoTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(0, CounterTeardown, "Teardown Was Unexpectedly Run");
+}
+
+void spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(0, CounterSuiteSetup, "Suite Setup Was Unexpectedly Run");
+}
+
+void spec_ThisTestPassesWhenNormalSetupRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(1, CounterSetup, "Normal Setup Wasn't Run");
+}
+
+void spec_ThisTestPassesWhenNormalTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(1, CounterTeardown, "Normal Teardown Wasn't Run");
+}
+
+void custtest_ThisTestPassesWhenCustomSetupRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(2, CounterSetup, "Custom Setup Wasn't Run");
+}
+
+void custtest_ThisTestPassesWhenCustomTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(2, CounterTeardown, "Custom Teardown Wasn't Run");
+}
+
+//void test_NewStyleCommentsShouldBeIgnored(void)
+//{
+//    TEST_ASSERT_FAIL("New Style Comments Should Be Ignored");
+//}
+
+void test_NotBeConfusedByLongComplicatedStrings(void)
+{
+    const char* crazyString = "GET / HTTP/1.1\r\nHost: 127.0.0.1:8081\r\nConnection: keep-alive\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36\r\nPostman-Token: 768c7149-c3fb-f704-71a2-63918d9195b2\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-GB,en-US;q=0.8,en;q=0.6\r\n\r\n";
+
+    TEST_ASSERT_EQUAL_STRING_MESSAGE(crazyString, crazyString, "These Strings Are The Same");
+}
+
+void test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings(void)
+{
+    TEST_ASSERT_TRUE_MESSAGE(1, "1 Should be True");
+}
+
+void test_StillNotBeConfusedByLongComplicatedStrings(void)
+{
+    const char* crazyString = "GET / HTTP/1.1\r\nHost: 127.0.0.1:8081\r\nConnection: keep-alive\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36\r\nPostman-Token: 768c7149-c3fb-f704-71a2-63918d9195b2\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-GB,en-US;q=0.8,en;q=0.6\r\n\r\n";
+
+    TEST_ASSERT_EQUAL_STRING_MESSAGE(crazyString, crazyString, "These Strings Are Still The Same");
+}
+
+void should_RunTestsStartingWithShouldByDefault(void)
+{
+    TEST_ASSERT_TRUE_MESSAGE(1, "1 Should be True");
+}
+
+TEST_CASE(25)
+TEST_CASE(125)
+TEST_CASE(5)
+void paratest_ShouldHandleParameterizedTests(int Num)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(0, (Num % 5), "All The Values Are Divisible By 5");
+}
+
+TEST_CASE(7)
+void paratest_ShouldHandleParameterizedTests2(int Num)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(7, Num, "The Only Call To This Passes");
+}
+
+void paratest_ShouldHandleNonParameterizedTestsWhenParameterizationValid(void)
+{
+    TEST_PASS();
+}
+
+TEST_CASE(17)
+void paratest_ShouldHandleParameterizedTestsThatFail(int Num)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(3, Num, "This call should fail");
+}
+
+#ifdef USE_CEXCEPTION
+void extest_ShouldHandleCExceptionInTest(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(1, CEXCEPTION_BEING_USED, "Should be pulling in CException");
+}
+#endif
+
+#ifdef USE_ANOTHER_MAIN
+int custom_main(void);
+
+int main(void)
+{
+    return custom_main();
+}
+#endif
+
+void suitetest_ThisTestPassesWhenCustomSuiteSetupAndTeardownRan(void)
+{
+    TEST_ASSERT_EQUAL_MESSAGE(1, CounterSuiteSetup, "Suite Setup Should Have Run");
+}
+
+void test_ShouldCallMockInitAndVerifyFunctionsForEachTest(void)
+{
+    int passes = (int)(Unity.NumberOfTests - Unity.TestFailures - Unity.TestIgnores);
+    TEST_ASSERT_EQUAL_MESSAGE(Unity.NumberOfTests,     mockMock_Init_Counter,    "Mock Init Should Be Called Once Per Test Started");
+    TEST_ASSERT_EQUAL_MESSAGE(passes,                  mockMock_Verify_Counter,  "Mock Verify Should Be Called Once Per Test Passed");
+    TEST_ASSERT_EQUAL_MESSAGE(Unity.NumberOfTests - 1, mockMock_Destroy_Counter, "Mock Destroy Should Be Called Once Per Test Completed");
+    TEST_ASSERT_EQUAL_MESSAGE(0,                       CMockMemFreeFinalCounter, "Mock MemFreeFinal Should Not Be Called Until End");
+}
+

test/testdata/testsample.c

@@ -1,68 +0,0 @@
-// This is just a sample test file to be used to test the generator script
-#ifndef TEST_SAMPLE_H
-#define TEST_SAMPLE_H
-
-#include <setjmp.h>
-#include "unity.h"
-#include "funky.h"
-#include "stanky.h"
-
-void setUp(void)
-{
-  CustomSetupStuff();
-}
-
-void tearDown(void)
-{
-  CustomTeardownStuff
-}
-
-//Yup, nice comment
-void test_TheFirstThingToTest(void)
-{
-    TEST_ASSERT(1);
-
-    TEST_ASSERT_TRUE(1);
-}
-
-/*
-void test_ShouldBeIgnored(void)
-{
-    DoesStuff();
-}
-*/
-
-//void test_ShouldAlsoNotBeTested(void)
-//{
-//    Call_An_Expect();
-//
-//    CallAFunction();
-//    test_CallAFunctionThatLooksLikeATest();
-//}
-
-void test_TheSecondThingToTest(void)
-{
-    uint8_t* crazyString = "GET / HTTP/1.1\r\nHost: 127.0.0.1:8081\r\nConnection: keep-alive\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36\r\nPostman-Token: 768c7149-c3fb-f704-71a2-63918d9195b2\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-GB,en-US;q=0.8,en;q=0.6\r\n\r\n";
-
-    Call_An_Expect();
-
-    CallAFunction();
-    test_CallAFunctionThatLooksLikeATest();
-}
-
-void test_TheThirdThingToTest(void)
-{
-    CallAFunction();
-}
-
-void test_TheFourthThingToTest(void)
-{
-    uint8_t* anotherString = "GET / HTTP/1.1\r\nHost: 127.0.0.1:8081\r\nConnection: keep-alive\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36\r\nPostman-Token: 768c7149-c3fb-f704-71a2-63918d9195b2\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-GB,en-US;q=0.8,en;q=0.6\r\n\r\n";
-
-    Call_An_Expect();
-
-    CallAFunction();
-    test_CallAFunctionThatLooksLikeATest();
-}
-
-#endif //TEST_SAMPLE_H

test/tests/test_generate_test_runner.rb

@@ -6,96 +6,720 @@
 
 require '../auto/generate_test_runner.rb'
 
-TEST_FILE = 'testdata/testsample.c'
-TEST_MOCK = 'testdata/mocksample.c'
-OUT_FILE  = 'build/testsample_'
-EXP_FILE  = 'expectdata/testsample_'
-
+$generate_test_runner_tests = 0
 $generate_test_runner_failures = 0
 
-def verify_output_equal(subtest)
-  expected = File.read(EXP_FILE + subtest + '.c').gsub(/\r\n/,"\n")
-  actual   = File.read(OUT_FILE + subtest + '.c').gsub(/\r\n/,"\n")
-  if (expected != actual)
-    report("    #{subtest}:FAIL")
-    $generate_test_runner_failures += 1
+OUT_FILE  = 'build/testsample_'
+
+RUNNER_TESTS = [
+  { :name => 'DefaultsThroughOptions',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :options => nil, #defaults
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'DefaultsThroughCommandLine',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :cmdline => "", #defaults
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'DefaultsThroughYAMLFile',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :cmdline => "", #defaults
+    :yaml => {}, #defaults
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'ShorterFilterOfJustTest',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "test",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'ShorterFilterOfJustShould',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "should",
+    },
+    :expected => {
+      :to_pass => [ 'should_RunTestsStartingWithShouldByDefault' ],
+      :to_fail => [  ],
+      :to_ignore => [  ],
+    }
+  },
+
+  { :name => 'ShorterFilterOfJustSpec',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "spec",
+    },
+    :expected => {
+      :to_pass => [ 'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [  ],
+      :to_ignore => [  ],
+    }
+  },
+
+  { :name => 'InjectIncludes',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :includes => ['Defs.h'],
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'ParameterizedThroughOptions',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "paratest",
+      :use_param_tests => true,
+    },
+    :expected => {
+      :to_pass => [ 'paratest_ShouldHandleParameterizedTests\(25\)',
+                    'paratest_ShouldHandleParameterizedTests\(125\)',
+                    'paratest_ShouldHandleParameterizedTests\(5\)',
+                    'paratest_ShouldHandleParameterizedTests2\(7\)',
+                    'paratest_ShouldHandleNonParameterizedTestsWhenParameterizationValid\(RUN_TEST_NO_ARGS\)',
+                  ],
+      :to_fail => [ 'paratest_ShouldHandleParameterizedTestsThatFail\(17\)' ],
+      :to_ignore => [ ],
+    }
+  },
+
+  { :name => 'ParameterizedThroughCommandLine',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :cmdline => " --test_prefix=\"paratest\" --use_param_tests=1",
+    :expected => {
+      :to_pass => [ 'paratest_ShouldHandleParameterizedTests\(25\)',
+                    'paratest_ShouldHandleParameterizedTests\(125\)',
+                    'paratest_ShouldHandleParameterizedTests\(5\)',
+                    'paratest_ShouldHandleParameterizedTests2\(7\)',
+                    'paratest_ShouldHandleNonParameterizedTestsWhenParameterizationValid\(RUN_TEST_NO_ARGS\)',
+                  ],
+      :to_fail => [ 'paratest_ShouldHandleParameterizedTestsThatFail\(17\)' ],
+      :to_ignore => [ ],
+    }
+  },
+
+  { :name => 'ParameterizedThroughCommandLineAndYaml',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :cmdline => "--use_param_tests=1",
+    :yaml => {
+      :test_prefix => "paratest"
+    },
+    :expected => {
+      :to_pass => [ 'paratest_ShouldHandleParameterizedTests\(25\)',
+                    'paratest_ShouldHandleParameterizedTests\(125\)',
+                    'paratest_ShouldHandleParameterizedTests\(5\)',
+                    'paratest_ShouldHandleParameterizedTests2\(7\)',
+                    'paratest_ShouldHandleNonParameterizedTestsWhenParameterizationValid\(RUN_TEST_NO_ARGS\)',
+                  ],
+      :to_fail => [ 'paratest_ShouldHandleParameterizedTestsThatFail\(17\)' ],
+      :to_ignore => [ ],
+    }
+  },
+
+  { :name => 'SupportCExceptionWhenRequested',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST', 'USE_CEXCEPTION'],
+    :options => {
+      :test_prefix => "extest",
+      :cexception => 1,
+    },
+    :expected => {
+      :to_pass => [ 'extest_ShouldHandleCExceptionInTest' ],
+      :to_fail => [ ],
+      :to_ignore => [ ],
+    }
+  },
+
+  { :name => 'CustomSetupAndTeardownThroughOptions',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "custtest|test",
+      :setup_name => "custom_setup",
+      :teardown_name => "custom_teardown",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'custtest_ThisTestPassesWhenCustomSetupRan',
+                    'custtest_ThisTestPassesWhenCustomTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'CustomSetupAndTeardownThroughCommandLine',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :cmdline => " --test_prefix=\"custtest|test\" --setup_name=\"custom_setup\" --teardown_name=\"custom_teardown\"",
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'custtest_ThisTestPassesWhenCustomSetupRan',
+                    'custtest_ThisTestPassesWhenCustomTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'CustomSetupAndTeardownThroughYaml',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :cmdline => " --test_prefix=\"custtest|test\"",
+    :yaml => {
+      :setup_name => "custom_setup",
+      :teardown_name => "custom_teardown",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'custtest_ThisTestPassesWhenCustomSetupRan',
+                    'custtest_ThisTestPassesWhenCustomTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'UseACustomMainFunction',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST', "USE_ANOTHER_MAIN"],
+    :options => {
+      :main_name => "custom_main",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'SupportCustomSuiteSetupAndTeardown',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :includes => ['Defs.h'],
+    :options => {
+      :test_prefix    => "suitetest|test",
+      :suite_setup    => "  CounterSuiteSetup = 1;",
+      :suite_teardown => "  return num_failures;",
+    },
+    :expected =>  {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'suitetest_ThisTestPassesWhenCustomSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'SupportMainExternDeclaration',
+    :testfile => 'testdata/testRunnerGenerator.c',
+    :testdefines => ['TEST'],
+    :includes => ['Defs.h'],
+    :options => {
+      :main_export_decl => "EXTERN_DECL",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+
+  #### WITH MOCKS ##########################################
+
+  { :name => 'DefaultsThroughOptions',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :options => nil, #defaults
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'DefaultsThroughCommandLine',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :cmdline => "", #defaults
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'DefaultsThroughYAMLFile',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :cmdline => "", #defaults
+    :yaml => {}, #defaults
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'ShorterFilterOfJustTest',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "test",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'ShorterFilterOfJustShould',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "should",
+    },
+    :expected => {
+      :to_pass => [ 'should_RunTestsStartingWithShouldByDefault' ],
+      :to_fail => [  ],
+      :to_ignore => [  ],
+    }
+  },
+
+  { :name => 'ShorterFilterOfJustSpec',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "spec",
+    },
+    :expected => {
+      :to_pass => [ 'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                  ],
+      :to_fail => [  ],
+      :to_ignore => [  ],
+    }
+  },
+
+  { :name => 'InjectIncludes',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :includes => ['Defs.h'],
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'ParameterizedThroughOptions',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "paratest",
+      :use_param_tests => true,
+    },
+    :expected => {
+      :to_pass => [ 'paratest_ShouldHandleParameterizedTests\(25\)',
+                    'paratest_ShouldHandleParameterizedTests\(125\)',
+                    'paratest_ShouldHandleParameterizedTests\(5\)',
+                    'paratest_ShouldHandleParameterizedTests2\(7\)',
+                    'paratest_ShouldHandleNonParameterizedTestsWhenParameterizationValid\(RUN_TEST_NO_ARGS\)',
+                  ],
+      :to_fail => [ 'paratest_ShouldHandleParameterizedTestsThatFail\(17\)' ],
+      :to_ignore => [ ],
+    }
+  },
+
+  { :name => 'ParameterizedThroughCommandLine',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :cmdline => " --test_prefix=\"paratest\" --use_param_tests=1",
+    :expected => {
+      :to_pass => [ 'paratest_ShouldHandleParameterizedTests\(25\)',
+                    'paratest_ShouldHandleParameterizedTests\(125\)',
+                    'paratest_ShouldHandleParameterizedTests\(5\)',
+                    'paratest_ShouldHandleParameterizedTests2\(7\)',
+                    'paratest_ShouldHandleNonParameterizedTestsWhenParameterizationValid\(RUN_TEST_NO_ARGS\)',
+                  ],
+      :to_fail => [ 'paratest_ShouldHandleParameterizedTestsThatFail\(17\)' ],
+      :to_ignore => [ ],
+    }
+  },
+
+  { :name => 'ParameterizedThroughCommandLineAndYaml',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :cmdline => "--use_param_tests=1",
+    :yaml => {
+      :test_prefix => "paratest"
+    },
+    :expected => {
+      :to_pass => [ 'paratest_ShouldHandleParameterizedTests\(25\)',
+                    'paratest_ShouldHandleParameterizedTests\(125\)',
+                    'paratest_ShouldHandleParameterizedTests\(5\)',
+                    'paratest_ShouldHandleParameterizedTests2\(7\)',
+                    'paratest_ShouldHandleNonParameterizedTestsWhenParameterizationValid\(RUN_TEST_NO_ARGS\)',
+                  ],
+      :to_fail => [ 'paratest_ShouldHandleParameterizedTestsThatFail\(17\)' ],
+      :to_ignore => [ ],
+    }
+  },
+
+  { :name => 'SupportCExceptionWhenRequested',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST', 'USE_CEXCEPTION'],
+    :options => {
+      :test_prefix => "extest",
+      :cexception => 1,
+    },
+    :expected => {
+      :to_pass => [ 'extest_ShouldHandleCExceptionInTest' ],
+      :to_fail => [ ],
+      :to_ignore => [ ],
+    }
+  },
+
+  { :name => 'CustomSetupAndTeardownThroughOptions',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :options => {
+      :test_prefix => "custtest|test",
+      :setup_name => "custom_setup",
+      :teardown_name => "custom_teardown",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'custtest_ThisTestPassesWhenCustomSetupRan',
+                    'custtest_ThisTestPassesWhenCustomTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'CustomSetupAndTeardownThroughCommandLine',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :cmdline => " --test_prefix=\"custtest|test\" --setup_name=\"custom_setup\" --teardown_name=\"custom_teardown\"",
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'custtest_ThisTestPassesWhenCustomSetupRan',
+                    'custtest_ThisTestPassesWhenCustomTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'CustomSetupAndTeardownThroughYaml',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :cmdline => " --test_prefix=\"custtest|test\"",
+    :yaml => {
+      :setup_name => "custom_setup",
+      :teardown_name => "custom_teardown",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'custtest_ThisTestPassesWhenCustomSetupRan',
+                    'custtest_ThisTestPassesWhenCustomTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'UseACustomMainFunction',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST', "USE_ANOTHER_MAIN"],
+    :options => {
+      :main_name => "custom_main",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'SupportCustomSuiteSetupAndTeardown',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :includes => ['Defs.h'],
+    :options => {
+      :test_prefix    => "suitetest|test",
+      :suite_setup    => "  CounterSuiteSetup = 1;",
+      :suite_teardown => "  return num_failures;",
+    },
+    :expected =>  {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'suitetest_ThisTestPassesWhenCustomSuiteSetupAndTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+
+  { :name => 'SupportMainExternDeclaration',
+    :testfile => 'testdata/testRunnerGeneratorWithMocks.c',
+    :testdefines => ['TEST'],
+    :includes => ['Defs.h'],
+    :options => {
+      :main_export_decl => "EXTERN_DECL",
+    },
+    :expected => {
+      :to_pass => [ 'test_ThisTestAlwaysPasses',
+                    'spec_ThisTestPassesWhenNormalSetupRan',
+                    'spec_ThisTestPassesWhenNormalTeardownRan',
+                    'test_NotBeConfusedByLongComplicatedStrings',
+                    'test_NotDisappearJustBecauseTheTestBeforeAndAfterHaveCrazyStrings',
+                    'test_StillNotBeConfusedByLongComplicatedStrings',
+                    'should_RunTestsStartingWithShouldByDefault',
+                    'spec_ThisTestPassesWhenNormalSuiteSetupAndTeardownRan',
+                    'test_ShouldCallMockInitAndVerifyFunctionsForEachTest',
+                  ],
+      :to_fail => [ 'test_ThisTestAlwaysFails' ],
+      :to_ignore => [ 'test_ThisTestAlwaysIgnored' ],
+    }
+  },
+]
+
+def runner_test(test, runner, expected, test_defines)
+  # Tack on TEST define for compiling unit tests
+  load_configuration($cfg_file)
+
+  #compile objects
+  obj_list = [
+    compile(runner, test_defines),
+    compile(test, test_defines),
+    compile('../src/unity.c', test_defines),
+  ]
+
+  # Link the test executable
+  test_base = File.basename(test, C_EXTENSION)
+  link_it(test_base, obj_list)
+
+  # Execute unit test and generate results file
+  simulator = build_simulator_fields
+  executable = $cfg['linker']['bin_files']['destination'] + test_base + $cfg['linker']['bin_files']['extension']
+  if simulator.nil?
+    cmd_str = executable
   else
-    report("    #{subtest}:PASS")
+    cmd_str = "#{simulator[:command]} #{simulator[:pre_support]} #{executable} #{simulator[:post_support]}"
   end
+  output = execute(cmd_str, true)
+
+  #compare to the expected pass/fail
+  allgood = expected[:to_pass].inject(true)      {|s,v| s && (/#{v}:PASS/ =~ output) }
+  allgood = expected[:to_fail].inject(allgood)   {|s,v| s && (/#{v}:FAIL/ =~ output) }
+  allgood = expected[:to_ignore].inject(allgood) {|s,v| s && (/#{v}:IGNORE/ =~ output) }
+  report output if (!allgood && !$verbose) #report failures if not already reporting everything
+  return allgood
 end
 
-should "GenerateARunnerByCreatingRunnerWithOptions" do
-  sets = { 'def'  => nil,
-           'new1' => { :plugins => [:cexception], :includes => ['one.h', 'two.h'], :enforce_strict_ordering => true },
-           'new2' => { :plugins => [:ignore], :suite_setup => "a_custom_setup();", :suite_teardown => "a_custom_teardown();" }
-  }
+RUNNER_TESTS.each do |testset|
+  testset_name = "Runner_#{testset[:testfile]}_#{testset[:name]}"
+  should testset_name do
+    runner_name = OUT_FILE + testset[:name] + '_runner.c'
 
-  sets.each_pair do |subtest, options|
-    UnityTestRunnerGenerator.new(options).run(TEST_FILE, OUT_FILE + subtest + '.c')
-    verify_output_equal(subtest)
-    UnityTestRunnerGenerator.new(options).run(TEST_MOCK, OUT_FILE + 'mock_' + subtest + '.c')
-    verify_output_equal('mock_' + subtest)
-  end
-end
+    #create a yaml file first if required
+    yaml_option = ""
+    if (testset[:yaml])
+      File.open("build/runner_options.yml",'w') {|f| f << { :unity => testset[:yaml] }.to_yaml }
+      yaml_option = "build/runner_options.yml"
+    end
 
-should "GenerateARunnerAlongWithAHeaderIfSpecified" do
-  sets = { 'head1' => { :header_file => "#{OUT_FILE}head1.h" } }
-  sets.each_pair do |subtest, options|
-    UnityTestRunnerGenerator.new(options).run(TEST_FILE, OUT_FILE + subtest + '.c')
-    verify_output_equal(subtest)
-  end
+    #run script via command line or through hash function call, as requested
+    if (testset[:cmdline])
+      cmdstr = "ruby ../auto/generate_test_runner.rb #{yaml_option} #{testset[:cmdline]} \"#{testset[:testfile]}\" \"#{runner_name}\""
+      `#{cmdstr}`
+    else
+      UnityTestRunnerGenerator.new(testset[:options]).run(testset[:testfile], runner_name)
+    end
 
-  sets = { 'head1' => { :header_file => "#{OUT_FILE}mock_head1.h" } }
-  sets.each_pair do |subtest, options|
-    UnityTestRunnerGenerator.new(options).run(TEST_MOCK, OUT_FILE + 'mock_' + subtest + '.c')
-    verify_output_equal('mock_' + subtest)
-  end
-end
-
-should "GenerateARunnerByRunningRunnerWithOptions" do
-  sets = { 'run1' => { :plugins => [:cexception], :includes => ['one.h', 'two.h'], :enforce_strict_ordering => true },
-           'run2' => { :plugins => [:ignore], :suite_setup => "a_custom_setup();", :suite_teardown => "a_custom_teardown();" }
-  }
-
-  sets.each_pair do |subtest, options|
-    UnityTestRunnerGenerator.new.run(TEST_FILE, OUT_FILE + subtest + '.c', options)
-    verify_output_equal(subtest)
-    UnityTestRunnerGenerator.new.run(TEST_MOCK, OUT_FILE + 'mock_' + subtest + '.c', options)
-    verify_output_equal('mock_' + subtest)
-  end
-end
-
-should "GenerateARunnerByPullingYamlOptions" do
-  subtest = 'yaml'
-  cmdstr = "ruby ../auto/generate_test_runner.rb testdata/sample.yml \"#{TEST_FILE}\" \"#{OUT_FILE + subtest + '.c'}\""
-  `#{cmdstr}`
-  verify_output_equal(subtest)
-
-  cmdstr = "ruby ../auto/generate_test_runner.rb testdata/sample.yml \"#{TEST_MOCK}\" \"#{OUT_FILE + 'mock_' + subtest + '.c'}\""
-  `#{cmdstr}`
-  verify_output_equal('mock_' + subtest)
-end
-
-should "GenerateARunnerByPullingCommandlineOptions" do
-  subtest = 'cmd'
-  cmdstr = "ruby ../auto/generate_test_runner.rb -cexception \"#{TEST_FILE}\" \"#{OUT_FILE + subtest + '.c'}\""
-  `#{cmdstr}`
-  verify_output_equal(subtest)
-
-  cmdstr = "ruby ../auto/generate_test_runner.rb -cexception \"#{TEST_MOCK}\" \"#{OUT_FILE + 'mock_' + subtest + '.c'}\""
-  `#{cmdstr}`
-  verify_output_equal('mock_' + subtest)
-end
-
-should "GenerateARunnerThatUsesParameterizedTests" do
-  sets = { 'param'  => { :plugins => [:ignore], :use_param_tests => true }
-  }
-
-  sets.each_pair do |subtest, options|
-    UnityTestRunnerGenerator.new(options).run(TEST_FILE, OUT_FILE + subtest + '.c')
-    verify_output_equal(subtest)
-    UnityTestRunnerGenerator.new(options).run(TEST_MOCK, OUT_FILE + 'mock_' + subtest + '.c')
-    verify_output_equal('mock_' + subtest)
+    #test the script against the specified test file and check results
+    if (runner_test(testset[:testfile], runner_name, testset[:expected], testset[:testdefines]))
+      report "#{testset_name}:PASS"
+    else
+      report "#{testset_name}:FAIL"
+      $generate_test_runner_failures += 1
+    end
+    $generate_test_runner_tests += 1
   end
 end