Validate if we the test suite runs both good and bad tests (#3811)

Also, remove the ansible-lint configuration file because we can rely on the default one

.github/linters/.ansible-lint.yml

@@ -1,27 +0,0 @@
----
-########################
-# Make output parsable #
-########################
-parseable: true
-
-################
-# Tags to skip #
-################
-skip_list:
-  - 'empty-string-compare'  # Allow compare to empty string
-  - '204'  # Allow string length greater than 160 chars
-  - 'no-changed-when'  # False positives for running command shells
-  - 'command-instead-of-module'  # Allow git commands for push, add, etc...
-  - 'command-instead-of-shell'  # Allow use of shell when you want
-  - 'no-handler'  # Allow step to run like handler
-  - 'unnamed-task'  # Allow tasks without a name
-
-#############
-# Use rules #
-#############
-use_default_rules: true
-
-#################
-# Set verbosity #
-#################
-verbosity: 2

lib/functions/worker.sh

@@ -39,6 +39,10 @@ function LintCodebase() {
   SKIP_FLAG=0
   INDEX=0
 
+  # We use these flags to check how many "bad" and "good" test cases we ran
+  BAD_TEST_CASES_COUNT=0
+  GOOD_TEST_CASES_COUNT=0
+
   ############################################################
   # Check to see if we need to go through array or all files #
   ############################################################
@@ -300,6 +304,9 @@ function LintCodebase() {
       # Check for if it was supposed to pass #
       ########################################
       if [[ ${FILE_STATUS} == "good" ]]; then
+        # Increase the good test cases count
+        (("GOOD_TEST_CASES_COUNT++"))
+
         ##############################
         # Check the shell for errors #
         ##############################
@@ -333,6 +340,10 @@ function LintCodebase() {
         #######################################
         # File status = bad, this should fail #
         #######################################
+
+        # Increase the bad test cases count
+        (("BAD_TEST_CASES_COUNT++"))
+
         ##############################
         # Check the shell for errors #
         ##############################
@@ -372,14 +383,30 @@ function LintCodebase() {
     )
   done
 
-  ##############################
+  if [ "${TEST_CASE_RUN}" = "true" ]; then
-  # Validate we ran some tests #
+
-  ##############################
+    debug "The ${LINTER_NAME} (linter: ${LINTER_NAME}) test suite has ${INDEX} test, of which ${BAD_TEST_CASES_COUNT} 'bad' (supposed to fail), ${GOOD_TEST_CASES_COUNT} 'good' (supposed to pass)."
-  if [ "${TEST_CASE_RUN}" = "true" ] && [ "${INDEX}" -eq 0 ]; then
+
-    #################################################
+    # Check if we ran at least one test
-    # We failed to find files and no tests were ran #
+    if [ "${INDEX}" -eq 0 ]; then
-    #################################################
+      error "Failed to find any tests ran for the Linter:[${LINTER_NAME}]!"
-    error "Failed to find any tests ran for the Linter:[${LINTER_NAME}]!"
+      fatal "Validate logic and that tests exist for linter: ${LINTER_NAME}"
-    fatal "Validate logic and that tests exist for linter: ${LINTER_NAME}"
+    fi
+
+    # Check if we ran 'bad' tests
+    if [ "${BAD_TEST_CASES_COUNT}" -eq 0 ]; then
+      if [ "${FILE_TYPE}" = "ANSIBLE" ]; then
+        debug "There are no 'bad' tests for ${FILE_TYPE}, but it's a corner case that we allow because ${LINTER_NAME} is supposed to lint entire directories and the test suite doesn't support this corner case for 'bad' tests yet."
+      else
+        error "Failed to find any tests that are expected to fail for the Linter:[${LINTER_NAME}]!"
+        fatal "Validate logic and that tests that are expected to fail exist for linter: ${LINTER_NAME}"
+      fi
+    fi
+
+    # Check if we ran 'good' tests
+    if [ "${GOOD_TEST_CASES_COUNT}" -eq 0 ]; then
+      error "Failed to find any tests that are expected to pass for the Linter:[${LINTER_NAME}]!"
+      fatal "Validate logic and that tests that are expected to pass exist for linter: ${LINTER_NAME}"
+    fi
   fi
 }