Validate response status codes (#9605)

This checks that the status code of the response is a 200.
It also makes sure the headers are verified for the query and update tests.

toolset/test_types/abstract_test_type.py

@@ -28,6 +28,7 @@ class AbstractTestType(metaclass=abc.ABCMeta):
         self.args = args
         self.headers = ""
         self.body = ""
+        self.status = None
 
         if accept_header is None:
             self.accept_header = self.accept('json')
@@ -64,7 +65,7 @@ class AbstractTestType(metaclass=abc.ABCMeta):
                 "A %s requires the benchmark_config.json to contain %s" %
                 (self.name, self.args))
 
-    def request_headers_and_body(self, url):
+    def request_headers_and_body_and_status(self, url):
         '''
         Downloads a URL and returns the HTTP response headers
         and body content as a tuple
@@ -76,7 +77,8 @@ class AbstractTestType(metaclass=abc.ABCMeta):
 
         self.headers = r.headers
         self.body = r.content
-        return self.headers, self.body
+        self.status = r.status_code
+        return self.headers, self.body, self.status
 
     def output_headers_and_body(self):
         log(str(self.headers))

toolset/test_types/cached-query/cached-query.py

@@ -1,5 +1,5 @@
 from toolset.test_types.abstract_test_type import AbstractTestType
-from toolset.test_types.verifications import verify_query_cases
+from toolset.test_types.verifications import verify_query_cases, verify_status, verify_headers
 
 
 class TestType(AbstractTestType):
@@ -39,6 +39,10 @@ class TestType(AbstractTestType):
                  "Route for cached queries must be at least 15 characters, found '{}' instead".format(self.cached_query_url),
                  url))
 
+        headers, body, status = self.request_headers_and_body_and_status(url)
+        problems += verify_status(self.request_headers_and_body_and_status, status, url)
+        problems += verify_headers(self.request_headers_and_body_and_status, headers, url, should_be='json')
+
         if len(problems) == 0:
             return [('pass', '', url + case) for case, _ in cases]
         else:

toolset/test_types/db/db.py

@@ -1,5 +1,5 @@
 from toolset.test_types.abstract_test_type import AbstractTestType
-from toolset.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object, verify_queries_count
+from toolset.test_types.verifications import basic_body_verification, verify_status, verify_headers, verify_randomnumber_object, verify_queries_count
 
 
 class TestType(AbstractTestType):
@@ -28,7 +28,7 @@ class TestType(AbstractTestType):
         expected_queries = repetitions * concurrency
 
         url = base_url + self.db_url
-        headers, body = self.request_headers_and_body(url)
+        headers, body, status = self.request_headers_and_body_and_status(url)
 
         response, problems = basic_body_verification(body, url)
 
@@ -62,7 +62,8 @@ class TestType(AbstractTestType):
 
         # Verify response content
         problems += verify_randomnumber_object(response, url)
-        problems += verify_headers(self.request_headers_and_body, headers, url, should_be='json')
+        problems += verify_status(self.request_headers_and_body_and_status, status, url)
+        problems += verify_headers(self.request_headers_and_body_and_status, headers, url, should_be='json')
 
         if len(problems) == 0:
             problems += verify_queries_count(self, "World", url, concurrency, repetitions, expected_queries, expected_queries)

toolset/test_types/fortune/fortune.py

@@ -30,7 +30,7 @@ class TestType(AbstractTestType):
         expected_rows = 12 * expected_queries
 
         url = base_url + self.fortune_url
-        headers, body = self.request_headers_and_body(url)
+        headers, body, status = self.request_headers_and_body_and_status(url)
 
         _, problems = basic_body_verification(body, url, is_json_check=False)
 
@@ -49,7 +49,7 @@ class TestType(AbstractTestType):
         (valid, diff) = parser.isValidFortune(self.name, body.decode())
 
         if valid:
-            problems += verify_headers(self.request_headers_and_body, headers, url, should_be='html')
+            problems += verify_headers(self.request_headers_and_body_and_status, headers, url, should_be='html')
             if len(problems) == 0:
                 problems += verify_queries_count(self, "fortune", url, concurrency, repetitions, expected_queries, expected_rows)
             if len(problems) == 0:

toolset/test_types/json/json.py

@@ -1,5 +1,5 @@
 from toolset.test_types.abstract_test_type import AbstractTestType
-from toolset.test_types.verifications import basic_body_verification, verify_headers, verify_helloworld_object
+from toolset.test_types.verifications import basic_body_verification, verify_status, verify_headers, verify_helloworld_object
 
 class TestType(AbstractTestType):
     def __init__(self, config):
@@ -23,7 +23,7 @@ class TestType(AbstractTestType):
         '''
 
         url = base_url + self.json_url
-        headers, body = self.request_headers_and_body(url)
+        headers, body, status = self.request_headers_and_body_and_status(url)
 
         response, problems = basic_body_verification(body, url)
 
@@ -38,7 +38,8 @@ class TestType(AbstractTestType):
             return problems
 
         problems += verify_helloworld_object(response, url)
-        problems += verify_headers(self.request_headers_and_body, headers, url, should_be='json')
+        problems += verify_status(self.request_headers_and_body_and_status, status, url)
+        problems += verify_headers(self.request_headers_and_body_and_status, headers, url, should_be='json')
 
         if len(problems) > 0:
             return problems

toolset/test_types/plaintext/plaintext.py

@@ -1,4 +1,4 @@
-from toolset.test_types.verifications import basic_body_verification, verify_headers
+from toolset.test_types.verifications import basic_body_verification, verify_status, verify_headers
 from toolset.test_types.abstract_test_type import AbstractTestType
 
 
@@ -15,7 +15,7 @@ class TestType(AbstractTestType):
 
     def verify(self, base_url):
         url = base_url + self.plaintext_url
-        headers, body = self.request_headers_and_body(url)
+        headers, body, status = self.request_headers_and_body_and_status(url)
 
         _, problems = basic_body_verification(body, url, is_json_check=False)
 
@@ -45,7 +45,8 @@ class TestType(AbstractTestType):
                   "This may negatively affect benchmark performance." %
                   extra_bytes), url))
 
-        problems += verify_headers(self.request_headers_and_body, headers, url, should_be='plaintext')
+        problems += verify_status(self.request_headers_and_body_and_status, status, url)
+        problems += verify_headers(self.request_headers_and_body_and_status, headers, url, should_be='plaintext')
 
         if len(problems) == 0:
             return [('pass', '', url)]

toolset/test_types/query/query.py

@@ -1,5 +1,5 @@
 from toolset.test_types.abstract_test_type import AbstractTestType
-from toolset.test_types.verifications import verify_query_cases
+from toolset.test_types.verifications import verify_query_cases, verify_status, verify_headers
 
 
 class TestType(AbstractTestType):
@@ -39,6 +39,10 @@ class TestType(AbstractTestType):
                  "Route for queries must be at least 9 characters, found '{}' instead".format(self.query_url),
                  url))
 
+        headers, body, status = self.request_headers_and_body_and_status(url)
+        problems += verify_status(self.request_headers_and_body_and_status, status, url)
+        problems += verify_headers(self.request_headers_and_body_and_status, headers, url, should_be='json')
+
         if len(problems) == 0:
             return [('pass', '', url + case) for case, _ in cases]
         else:

toolset/test_types/update/update.py

@@ -1,5 +1,5 @@
 from toolset.test_types.abstract_test_type import AbstractTestType
-from toolset.test_types.verifications import verify_query_cases
+from toolset.test_types.verifications import verify_query_cases, verify_status, verify_headers
 
 
 class TestType(AbstractTestType):
@@ -38,6 +38,10 @@ class TestType(AbstractTestType):
                  "Route for update must be at least 8 characters, found '{}' instead".format(self.update_url),
                  url))
 
+        headers, body, status = self.request_headers_and_body_and_status(url)
+        problems += verify_status(self.request_headers_and_body_and_status, status, url)
+        problems += verify_headers(self.request_headers_and_body_and_status, headers, url, should_be='json')
+
         if len(problems) == 0:
             return [('pass', '', url + case) for (case, _) in cases]
         else:

toolset/test_types/verifications.py

@@ -41,8 +41,23 @@ def basic_body_verification(body, url, is_json_check=True):
     # they do not need or expect a dict back
     return None, []
 
+def verify_status(request_headers_and_body_and_status, status, url, expected_status=200):
+    '''
+    Verifies the status code of a framework response
+    '''
+
+    problems = []
+
+    if status is not expected_status:
+        problems.append((
+            'fail',
+            'Invalid response status, found \"%s\", did not match \"%s\"'
+            % (status, expected_status), url))
+
+    return problems
+
 
-def verify_headers(request_headers_and_body, headers, url, should_be='json'):
+def verify_headers(request_headers_and_body_and_status, headers, url, should_be='json'):
     '''
     Verifies the headers of a framework response
     param `should_be` is a switch for the three acceptable content types
@@ -76,7 +91,7 @@ def verify_headers(request_headers_and_body, headers, url, should_be='json'):
     # Verify response content
     # Make sure that the date object isn't cached
     sleep(3)
-    second_headers, body2 = request_headers_and_body(url)
+    second_headers, body2, status2 = request_headers_and_body_and_status(url)
     second_date = second_headers.get('Date')
 
     date2 = second_headers.get('Date')
@@ -349,7 +364,7 @@ def verify_query_cases(self, cases, url, check_updates=False):
 
     for q, max_infraction in cases:
         case_url = url + q
-        headers, body = self.request_headers_and_body(case_url)
+        headers, body, status = self.request_headers_and_body_and_status(case_url)
 
         try:
             queries = int(q)  # drops down for 'foo' and ''
@@ -363,7 +378,7 @@ def verify_query_cases(self, cases, url, check_updates=False):
 
             problems += verify_randomnumber_list(expected_len, headers, body,
                                                  case_url, max_infraction)
-            problems += verify_headers(self.request_headers_and_body, headers, case_url)
+            problems += verify_headers(self.request_headers_and_body_and_status, headers, case_url)
 
             # Only check update changes if we are doing an Update verification and if we're testing
             # the highest number of queries, to ensure that we don't accidentally FAIL for a query
@@ -393,7 +408,7 @@ def verify_query_cases(self, cases, url, check_updates=False):
                 # parameter input
                 problems += verify_randomnumber_list(
                     expected_len, headers, body, case_url, max_infraction)
-                problems += verify_headers(self.request_headers_and_body, headers, case_url)
+                problems += verify_headers(self.request_headers_and_body_and_status, headers, case_url)
 
     if hasattr(self, 'database'):
         # verify the number of queries and rows read for 20 queries, with a concurrency level of 512, with 2 repetitions