Disable "auto" selector for IS-04-04, IS-05-03, IS-09-02 & BCP-003-01

nmostesting/ControllerTest.py

@@ -76,15 +76,13 @@ class ControllerTest(GenericTest):
     """
     Testing initial set up of new test suite for controller testing
     """
-    def __init__(self, apis, registries, node, dns_server, disable_auto=True):
-        # Remove the spec_path as there are no corresponding GitHub repos for Controller Tests
-        apis[CONTROLLER_TEST_API_KEY].pop("spec_path", None)
+    def __init__(self, apis, registries, node, dns_server, **kwargs):
         if CONFIG.ENABLE_HTTPS:
             # Comms with Testing Facade are http only
             apis[CONTROLLER_TEST_API_KEY]["base_url"] \
                 = apis[CONTROLLER_TEST_API_KEY]["base_url"].replace("https", "http")
             apis[CONTROLLER_TEST_API_KEY]["url"] = apis[CONTROLLER_TEST_API_KEY]["url"].replace("https", "http")
-        GenericTest.__init__(self, apis, disable_auto=disable_auto)
+        GenericTest.__init__(self, apis, **kwargs)
         self.authorization = False
         self.primary_registry = registries[1]
         self.node = node

nmostesting/GenericTest.py

@@ -86,7 +86,7 @@ def __init__(self, apis, omit_paths=None, disable_auto=False, auths=None, **kwar
         test = Test("Test initialisation")
 
         for api_name, api_data in self.apis.items():
-            if "spec_path" not in api_data or api_data["version"] is None:
+            if self.disable_auto or "spec_path" not in api_data or api_data["version"] is None:
                 continue
 
             repo = git.Repo(api_data["spec_path"])
@@ -122,7 +122,7 @@ def __init__(self, apis, omit_paths=None, disable_auto=False, auths=None, **kwar
     def parse_RAML(self):
         """Create a Specification object for each API defined in this object"""
         for api in self.apis:
-            if "raml" not in self.apis[api]:
+            if self.disable_auto or "raml" not in self.apis[api]:
                 continue
             raml_path = os.path.join(self.apis[api]["spec_path"] + '/APIs/' + self.apis[api]["raml"])
             self.apis[api]["spec"] = Specification(raml_path)

nmostesting/NMOSTesting.py

@@ -221,6 +221,7 @@
             "api_key": "query",
             "disable_fields": ["host", "port"]
         }],
+        "disable_auto": True,
         "class": IS0404Test.IS0404Test
     },
     "IS-05-01": {
@@ -256,6 +257,7 @@
             "api_key": "connection",
             "disable_fields": ["host", "port"]
         }],
+        "disable_auto": True,
         "class": IS0503Test.IS0503Test
     },
     "IS-06-01": {
@@ -329,6 +331,7 @@
             "api_key": "system",
             "disable_fields": ["host", "port"]
         }],
+        "disable_auto": True,
         "class": IS0902Test.IS0902Test
     },
     # IS-10 testing is disabled until testing can be refactored to deal with commercial servers
@@ -346,6 +349,7 @@
             "spec_key": "bcp-003-01",
             "api_key": "secure"
         }],
+        "disable_auto": True,
         "class": BCP00301Test.BCP00301Test
     },
     "BCP-006-01-01": {
@@ -382,12 +386,11 @@
 }
 
 
-def enumerate_tests(class_def, describe=False):
-    if describe:
-        tests = ["all: Runs all tests in the suite",
-                 "auto: Basic API tests derived directly from the specification RAML"]
-    else:
-        tests = ["all", "auto"]
+def enumerate_tests(class_def, describe=False, disable_auto=False):
+    tests = ["all: Runs all tests in the suite" if describe else "all"]
+    if not disable_auto:
+        tests.append("auto: Basic API tests derived directly from the specification RAML" if describe else "auto")
+
     for method_name in dir(class_def):
         if method_name.startswith("test_"):
             method = getattr(class_def, method_name)
@@ -453,8 +456,12 @@ class DataForm(Form):
     for test_id in TEST_DEFINITIONS:
         test_data[test_id] = copy.deepcopy(TEST_DEFINITIONS[test_id])
         test_data[test_id].pop("class")
-        test_data[test_id]["test_methods"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"])
-        test_data[test_id]["test_descriptions"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], describe=True)
+        test_data[test_id]["test_methods"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"],
+                                                             disable_auto=TEST_DEFINITIONS[test_id]
+                                                             .get("disable_auto"))
+        test_data[test_id]["test_descriptions"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], describe=True,
+                                                                  disable_auto=TEST_DEFINITIONS[test_id]
+                                                                  .get("disable_auto"))
 
     hidden_options = HiddenField(default=max_endpoints)
     hidden_tests = HiddenField(default=json.dumps(test_data))
@@ -617,7 +624,8 @@ def run_tests(test, endpoints, test_selection=["all"]):
                                      registries=REGISTRIES,
                                      node=NODE,
                                      dns_server=DNS_SERVER,
-                                     auths=[PRIMARY_AUTH, SECONDARY_AUTH])
+                                     auths=[PRIMARY_AUTH, SECONDARY_AUTH],
+                                     disable_auto=test_def.get("disable_auto"))
 
         core_app.config['TEST_ACTIVE'] = time.time()
         try:
@@ -839,11 +847,14 @@ def validate_args(args, access_type="cli"):
             msg = "ERROR: The requested test suite '{}' does not exist".format(args.suite)
             return_type = ExitCodes.ERROR
         elif args.list_tests:
-            tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"])
+            tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"],
+                                    disable_auto=TEST_DEFINITIONS[args.suite].get("disable_auto"))
             for test_name in tests:
                 msg += test_name + '\n'
         elif args.describe_tests:
-            tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"], describe=True)
+            tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"],
+                                    disable_auto=TEST_DEFINITIONS[args.suite].get("disable_auto"),
+                                    describe=True)
             for test_description in tests:
                 msg += test_description + '\n'
         elif getattr(args, "selection", "all") not in enumerate_tests(TEST_DEFINITIONS[args.suite]["class"]):

nmostesting/suites/BCP00301Test.py

@@ -31,7 +31,7 @@ class BCP00301Test(GenericTest):
     Runs BCP-003-01-Test
     """
     def __init__(self, apis, **kwargs):
-        GenericTest.__init__(self, apis)
+        GenericTest.__init__(self, apis, **kwargs)
         if not CONFIG.ENABLE_HTTPS:
             raise NMOSInitException("BCP-003-01 can only be tested when ENABLE_HTTPS is set to True in UserConfig.py")
         self.authorization = False  # Don't send tokens in every request

nmostesting/suites/IS0404Test.py

@@ -29,8 +29,8 @@ class IS0404Test(ControllerTest):
     """
     Testing initial set up of new test suite for controller testing
     """
-    def __init__(self, apis, registries, node, dns_server, **kwargs):
-        ControllerTest.__init__(self, apis, registries, node, dns_server)
+    def __init__(self, apis, **kwargs):
+        ControllerTest.__init__(self, apis, **kwargs)
 
     def set_up_tests(self):
         # Sender initial details

nmostesting/suites/IS0503Test.py

@@ -28,8 +28,8 @@ class IS0503Test(ControllerTest):
     """
     Testing initial set up of new test suite for controller testing
     """
-    def __init__(self, apis, registries, node, dns_server, **kwargs):
-        ControllerTest.__init__(self, apis, registries, node, dns_server)
+    def __init__(self, apis, **kwargs):
+        ControllerTest.__init__(self, apis, **kwargs)
 
     def set_up_tests(self):
         # Sender initial details

nmostesting/suites/IS0902Test.py

@@ -30,7 +30,7 @@ class IS0902Test(GenericTest):
     Runs IS-09-02-Test
     """
     def __init__(self, apis, systems, dns_server, **kwargs):
-        GenericTest.__init__(self, apis, disable_auto=True)
+        GenericTest.__init__(self, apis, **kwargs)
         self.authorization = False  # System API doesn't use auth, so don't send tokens in every request
         self.invalid_system = systems[0]
         self.primary_system = systems[1]