From 5993b66bca98247d7aa2e054bc74bc2c8951942e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Nov 2023 05:28:07 +0100 Subject: [PATCH] Add checking of all endpoints defined in FTL but not in the OpenAPI specs and vice versa Signed-off-by: DL6ER --- test/api/checkAPI.py | 37 ++++++++++----- test/api/libs/openAPI.py | 2 +- test/api/libs/responseVerifyer.py | 76 ++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 14 deletions(-) diff --git a/test/api/checkAPI.py b/test/api/checkAPI.py index 0d8fcd13..b315719a 100644 --- a/test/api/checkAPI.py +++ b/test/api/checkAPI.py @@ -48,7 +48,7 @@ if __name__ == "__main__": # Check if endpoints that are in both FTL and OpenAPI specs match # and have the same response format. Also verify that the examples # matches the OpenAPI specs. - print("Verifying the individual endpoint properties...") + print("Verifying the individual OpenAPI endpoint properties...") teleporter = None for path in openapi.endpoints["get"]: # We do not check the action endpoints as they'd trigger @@ -56,19 +56,32 @@ if __name__ == "__main__": # gravity, stutting down the system, etc. if path.startswith("/api/action"): continue - verifyer = ResponseVerifyer(ftl, openapi) - errors = verifyer.verify_endpoint(path) - if verifyer.teleporter_archive is not None: - teleporter = verifyer.teleporter_archive - if len(errors) == 0: - print(" GET " + path + " (" + verifyer.auth_method + " auth): OK") - else: - print(" GET " + path + " (" + verifyer.auth_method + " auth):") - for error in errors: - print(" - " + error) - errs[2] += len(errors) + with ResponseVerifyer(ftl, openapi) as verifyer: + errors = verifyer.verify_endpoint(path) + if verifyer.teleporter_archive is not None: + teleporter = verifyer.teleporter_archive + if len(errors) == 0: + print(" GET " + path + " (" + verifyer.auth_method + " auth): OK") + else: + print(" GET " + path + " (" + verifyer.auth_method + " auth):") + for error in errors: + print(" - " + error) + errs[2] += len(errors) print("") + # Verify that all the endpoint defined by GET /api/endpoints are documented + # and that there are no undocumented endpoints + print("Verifying the /api/endpoints endpoint...") + verifyer = ResponseVerifyer(ftl, openapi) + errors = verifyer.verify_endpoints() + if len(errors) == 0: + print(" GET /api/endpoints: OK") + else: + print(" Errors:") + for error in errors: + print(" - " + error) + errs[2] += len(errors) + # Verify FTL Teleporter import print("Verifying FTL Teleporter import...") verifyer = ResponseVerifyer(ftl, openapi) diff --git a/test/api/libs/openAPI.py b/test/api/libs/openAPI.py index d42be393..0cec4596 100644 --- a/test/api/libs/openAPI.py +++ b/test/api/libs/openAPI.py @@ -15,7 +15,7 @@ import json class openApi(): # List of methods we want to extract - METHODS = ["get", "post", "put", "delete"] + METHODS = ["get", "post", "put", "patch", "delete"] def __init__(self, base_path: str, api_root: str = "/api") -> None: # Store arguments diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index 77323beb..090717a7 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -34,6 +34,14 @@ class ResponseVerifyer(): self.errors = [] + def __enter__(self): + return self + + + def __exit__(self, exc_type, exc_value, traceback): + return + + def flatten_dict(self, d: MutableMapping, parent_key: str = '', sep: str ='.') -> MutableMapping: items = [] # Iterate over all items in the dictionary @@ -67,7 +75,7 @@ class ResponseVerifyer(): return self.errors # Get YAML response schema and examples (if applicable) - expected_mimetype = True + expected_mimetype = None # Assign random authentication method so we can test them all authentication_method = random.choice([a for a in AuthenticationMethods]) # Check if the expected response is defined in the API specs @@ -86,6 +94,11 @@ class ResponseVerifyer(): authentication_method = AuthenticationMethods.HEADER YAMLresponseSchema = None YAMLresponseExamples = None + elif 'text/html' in content: + expected_mimetype = 'text/html' + jsonData = content[expected_mimetype] + YAMLresponseSchema = None + YAMLresponseExamples = None else: # No response defined return self.errors @@ -166,6 +179,17 @@ class ResponseVerifyer(): # Store Teleporter archive for later use self.teleporter_archive = FTLresponse + elif expected_mimetype == "text/html": + # Decode the response if it is bytes + if type(FTLresponse) is bytes: + FTLresponse = FTLresponse.decode("utf-8") + elif type(FTLresponse) is not str: + self.errors.append("FTL's response is neither bytes nor string") + # Check if the document starts with either "" or + # "" (case-insensitive) + r = FTLresponse.lower() + if not r.startswith("") and not r.startswith(""): + self.errors.append("FTL's response does not start with or ") else: self.errors.append("Checker script does not know how to check for mimetype \"" + expected_mimetype + "\"") @@ -328,3 +352,53 @@ class ResponseVerifyer(): self.errors.append(f"FTL's reply ({str(ftl_type)}) does not match defined type ({yaml_type}) in {flat_path}") return False return all_okay + + + def verify_endpoints(self): + # Get FTL response + authentication_method = random.choice([a for a in AuthenticationMethods]) + FTLresponse = self.ftl.GET("/api/endpoints", authenticate = authentication_method) + if FTLresponse is None: + self.errors.append("No response from FTL API") + return self.errors + + # Construct full URI to check (this is what we specify in OpenAPI specs) + for method in FTLresponse['endpoints']: + for endpoint in FTLresponse['endpoints'][method]: + endpoint["full_uri"] = endpoint["uri"] + endpoint["parameters"] # type: str + # If the endpoint starts with /api, remove this part (it is not + # part of the YAML specs) + if endpoint["full_uri"].startswith("/api"): + endpoint["full_uri"] = endpoint["full_uri"][4:] + + # Do the same for the specified endpoints in the OpenAPI specs + openapi = {} + for endpoint in self.openapi.paths: + openapi[endpoint] = {} + for method in self.openapi.paths[endpoint]: + if method not in self.openapi.METHODS: + # Skip keys like "parameters" and "summary" + continue + #if "parameters" in self.openapi.paths[endpoint][method]: + # # Construct full URI to check (this is what we specify in OpenAPI specs) + # for param in self.openapi.paths[endpoint][method]["parameters"]: + # if param["in"] == "query": + # endpoint += "?" + param["name"] + "=" + urllib.parse.quote_plus(str(param["example"])) + openapi[endpoint][method] = endpoint + + # Check if FTL reports endpoints not defined in the API specs + for method in FTLresponse['endpoints']: + for endpoint in FTLresponse['endpoints'][method]: + m = method.upper() # type: str + if endpoint["full_uri"] not in self.openapi.paths or method not in self.openapi.paths[endpoint["full_uri"]]: + self.errors.append("Endpoint " + m + " " + endpoint["full_uri"] + " not found in the OpenAPI specs") + + # Check if all endpoints defined in the API specs are also defined in FTL + for endpoint in openapi: + for method in openapi[endpoint]: + full_uris = [ep["full_uri"] for ep in FTLresponse['endpoints'][method]] # type: list[str] + if endpoint not in full_uris: + m = method.upper() # type: str + self.errors.append("Endpoint " + m + " " + endpoint + " not found in FTL endpoints") + + return self.errors