diff --git a/README.md b/README.md index 38f5961..7b9297f 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Modify the config\example.ini file to enter the system details under below secti | online_profiles | boolean | Whether to download online profiles | | debugging | boolean | Whether to print debug to log | | required_profiles_dir | string | Option to set the root folder of required profiles | +| collectionlimit | string | Sets a limit to links gathered from collections by type, e.g. `ComputerSystem 20` limits ComputerSystemCollection to 20 links | ### Payload options diff --git a/config/example.ini b/config/example.ini index e3fc9b3..b81cd57 100644 --- a/config/example.ini +++ b/config/example.ini @@ -18,4 +18,5 @@ logdir = ./logs oemcheck = True online_profiles = True debugging = False +collectionlimit = LogEntry 20 diff --git a/redfish_interop_validator/RedfishInteropValidator.py b/redfish_interop_validator/RedfishInteropValidator.py index 72d5bc5..8430d8d 100644 --- a/redfish_interop_validator/RedfishInteropValidator.py +++ b/redfish_interop_validator/RedfishInteropValidator.py @@ -74,6 +74,7 @@ def main(argslist=None, configfile=None): argget.add_argument('--nooemcheck', action='store_false', dest='oemcheck', help='Don\'t check OEM items') argget.add_argument('--debugging', action="store_true", help='Output debug statements to text log, otherwise it only uses INFO') argget.add_argument('--required_profiles_dir', type=str, help='root directory for required profiles') + argget.add_argument('--collectionlimit', type=str, default=['LogEntry', '20'], help='apply a limit to collections (format: RESOURCE1 COUNT1 RESOURCE2 COUNT2...)', nargs='+') # Config information unique to Interop Validator argget.add_argument('profile', type=str, default='sample.json', nargs='+', help='interop profile with which to validate service against') @@ -135,6 +136,10 @@ def main(argslist=None, configfile=None): my_logger.error('IP is missing ip/host') return 1, None, 'IP Incomplete' + if len(args.collectionlimit) % 2 != 0: + my_logger.error('Collection Limit requires two arguments per entry (ResourceType Count)') + return 1, None, 'Collection Limit Incomplete' + # Start printing config details, remove redundant/private info from print my_logger.info('Target URI: ' + args.ip) my_logger.info('\n'.join( diff --git a/redfish_interop_validator/config.py b/redfish_interop_validator/config.py index 61cb996..7fdee15 100644 --- a/redfish_interop_validator/config.py +++ b/redfish_interop_validator/config.py @@ -11,7 +11,7 @@ config_struct = { 'Tool': ['verbose'], 'Host': ['ip', 'username', 'password', 'description', 'forceauth', 'authtype', 'token'], - 'Validator': ['payload', 'logdir', 'oemcheck', 'debugging'] + 'Validator': ['payload', 'logdir', 'oemcheck', 'debugging', 'required_profiles_dir', 'collectionlimit'] } config_options = [x for name in config_struct for x in config_struct[name]] diff --git a/redfish_interop_validator/traverseInterop.py b/redfish_interop_validator/traverseInterop.py index 58243c6..ece76a8 100644 --- a/redfish_interop_validator/traverseInterop.py +++ b/redfish_interop_validator/traverseInterop.py @@ -77,6 +77,15 @@ def __init__(self, my_config): self.config['certificatebundle'] = None self.config['timeout'] = 10 + # NOTE: this is a validator limitation. maybe move this to its own config inside validateResource + if self.config['collectionlimit']: + total_len = len(self.config['collectionlimit']) / 2 + limit_string = ' '.join(self.config['collectionlimit']) + limit_array = [tuple(found_item.split(' ')) for found_item in re.findall(r"[A-Za-z]+ [0-9]+", limit_string)] + if len(limit_array) != total_len: + raise ValueError('Collection Limit array seems malformed, use format: RESOURCE1 COUNT1 RESOURCE2 COUNT2)...') + self.config['collectionlimit'] = {x[0]: int(x[1]) for x in limit_array} + # httpprox = config['httpproxy'] # httpsprox = config['httpsproxy'] # self.proxies['http'] = httpprox if httpprox != "" else None diff --git a/redfish_interop_validator/validateResource.py b/redfish_interop_validator/validateResource.py index fe15cd5..5a2e874 100644 --- a/redfish_interop_validator/validateResource.py +++ b/redfish_interop_validator/validateResource.py @@ -19,7 +19,6 @@ def filter(self, rec): fmt = logging.Formatter('%(levelname)s - %(message)s') - def create_logging_capture(this_logger): errorMessages = StringIO() warnMessages = StringIO() @@ -120,11 +119,17 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem oemcheck = traverseInterop.config.get('oemcheck', True) + collection_limit = traverseInterop.config.get('collectionlimit', {'LogEntry': 20}) + if SchemaType not in profile_resources: my_logger.verbose1('Visited {}, type {}'.format(URI, SchemaType)) # Get all links available - links = getURIsInProperty(jsondata, uriName, oemcheck) - return True, counts, results, links, resource_obj + links, limited_links = getURIsInProperty(jsondata, uriName, oemcheck, collection_limit) + return True, counts, results, (links, limited_links), resource_obj + + if '_count' not in profile_resources[SchemaType]: + profile_resources[SchemaType]['_count'] = 0 + profile_resources[SchemaType]['_count'] += 1 # Verify odata_id properly resolves to its parent if holding fragment odata_id = resource_obj.jsondata.get('@odata.id', '') @@ -170,7 +175,7 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem my_logger.info('%s, %s\n', SchemaFullType, counts) # Get all links available - links = getURIsInProperty(resource_obj.jsondata, uriName, oemcheck) + links, limited_links = getURIsInProperty(resource_obj.jsondata, uriName, oemcheck, collection_limit) results[uriName]['warns'], results[uriName]['errors'] = get_my_capture(my_logger, whandler), get_my_capture(my_logger, ehandler) @@ -184,34 +189,45 @@ def validateSingleURI(URI, profile, uriName='', expectedType=None, expectedSchem for msg in results[uriName]['messages']: msg.parent_results = results - return True, counts, results, links, resource_obj + return True, counts, results, (links, limited_links), resource_obj import re urlCheck = re.compile(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+") allowable_annotations = ['@odata.id'] -def getURIsInProperty(property, name='Root', oemcheck=True): - my_links = {} +def getURIsInProperty(property, name='Root', oemcheck=True, collection_limit={}): + my_links, limited_links = {}, {} # Return nothing if we are Oem if not oemcheck and name == 'Oem': - return my_links + return my_links, limited_links if isinstance(property, dict): - for x, y in property.items(): - if '@' in x and x.lower() not in allowable_annotations: + for sub_name, value in property.items(): + if '@' in sub_name and sub_name.lower() not in allowable_annotations: continue - if isinstance(y, str) and x.lower() in ['@odata.id']: - my_link = getURIfromOdata(y) + if isinstance(value, str) and sub_name.lower() in ['@odata.id']: + my_link = getURIfromOdata(value) if my_link: if '/Oem/' not in my_link: my_links[name] = my_link if '/Oem/' in my_link and oemcheck: my_links[name] = my_link else: - my_links.update(getURIsInProperty(y, "{}:{}".format(name, x), oemcheck)) + new_links, new_limited_links = getURIsInProperty(value, "{}:{}".format(name, sub_name), oemcheck) + limited_links.update(new_limited_links) + parent_type = property.get('@odata.type', '') + if sub_name == 'Members' and 'Collection' in parent_type: + my_type = getType(parent_type).split('Collection')[0] + if my_type in collection_limit: + new_limited_links = {x: new_links[x] for x in list(new_links.keys())[collection_limit[my_type]:]} + new_links = {x: new_links[x] for x in list(new_links.keys())[:collection_limit[my_type]]} + limited_links.update(new_limited_links) + my_links.update(new_links) if isinstance(property, list): for n, x in enumerate(property): - my_links.update(getURIsInProperty(x, "{}#{}".format(name, n), oemcheck)) - return my_links + new_links, new_limited_links = getURIsInProperty(x, "{}#{}".format(name, n), oemcheck) + limited_links.update(new_limited_links) + my_links.update(new_links) + return my_links, limited_links def getURIfromOdata(property): if '.json' not in property[:-5].lower(): @@ -238,6 +254,10 @@ def validateURITree(URI, profile, uriName, expectedType=None, expectedSchema=Non # Validate top URI validateSuccess, counts, results, links, resource_obj = \ validateSingleURI(URI, profile, uriName, expectedType, expectedSchema, expectedJson) + + links, limited_links = links + for skipped_link in limited_links: + allLinks.add(limited_links[skipped_link]) if resource_obj: SchemaType = getType(resource_obj.jsondata.get('@odata.type', 'NoType')) @@ -278,7 +298,7 @@ def validateURITree(URI, profile, uriName, expectedType=None, expectedSchema=Non # NOTE: unable to determine autoexpanded resources without Schema else: - linkSuccess, linkCounts, linkResults, innerLinks, linkobj = \ + linkSuccess, linkCounts, linkResults, inner_links, linkobj = \ validateSingleURI(link, profile, linkName, parent=parent) allLinks.add(link.rstrip('/')) @@ -289,7 +309,12 @@ def validateURITree(URI, profile, uriName, expectedType=None, expectedSchema=Non if not linkSuccess: continue - innerLinksTuple = [(link, innerLinks[link], linkobj) for link in innerLinks] + inner_links, inner_limited_links = inner_links + + for skipped_link in inner_limited_links: + allLinks.add(inner_limited_links[skipped_link]) + + innerLinksTuple = [(link, inner_links[link], linkobj) for link in inner_links] newLinks.extend(innerLinksTuple) SchemaType = getType(linkobj.jsondata.get('@odata.type', 'NoType'))