From 67defaea1dfe8c920028e458239310d078fd0d9d Mon Sep 17 00:00:00 2001 From: kntrain Date: Thu, 7 Jan 2021 15:35:40 +0100 Subject: [PATCH 01/14] -added additional show options -replaced run_raw call with file reference in wrapper --- aws_list_all/__main__.py | 11 ++++- aws_list_all/listing.py | 98 +++++++++++++++++++++++++++++----------- aws_list_all/query.py | 30 ++++++++++-- 3 files changed, 106 insertions(+), 33 deletions(-) diff --git a/aws_list_all/__main__.py b/aws_list_all/__main__.py index 15a87c9..9bedfbd 100755 --- a/aws_list_all/__main__.py +++ b/aws_list_all/__main__.py @@ -83,6 +83,9 @@ def main(): ) show.add_argument('listingfile', nargs='*', help='listing file(s) to load and print') show.add_argument('-v', '--verbose', action='count', help='print given listing files with detailed info') + show.add_argument('-n', '--not_found', default=False, type=bool, help='additionally print listing files of resources not found') + show.add_argument('-e', '--errors', default=False, type=bool, help='additionally print listing files of resources where queries resulted in errors') + show.add_argument('-d', '--denied', default=False, type=bool, help='additionally print listing files of resources with "missing permission" errors') # Introspection debugging is not the main function. So we put it all into a subcommand. introspect = subparsers.add_parser( @@ -161,7 +164,13 @@ def main(): elif args.command == 'show': if args.listingfile: increase_limit_nofiles() - do_list_files(args.listingfile, verbose=args.verbose or 0) + do_list_files( + args.listingfile, + verbose=args.verbose or 0, + not_found=args.not_found, + errors=args.errors, + denied=args.denied + ) else: show.print_help() return 1 diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index 82e5912..c94d300 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -1,3 +1,4 @@ +import json import pprint import boto3 @@ -79,14 +80,58 @@ def run_raw_listing_operation(service, region, operation, profile): return getattr(client, api_to_method_mapping[operation])(**parameters) +class ListingFile(object): + + def __init__(self, input, directory='./'): + self.input = input + self.directory = directory + + @property + def resources(self): + response = self.input.resources + + # Special handling for service-level kms keys; derived from alias name. + if self.input.service == 'kms' and self.input.operation == 'ListKeys': + aliases_file = '{}_{}_{}_{}.json'.format(self.input.service, 'ListAliases', self.input.region, self.input.profile) + aliases_file = self.directory + aliases_file + aliases_listing = Listing.from_json(json.load(open(aliases_file, 'rb'))) + list_aliases = aliases_listing.response + service_key_ids = [ + k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) + if k.get('AliasName').lower().startswith('alias/aws') + ] + response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] + + # Filter default Internet Gateways + if self.input.service == 'ec2' and self.input.operation == 'DescribeInternetGateways': + vpcs_file = '{}_{}_{}_{}.json'.format(self.input.service, 'DescribeVpcs', self.input.region, self.input.profile) + vpcs_file = self.directory + vpcs_file + vpcs_listing = Listing.from_json(json.load(open(vpcs_file, 'rb'))) + describe_vpcs = getattr(vpcs_listing, 'response') + vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} + internet_gateways = [] + for ig in response['InternetGateways']: + attachments = ig.get('Attachments', []) + # more than one, it cannot be default. + if len(attachments) != 1: + continue + vpc = attachments[0].get('VpcId') + if not vpcs.get(vpc, {}).get('IsDefault', False): + internet_gateways.append(ig) + response['InternetGateways'] = internet_gateways + + return response + + class Listing(object): """Represents a listing operation on an AWS service and its result""" - def __init__(self, service, region, operation, response, profile): + def __init__(self, service, region, operation, response, profile, error=''): self.service = service self.region = region self.operation = operation self.response = response self.profile = profile + self.error = error def to_json(self): return { @@ -95,6 +140,7 @@ def to_json(self): 'profile': self.profile, 'operation': self.operation, 'response': self.response, + 'error': self.error, } @classmethod @@ -104,7 +150,8 @@ def from_json(cls, data): region=data.get('region'), profile=data.get('profile'), operation=data.get('operation'), - response=data.get('response') + response=data.get('response'), + error=data.get('error') ) @property @@ -132,13 +179,15 @@ def __str__(self): def acquire(cls, service, region, operation, profile): """Acquire the given listing by making an AWS request""" response = run_raw_listing_operation(service, region, operation, profile) - if response['ResponseMetadata']['HTTPStatusCode'] != 200: - raise Exception('Bad AWS HTTP Status Code', response) + # if response['ResponseMetadata']['HTTPStatusCode'] != 200: + # raise Exception('Bad AWS HTTP Status Code', response) return cls(service, region, operation, response, profile) @property def resources(self): # pylint:disable=too-many-branches """Transform the response data into a dict of resource names to resource listings""" + if not(self.response): + return self.response.copy() response = self.response.copy() complete = True @@ -231,15 +280,6 @@ def resources(self): # pylint:disable=too-many-branches if not alias.get('AliasName').lower().startswith('alias/aws') ] - # Special handling for service-level kms keys; derived from alias name. - if self.service == 'kms' and self.operation == 'ListKeys': - list_aliases = run_raw_listing_operation(self.service, self.region, 'ListAliases', self.profile) - service_key_ids = [ - k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) - if k.get('AliasName').lower().startswith('alias/aws') - ] - response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] - # Filter PUBLIC images from appstream if self.service == 'appstream' and self.operation == 'DescribeImages': response['Images'] = [ @@ -336,20 +376,20 @@ def resources(self): # pylint:disable=too-many-branches if self.service == 'ec2' and self.operation == 'DescribeNetworkAcls': response['NetworkAcls'] = [nacl for nacl in response['NetworkAcls'] if not nacl['IsDefault']] - # Filter default Internet Gateways - if self.service == 'ec2' and self.operation == 'DescribeInternetGateways': - describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile) - vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} - internet_gateways = [] - for ig in response['InternetGateways']: - attachments = ig.get('Attachments', []) - # more than one, it cannot be default. - if len(attachments) != 1: - continue - vpc = attachments[0].get('VpcId') - if not vpcs.get(vpc, {}).get('IsDefault', False): - internet_gateways.append(ig) - response['InternetGateways'] = internet_gateways + # # Filter default Internet Gateways + # if self.service == 'ec2' and self.operation == 'DescribeInternetGateways': + # describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile) + # vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} + # internet_gateways = [] + # for ig in response['InternetGateways']: + # attachments = ig.get('Attachments', []) + # # more than one, it cannot be default. + # if len(attachments) != 1: + # continue + # vpc = attachments[0].get('VpcId') + # if not vpcs.get(vpc, {}).get('IsDefault', False): + # internet_gateways.append(ig) + # response['InternetGateways'] = internet_gateways # Filter Public images from ec2.fpga images if self.service == 'ec2' and self.operation == 'DescribeFpgaImages': @@ -379,4 +419,8 @@ def resources(self): # pylint:disable=too-many-branches if not complete: response['truncated'] = [True] + # if self.operation == 'DescribeInternetGateways': + # print('RIGHT HERE!') + # print(response.values()) + return response diff --git a/aws_list_all/query.py b/aws_list_all/query.py index 337906a..3157f64 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -12,7 +12,7 @@ from traceback import print_exc from .introspection import get_listing_operations, get_regions_for_service -from .listing import Listing +from .listing import Listing, ListingFile RESULT_NOTHING = '---' RESULT_SOMETHING = '+++' @@ -239,9 +239,9 @@ def acquire_listing(verbose, what): if verbose > 1: print(what, '...request successful') print("timing [success]:", duration, what) - if listing.resource_total_count > 0: - with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: + with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: json.dump(listing.to_json(), jsonfile, default=datetime.isoformat) + if listing.resource_total_count > 0: return (RESULT_SOMETHING, service, region, operation, profile, ', '.join(listing.resource_types)) else: return (RESULT_NOTHING, service, region, operation, profile, ', '.join(listing.resource_types)) @@ -266,19 +266,39 @@ def acquire_listing(verbose, what): if not_available_string in str(exc): result_type = RESULT_NOTHING + listing = Listing(service, region, operation, {}, profile, result_type) + with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: + json.dump(listing.to_json(), jsonfile, default=datetime.isoformat) return (result_type, service, region, operation, profile, repr(exc)) -def do_list_files(filenames, verbose=0): +def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=False): """Print out a rudimentary summary of the Listing objects contained in the given files""" + dir = filenames[0][:filenames[0].rfind('/') + 1] for listing_filename in filenames: listing = Listing.from_json(json.load(open(listing_filename, 'rb'))) - resources = listing.resources + listing_entry = ListingFile(listing, dir) + resources = listing_entry.resources + truncated = False + was_denied = False if 'truncated' in resources: truncated = resources['truncated'] del resources['truncated'] + if listing.error == RESULT_NO_ACCESS: + was_denied = True + if not resources: + print(listing.service, listing.region, listing.operation, 'MISSING PERMISSION', '0') + if listing.error == RESULT_ERROR and errors: + print(listing.service, listing.region, listing.operation, 'ERROR', '0') + for resource_type, value in resources.items(): + if not not_found and len(value) == 0 and not was_denied: + continue + if not denied and was_denied: + continue + if was_denied: + resource_type = 'MISSING PERMISSION' len_string = '> {}'.format(len(value)) if truncated else str(len(value)) print(listing.service, listing.region, listing.operation, resource_type, len_string) if verbose > 0: From faec31c5abdcd5ace26c40b91bf7d0c1b90c10b5 Mon Sep 17 00:00:00 2001 From: kntrain Date: Thu, 7 Jan 2021 16:17:40 +0100 Subject: [PATCH 02/14] -moved modifications in resources to filter class --- aws_list_all/listing.py | 265 +++++++------------------- aws_list_all/resource_filter.py | 318 ++++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+), 203 deletions(-) create mode 100644 aws_list_all/resource_filter.py diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index c94d300..265b353 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -4,6 +4,7 @@ import boto3 from .client import get_client +from .resource_filter import * PARAMETERS = { 'cloudfront': { @@ -193,188 +194,67 @@ def resources(self): # pylint:disable=too-many-branches del response['ResponseMetadata'] - # Transmogrify strange cloudfront results into standard AWS format - if self.service == 'cloudfront': - assert len(response.keys()) == 1, 'Unexpected cloudfront response: {}'.format(response) - key = list(response.keys())[0][:-len('List')] - response = list(response.values())[0] - response[key] = response.get('Items', []) - - # medialive List* things sends a next token; remove if no channels/lists - if self.service == 'medialive': - if self.operation == 'ListChannels' and not response['Channels']: - if 'Channels' in response: - del response['Channels'] - if 'NextToken' in response: - del response['NextToken'] - if self.operation == 'ListInputs' and not response['Inputs']: - if 'Inputs' in response: - del response['Inputs'] - if 'NextToken' in response: - del response['NextToken'] - - # ssm ListCommands sends a next token; remove if no channels - if self.service == 'ssm' and self.operation == 'ListCommands': - if 'NextToken' in response and not response['Commands']: - del response['NextToken'] - - # SNS ListSubscriptions always sends a next token... - if self.service == 'sns' and self.operation == 'ListSubscriptions': - del response['NextToken'] - - # Athena has a "primary" work group that is always present - if self.service == 'athena' and self.operation == 'ListWorkGroups': - response['WorkGroups'] = [wg for wg in response.get('WorkGroups', []) if wg['Name'] != 'primary'] - - # Remove default event buses - if self.service == 'events' and self.operation == 'ListEventBuses': - response['EventBuses'] = [wg for wg in response.get('EventBuses', []) if wg['Name'] != 'default'] - - # XRay has a "Default" group that is always present - if self.service == 'xray' and self.operation == 'GetGroups': - response['Groups'] = [wg for wg in response.get('Groups', []) if wg['GroupName'] != 'Default'] - - if self.service == 'route53resolver': - if self.operation == 'ListResolverRules': - response['ResolverRules'] = [ - rule for rule in response.get('ResolverRules', []) - if rule['Id'] != 'rslvr-autodefined-rr-internet-resolver' - ] - if self.operation == 'ListResolverRuleAssociations': - response['ResolverRuleAssociations'] = [ - rule for rule in response.get('ResolverRuleAssociations', []) - if rule['ResolverRuleId'] != 'rslvr-autodefined-rr-internet-resolver' - ] - - if 'Count' in response: - if 'MaxResults' in response: - if response['MaxResults'] <= response['Count']: - complete = False - del response['MaxResults'] - del response['Count'] - - if 'Quantity' in response: - if 'MaxItems' in response: - if response['MaxItems'] <= response['Quantity']: - complete = False - del response['MaxItems'] - del response['Quantity'] - - for neutral_thing in ('MaxItems', 'MaxResults', 'Quantity'): - if neutral_thing in response: - del response[neutral_thing] - - for bad_thing in ( - 'hasMoreResults', 'IsTruncated', 'Truncated', 'HasMoreApplications', 'HasMoreDeliveryStreams', - 'HasMoreStreams', 'NextToken', 'NextMarker', 'nextMarker', 'Marker' - ): - if bad_thing in response: - if response[bad_thing]: - complete = False - del response[bad_thing] - - # Special handling for Aliases in kms, there are some reserved AWS-managed aliases. - if self.service == 'kms' and self.operation == 'ListAliases': - response['Aliases'] = [ - alias for alias in response.get('Aliases', []) - if not alias.get('AliasName').lower().startswith('alias/aws') - ] - - # Filter PUBLIC images from appstream - if self.service == 'appstream' and self.operation == 'DescribeImages': - response['Images'] = [ - image for image in response.get('Images', []) if not image.get('Visibility', 'PRIVATE') == 'PUBLIC' - ] - - # This API returns a dict instead of a list - if self.service == 'cloudsearch' and self.operation == 'ListDomainNames': - response['DomainNames'] = list(response['DomainNames'].items()) - - # Only list CloudTrail trails in own/Home Region - if self.service == 'cloudtrail' and self.operation == 'DescribeTrails': - response['trailList'] = [ - trail for trail in response['trailList'] - if trail.get('HomeRegion') == self.region or not trail.get('IsMultiRegionTrail') - ] - - # Remove AWS-default cloudwatch metrics - if self.service == 'cloudwatch' and self.operation == 'ListMetrics': - response['Metrics'] = [ - metric for metric in response['Metrics'] if not metric.get('Namespace').startswith('AWS/') - ] - - # Remove AWS supplied policies - if self.service == 'iam' and self.operation == 'ListPolicies': - response['Policies'] = [ - policy for policy in response['Policies'] if not policy['Arn'].startswith('arn:aws:iam::aws:') - ] - - # Owner Info is not necessary - if self.service == 's3' and self.operation == 'ListBuckets': - del response['Owner'] - - # Remove failures from ecs/DescribeClusters - if self.service == 'ecs' and self.operation == 'DescribeClusters': - if 'failures' in response: - del response['failures'] - - # This API returns a dict instead of a list - if self.service == 'pinpoint' and self.operation == 'GetApps': - response['ApplicationsResponse'] = response.get('ApplicationsResponse', {}).get('Items', []) - - # Remove AWS-defined Baselines - if self.service == 'ssm' and self.operation == 'DescribePatchBaselines': - response['BaselineIdentities'] = [ - line for line in response['BaselineIdentities'] if not line['BaselineName'].startswith('AWS-') - ] - - # Remove default DB Security Group - if self.service in 'rds' and self.operation == 'DescribeDBSecurityGroups': - response['DBSecurityGroups'] = [ - group for group in response['DBSecurityGroups'] if group['DBSecurityGroupName'] != 'default' - ] - - # Remove default DB Parameter Groups - if self.service in ('rds', 'neptune', 'docdb') and self.operation in 'DescribeDBParameterGroups': - response['DBParameterGroups'] = [ - group for group in response['DBParameterGroups'] - if not group['DBParameterGroupName'].startswith('default.') - ] - - # Remove default DB Cluster Parameter Groups - if self.service in ('rds', 'neptune', 'docdb') and self.operation in 'DescribeDBClusterParameterGroups': - response['DBClusterParameterGroups'] = [ - group for group in response['DBClusterParameterGroups'] - if not group['DBClusterParameterGroupName'].startswith('default.') - ] - - # Remove default DB Option Groups - if self.service == 'rds' and self.operation == 'DescribeOptionGroups': - response['OptionGroupsList'] = [ - group for group in response['OptionGroupsList'] if not group['OptionGroupName'].startswith('default:') - ] - - # Filter default VPCs - if self.service == 'ec2' and self.operation == 'DescribeVpcs': - response['Vpcs'] = [vpc for vpc in response['Vpcs'] if not vpc['IsDefault']] - - # Filter default Subnets - if self.service == 'ec2' and self.operation == 'DescribeSubnets': - response['Subnets'] = [net for net in response['Subnets'] if not net['DefaultForAz']] - - # Filter default SGs - if self.service == 'ec2' and self.operation == 'DescribeSecurityGroups': - response['SecurityGroups'] = [sg for sg in response['SecurityGroups'] if sg['GroupName'] != 'default'] - - # Filter main route tables - if self.service == 'ec2' and self.operation == 'DescribeRouteTables': - response['RouteTables'] = [ - rt for rt in response['RouteTables'] if not any(x['Main'] for x in rt['Associations']) - ] - - # Filter default Network ACLs - if self.service == 'ec2' and self.operation == 'DescribeNetworkAcls': - response['NetworkAcls'] = [nacl for nacl in response['NetworkAcls'] if not nacl['IsDefault']] + cloudfront_filter = CloudfrontFilter() + medialive_filter = MedialiveFilter() + ssmListCommands_filter = SSMListCommandsFilter() + snsListSubscriptions_filter = SNSListSubscriptionsFilter() + athenaWorkGroups_filter = AthenaWorkGroupsFilter() + listEventBuses_filter = ListEventBusesFilter() + xRayGroups_filter = XRayGroupsFilter() + kmsListAliases_filter = KMSListAliasesFilter() + appstreamImages_filter = AppstreamImagesFilter() + cloudsearch_filter = CloudsearchFilter() + cloudTrail_filter = CloudTrailFilter() + cloudWatch_filter = CloudWatchFilter() + iamPolicies_filter = IAMPoliciesFilter() + s3Owner_filter = S3OwnerFilter() + ecsClustersFailure_filter = ECSClustersFailureFilter() + pinpointGetApps_filter = PinpointGetAppsFilter() + ssmBaselines_filter = SSMBaselinesFilter() + dbSecurityGroups_filter = DBSecurityGroupsFilter() + dbParameterGroups_filter = DBParameterGroupsFilter() + dbClusterParameterGroups_filter = DBClusterParameterGroupsFilter() + dbOptionGroups_filter = DBOptionGroupsFilter() + ec2VPC_filter = EC2VPCFilter() + ec2Subnets_filter = EC2SubnetsFilter() + ec2SecurityGroups_filter = EC2SecurityGroupsFilter() + ec2RouteTables_filter = EC2RouteTablesFilter() + ec2NetworkAcls_filter = EC2NetworkAclsFilter() + ec2FpgaImgaes_filter = EC2FpgaImgaesFilter() + workmailDeletedOrganizations_filter = WorkmailDeletedOrganizationsFilter() + elasticacheSubnetGroups_filter = ElasticacheSubnetGroupsFilter() + nextToken_filter = NextTokenFilter() + + cloudfront_filter.execute(self, response) + medialive_filter.execute(self, response) + ssmListCommands_filter.execute(self, response) + snsListSubscriptions_filter.execute(self, response) + athenaWorkGroups_filter.execute(self, response) + listEventBuses_filter.execute(self, response) + xRayGroups_filter.execute(self, response) + kmsListAliases_filter.execute(self, response) + appstreamImages_filter.execute(self, response) + cloudsearch_filter.execute(self, response) + cloudTrail_filter.execute(self, response) + cloudWatch_filter.execute(self, response) + iamPolicies_filter.execute(self, response) + s3Owner_filter.execute(self, response) + ecsClustersFailure_filter.execute(self, response) + pinpointGetApps_filter.execute(self, response) + ssmBaselines_filter.execute(self, response) + dbSecurityGroups_filter.execute(self, response) + dbParameterGroups_filter.execute(self, response) + dbClusterParameterGroups_filter.execute(self, response) + dbOptionGroups_filter.execute(self, response) + ec2VPC_filter.execute(self, response) + ec2Subnets_filter.execute(self, response) + ec2SecurityGroups_filter.execute(self, response) + ec2RouteTables_filter.execute(self, response) + ec2NetworkAcls_filter.execute(self, response) + ec2FpgaImgaes_filter.execute(self, response) + workmailDeletedOrganizations_filter.execute(self, response) + elasticacheSubnetGroups_filter.execute(self, response) + nextToken_filter.execute(self, response) # # Filter default Internet Gateways # if self.service == 'ec2' and self.operation == 'DescribeInternetGateways': @@ -391,27 +271,6 @@ def resources(self): # pylint:disable=too-many-branches # internet_gateways.append(ig) # response['InternetGateways'] = internet_gateways - # Filter Public images from ec2.fpga images - if self.service == 'ec2' and self.operation == 'DescribeFpgaImages': - response['FpgaImages'] = [image for image in response.get('FpgaImages', []) if not image.get('Public')] - - # Remove deleted Organizations - if self.service == 'workmail' and self.operation == 'ListOrganizations': - response['OrganizationSummaries'] = [ - s for s in response.get('OrganizationSummaries', []) if not s.get('State') == 'Deleted' - ] - - if self.service == 'elasticache' and self.operation == 'DescribeCacheSubnetGroups': - response['CacheSubnetGroups'] = [ - g for g in response.get('CacheSubnetGroups', []) if g.get('CacheSubnetGroupName') != 'default' - ] - - # interpret nextToken in several services - if (self.service, self.operation) in (('inspector', 'ListFindings'), ('logs', 'DescribeLogGroups')): - if response.get('nextToken'): - complete = False - del response['nextToken'] - for key, value in response.items(): if not isinstance(value, list): raise Exception('No listing: {} is no list:'.format(key), response) diff --git a/aws_list_all/resource_filter.py b/aws_list_all/resource_filter.py new file mode 100644 index 0000000..59bc164 --- /dev/null +++ b/aws_list_all/resource_filter.py @@ -0,0 +1,318 @@ +import json +import pprint + +import boto3 + +from .client import get_client + +class ResourceFilter: + def execute(self, listing, response): + pass + +class CloudfrontFilter: + def execute(self, listing, response): + # Transmogrify strange cloudfront results into standard AWS format + if listing.service == 'cloudfront': + assert len(response.keys()) == 1, 'Unexpected cloudfront response: {}'.format(response) + key = list(response.keys())[0][:-len('List')] + response = list(response.values())[0] + response[key] = response.get('Items', []) + +class MedialiveFilter: + def execute(self, listing, response): + # medialive List* things sends a next token; remove if no channels/lists + if listing.service == 'medialive': + if listing.operation == 'ListChannels' and not response['Channels']: + if 'Channels' in response: + del response['Channels'] + if 'NextToken' in response: + del response['NextToken'] + if listing.operation == 'ListInputs' and not response['Inputs']: + if 'Inputs' in response: + del response['Inputs'] + if 'NextToken' in response: + del response['NextToken'] + +class SSMListCommandsFilter: + def execute(self, listing, response): + # ssm ListCommands sends a next token; remove if no channels + if listing.service == 'ssm' and listing.operation == 'ListCommands': + if 'NextToken' in response and not response['Commands']: + del response['NextToken'] + +class SNSListSubscriptionsFilter: + def execute(self, listing, response): + # SNS ListSubscriptions always sends a next token... + if listing.service == 'sns' and listing.operation == 'ListSubscriptions': + del response['NextToken'] + +class AthenaWorkGroupsFilter: + def execute(self, listing, response): + # Athena has a "primary" work group that is always present + if listing.service == 'athena' and listing.operation == 'ListWorkGroups': + response['WorkGroups'] = [wg for wg in response.get('WorkGroups', []) if wg['Name'] != 'primary'] + +class ListEventBusesFilter: + def execute(self, listing, response): + # Remove default event buses + if listing.service == 'events' and listing.operation == 'ListEventBuses': + response['EventBuses'] = [wg for wg in response.get('EventBuses', []) if wg['Name'] != 'default'] + +class XRayGroupsFilter: + def execute(self, listing, response): + # XRay has a "Default" group that is always present + if listing.service == 'xray' and listing.operation == 'GetGroups': + response['Groups'] = [wg for wg in response.get('Groups', []) if wg['GroupName'] != 'Default'] + + if listing.service == 'route53resolver': + if listing.operation == 'ListResolverRules': + response['ResolverRules'] = [ + rule for rule in response.get('ResolverRules', []) + if rule['Id'] != 'rslvr-autodefined-rr-internet-resolver' + ] + if listing.operation == 'ListResolverRuleAssociations': + response['ResolverRuleAssociations'] = [ + rule for rule in response.get('ResolverRuleAssociations', []) + if rule['ResolverRuleId'] != 'rslvr-autodefined-rr-internet-resolver' + ] + + if 'Count' in response: + if 'MaxResults' in response: + if response['MaxResults'] <= response['Count']: + complete = False + del response['MaxResults'] + del response['Count'] + + if 'Quantity' in response: + if 'MaxItems' in response: + if response['MaxItems'] <= response['Quantity']: + complete = False + del response['MaxItems'] + del response['Quantity'] + + for neutral_thing in ('MaxItems', 'MaxResults', 'Quantity'): + if neutral_thing in response: + del response[neutral_thing] + + for bad_thing in ( + 'hasMoreResults', 'IsTruncated', 'Truncated', 'HasMoreApplications', 'HasMoreDeliveryStreams', + 'HasMoreStreams', 'NextToken', 'NextMarker', 'nextMarker', 'Marker' + ): + if bad_thing in response: + if response[bad_thing]: + complete = False + del response[bad_thing] + +class KMSListAliasesFilter: + def execute(self, listing, response): + # Special handling for Aliases in kms, there are some reserved AWS-managed aliases. + if listing.service == 'kms' and listing.operation == 'ListAliases': + response['Aliases'] = [ + alias for alias in response.get('Aliases', []) + if not alias.get('AliasName').lower().startswith('alias/aws') + ] + +# class KMSListKeysFilter: +# def __init__(self, directory): +# self.directory = directory + +# def execute(self, listing, response): +# # Special handling for service-level kms keys; derived from alias name. +# if listing.input.service == 'kms' and listing.input.operation == 'ListKeys': +# #list_aliases = run_raw_listing_operation(self.service, self.region, 'ListAliases', self.profile) +# aliases_file = '{}_{}_{}_{}.json'.format(listing.input.service, 'ListAliases', listing.input.region, listing.input.profile) +# aliases_file = self.directory + aliases_file +# aliases_listing = Listing.from_json(json.load(open(aliases_file, 'rb'))) +# list_aliases = aliases_listing.response +# service_key_ids = [ +# k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) +# if k.get('AliasName').lower().startswith('alias/aws') +# ] +# response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] + +class AppstreamImagesFilter: + def execute(self, listing, response): + # Filter PUBLIC images from appstream + if listing.service == 'appstream' and listing.operation == 'DescribeImages': + response['Images'] = [ + image for image in response.get('Images', []) if not image.get('Visibility', 'PRIVATE') == 'PUBLIC' + ] + +class CloudsearchFilter: + def execute(self, listing, response): + # This API returns a dict instead of a list + if listing.service == 'cloudsearch' and listing.operation == 'ListDomainNames': + response['DomainNames'] = list(response['DomainNames'].items()) + +class CloudTrailFilter: + def execute(self, listing, response): + # Only list CloudTrail trails in own/Home Region + if listing.service == 'cloudtrail' and listing.operation == 'DescribeTrails': + response['trailList'] = [ + trail for trail in response['trailList'] + if trail.get('HomeRegion') == self.region or not trail.get('IsMultiRegionTrail') + ] + +class CloudWatchFilter: + def execute(self, listing, response): + # Remove AWS-default cloudwatch metrics + if listing.service == 'cloudwatch' and listing.operation == 'ListMetrics': + response['Metrics'] = [ + metric for metric in response['Metrics'] if not metric.get('Namespace').startswith('AWS/') + ] + +class IAMPoliciesFilter: + def execute(self, listing, response): + # Remove AWS supplied policies + if listing.service == 'iam' and listing.operation == 'ListPolicies': + response['Policies'] = [ + policy for policy in response['Policies'] if not policy['Arn'].startswith('arn:aws:iam::aws:') + ] + +class S3OwnerFilter: + def execute(self, listing, response): + # Owner Info is not necessary + if listing.service == 's3' and listing.operation == 'ListBuckets': + del response['Owner'] + +class ECSClustersFailureFilter: + def execute(self, listing, response): + # Remove failures from ecs/DescribeClusters + if listing.service == 'ecs' and listing.operation == 'DescribeClusters': + if 'failures' in response: + del response['failures'] + +class PinpointGetAppsFilter: + def execute(self, listing, response): + # This API returns a dict instead of a list + if listing.service == 'pinpoint' and listing.operation == 'GetApps': + response['ApplicationsResponse'] = response.get('ApplicationsResponse', {}).get('Items', []) + +class SSMBaselinesFilter: + def execute(self, listing, response): + # Remove AWS-defined Baselines + if listing.service == 'ssm' and listing.operation == 'DescribePatchBaselines': + response['BaselineIdentities'] = [ + line for line in response['BaselineIdentities'] if not line['BaselineName'].startswith('AWS-') + ] + +class DBSecurityGroupsFilter: + def execute(self, listing, response): + # Remove default DB Security Group + if listing.service in 'rds' and listing.operation == 'DescribeDBSecurityGroups': + response['DBSecurityGroups'] = [ + group for group in response['DBSecurityGroups'] if group['DBSecurityGroupName'] != 'default' + ] + +class DBParameterGroupsFilter: + def execute(self, listing, response): + # Remove default DB Parameter Groups + if listing.service in ('rds', 'neptune', 'docdb') and listing.operation in 'DescribeDBParameterGroups': + response['DBParameterGroups'] = [ + group for group in response['DBParameterGroups'] + if not group['DBParameterGroupName'].startswith('default.') + ] + +class DBClusterParameterGroupsFilter: + def execute(self, listing, response): + # Remove default DB Cluster Parameter Groups + if listing.service in ('rds', 'neptune', 'docdb') and listing.operation in 'DescribeDBClusterParameterGroups': + response['DBClusterParameterGroups'] = [ + group for group in response['DBClusterParameterGroups'] + if not group['DBClusterParameterGroupName'].startswith('default.') + ] + +class DBOptionGroupsFilter: + def execute(self, listing, response): + # Remove default DB Option Groups + if listing.service == 'rds' and listing.operation == 'DescribeOptionGroups': + response['OptionGroupsList'] = [ + group for group in response['OptionGroupsList'] if not group['OptionGroupName'].startswith('default:') + ] + +class EC2VPCFilter: + def execute(self, listing, response): + # Filter default VPCs + if listing.service == 'ec2' and listing.operation == 'DescribeVpcs': + response['Vpcs'] = [vpc for vpc in response['Vpcs'] if not vpc['IsDefault']] + +class EC2SubnetsFilter: + def execute(self, listing, response): + # Filter default Subnets + if listing.service == 'ec2' and listing.operation == 'DescribeSubnets': + response['Subnets'] = [net for net in response['Subnets'] if not net['DefaultForAz']] + +class EC2SecurityGroupsFilter: + def execute(self, listing, response): + # Filter default SGs + if listing.service == 'ec2' and listing.operation == 'DescribeSecurityGroups': + response['SecurityGroups'] = [sg for sg in response['SecurityGroups'] if sg['GroupName'] != 'default'] + +class EC2RouteTablesFilter: + def execute(self, listing, response): + # Filter main route tables + if listing.service == 'ec2' and listing.operation == 'DescribeRouteTables': + response['RouteTables'] = [ + rt for rt in response['RouteTables'] if not any(x['Main'] for x in rt['Associations']) + ] + +class EC2NetworkAclsFilter: + def execute(self, listing, response): + # Filter default Network ACLs + if listing.service == 'ec2' and listing.operation == 'DescribeNetworkAcls': + response['NetworkAcls'] = [nacl for nacl in response['NetworkAcls'] if not nacl['IsDefault']] + +# class EC2InternetGatewaysFilter: +# def __init__(self, directory): +# self.directory = directory + +# def execute(self, listing, response): +# # Filter default Internet Gateways +# if listing.input.service == 'ec2' and listing.input.operation == 'DescribeInternetGateways': +# #describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile) +# vpcs_file = '{}_{}_{}_{}.json'.format(listing.input.service, 'DescribeVpcs', listing.input.region, listing.input.profile) +# vpcs_file = self.directory + vpcs_file +# vpcs_listing = Listing.from_json(json.load(open(vpcs_file, 'rb'))) +# describe_vpcs = vpcs_listing.response +# vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} +# internet_gateways = [] +# # print(self.response) +# # print(response) +# for ig in response['InternetGateways']: +# attachments = ig.get('Attachments', []) +# # more than one, it cannot be default. +# if len(attachments) != 1: +# continue +# vpc = attachments[0].get('VpcId') +# if not vpcs.get(vpc, {}).get('IsDefault', False): +# internet_gateways.append(ig) +# response['InternetGateways'] = internet_gateways + +class EC2FpgaImgaesFilter: + def execute(self, listing, response): + # Filter Public images from ec2.fpga images + if listing.service == 'ec2' and listing.operation == 'DescribeFpgaImages': + response['FpgaImages'] = [image for image in response.get('FpgaImages', []) if not image.get('Public')] + +class WorkmailDeletedOrganizationsFilter: + def execute(self, listing, response): + # Remove deleted Organizations + if listing.service == 'workmail' and listing.operation == 'ListOrganizations': + response['OrganizationSummaries'] = [ + s for s in response.get('OrganizationSummaries', []) if not s.get('State') == 'Deleted' + ] + +class ElasticacheSubnetGroupsFilter: + def execute(self, listing, response): + if listing.service == 'elasticache' and listing.operation == 'DescribeCacheSubnetGroups': + response['CacheSubnetGroups'] = [ + g for g in response.get('CacheSubnetGroups', []) if g.get('CacheSubnetGroupName') != 'default' + ] + +class NextTokenFilter: + def execute(self, listing, response): + # interpret nextToken in several services + if (listing.service, listing.operation) in (('inspector', 'ListFindings'), ('logs', 'DescribeLogGroups')): + if response.get('nextToken'): + complete = False + del response['nextToken'] \ No newline at end of file From 889c555e694ed1254866aeda17dc6e15a234e0d7 Mon Sep 17 00:00:00 2001 From: kntrain Date: Thu, 14 Jan 2021 03:18:31 +0100 Subject: [PATCH 03/14] -fixed restructuring issues -moved resources execution to apply_filter -added command line option for resource filters --- aws_list_all/__main__.py | 18 +++- aws_list_all/apply_filter.py | 163 ++++++++++++++++++++++++++++++++ aws_list_all/listing.py | 113 +++++++++------------- aws_list_all/query.py | 22 +++-- aws_list_all/resource_filter.py | 118 +++++++++++++---------- 5 files changed, 307 insertions(+), 127 deletions(-) create mode 100644 aws_list_all/apply_filter.py diff --git a/aws_list_all/__main__.py b/aws_list_all/__main__.py index 9bedfbd..2b4854d 100755 --- a/aws_list_all/__main__.py +++ b/aws_list_all/__main__.py @@ -72,6 +72,12 @@ def main(): action='append', help='Restrict querying to the given operation (can be specified multiple times)' ) + query.add_argument( + '-u', + '--unfilter', + action='append', + help='Exclude given default-value filter from being applied (can be specified multiple times)' + ) query.add_argument('-p', '--parallel', default=32, type=int, help='Number of request to do in parallel') query.add_argument('-d', '--directory', default='.', help='Directory to save result listings to') query.add_argument('-v', '--verbose', action='count', help='Print detailed info during run') @@ -86,6 +92,12 @@ def main(): show.add_argument('-n', '--not_found', default=False, type=bool, help='additionally print listing files of resources not found') show.add_argument('-e', '--errors', default=False, type=bool, help='additionally print listing files of resources where queries resulted in errors') show.add_argument('-d', '--denied', default=False, type=bool, help='additionally print listing files of resources with "missing permission" errors') + show.add_argument( + '-u', + '--unfilter', + action='append', + help='Exclude given default-value filter from being applied (can be specified multiple times)' + ) # Introspection debugging is not the main function. So we put it all into a subcommand. introspect = subparsers.add_parser( @@ -159,7 +171,8 @@ def main(): args.operation, verbose=args.verbose or 0, parallel=args.parallel, - selected_profile=args.profile + selected_profile=args.profile, + unfilter=args.unfilter ) elif args.command == 'show': if args.listingfile: @@ -169,7 +182,8 @@ def main(): verbose=args.verbose or 0, not_found=args.not_found, errors=args.errors, - denied=args.denied + denied=args.denied, + unfilter=args.unfilter ) else: show.print_help() diff --git a/aws_list_all/apply_filter.py b/aws_list_all/apply_filter.py new file mode 100644 index 0000000..9cf2c83 --- /dev/null +++ b/aws_list_all/apply_filter.py @@ -0,0 +1,163 @@ +import json +import os +import sys + +from .resource_filter import * + +def apply_filters(listing, unfilter, response, complete): + unfilterList = [] + if unfilter is not None: + if isinstance(unfilter, list): + unfilterList = unfilter + else: + unfilterList.append(unfilter) + apply_complete = complete + + if not('cloudfront' in unfilterList): + filter = CloudfrontFilter() + filter.execute(listing, response) + + if not('medialive' in unfilterList): + filter = MedialiveFilter() + filter.execute(listing, response) + + if not('ssmListCommands' in unfilterList): + filter = SSMListCommandsFilter() + filter.execute(listing, response) + + if not('snsListSubscriptions' in unfilterList): + filter = SNSListSubscriptionsFilter() + filter.execute(listing, response) + + if not('athenaWorkGroups' in unfilterList): + filter = AthenaWorkGroupsFilter() + filter.execute(listing, response) + + if not('listEventBuses' in unfilterList): + filter = ListEventBusesFilter() + filter.execute(listing, response) + + if not('xRayGroups' in unfilterList): + filter = XRayGroupsFilter() + filter.execute(listing, response) + + if not('route53Resolver' in unfilterList): + filter = Route53ResolverFilter() + filter.execute(listing, response) + + if not('count' in unfilterList): + filter = CountFilter(apply_complete) + filter.execute(listing, response) + apply_complete = getattr(filter, 'complete') + + if not('quantity' in unfilterList): + filter = QuantityFilter(apply_complete) + filter.execute(listing, response) + apply_complete = getattr(filter, 'complete') + + if not('neutralThing' in unfilterList): + filter = NeutralThingFilter() + filter.execute(listing, response) + + if not('badThing' in unfilterList): + filter = BadThingFilter(apply_complete) + filter.execute(listing, response) + apply_complete = getattr(filter, 'complete') + + if not('kmsListAliases' in unfilterList): + filter = KMSListAliasesFilter() + filter.execute(listing, response) + + if not('appstreamImages' in unfilterList): + filter = AppstreamImagesFilter() + filter.execute(listing, response) + + if not('cloudsearch' in unfilterList): + filter = CloudsearchFilter() + filter.execute(listing, response) + + if not('cloudTrail' in unfilterList): + filter = CloudTrailFilter() + filter.execute(listing, response) + + if not('cloudWatch' in unfilterList): + filter = CloudWatchFilter() + filter.execute(listing, response) + + if not('iamPolicies' in unfilterList): + filter = IAMPoliciesFilter() + filter.execute(listing, response) + + if not('s3Owner' in unfilterList): + filter = S3OwnerFilter() + filter.execute(listing, response) + + if not('ecsClustersFailure' in unfilterList): + filter = ECSClustersFailureFilter() + filter.execute(listing, response) + + if not('pinpointGetApps' in unfilterList): + filter = PinpointGetAppsFilter() + filter.execute(listing, response) + + if not('ssmBaselines' in unfilterList): + filter = SSMBaselinesFilter() + filter.execute(listing, response) + + if not('dbSecurityGroups' in unfilterList): + filter = DBSecurityGroupsFilter() + filter.execute(listing, response) + + if not('dbParameterGroups' in unfilterList): + filter = DBParameterGroupsFilter() + filter.execute(listing, response) + + if not('dbClusterParameterGroups' in unfilterList): + filter = DBClusterParameterGroupsFilter() + filter.execute(listing, response) + + if not('dbOptionGroups' in unfilterList): + filter = DBOptionGroupsFilter() + filter.execute(listing, response) + + if not('ec2VPC' in unfilterList): + filter = EC2VPCFilter() + filter.execute(listing, response) + + if not('ec2Subnets' in unfilterList): + filter = EC2SubnetsFilter() + filter.execute(listing, response) + + if not('ec2SecurityGroups' in unfilterList): + filter = EC2SecurityGroupsFilter() + filter.execute(listing, response) + + if not('ec2RouteTables' in unfilterList): + filter = EC2RouteTablesFilter() + filter.execute(listing, response) + + if not('ec2NetworkAcls' in unfilterList): + filter = EC2NetworkAclsFilter() + filter.execute(listing, response) + + if not('ec2FpgaImages' in unfilterList): + filter = EC2FpgaImagesFilter() + filter.execute(listing, response) + + if not('workmailDeletedOrganizations' in unfilterList): + filter = WorkmailDeletedOrganizationsFilter() + filter.execute(listing, response) + + if not('elasticacheSubnetGroups' in unfilterList): + filter = ElasticacheSubnetGroupsFilter() + filter.execute(listing, response) + + if not('nextToken' in unfilterList): + filter = NextTokenFilter(apply_complete) + filter.execute(listing, response) + apply_complete = getattr(filter, 'complete') + + return apply_complete + + + diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index 265b353..898bca9 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -3,6 +3,7 @@ import boto3 +from .apply_filter import apply_filters from .client import get_client from .resource_filter import * @@ -83,13 +84,47 @@ def run_raw_listing_operation(service, region, operation, profile): class ListingFile(object): - def __init__(self, input, directory='./'): + def __init__(self, input, directory='./', unfilter=()): self.input = input self.directory = directory + self.unfilter = unfilter + + @property + def resource_types(self): + """The list of resource types (Keys with list content) in the response""" + return list(self.resources.keys()) + + @property + def resource_total_count(self): + """The estimated total count of resources - can be incomplete""" + return sum(len(v) for v in self.resources.values()) + + def export_resources(self, filename): + """Export the result to the given JSON file""" + with open(filename, 'w') as outfile: + outfile.write(pprint.pformat(self.resources).encode('utf-8')) + + def __str__(self): + opdesc = '{} {} {} {}'.format(self.input.service, self.input.region, self.input.operation, self.input.profile) + if len(self.resource_types) == 0 or self.resource_total_count == 0: + return '{} (no resources found)'.format(opdesc) + return opdesc + ', '.join('#{}: {}'.format(key, len(listing)) for key, listing in self.resources.items()) @property def resources(self): - response = self.input.resources + """Transform the response data into a dict of resource names to resource listings""" + if not(self.input.response): + return self.input.response.copy() + response = self.input.response.copy() + complete = True + del response['ResponseMetadata'] + + complete = apply_filters(self.input, self.unfilter, response, complete) + #response = self.input.resources + kmsListKeys_filter = KMSListKeysFilter(self.directory) + ec2InternetGateways_filter = EC2InternetGatewaysFilter(self.directory) + kmsListKeys_filter.execute(self.input, response) + ec2InternetGateways_filter.execute(self.input, response) # Special handling for service-level kms keys; derived from alias name. if self.input.service == 'kms' and self.input.operation == 'ListKeys': @@ -121,6 +156,13 @@ def resources(self): internet_gateways.append(ig) response['InternetGateways'] = internet_gateways + for key, value in response.items(): + if not isinstance(value, list): + raise Exception('No listing: {} is no list:'.format(key), response) + + if not complete: + response['truncated'] = [True] + return response @@ -193,68 +235,7 @@ def resources(self): # pylint:disable=too-many-branches complete = True del response['ResponseMetadata'] - - cloudfront_filter = CloudfrontFilter() - medialive_filter = MedialiveFilter() - ssmListCommands_filter = SSMListCommandsFilter() - snsListSubscriptions_filter = SNSListSubscriptionsFilter() - athenaWorkGroups_filter = AthenaWorkGroupsFilter() - listEventBuses_filter = ListEventBusesFilter() - xRayGroups_filter = XRayGroupsFilter() - kmsListAliases_filter = KMSListAliasesFilter() - appstreamImages_filter = AppstreamImagesFilter() - cloudsearch_filter = CloudsearchFilter() - cloudTrail_filter = CloudTrailFilter() - cloudWatch_filter = CloudWatchFilter() - iamPolicies_filter = IAMPoliciesFilter() - s3Owner_filter = S3OwnerFilter() - ecsClustersFailure_filter = ECSClustersFailureFilter() - pinpointGetApps_filter = PinpointGetAppsFilter() - ssmBaselines_filter = SSMBaselinesFilter() - dbSecurityGroups_filter = DBSecurityGroupsFilter() - dbParameterGroups_filter = DBParameterGroupsFilter() - dbClusterParameterGroups_filter = DBClusterParameterGroupsFilter() - dbOptionGroups_filter = DBOptionGroupsFilter() - ec2VPC_filter = EC2VPCFilter() - ec2Subnets_filter = EC2SubnetsFilter() - ec2SecurityGroups_filter = EC2SecurityGroupsFilter() - ec2RouteTables_filter = EC2RouteTablesFilter() - ec2NetworkAcls_filter = EC2NetworkAclsFilter() - ec2FpgaImgaes_filter = EC2FpgaImgaesFilter() - workmailDeletedOrganizations_filter = WorkmailDeletedOrganizationsFilter() - elasticacheSubnetGroups_filter = ElasticacheSubnetGroupsFilter() - nextToken_filter = NextTokenFilter() - - cloudfront_filter.execute(self, response) - medialive_filter.execute(self, response) - ssmListCommands_filter.execute(self, response) - snsListSubscriptions_filter.execute(self, response) - athenaWorkGroups_filter.execute(self, response) - listEventBuses_filter.execute(self, response) - xRayGroups_filter.execute(self, response) - kmsListAliases_filter.execute(self, response) - appstreamImages_filter.execute(self, response) - cloudsearch_filter.execute(self, response) - cloudTrail_filter.execute(self, response) - cloudWatch_filter.execute(self, response) - iamPolicies_filter.execute(self, response) - s3Owner_filter.execute(self, response) - ecsClustersFailure_filter.execute(self, response) - pinpointGetApps_filter.execute(self, response) - ssmBaselines_filter.execute(self, response) - dbSecurityGroups_filter.execute(self, response) - dbParameterGroups_filter.execute(self, response) - dbClusterParameterGroups_filter.execute(self, response) - dbOptionGroups_filter.execute(self, response) - ec2VPC_filter.execute(self, response) - ec2Subnets_filter.execute(self, response) - ec2SecurityGroups_filter.execute(self, response) - ec2RouteTables_filter.execute(self, response) - ec2NetworkAcls_filter.execute(self, response) - ec2FpgaImgaes_filter.execute(self, response) - workmailDeletedOrganizations_filter.execute(self, response) - elasticacheSubnetGroups_filter.execute(self, response) - nextToken_filter.execute(self, response) + #complete = apply_filters(self, response, complete) # # Filter default Internet Gateways # if self.service == 'ec2' and self.operation == 'DescribeInternetGateways': @@ -278,8 +259,4 @@ def resources(self): # pylint:disable=too-many-branches if not complete: response['truncated'] = [True] - # if self.operation == 'DescribeInternetGateways': - # print('RIGHT HERE!') - # print(response.values()) - return response diff --git a/aws_list_all/query.py b/aws_list_all/query.py index 3157f64..3be9937 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -1,6 +1,8 @@ from __future__ import print_function +import codecs import json +import os import sys import contextlib from collections import defaultdict @@ -195,7 +197,7 @@ NOT_AVAILABLE_STRINGS = NOT_AVAILABLE_FOR_REGION_STRINGS + NOT_AVAILABLE_FOR_ACCOUNT_STRINGS -def do_query(services, selected_regions=(), selected_operations=(), verbose=0, parallel=32, selected_profile=None): +def do_query(services, selected_regions=(), selected_operations=(), verbose=0, parallel=32, selected_profile=None, unfilter=()): """For the given services, execute all selected operations (default: all) in selected regions (default: all)""" to_run = [] @@ -207,7 +209,7 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p region_name = region or 'n/a' print('Service: {: <28} | Region: {:<15} | Operation: {}'.format(service, region_name, operation)) - to_run.append([service, region, operation, selected_profile]) + to_run.append([service, region, operation, selected_profile, unfilter]) shuffle(to_run) # Distribute requests across endpoints results_by_type = defaultdict(list) print('...done. Executing queries...') @@ -229,22 +231,23 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p def acquire_listing(verbose, what): """Given a service, region and operation execute the operation, serialize and save the result and return a tuple of strings describing the result.""" - service, region, operation, profile = what + service, region, operation, profile, unfilter = what start_time = time() try: if verbose > 1: print(what, 'starting request...') listing = Listing.acquire(service, region, operation, profile) + listingFile = ListingFile(listing, './', unfilter) #os.getcwd() + '/' duration = time() - start_time if verbose > 1: print(what, '...request successful') print("timing [success]:", duration, what) with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: json.dump(listing.to_json(), jsonfile, default=datetime.isoformat) - if listing.resource_total_count > 0: - return (RESULT_SOMETHING, service, region, operation, profile, ', '.join(listing.resource_types)) + if listingFile.resource_total_count > 0: + return (RESULT_SOMETHING, service, region, operation, profile, ', '.join(listingFile.resource_types)) else: - return (RESULT_NOTHING, service, region, operation, profile, ', '.join(listing.resource_types)) + return (RESULT_NOTHING, service, region, operation, profile, ', '.join(listingFile.resource_types)) except Exception as exc: # pylint:disable=broad-except duration = time() - start_time if verbose > 1: @@ -272,12 +275,12 @@ def acquire_listing(verbose, what): return (result_type, service, region, operation, profile, repr(exc)) -def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=False): +def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=False, unfilter=()): """Print out a rudimentary summary of the Listing objects contained in the given files""" dir = filenames[0][:filenames[0].rfind('/') + 1] for listing_filename in filenames: listing = Listing.from_json(json.load(open(listing_filename, 'rb'))) - listing_entry = ListingFile(listing, dir) + listing_entry = ListingFile(listing, dir, unfilter) resources = listing_entry.resources truncated = False @@ -287,7 +290,7 @@ def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=Fa del resources['truncated'] if listing.error == RESULT_NO_ACCESS: was_denied = True - if not resources: + if not resources and denied: print(listing.service, listing.region, listing.operation, 'MISSING PERMISSION', '0') if listing.error == RESULT_ERROR and errors: print(listing.service, listing.region, listing.operation, 'ERROR', '0') @@ -333,3 +336,4 @@ def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=Fa print(' - ', item) if truncated: print(' - ... (more items, query truncated)') + diff --git a/aws_list_all/resource_filter.py b/aws_list_all/resource_filter.py index 59bc164..71151d5 100644 --- a/aws_list_all/resource_filter.py +++ b/aws_list_all/resource_filter.py @@ -64,6 +64,8 @@ def execute(self, listing, response): if listing.service == 'xray' and listing.operation == 'GetGroups': response['Groups'] = [wg for wg in response.get('Groups', []) if wg['GroupName'] != 'Default'] +class Route53ResolverFilter: + def execute(self, listing, response): if listing.service == 'route53resolver': if listing.operation == 'ListResolverRules': response['ResolverRules'] = [ @@ -76,31 +78,48 @@ def execute(self, listing, response): if rule['ResolverRuleId'] != 'rslvr-autodefined-rr-internet-resolver' ] +class CountFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): if 'Count' in response: if 'MaxResults' in response: if response['MaxResults'] <= response['Count']: - complete = False + self.complete = False del response['MaxResults'] del response['Count'] +class QuantityFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): if 'Quantity' in response: if 'MaxItems' in response: if response['MaxItems'] <= response['Quantity']: - complete = False + self.complete = False del response['MaxItems'] del response['Quantity'] +class NeutralThingFilter: + def execute(self, listing, response): for neutral_thing in ('MaxItems', 'MaxResults', 'Quantity'): if neutral_thing in response: del response[neutral_thing] +class BadThingFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): for bad_thing in ( 'hasMoreResults', 'IsTruncated', 'Truncated', 'HasMoreApplications', 'HasMoreDeliveryStreams', 'HasMoreStreams', 'NextToken', 'NextMarker', 'nextMarker', 'Marker' ): if bad_thing in response: if response[bad_thing]: - complete = False + self.complete = False del response[bad_thing] class KMSListAliasesFilter: @@ -112,23 +131,23 @@ def execute(self, listing, response): if not alias.get('AliasName').lower().startswith('alias/aws') ] -# class KMSListKeysFilter: -# def __init__(self, directory): -# self.directory = directory - -# def execute(self, listing, response): -# # Special handling for service-level kms keys; derived from alias name. -# if listing.input.service == 'kms' and listing.input.operation == 'ListKeys': -# #list_aliases = run_raw_listing_operation(self.service, self.region, 'ListAliases', self.profile) -# aliases_file = '{}_{}_{}_{}.json'.format(listing.input.service, 'ListAliases', listing.input.region, listing.input.profile) -# aliases_file = self.directory + aliases_file -# aliases_listing = Listing.from_json(json.load(open(aliases_file, 'rb'))) -# list_aliases = aliases_listing.response -# service_key_ids = [ -# k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) -# if k.get('AliasName').lower().startswith('alias/aws') -# ] -# response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] +class KMSListKeysFilter: + def __init__(self, directory): + self.directory = directory + + def execute(self, listing, response): + # Special handling for service-level kms keys; derived from alias name. + if listing.service == 'kms' and listing.operation == 'ListKeys': + #list_aliases = run_raw_listing_operation(self.service, self.region, 'ListAliases', self.profile) + aliases_file = '{}_{}_{}_{}.json'.format(listing.service, 'ListAliases', listing.region, listing.profile) + aliases_file = self.directory + aliases_file + aliases_listing = Listing.from_json(json.load(open(aliases_file, 'rb'))) + list_aliases = aliases_listing.response + service_key_ids = [ + k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) + if k.get('AliasName').lower().startswith('alias/aws') + ] + response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] class AppstreamImagesFilter: def execute(self, listing, response): @@ -262,33 +281,33 @@ def execute(self, listing, response): if listing.service == 'ec2' and listing.operation == 'DescribeNetworkAcls': response['NetworkAcls'] = [nacl for nacl in response['NetworkAcls'] if not nacl['IsDefault']] -# class EC2InternetGatewaysFilter: -# def __init__(self, directory): -# self.directory = directory - -# def execute(self, listing, response): -# # Filter default Internet Gateways -# if listing.input.service == 'ec2' and listing.input.operation == 'DescribeInternetGateways': -# #describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile) -# vpcs_file = '{}_{}_{}_{}.json'.format(listing.input.service, 'DescribeVpcs', listing.input.region, listing.input.profile) -# vpcs_file = self.directory + vpcs_file -# vpcs_listing = Listing.from_json(json.load(open(vpcs_file, 'rb'))) -# describe_vpcs = vpcs_listing.response -# vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} -# internet_gateways = [] -# # print(self.response) -# # print(response) -# for ig in response['InternetGateways']: -# attachments = ig.get('Attachments', []) -# # more than one, it cannot be default. -# if len(attachments) != 1: -# continue -# vpc = attachments[0].get('VpcId') -# if not vpcs.get(vpc, {}).get('IsDefault', False): -# internet_gateways.append(ig) -# response['InternetGateways'] = internet_gateways - -class EC2FpgaImgaesFilter: +class EC2InternetGatewaysFilter: + def __init__(self, directory): + self.directory = directory + + def execute(self, listing, response): + # Filter default Internet Gateways + if listing.service == 'ec2' and listing.operation == 'DescribeInternetGateways': + #describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile) + vpcs_file = '{}_{}_{}_{}.json'.format(listing.service, 'DescribeVpcs', listing.region, listing.profile) + vpcs_file = self.directory + vpcs_file + vpcs_listing = Listing.from_json(json.load(open(vpcs_file, 'rb'))) + describe_vpcs = vpcs_listing.response + vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} + internet_gateways = [] + # print(self.response) + # print(response) + for ig in response['InternetGateways']: + attachments = ig.get('Attachments', []) + # more than one, it cannot be default. + if len(attachments) != 1: + continue + vpc = attachments[0].get('VpcId') + if not vpcs.get(vpc, {}).get('IsDefault', False): + internet_gateways.append(ig) + response['InternetGateways'] = internet_gateways + +class EC2FpgaImagesFilter: def execute(self, listing, response): # Filter Public images from ec2.fpga images if listing.service == 'ec2' and listing.operation == 'DescribeFpgaImages': @@ -310,9 +329,12 @@ def execute(self, listing, response): ] class NextTokenFilter: + def __init__(self, complete): + self.complete = complete + def execute(self, listing, response): # interpret nextToken in several services if (listing.service, listing.operation) in (('inspector', 'ListFindings'), ('logs', 'DescribeLogGroups')): if response.get('nextToken'): - complete = False + self.complete = False del response['nextToken'] \ No newline at end of file From d7a4a218dd9913b40d04565df1d682b2dd8eec68 Mon Sep 17 00:00:00 2001 From: kntrain Date: Thu, 14 Jan 2021 12:31:11 +0100 Subject: [PATCH 04/14] -removed call to file-reference filter in listing --- aws_list_all/apply_filter.py | 17 +++++++++++------ aws_list_all/listing.py | 12 ++++-------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/aws_list_all/apply_filter.py b/aws_list_all/apply_filter.py index 9cf2c83..94f4884 100644 --- a/aws_list_all/apply_filter.py +++ b/aws_list_all/apply_filter.py @@ -5,12 +5,7 @@ from .resource_filter import * def apply_filters(listing, unfilter, response, complete): - unfilterList = [] - if unfilter is not None: - if isinstance(unfilter, list): - unfilterList = unfilter - else: - unfilterList.append(unfilter) + unfilterList = convert_unfilterList(unfilter) apply_complete = complete if not('cloudfront' in unfilterList): @@ -159,5 +154,15 @@ def apply_filters(listing, unfilter, response, complete): return apply_complete +def convert_unfilterList(unfilter): + unfilterList = [] + if unfilter is not None: + if isinstance(unfilter, list): + unfilterList = unfilter + else: + unfilterList.append(unfilter) + + return unfilterList + diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index 898bca9..12ae872 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -3,7 +3,7 @@ import boto3 -from .apply_filter import apply_filters +from .apply_filter import apply_filters, convert_unfilterList from .client import get_client from .resource_filter import * @@ -120,14 +120,10 @@ def resources(self): del response['ResponseMetadata'] complete = apply_filters(self.input, self.unfilter, response, complete) - #response = self.input.resources - kmsListKeys_filter = KMSListKeysFilter(self.directory) - ec2InternetGateways_filter = EC2InternetGatewaysFilter(self.directory) - kmsListKeys_filter.execute(self.input, response) - ec2InternetGateways_filter.execute(self.input, response) + unfilterList = convert_unfilterList(self.unfilter) # Special handling for service-level kms keys; derived from alias name. - if self.input.service == 'kms' and self.input.operation == 'ListKeys': + if 'kmsListKeys' not in unfilterList and self.input.service == 'kms' and self.input.operation == 'ListKeys': aliases_file = '{}_{}_{}_{}.json'.format(self.input.service, 'ListAliases', self.input.region, self.input.profile) aliases_file = self.directory + aliases_file aliases_listing = Listing.from_json(json.load(open(aliases_file, 'rb'))) @@ -139,7 +135,7 @@ def resources(self): response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] # Filter default Internet Gateways - if self.input.service == 'ec2' and self.input.operation == 'DescribeInternetGateways': + if 'ec2InternetGateways' not in unfilterList and self.input.service == 'ec2' and self.input.operation == 'DescribeInternetGateways': vpcs_file = '{}_{}_{}_{}.json'.format(self.input.service, 'DescribeVpcs', self.input.region, self.input.profile) vpcs_file = self.directory + vpcs_file vpcs_listing = Listing.from_json(json.load(open(vpcs_file, 'rb'))) From f2d06fcd40343b98e7ead713cb49a4c606e3d034 Mon Sep 17 00:00:00 2001 From: kntrain Date: Mon, 25 Jan 2021 03:17:20 +0100 Subject: [PATCH 05/14] -renamed Listing -> RawListing -renamed ListingFile -> FilteredListing -moved non-default filters to fixing_filter.py --- aws_list_all/apply_filter.py | 3 ++ aws_list_all/fixing_filter.py | 50 +++++++++++++++++++++++++++++++++ aws_list_all/listing.py | 41 ++++----------------------- aws_list_all/query.py | 12 ++++---- aws_list_all/resource_filter.py | 44 ----------------------------- 5 files changed, 65 insertions(+), 85 deletions(-) create mode 100644 aws_list_all/fixing_filter.py diff --git a/aws_list_all/apply_filter.py b/aws_list_all/apply_filter.py index 94f4884..dd9e932 100644 --- a/aws_list_all/apply_filter.py +++ b/aws_list_all/apply_filter.py @@ -2,9 +2,11 @@ import os import sys +from .fixing_filter import * from .resource_filter import * def apply_filters(listing, unfilter, response, complete): + """Apply filters to remove default resources from the response""" unfilterList = convert_unfilterList(unfilter) apply_complete = complete @@ -155,6 +157,7 @@ def apply_filters(listing, unfilter, response, complete): return apply_complete def convert_unfilterList(unfilter): + """Check if unfilter parameter is a list or single argument and return list of it""" unfilterList = [] if unfilter is not None: if isinstance(unfilter, list): diff --git a/aws_list_all/fixing_filter.py b/aws_list_all/fixing_filter.py new file mode 100644 index 0000000..5da0221 --- /dev/null +++ b/aws_list_all/fixing_filter.py @@ -0,0 +1,50 @@ +import json +import pprint + +import boto3 + +from .client import get_client + +class CountFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): + if 'Count' in response: + if 'MaxResults' in response: + if response['MaxResults'] <= response['Count']: + self.complete = False + del response['MaxResults'] + del response['Count'] + +class QuantityFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): + if 'Quantity' in response: + if 'MaxItems' in response: + if response['MaxItems'] <= response['Quantity']: + self.complete = False + del response['MaxItems'] + del response['Quantity'] + +class NeutralThingFilter: + def execute(self, listing, response): + for neutral_thing in ('MaxItems', 'MaxResults', 'Quantity'): + if neutral_thing in response: + del response[neutral_thing] + +class BadThingFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): + for bad_thing in ( + 'hasMoreResults', 'IsTruncated', 'Truncated', 'HasMoreApplications', 'HasMoreDeliveryStreams', + 'HasMoreStreams', 'NextToken', 'NextMarker', 'nextMarker', 'Marker' + ): + if bad_thing in response: + if response[bad_thing]: + self.complete = False + del response[bad_thing] \ No newline at end of file diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index 12ae872..4067f42 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -82,7 +82,7 @@ def run_raw_listing_operation(service, region, operation, profile): return getattr(client, api_to_method_mapping[operation])(**parameters) -class ListingFile(object): +class FilteredListing(object): def __init__(self, input, directory='./', unfilter=()): self.input = input @@ -126,7 +126,7 @@ def resources(self): if 'kmsListKeys' not in unfilterList and self.input.service == 'kms' and self.input.operation == 'ListKeys': aliases_file = '{}_{}_{}_{}.json'.format(self.input.service, 'ListAliases', self.input.region, self.input.profile) aliases_file = self.directory + aliases_file - aliases_listing = Listing.from_json(json.load(open(aliases_file, 'rb'))) + aliases_listing = RawListing.from_json(json.load(open(aliases_file, 'rb'))) list_aliases = aliases_listing.response service_key_ids = [ k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) @@ -138,8 +138,9 @@ def resources(self): if 'ec2InternetGateways' not in unfilterList and self.input.service == 'ec2' and self.input.operation == 'DescribeInternetGateways': vpcs_file = '{}_{}_{}_{}.json'.format(self.input.service, 'DescribeVpcs', self.input.region, self.input.profile) vpcs_file = self.directory + vpcs_file - vpcs_listing = Listing.from_json(json.load(open(vpcs_file, 'rb'))) - describe_vpcs = getattr(vpcs_listing, 'response') + # Sometimes 'No JSON Object' or 'Directory not found' + vpcs_listing = RawListing.from_json(json.load(open(vpcs_file, 'rb'))) + describe_vpcs = vpcs_listing.response vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} internet_gateways = [] for ig in response['InternetGateways']: @@ -162,7 +163,7 @@ def resources(self): return response -class Listing(object): +class RawListing(object): """Represents a listing operation on an AWS service and its result""" def __init__(self, service, region, operation, response, profile, error=''): self.service = service @@ -193,21 +194,6 @@ def from_json(cls, data): error=data.get('error') ) - @property - def resource_types(self): - """The list of resource types (Keys with list content) in the response""" - return list(self.resources.keys()) - - @property - def resource_total_count(self): - """The estimated total count of resources - can be incomplete""" - return sum(len(v) for v in self.resources.values()) - - def export_resources(self, filename): - """Export the result to the given JSON file""" - with open(filename, 'w') as outfile: - outfile.write(pprint.pformat(self.resources).encode('utf-8')) - def __str__(self): opdesc = '{} {} {} {}'.format(self.service, self.region, self.operation, self.profile) if len(self.resource_types) == 0 or self.resource_total_count == 0: @@ -233,21 +219,6 @@ def resources(self): # pylint:disable=too-many-branches del response['ResponseMetadata'] #complete = apply_filters(self, response, complete) - # # Filter default Internet Gateways - # if self.service == 'ec2' and self.operation == 'DescribeInternetGateways': - # describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile) - # vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} - # internet_gateways = [] - # for ig in response['InternetGateways']: - # attachments = ig.get('Attachments', []) - # # more than one, it cannot be default. - # if len(attachments) != 1: - # continue - # vpc = attachments[0].get('VpcId') - # if not vpcs.get(vpc, {}).get('IsDefault', False): - # internet_gateways.append(ig) - # response['InternetGateways'] = internet_gateways - for key, value in response.items(): if not isinstance(value, list): raise Exception('No listing: {} is no list:'.format(key), response) diff --git a/aws_list_all/query.py b/aws_list_all/query.py index 3be9937..179cddd 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -14,7 +14,7 @@ from traceback import print_exc from .introspection import get_listing_operations, get_regions_for_service -from .listing import Listing, ListingFile +from .listing import RawListing, FilteredListing RESULT_NOTHING = '---' RESULT_SOMETHING = '+++' @@ -236,8 +236,8 @@ def acquire_listing(verbose, what): try: if verbose > 1: print(what, 'starting request...') - listing = Listing.acquire(service, region, operation, profile) - listingFile = ListingFile(listing, './', unfilter) #os.getcwd() + '/' + listing = RawListing.acquire(service, region, operation, profile) + listingFile = FilteredListing(listing, './', unfilter) #os.getcwd() + '/' duration = time() - start_time if verbose > 1: print(what, '...request successful') @@ -269,7 +269,7 @@ def acquire_listing(verbose, what): if not_available_string in str(exc): result_type = RESULT_NOTHING - listing = Listing(service, region, operation, {}, profile, result_type) + listing = RawListing(service, region, operation, {}, profile, result_type) with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: json.dump(listing.to_json(), jsonfile, default=datetime.isoformat) return (result_type, service, region, operation, profile, repr(exc)) @@ -279,8 +279,8 @@ def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=Fa """Print out a rudimentary summary of the Listing objects contained in the given files""" dir = filenames[0][:filenames[0].rfind('/') + 1] for listing_filename in filenames: - listing = Listing.from_json(json.load(open(listing_filename, 'rb'))) - listing_entry = ListingFile(listing, dir, unfilter) + listing = RawListing.from_json(json.load(open(listing_filename, 'rb'))) + listing_entry = FilteredListing(listing, dir, unfilter) resources = listing_entry.resources truncated = False diff --git a/aws_list_all/resource_filter.py b/aws_list_all/resource_filter.py index 71151d5..938a8aa 100644 --- a/aws_list_all/resource_filter.py +++ b/aws_list_all/resource_filter.py @@ -78,50 +78,6 @@ def execute(self, listing, response): if rule['ResolverRuleId'] != 'rslvr-autodefined-rr-internet-resolver' ] -class CountFilter: - def __init__(self, complete): - self.complete = complete - - def execute(self, listing, response): - if 'Count' in response: - if 'MaxResults' in response: - if response['MaxResults'] <= response['Count']: - self.complete = False - del response['MaxResults'] - del response['Count'] - -class QuantityFilter: - def __init__(self, complete): - self.complete = complete - - def execute(self, listing, response): - if 'Quantity' in response: - if 'MaxItems' in response: - if response['MaxItems'] <= response['Quantity']: - self.complete = False - del response['MaxItems'] - del response['Quantity'] - -class NeutralThingFilter: - def execute(self, listing, response): - for neutral_thing in ('MaxItems', 'MaxResults', 'Quantity'): - if neutral_thing in response: - del response[neutral_thing] - -class BadThingFilter: - def __init__(self, complete): - self.complete = complete - - def execute(self, listing, response): - for bad_thing in ( - 'hasMoreResults', 'IsTruncated', 'Truncated', 'HasMoreApplications', 'HasMoreDeliveryStreams', - 'HasMoreStreams', 'NextToken', 'NextMarker', 'nextMarker', 'Marker' - ): - if bad_thing in response: - if response[bad_thing]: - self.complete = False - del response[bad_thing] - class KMSListAliasesFilter: def execute(self, listing, response): # Special handling for Aliases in kms, there are some reserved AWS-managed aliases. From 65a47882be661b20e9d51f5c79887190beacd39a Mon Sep 17 00:00:00 2001 From: kntrain Date: Fri, 29 Jan 2021 22:25:41 +0100 Subject: [PATCH 06/14] changed fixing_filters to always apply --- aws_list_all/apply_filter.py | 33 ++++++++++++++------------------- aws_list_all/fixing_filter.py | 13 ++++++++++++- aws_list_all/resource_filter.py | 13 +------------ 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/aws_list_all/apply_filter.py b/aws_list_all/apply_filter.py index dd9e932..ef2c5af 100644 --- a/aws_list_all/apply_filter.py +++ b/aws_list_all/apply_filter.py @@ -42,24 +42,20 @@ def apply_filters(listing, unfilter, response, complete): filter = Route53ResolverFilter() filter.execute(listing, response) - if not('count' in unfilterList): - filter = CountFilter(apply_complete) - filter.execute(listing, response) - apply_complete = getattr(filter, 'complete') + filter = CountFilter(apply_complete) + filter.execute(listing, response) + apply_complete = getattr(filter, 'complete') - if not('quantity' in unfilterList): - filter = QuantityFilter(apply_complete) - filter.execute(listing, response) - apply_complete = getattr(filter, 'complete') + filter = QuantityFilter(apply_complete) + filter.execute(listing, response) + apply_complete = getattr(filter, 'complete') - if not('neutralThing' in unfilterList): - filter = NeutralThingFilter() - filter.execute(listing, response) + filter = NeutralThingFilter() + filter.execute(listing, response) - if not('badThing' in unfilterList): - filter = BadThingFilter(apply_complete) - filter.execute(listing, response) - apply_complete = getattr(filter, 'complete') + filter = BadThingFilter(apply_complete) + filter.execute(listing, response) + apply_complete = getattr(filter, 'complete') if not('kmsListAliases' in unfilterList): filter = KMSListAliasesFilter() @@ -149,10 +145,9 @@ def apply_filters(listing, unfilter, response, complete): filter = ElasticacheSubnetGroupsFilter() filter.execute(listing, response) - if not('nextToken' in unfilterList): - filter = NextTokenFilter(apply_complete) - filter.execute(listing, response) - apply_complete = getattr(filter, 'complete') + filter = NextTokenFilter(apply_complete) + filter.execute(listing, response) + apply_complete = getattr(filter, 'complete') return apply_complete diff --git a/aws_list_all/fixing_filter.py b/aws_list_all/fixing_filter.py index 5da0221..f6feb97 100644 --- a/aws_list_all/fixing_filter.py +++ b/aws_list_all/fixing_filter.py @@ -47,4 +47,15 @@ def execute(self, listing, response): if bad_thing in response: if response[bad_thing]: self.complete = False - del response[bad_thing] \ No newline at end of file + del response[bad_thing] + +class NextTokenFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): + # interpret nextToken in several services + if (listing.service, listing.operation) in (('inspector', 'ListFindings'), ('logs', 'DescribeLogGroups')): + if response.get('nextToken'): + self.complete = False + del response['nextToken'] \ No newline at end of file diff --git a/aws_list_all/resource_filter.py b/aws_list_all/resource_filter.py index 938a8aa..a687d37 100644 --- a/aws_list_all/resource_filter.py +++ b/aws_list_all/resource_filter.py @@ -282,15 +282,4 @@ def execute(self, listing, response): if listing.service == 'elasticache' and listing.operation == 'DescribeCacheSubnetGroups': response['CacheSubnetGroups'] = [ g for g in response.get('CacheSubnetGroups', []) if g.get('CacheSubnetGroupName') != 'default' - ] - -class NextTokenFilter: - def __init__(self, complete): - self.complete = complete - - def execute(self, listing, response): - # interpret nextToken in several services - if (listing.service, listing.operation) in (('inspector', 'ListFindings'), ('logs', 'DescribeLogGroups')): - if response.get('nextToken'): - self.complete = False - del response['nextToken'] \ No newline at end of file + ] \ No newline at end of file From c92bb38cf49d6558524410e6c30da497fc0a4af6 Mon Sep 17 00:00:00 2001 From: kntrain Date: Wed, 10 Mar 2021 15:12:08 +0100 Subject: [PATCH 07/14] - added handling in case special ops are specified --- aws_list_all/apply_filter.py | 3 ++- aws_list_all/query.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/aws_list_all/apply_filter.py b/aws_list_all/apply_filter.py index ef2c5af..ea13b63 100644 --- a/aws_list_all/apply_filter.py +++ b/aws_list_all/apply_filter.py @@ -6,7 +6,8 @@ from .resource_filter import * def apply_filters(listing, unfilter, response, complete): - """Apply filters to remove default resources from the response""" + """Apply filters for operations to be handled in a special way or + to remove default resources from the response""" unfilterList = convert_unfilterList(unfilter) apply_complete = complete diff --git a/aws_list_all/query.py b/aws_list_all/query.py index 179cddd..e61e537 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -201,6 +201,7 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p """For the given services, execute all selected operations (default: all) in selected regions (default: all)""" to_run = [] + selected_operations = check_special_ops(selected_operations) print('Building set of queries to execute...') for service in services: for region in get_regions_for_service(service, selected_regions): @@ -209,6 +210,7 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p region_name = region or 'n/a' print('Service: {: <28} | Region: {:<15} | Operation: {}'.format(service, region_name, operation)) + to_run.append([service, region, operation, selected_profile, unfilter]) shuffle(to_run) # Distribute requests across endpoints results_by_type = defaultdict(list) @@ -337,3 +339,14 @@ def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=Fa if truncated: print(' - ... (more items, query truncated)') +def check_special_ops(sel_ops): + """Special operations require other operations to be queried first""" + if sel_ops is None: + return sel_ops + else: + sel_oplist = list(sel_ops) + if 'ListKeys' in sel_oplist: + sel_oplist.insert(sel_oplist.index('ListKeys'), 'ListAliases') + if 'DescribeInternetGateways' in sel_oplist: + sel_oplist.insert(sel_oplist.index('DescribeInternetGateways'), 'DescribeVpcs') + return tuple(sel_oplist) From 5d120bb6206babba594d812b24a8cf4334ffd92e Mon Sep 17 00:00:00 2001 From: kntrain Date: Wed, 10 Mar 2021 15:15:08 +0100 Subject: [PATCH 08/14] - removed empty line --- aws_list_all/query.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aws_list_all/query.py b/aws_list_all/query.py index e61e537..0e330ef 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -210,7 +210,6 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p region_name = region or 'n/a' print('Service: {: <28} | Region: {:<15} | Operation: {}'.format(service, region_name, operation)) - to_run.append([service, region, operation, selected_profile, unfilter]) shuffle(to_run) # Distribute requests across endpoints results_by_type = defaultdict(list) From 8e401895b6e308dd5760cab97957a9746c752cea Mon Sep 17 00:00:00 2001 From: kntrain Date: Wed, 10 Mar 2021 15:36:37 +0100 Subject: [PATCH 09/14] - added unfilter options description --- README.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.rst b/README.rst index afafdd4..cc59cd3 100644 --- a/README.rst +++ b/README.rst @@ -53,6 +53,40 @@ region, and operation used to find them. They can be dumped with:: aws-list-all show data/ec2_* aws-list-all show --verbose data/ec2_DescribeSecurityGroups_eu-west-1.json +Special treatment and removal of default resources which are performed by default during +data handling can be omitted with --unfilter and following arguments: +- cloudfront +- medialive +- ssmListCommands +- snsListSubscriptions +- athenaWorkGroups +- listEventBuses +- xRayGroups +- route53Resolver +- kmsListAliases +- appstreamImages +- cloudsearch +- cloudTrail +- cloudWatch +- iamPolicies +- s3Owner +- ecsClustersFailure +- pinpointGetApps +- ssmBaselines +- dbSecurityGroups +- dbParameterGroups +- dbClusterParameterGroups +- dbOptionGroups +- ec2VPC +- ec2Subnets +- ec2SecurityGroups +- ec2RouteTables +- ec2NetworkAcls +- ec2FpgaImages +- workmailDeletedOrganizations +- elasticacheSubnetGroups + + How do I really list everything? ------------------------------------------------ From 975320943afbd98034f723df428adb8aea6b2dd2 Mon Sep 17 00:00:00 2001 From: kntrain Date: Tue, 16 Mar 2021 22:48:23 +0100 Subject: [PATCH 10/14] Fix dependent ops and show command flags Bugs --- aws_list_all/__main__.py | 6 +++--- aws_list_all/query.py | 40 +++++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/aws_list_all/__main__.py b/aws_list_all/__main__.py index 2b4854d..080e1b2 100755 --- a/aws_list_all/__main__.py +++ b/aws_list_all/__main__.py @@ -89,9 +89,9 @@ def main(): ) show.add_argument('listingfile', nargs='*', help='listing file(s) to load and print') show.add_argument('-v', '--verbose', action='count', help='print given listing files with detailed info') - show.add_argument('-n', '--not_found', default=False, type=bool, help='additionally print listing files of resources not found') - show.add_argument('-e', '--errors', default=False, type=bool, help='additionally print listing files of resources where queries resulted in errors') - show.add_argument('-d', '--denied', default=False, type=bool, help='additionally print listing files of resources with "missing permission" errors') + show.add_argument('-n', '--not_found', default=False, action='store_true', help='additionally print listing files of resources not found') + show.add_argument('-e', '--errors', default=False, action='store_true', help='additionally print listing files of resources where queries resulted in errors') + show.add_argument('-d', '--denied', default=False, action='store_true', help='additionally print listing files of resources with "missing permission" errors') show.add_argument( '-u', '--unfilter', diff --git a/aws_list_all/query.py b/aws_list_all/query.py index 0e330ef..f11f774 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -21,6 +21,11 @@ RESULT_ERROR = '!!!' RESULT_NO_ACCESS = '>:|' +DEPENDENT_OPERATIONS = { + 'ListKeys' : 'ListAliases', + 'DescribeInternetGateways' : 'DescribeVpcs', +} + # List of requests with legitimate, persistent errors that indicate that no listable resources are present. # # If the request would never return listable resources, it should not be done and be listed in one of the lists @@ -201,7 +206,7 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p """For the given services, execute all selected operations (default: all) in selected regions (default: all)""" to_run = [] - selected_operations = check_special_ops(selected_operations) + dependencies = {} print('Building set of queries to execute...') for service in services: for region in get_regions_for_service(service, selected_regions): @@ -209,11 +214,26 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p if verbose > 0: region_name = region or 'n/a' print('Service: {: <28} | Region: {:<15} | Operation: {}'.format(service, region_name, operation)) + if operation in DEPENDENT_OPERATIONS: + dependencies[DEPENDENT_OPERATIONS[operation], region] = [service, region, DEPENDENT_OPERATIONS[operation], selected_profile, unfilter] + if operation in DEPENDENT_OPERATIONS.values(): + dependencies[operation, region] = [service, region, operation, selected_profile, unfilter] + continue to_run.append([service, region, operation, selected_profile, unfilter]) shuffle(to_run) # Distribute requests across endpoints results_by_type = defaultdict(list) print('...done. Executing queries...') + + results_by_type = execute_query(dependencies.values(), verbose, parallel, results_by_type) + results_by_type = execute_query(to_run, verbose, parallel, results_by_type) + print('...done') + for result_type in (RESULT_NOTHING, RESULT_SOMETHING, RESULT_NO_ACCESS, RESULT_ERROR): + for result in sorted(results_by_type[result_type]): + print(*result) + + +def execute_query(to_run, verbose, parallel, results_by_type): # the `with` block is a workaround for a bug: https://bugs.python.org/issue35629 with contextlib.closing(ThreadPool(parallel)) as pool: for result in pool.imap_unordered(partial(acquire_listing, verbose), to_run): @@ -223,10 +243,7 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p else: print(result[0][-1], end='') sys.stdout.flush() - print('...done') - for result_type in (RESULT_NOTHING, RESULT_SOMETHING, RESULT_NO_ACCESS, RESULT_ERROR): - for result in sorted(results_by_type[result_type]): - print(*result) + return results_by_type def acquire_listing(verbose, what): @@ -295,7 +312,7 @@ def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=Fa print(listing.service, listing.region, listing.operation, 'MISSING PERMISSION', '0') if listing.error == RESULT_ERROR and errors: print(listing.service, listing.region, listing.operation, 'ERROR', '0') - + for resource_type, value in resources.items(): if not not_found and len(value) == 0 and not was_denied: continue @@ -338,14 +355,3 @@ def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=Fa if truncated: print(' - ... (more items, query truncated)') -def check_special_ops(sel_ops): - """Special operations require other operations to be queried first""" - if sel_ops is None: - return sel_ops - else: - sel_oplist = list(sel_ops) - if 'ListKeys' in sel_oplist: - sel_oplist.insert(sel_oplist.index('ListKeys'), 'ListAliases') - if 'DescribeInternetGateways' in sel_oplist: - sel_oplist.insert(sel_oplist.index('DescribeInternetGateways'), 'DescribeVpcs') - return tuple(sel_oplist) From aefb2a5e87802aaafd037eba6404a80886c8363d Mon Sep 17 00:00:00 2001 From: kntrain Date: Thu, 25 Mar 2021 23:59:05 +0100 Subject: [PATCH 11/14] Remove stray comment and unfilter-list conversion --- aws_list_all/apply_filter.py | 21 +++++---------------- aws_list_all/listing.py | 11 +++++++---- aws_list_all/query.py | 5 +++-- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/aws_list_all/apply_filter.py b/aws_list_all/apply_filter.py index ea13b63..45c2bec 100644 --- a/aws_list_all/apply_filter.py +++ b/aws_list_all/apply_filter.py @@ -5,10 +5,9 @@ from .fixing_filter import * from .resource_filter import * -def apply_filters(listing, unfilter, response, complete): +def apply_filters(listing, unfilterList, response, complete): """Apply filters for operations to be handled in a special way or to remove default resources from the response""" - unfilterList = convert_unfilterList(unfilter) apply_complete = complete if not('cloudfront' in unfilterList): @@ -45,18 +44,18 @@ def apply_filters(listing, unfilter, response, complete): filter = CountFilter(apply_complete) filter.execute(listing, response) - apply_complete = getattr(filter, 'complete') + apply_complete = filter.complete filter = QuantityFilter(apply_complete) filter.execute(listing, response) - apply_complete = getattr(filter, 'complete') + apply_complete = filter.complete filter = NeutralThingFilter() filter.execute(listing, response) filter = BadThingFilter(apply_complete) filter.execute(listing, response) - apply_complete = getattr(filter, 'complete') + apply_complete = filter.complete if not('kmsListAliases' in unfilterList): filter = KMSListAliasesFilter() @@ -148,20 +147,10 @@ def apply_filters(listing, unfilter, response, complete): filter = NextTokenFilter(apply_complete) filter.execute(listing, response) - apply_complete = getattr(filter, 'complete') + apply_complete = filter.complete return apply_complete -def convert_unfilterList(unfilter): - """Check if unfilter parameter is a list or single argument and return list of it""" - unfilterList = [] - if unfilter is not None: - if isinstance(unfilter, list): - unfilterList = unfilter - else: - unfilterList.append(unfilter) - - return unfilterList diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index 4067f42..3e68e98 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -3,7 +3,7 @@ import boto3 -from .apply_filter import apply_filters, convert_unfilterList +from .apply_filter import apply_filters from .client import get_client from .resource_filter import * @@ -120,12 +120,15 @@ def resources(self): del response['ResponseMetadata'] complete = apply_filters(self.input, self.unfilter, response, complete) - unfilterList = convert_unfilterList(self.unfilter) + unfilterList = self.unfilter + + # Following two if-blocks rely on certain JSON-files being present, hence the DEPENDENT_OPERATIONS + # in query.py. May need rework to function without dependencies. # Special handling for service-level kms keys; derived from alias name. if 'kmsListKeys' not in unfilterList and self.input.service == 'kms' and self.input.operation == 'ListKeys': aliases_file = '{}_{}_{}_{}.json'.format(self.input.service, 'ListAliases', self.input.region, self.input.profile) - aliases_file = self.directory + aliases_file + aliases_file = self.directory + '/' + aliases_file aliases_listing = RawListing.from_json(json.load(open(aliases_file, 'rb'))) list_aliases = aliases_listing.response service_key_ids = [ @@ -137,7 +140,7 @@ def resources(self): # Filter default Internet Gateways if 'ec2InternetGateways' not in unfilterList and self.input.service == 'ec2' and self.input.operation == 'DescribeInternetGateways': vpcs_file = '{}_{}_{}_{}.json'.format(self.input.service, 'DescribeVpcs', self.input.region, self.input.profile) - vpcs_file = self.directory + vpcs_file + vpcs_file = self.directory + '/' + vpcs_file # Sometimes 'No JSON Object' or 'Directory not found' vpcs_listing = RawListing.from_json(json.load(open(vpcs_file, 'rb'))) describe_vpcs = vpcs_listing.response diff --git a/aws_list_all/query.py b/aws_list_all/query.py index f11f774..ed3d3cf 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -15,6 +15,7 @@ from .introspection import get_listing_operations, get_regions_for_service from .listing import RawListing, FilteredListing +from os.path import dirname; RESULT_NOTHING = '---' RESULT_SOMETHING = '+++' @@ -255,7 +256,7 @@ def acquire_listing(verbose, what): if verbose > 1: print(what, 'starting request...') listing = RawListing.acquire(service, region, operation, profile) - listingFile = FilteredListing(listing, './', unfilter) #os.getcwd() + '/' + listingFile = FilteredListing(listing, './', unfilter) duration = time() - start_time if verbose > 1: print(what, '...request successful') @@ -295,7 +296,7 @@ def acquire_listing(verbose, what): def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=False, unfilter=()): """Print out a rudimentary summary of the Listing objects contained in the given files""" - dir = filenames[0][:filenames[0].rfind('/') + 1] + dir = dirname(filenames[0]) for listing_filename in filenames: listing = RawListing.from_json(json.load(open(listing_filename, 'rb'))) listing_entry = FilteredListing(listing, dir, unfilter) From 3d0565fd27c8ceb8ce48f53f43ce9d38e97de08e Mon Sep 17 00:00:00 2001 From: kntrain Date: Mon, 29 Mar 2021 21:03:39 +0200 Subject: [PATCH 12/14] Add error-handling to JSON-file dependent function --- aws_list_all/listing.py | 72 +++++++++++++++++++++++++---------------- aws_list_all/query.py | 6 +++- 2 files changed, 49 insertions(+), 29 deletions(-) diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index 3e68e98..1994359 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -2,11 +2,15 @@ import pprint import boto3 +import json +from datetime import datetime from .apply_filter import apply_filters from .client import get_client from .resource_filter import * +RESULT_TYPE_LENGTH = 3 + PARAMETERS = { 'cloudfront': { 'ListCachePolicies': { @@ -84,10 +88,10 @@ def run_raw_listing_operation(service, region, operation, profile): class FilteredListing(object): - def __init__(self, input, directory='./', unfilter=()): + def __init__(self, input, directory='./', unfilter=None): self.input = input self.directory = directory - self.unfilter = unfilter + self.unfilter = [] if unfilter is None else unfilter @property def resource_types(self): @@ -124,37 +128,42 @@ def resources(self): # Following two if-blocks rely on certain JSON-files being present, hence the DEPENDENT_OPERATIONS # in query.py. May need rework to function without dependencies. - + # Special handling for service-level kms keys; derived from alias name. if 'kmsListKeys' not in unfilterList and self.input.service == 'kms' and self.input.operation == 'ListKeys': - aliases_file = '{}_{}_{}_{}.json'.format(self.input.service, 'ListAliases', self.input.region, self.input.profile) - aliases_file = self.directory + '/' + aliases_file - aliases_listing = RawListing.from_json(json.load(open(aliases_file, 'rb'))) - list_aliases = aliases_listing.response - service_key_ids = [ - k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) - if k.get('AliasName').lower().startswith('alias/aws') - ] - response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] + try: + aliases_file = '{}_{}_{}_{}.json'.format(self.input.service, 'ListAliases', self.input.region, self.input.profile) + aliases_file = self.directory + '/' + aliases_file + aliases_listing = RawListing.from_json(json.load(open(aliases_file, 'rb'))) + list_aliases = aliases_listing.response + service_key_ids = [ + k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) + if k.get('AliasName').lower().startswith('alias/aws') + ] + response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] + except Exception as exc: + self.input.error = repr(exc) # Filter default Internet Gateways if 'ec2InternetGateways' not in unfilterList and self.input.service == 'ec2' and self.input.operation == 'DescribeInternetGateways': - vpcs_file = '{}_{}_{}_{}.json'.format(self.input.service, 'DescribeVpcs', self.input.region, self.input.profile) - vpcs_file = self.directory + '/' + vpcs_file - # Sometimes 'No JSON Object' or 'Directory not found' - vpcs_listing = RawListing.from_json(json.load(open(vpcs_file, 'rb'))) - describe_vpcs = vpcs_listing.response - vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} - internet_gateways = [] - for ig in response['InternetGateways']: - attachments = ig.get('Attachments', []) - # more than one, it cannot be default. - if len(attachments) != 1: - continue - vpc = attachments[0].get('VpcId') - if not vpcs.get(vpc, {}).get('IsDefault', False): - internet_gateways.append(ig) - response['InternetGateways'] = internet_gateways + try: + vpcs_file = '{}_{}_{}_{}.json'.format(self.input.service, 'DescribeVpcs', self.input.region, self.input.profile) + vpcs_file = self.directory + '/' + vpcs_file + vpcs_listing = RawListing.from_json(json.load(open(vpcs_file, 'rb'))) + describe_vpcs = vpcs_listing.response + vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} + internet_gateways = [] + for ig in response['InternetGateways']: + attachments = ig.get('Attachments', []) + # more than one, it cannot be default. + if len(attachments) != 1: + continue + vpc = attachments[0].get('VpcId') + if not vpcs.get(vpc, {}).get('IsDefault', False): + internet_gateways.append(ig) + response['InternetGateways'] = internet_gateways + except Exception as exc: + self.input.error = repr(exc) for key, value in response.items(): if not isinstance(value, list): @@ -163,6 +172,13 @@ def resources(self): if not complete: response['truncated'] = [True] + # Update JSON-file in case an error occurred during resources-processing + if len(self.input.error) > RESULT_TYPE_LENGTH: + self.input.error = '!!!' + with open('{}_{}_{}_{}.json'.format( + self.input.service, self.input.operation, self.input.region, self.input.profile), 'w') as jsonfile: + json.dump(self.input.to_json(), jsonfile, default=datetime.isoformat) + return response diff --git a/aws_list_all/query.py b/aws_list_all/query.py index ed3d3cf..767f88c 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -263,7 +263,11 @@ def acquire_listing(verbose, what): print("timing [success]:", duration, what) with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: json.dump(listing.to_json(), jsonfile, default=datetime.isoformat) - if listingFile.resource_total_count > 0: + + resource_count = listingFile.resource_total_count + if listingFile.input.error == RESULT_ERROR: + return (RESULT_ERROR, service, region, operation, profile, 'Error(Error during processing of resources)') + if resource_count > 0: return (RESULT_SOMETHING, service, region, operation, profile, ', '.join(listingFile.resource_types)) else: return (RESULT_NOTHING, service, region, operation, profile, ', '.join(listingFile.resource_types)) From 7e2cde7ab15daa4d2f09b8ea998aa9483656d942 Mon Sep 17 00:00:00 2001 From: kntrain Date: Fri, 2 Apr 2021 18:28:28 +0200 Subject: [PATCH 13/14] Fix wrong do_list_files behaviour for errors --- aws_list_all/query.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws_list_all/query.py b/aws_list_all/query.py index 767f88c..1abcb67 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -317,6 +317,8 @@ def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=Fa print(listing.service, listing.region, listing.operation, 'MISSING PERMISSION', '0') if listing.error == RESULT_ERROR and errors: print(listing.service, listing.region, listing.operation, 'ERROR', '0') + if listing.error == RESULT_ERROR and listing_entry.resource_total_count > 0: + continue for resource_type, value in resources.items(): if not not_found and len(value) == 0 and not was_denied: From 2055571887e4c1f6879f7a360180fa2fc0e5ed00 Mon Sep 17 00:00:00 2001 From: kntrain Date: Sat, 3 Apr 2021 00:18:36 +0200 Subject: [PATCH 14/14] Add ResultListing as return Obj of acquire_listing --- aws_list_all/listing.py | 14 ++++++++++++++ aws_list_all/query.py | 18 +++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index 1994359..26a8156 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -246,3 +246,17 @@ def resources(self): # pylint:disable=too-many-branches response['truncated'] = [True] return response + + +class ResultListing(object): + """Represents a listing result summary acquired from the function acquire_listing""" + def __init__(self, input, result_type, details): + self.input = input + self.result_type = result_type + self.details = details + + @property + def to_tuple(self): + """Return a tiple of strings describing the result of an executed query""" + return (self.result_type, self.input.service, self.input.region, + self.input.operation, self.input.profile, self.details) diff --git a/aws_list_all/query.py b/aws_list_all/query.py index 1abcb67..0f882a7 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -14,7 +14,7 @@ from traceback import print_exc from .introspection import get_listing_operations, get_regions_for_service -from .listing import RawListing, FilteredListing +from .listing import RawListing, FilteredListing, ResultListing from os.path import dirname; RESULT_NOTHING = '---' @@ -231,18 +231,18 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p print('...done') for result_type in (RESULT_NOTHING, RESULT_SOMETHING, RESULT_NO_ACCESS, RESULT_ERROR): for result in sorted(results_by_type[result_type]): - print(*result) + print(*result.to_tuple) def execute_query(to_run, verbose, parallel, results_by_type): # the `with` block is a workaround for a bug: https://bugs.python.org/issue35629 with contextlib.closing(ThreadPool(parallel)) as pool: for result in pool.imap_unordered(partial(acquire_listing, verbose), to_run): - results_by_type[result[0]].append(result) + results_by_type[result.result_type].append(result) if verbose > 1: - print('ExecutedQueryResult: {}'.format(result)) + print('ExecutedQueryResult: {}'.format(result.to_tuple)) else: - print(result[0][-1], end='') + print(result.to_tuple[0][-1], end='') sys.stdout.flush() return results_by_type @@ -266,11 +266,11 @@ def acquire_listing(verbose, what): resource_count = listingFile.resource_total_count if listingFile.input.error == RESULT_ERROR: - return (RESULT_ERROR, service, region, operation, profile, 'Error(Error during processing of resources)') + return ResultListing(listing, RESULT_ERROR, 'Error(Error during processing of resources)') if resource_count > 0: - return (RESULT_SOMETHING, service, region, operation, profile, ', '.join(listingFile.resource_types)) + return ResultListing(listing, RESULT_SOMETHING, ', '.join(listingFile.resource_types)) else: - return (RESULT_NOTHING, service, region, operation, profile, ', '.join(listingFile.resource_types)) + return ResultListing(listing, RESULT_NOTHING, ', '.join(listingFile.resource_types)) except Exception as exc: # pylint:disable=broad-except duration = time() - start_time if verbose > 1: @@ -295,7 +295,7 @@ def acquire_listing(verbose, what): listing = RawListing(service, region, operation, {}, profile, result_type) with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: json.dump(listing.to_json(), jsonfile, default=datetime.isoformat) - return (result_type, service, region, operation, profile, repr(exc)) + return ResultListing(listing, result_type, repr(exc)) def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=False, unfilter=()):