diff --git a/README.rst b/README.rst index afafdd4..cc59cd3 100644 --- a/README.rst +++ b/README.rst @@ -53,6 +53,40 @@ region, and operation used to find them. They can be dumped with:: aws-list-all show data/ec2_* aws-list-all show --verbose data/ec2_DescribeSecurityGroups_eu-west-1.json +Special treatment and removal of default resources which are performed by default during +data handling can be omitted with --unfilter and following arguments: +- cloudfront +- medialive +- ssmListCommands +- snsListSubscriptions +- athenaWorkGroups +- listEventBuses +- xRayGroups +- route53Resolver +- kmsListAliases +- appstreamImages +- cloudsearch +- cloudTrail +- cloudWatch +- iamPolicies +- s3Owner +- ecsClustersFailure +- pinpointGetApps +- ssmBaselines +- dbSecurityGroups +- dbParameterGroups +- dbClusterParameterGroups +- dbOptionGroups +- ec2VPC +- ec2Subnets +- ec2SecurityGroups +- ec2RouteTables +- ec2NetworkAcls +- ec2FpgaImages +- workmailDeletedOrganizations +- elasticacheSubnetGroups + + How do I really list everything? ------------------------------------------------ diff --git a/aws_list_all/__main__.py b/aws_list_all/__main__.py index d625829..25de924 100755 --- a/aws_list_all/__main__.py +++ b/aws_list_all/__main__.py @@ -72,6 +72,12 @@ def main(): action='append', help='Restrict querying to the given operation (can be specified multiple times)' ) + query.add_argument( + '-u', + '--unfilter', + action='append', + help='Exclude given default-value filter from being applied (can be specified multiple times)' + ) query.add_argument('-p', '--parallel', default=32, type=int, help='Number of request to do in parallel') query.add_argument('-d', '--directory', default='.', help='Directory to save result listings to') query.add_argument('-v', '--verbose', action='count', help='Print detailed info during run') @@ -83,6 +89,15 @@ def main(): ) show.add_argument('listingfile', nargs='*', help='listing file(s) to load and print') show.add_argument('-v', '--verbose', action='count', help='print given listing files with detailed info') + show.add_argument('-n', '--not_found', default=False, action='store_true', help='additionally print listing files of resources not found') + show.add_argument('-e', '--errors', default=False, action='store_true', help='additionally print listing files of resources where queries resulted in errors') + show.add_argument('-d', '--denied', default=False, action='store_true', help='additionally print listing files of resources with "missing permission" errors') + show.add_argument( + '-u', + '--unfilter', + action='append', + help='Exclude given default-value filter from being applied (can be specified multiple times)' + ) # Introspection debugging is not the main function. So we put it all into a subcommand. introspect = subparsers.add_parser( @@ -157,12 +172,20 @@ def main(): args.operation, verbose=args.verbose or 0, parallel=args.parallel, - selected_profile=args.profile + selected_profile=args.profile, + unfilter=args.unfilter ) elif args.command == 'show': if args.listingfile: increase_limit_nofiles() - do_list_files(args.listingfile, verbose=args.verbose or 0) + do_list_files( + args.listingfile, + verbose=args.verbose or 0, + not_found=args.not_found, + errors=args.errors, + denied=args.denied, + unfilter=args.unfilter + ) else: show.print_help() return 1 diff --git a/aws_list_all/apply_filter.py b/aws_list_all/apply_filter.py new file mode 100644 index 0000000..45c2bec --- /dev/null +++ b/aws_list_all/apply_filter.py @@ -0,0 +1,156 @@ +import json +import os +import sys + +from .fixing_filter import * +from .resource_filter import * + +def apply_filters(listing, unfilterList, response, complete): + """Apply filters for operations to be handled in a special way or + to remove default resources from the response""" + apply_complete = complete + + if not('cloudfront' in unfilterList): + filter = CloudfrontFilter() + filter.execute(listing, response) + + if not('medialive' in unfilterList): + filter = MedialiveFilter() + filter.execute(listing, response) + + if not('ssmListCommands' in unfilterList): + filter = SSMListCommandsFilter() + filter.execute(listing, response) + + if not('snsListSubscriptions' in unfilterList): + filter = SNSListSubscriptionsFilter() + filter.execute(listing, response) + + if not('athenaWorkGroups' in unfilterList): + filter = AthenaWorkGroupsFilter() + filter.execute(listing, response) + + if not('listEventBuses' in unfilterList): + filter = ListEventBusesFilter() + filter.execute(listing, response) + + if not('xRayGroups' in unfilterList): + filter = XRayGroupsFilter() + filter.execute(listing, response) + + if not('route53Resolver' in unfilterList): + filter = Route53ResolverFilter() + filter.execute(listing, response) + + filter = CountFilter(apply_complete) + filter.execute(listing, response) + apply_complete = filter.complete + + filter = QuantityFilter(apply_complete) + filter.execute(listing, response) + apply_complete = filter.complete + + filter = NeutralThingFilter() + filter.execute(listing, response) + + filter = BadThingFilter(apply_complete) + filter.execute(listing, response) + apply_complete = filter.complete + + if not('kmsListAliases' in unfilterList): + filter = KMSListAliasesFilter() + filter.execute(listing, response) + + if not('appstreamImages' in unfilterList): + filter = AppstreamImagesFilter() + filter.execute(listing, response) + + if not('cloudsearch' in unfilterList): + filter = CloudsearchFilter() + filter.execute(listing, response) + + if not('cloudTrail' in unfilterList): + filter = CloudTrailFilter() + filter.execute(listing, response) + + if not('cloudWatch' in unfilterList): + filter = CloudWatchFilter() + filter.execute(listing, response) + + if not('iamPolicies' in unfilterList): + filter = IAMPoliciesFilter() + filter.execute(listing, response) + + if not('s3Owner' in unfilterList): + filter = S3OwnerFilter() + filter.execute(listing, response) + + if not('ecsClustersFailure' in unfilterList): + filter = ECSClustersFailureFilter() + filter.execute(listing, response) + + if not('pinpointGetApps' in unfilterList): + filter = PinpointGetAppsFilter() + filter.execute(listing, response) + + if not('ssmBaselines' in unfilterList): + filter = SSMBaselinesFilter() + filter.execute(listing, response) + + if not('dbSecurityGroups' in unfilterList): + filter = DBSecurityGroupsFilter() + filter.execute(listing, response) + + if not('dbParameterGroups' in unfilterList): + filter = DBParameterGroupsFilter() + filter.execute(listing, response) + + if not('dbClusterParameterGroups' in unfilterList): + filter = DBClusterParameterGroupsFilter() + filter.execute(listing, response) + + if not('dbOptionGroups' in unfilterList): + filter = DBOptionGroupsFilter() + filter.execute(listing, response) + + if not('ec2VPC' in unfilterList): + filter = EC2VPCFilter() + filter.execute(listing, response) + + if not('ec2Subnets' in unfilterList): + filter = EC2SubnetsFilter() + filter.execute(listing, response) + + if not('ec2SecurityGroups' in unfilterList): + filter = EC2SecurityGroupsFilter() + filter.execute(listing, response) + + if not('ec2RouteTables' in unfilterList): + filter = EC2RouteTablesFilter() + filter.execute(listing, response) + + if not('ec2NetworkAcls' in unfilterList): + filter = EC2NetworkAclsFilter() + filter.execute(listing, response) + + if not('ec2FpgaImages' in unfilterList): + filter = EC2FpgaImagesFilter() + filter.execute(listing, response) + + if not('workmailDeletedOrganizations' in unfilterList): + filter = WorkmailDeletedOrganizationsFilter() + filter.execute(listing, response) + + if not('elasticacheSubnetGroups' in unfilterList): + filter = ElasticacheSubnetGroupsFilter() + filter.execute(listing, response) + + filter = NextTokenFilter(apply_complete) + filter.execute(listing, response) + apply_complete = filter.complete + + return apply_complete + + + + diff --git a/aws_list_all/fixing_filter.py b/aws_list_all/fixing_filter.py new file mode 100644 index 0000000..f6feb97 --- /dev/null +++ b/aws_list_all/fixing_filter.py @@ -0,0 +1,61 @@ +import json +import pprint + +import boto3 + +from .client import get_client + +class CountFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): + if 'Count' in response: + if 'MaxResults' in response: + if response['MaxResults'] <= response['Count']: + self.complete = False + del response['MaxResults'] + del response['Count'] + +class QuantityFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): + if 'Quantity' in response: + if 'MaxItems' in response: + if response['MaxItems'] <= response['Quantity']: + self.complete = False + del response['MaxItems'] + del response['Quantity'] + +class NeutralThingFilter: + def execute(self, listing, response): + for neutral_thing in ('MaxItems', 'MaxResults', 'Quantity'): + if neutral_thing in response: + del response[neutral_thing] + +class BadThingFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): + for bad_thing in ( + 'hasMoreResults', 'IsTruncated', 'Truncated', 'HasMoreApplications', 'HasMoreDeliveryStreams', + 'HasMoreStreams', 'NextToken', 'NextMarker', 'nextMarker', 'Marker' + ): + if bad_thing in response: + if response[bad_thing]: + self.complete = False + del response[bad_thing] + +class NextTokenFilter: + def __init__(self, complete): + self.complete = complete + + def execute(self, listing, response): + # interpret nextToken in several services + if (listing.service, listing.operation) in (('inspector', 'ListFindings'), ('logs', 'DescribeLogGroups')): + if response.get('nextToken'): + self.complete = False + del response['nextToken'] \ No newline at end of file diff --git a/aws_list_all/listing.py b/aws_list_all/listing.py index 82e5912..26a8156 100644 --- a/aws_list_all/listing.py +++ b/aws_list_all/listing.py @@ -1,8 +1,15 @@ +import json import pprint import boto3 +import json +from datetime import datetime +from .apply_filter import apply_filters from .client import get_client +from .resource_filter import * + +RESULT_TYPE_LENGTH = 3 PARAMETERS = { 'cloudfront': { @@ -79,14 +86,111 @@ def run_raw_listing_operation(service, region, operation, profile): return getattr(client, api_to_method_mapping[operation])(**parameters) -class Listing(object): +class FilteredListing(object): + + def __init__(self, input, directory='./', unfilter=None): + self.input = input + self.directory = directory + self.unfilter = [] if unfilter is None else unfilter + + @property + def resource_types(self): + """The list of resource types (Keys with list content) in the response""" + return list(self.resources.keys()) + + @property + def resource_total_count(self): + """The estimated total count of resources - can be incomplete""" + return sum(len(v) for v in self.resources.values()) + + def export_resources(self, filename): + """Export the result to the given JSON file""" + with open(filename, 'w') as outfile: + outfile.write(pprint.pformat(self.resources).encode('utf-8')) + + def __str__(self): + opdesc = '{} {} {} {}'.format(self.input.service, self.input.region, self.input.operation, self.input.profile) + if len(self.resource_types) == 0 or self.resource_total_count == 0: + return '{} (no resources found)'.format(opdesc) + return opdesc + ', '.join('#{}: {}'.format(key, len(listing)) for key, listing in self.resources.items()) + + @property + def resources(self): + """Transform the response data into a dict of resource names to resource listings""" + if not(self.input.response): + return self.input.response.copy() + response = self.input.response.copy() + complete = True + del response['ResponseMetadata'] + + complete = apply_filters(self.input, self.unfilter, response, complete) + unfilterList = self.unfilter + + # Following two if-blocks rely on certain JSON-files being present, hence the DEPENDENT_OPERATIONS + # in query.py. May need rework to function without dependencies. + + # Special handling for service-level kms keys; derived from alias name. + if 'kmsListKeys' not in unfilterList and self.input.service == 'kms' and self.input.operation == 'ListKeys': + try: + aliases_file = '{}_{}_{}_{}.json'.format(self.input.service, 'ListAliases', self.input.region, self.input.profile) + aliases_file = self.directory + '/' + aliases_file + aliases_listing = RawListing.from_json(json.load(open(aliases_file, 'rb'))) + list_aliases = aliases_listing.response + service_key_ids = [ + k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) + if k.get('AliasName').lower().startswith('alias/aws') + ] + response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] + except Exception as exc: + self.input.error = repr(exc) + + # Filter default Internet Gateways + if 'ec2InternetGateways' not in unfilterList and self.input.service == 'ec2' and self.input.operation == 'DescribeInternetGateways': + try: + vpcs_file = '{}_{}_{}_{}.json'.format(self.input.service, 'DescribeVpcs', self.input.region, self.input.profile) + vpcs_file = self.directory + '/' + vpcs_file + vpcs_listing = RawListing.from_json(json.load(open(vpcs_file, 'rb'))) + describe_vpcs = vpcs_listing.response + vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} + internet_gateways = [] + for ig in response['InternetGateways']: + attachments = ig.get('Attachments', []) + # more than one, it cannot be default. + if len(attachments) != 1: + continue + vpc = attachments[0].get('VpcId') + if not vpcs.get(vpc, {}).get('IsDefault', False): + internet_gateways.append(ig) + response['InternetGateways'] = internet_gateways + except Exception as exc: + self.input.error = repr(exc) + + for key, value in response.items(): + if not isinstance(value, list): + raise Exception('No listing: {} is no list:'.format(key), response) + + if not complete: + response['truncated'] = [True] + + # Update JSON-file in case an error occurred during resources-processing + if len(self.input.error) > RESULT_TYPE_LENGTH: + self.input.error = '!!!' + with open('{}_{}_{}_{}.json'.format( + self.input.service, self.input.operation, self.input.region, self.input.profile), 'w') as jsonfile: + json.dump(self.input.to_json(), jsonfile, default=datetime.isoformat) + + return response + + +class RawListing(object): """Represents a listing operation on an AWS service and its result""" - def __init__(self, service, region, operation, response, profile): + def __init__(self, service, region, operation, response, profile, error=''): self.service = service self.region = region self.operation = operation self.response = response self.profile = profile + self.error = error def to_json(self): return { @@ -95,6 +199,7 @@ def to_json(self): 'profile': self.profile, 'operation': self.operation, 'response': self.response, + 'error': self.error, } @classmethod @@ -104,24 +209,10 @@ def from_json(cls, data): region=data.get('region'), profile=data.get('profile'), operation=data.get('operation'), - response=data.get('response') + response=data.get('response'), + error=data.get('error') ) - @property - def resource_types(self): - """The list of resource types (Keys with list content) in the response""" - return list(self.resources.keys()) - - @property - def resource_total_count(self): - """The estimated total count of resources - can be incomplete""" - return sum(len(v) for v in self.resources.values()) - - def export_resources(self, filename): - """Export the result to the given JSON file""" - with open(filename, 'w') as outfile: - outfile.write(pprint.pformat(self.resources).encode('utf-8')) - def __str__(self): opdesc = '{} {} {} {}'.format(self.service, self.region, self.operation, self.profile) if len(self.resource_types) == 0 or self.resource_total_count == 0: @@ -132,245 +223,20 @@ def __str__(self): def acquire(cls, service, region, operation, profile): """Acquire the given listing by making an AWS request""" response = run_raw_listing_operation(service, region, operation, profile) - if response['ResponseMetadata']['HTTPStatusCode'] != 200: - raise Exception('Bad AWS HTTP Status Code', response) + # if response['ResponseMetadata']['HTTPStatusCode'] != 200: + # raise Exception('Bad AWS HTTP Status Code', response) return cls(service, region, operation, response, profile) @property def resources(self): # pylint:disable=too-many-branches """Transform the response data into a dict of resource names to resource listings""" + if not(self.response): + return self.response.copy() response = self.response.copy() complete = True del response['ResponseMetadata'] - - # Transmogrify strange cloudfront results into standard AWS format - if self.service == 'cloudfront': - assert len(response.keys()) == 1, 'Unexpected cloudfront response: {}'.format(response) - key = list(response.keys())[0][:-len('List')] - response = list(response.values())[0] - response[key] = response.get('Items', []) - - # medialive List* things sends a next token; remove if no channels/lists - if self.service == 'medialive': - if self.operation == 'ListChannels' and not response['Channels']: - if 'Channels' in response: - del response['Channels'] - if 'NextToken' in response: - del response['NextToken'] - if self.operation == 'ListInputs' and not response['Inputs']: - if 'Inputs' in response: - del response['Inputs'] - if 'NextToken' in response: - del response['NextToken'] - - # ssm ListCommands sends a next token; remove if no channels - if self.service == 'ssm' and self.operation == 'ListCommands': - if 'NextToken' in response and not response['Commands']: - del response['NextToken'] - - # SNS ListSubscriptions always sends a next token... - if self.service == 'sns' and self.operation == 'ListSubscriptions': - del response['NextToken'] - - # Athena has a "primary" work group that is always present - if self.service == 'athena' and self.operation == 'ListWorkGroups': - response['WorkGroups'] = [wg for wg in response.get('WorkGroups', []) if wg['Name'] != 'primary'] - - # Remove default event buses - if self.service == 'events' and self.operation == 'ListEventBuses': - response['EventBuses'] = [wg for wg in response.get('EventBuses', []) if wg['Name'] != 'default'] - - # XRay has a "Default" group that is always present - if self.service == 'xray' and self.operation == 'GetGroups': - response['Groups'] = [wg for wg in response.get('Groups', []) if wg['GroupName'] != 'Default'] - - if self.service == 'route53resolver': - if self.operation == 'ListResolverRules': - response['ResolverRules'] = [ - rule for rule in response.get('ResolverRules', []) - if rule['Id'] != 'rslvr-autodefined-rr-internet-resolver' - ] - if self.operation == 'ListResolverRuleAssociations': - response['ResolverRuleAssociations'] = [ - rule for rule in response.get('ResolverRuleAssociations', []) - if rule['ResolverRuleId'] != 'rslvr-autodefined-rr-internet-resolver' - ] - - if 'Count' in response: - if 'MaxResults' in response: - if response['MaxResults'] <= response['Count']: - complete = False - del response['MaxResults'] - del response['Count'] - - if 'Quantity' in response: - if 'MaxItems' in response: - if response['MaxItems'] <= response['Quantity']: - complete = False - del response['MaxItems'] - del response['Quantity'] - - for neutral_thing in ('MaxItems', 'MaxResults', 'Quantity'): - if neutral_thing in response: - del response[neutral_thing] - - for bad_thing in ( - 'hasMoreResults', 'IsTruncated', 'Truncated', 'HasMoreApplications', 'HasMoreDeliveryStreams', - 'HasMoreStreams', 'NextToken', 'NextMarker', 'nextMarker', 'Marker' - ): - if bad_thing in response: - if response[bad_thing]: - complete = False - del response[bad_thing] - - # Special handling for Aliases in kms, there are some reserved AWS-managed aliases. - if self.service == 'kms' and self.operation == 'ListAliases': - response['Aliases'] = [ - alias for alias in response.get('Aliases', []) - if not alias.get('AliasName').lower().startswith('alias/aws') - ] - - # Special handling for service-level kms keys; derived from alias name. - if self.service == 'kms' and self.operation == 'ListKeys': - list_aliases = run_raw_listing_operation(self.service, self.region, 'ListAliases', self.profile) - service_key_ids = [ - k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) - if k.get('AliasName').lower().startswith('alias/aws') - ] - response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] - - # Filter PUBLIC images from appstream - if self.service == 'appstream' and self.operation == 'DescribeImages': - response['Images'] = [ - image for image in response.get('Images', []) if not image.get('Visibility', 'PRIVATE') == 'PUBLIC' - ] - - # This API returns a dict instead of a list - if self.service == 'cloudsearch' and self.operation == 'ListDomainNames': - response['DomainNames'] = list(response['DomainNames'].items()) - - # Only list CloudTrail trails in own/Home Region - if self.service == 'cloudtrail' and self.operation == 'DescribeTrails': - response['trailList'] = [ - trail for trail in response['trailList'] - if trail.get('HomeRegion') == self.region or not trail.get('IsMultiRegionTrail') - ] - - # Remove AWS-default cloudwatch metrics - if self.service == 'cloudwatch' and self.operation == 'ListMetrics': - response['Metrics'] = [ - metric for metric in response['Metrics'] if not metric.get('Namespace').startswith('AWS/') - ] - - # Remove AWS supplied policies - if self.service == 'iam' and self.operation == 'ListPolicies': - response['Policies'] = [ - policy for policy in response['Policies'] if not policy['Arn'].startswith('arn:aws:iam::aws:') - ] - - # Owner Info is not necessary - if self.service == 's3' and self.operation == 'ListBuckets': - del response['Owner'] - - # Remove failures from ecs/DescribeClusters - if self.service == 'ecs' and self.operation == 'DescribeClusters': - if 'failures' in response: - del response['failures'] - - # This API returns a dict instead of a list - if self.service == 'pinpoint' and self.operation == 'GetApps': - response['ApplicationsResponse'] = response.get('ApplicationsResponse', {}).get('Items', []) - - # Remove AWS-defined Baselines - if self.service == 'ssm' and self.operation == 'DescribePatchBaselines': - response['BaselineIdentities'] = [ - line for line in response['BaselineIdentities'] if not line['BaselineName'].startswith('AWS-') - ] - - # Remove default DB Security Group - if self.service in 'rds' and self.operation == 'DescribeDBSecurityGroups': - response['DBSecurityGroups'] = [ - group for group in response['DBSecurityGroups'] if group['DBSecurityGroupName'] != 'default' - ] - - # Remove default DB Parameter Groups - if self.service in ('rds', 'neptune', 'docdb') and self.operation in 'DescribeDBParameterGroups': - response['DBParameterGroups'] = [ - group for group in response['DBParameterGroups'] - if not group['DBParameterGroupName'].startswith('default.') - ] - - # Remove default DB Cluster Parameter Groups - if self.service in ('rds', 'neptune', 'docdb') and self.operation in 'DescribeDBClusterParameterGroups': - response['DBClusterParameterGroups'] = [ - group for group in response['DBClusterParameterGroups'] - if not group['DBClusterParameterGroupName'].startswith('default.') - ] - - # Remove default DB Option Groups - if self.service == 'rds' and self.operation == 'DescribeOptionGroups': - response['OptionGroupsList'] = [ - group for group in response['OptionGroupsList'] if not group['OptionGroupName'].startswith('default:') - ] - - # Filter default VPCs - if self.service == 'ec2' and self.operation == 'DescribeVpcs': - response['Vpcs'] = [vpc for vpc in response['Vpcs'] if not vpc['IsDefault']] - - # Filter default Subnets - if self.service == 'ec2' and self.operation == 'DescribeSubnets': - response['Subnets'] = [net for net in response['Subnets'] if not net['DefaultForAz']] - - # Filter default SGs - if self.service == 'ec2' and self.operation == 'DescribeSecurityGroups': - response['SecurityGroups'] = [sg for sg in response['SecurityGroups'] if sg['GroupName'] != 'default'] - - # Filter main route tables - if self.service == 'ec2' and self.operation == 'DescribeRouteTables': - response['RouteTables'] = [ - rt for rt in response['RouteTables'] if not any(x['Main'] for x in rt['Associations']) - ] - - # Filter default Network ACLs - if self.service == 'ec2' and self.operation == 'DescribeNetworkAcls': - response['NetworkAcls'] = [nacl for nacl in response['NetworkAcls'] if not nacl['IsDefault']] - - # Filter default Internet Gateways - if self.service == 'ec2' and self.operation == 'DescribeInternetGateways': - describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile) - vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} - internet_gateways = [] - for ig in response['InternetGateways']: - attachments = ig.get('Attachments', []) - # more than one, it cannot be default. - if len(attachments) != 1: - continue - vpc = attachments[0].get('VpcId') - if not vpcs.get(vpc, {}).get('IsDefault', False): - internet_gateways.append(ig) - response['InternetGateways'] = internet_gateways - - # Filter Public images from ec2.fpga images - if self.service == 'ec2' and self.operation == 'DescribeFpgaImages': - response['FpgaImages'] = [image for image in response.get('FpgaImages', []) if not image.get('Public')] - - # Remove deleted Organizations - if self.service == 'workmail' and self.operation == 'ListOrganizations': - response['OrganizationSummaries'] = [ - s for s in response.get('OrganizationSummaries', []) if not s.get('State') == 'Deleted' - ] - - if self.service == 'elasticache' and self.operation == 'DescribeCacheSubnetGroups': - response['CacheSubnetGroups'] = [ - g for g in response.get('CacheSubnetGroups', []) if g.get('CacheSubnetGroupName') != 'default' - ] - - # interpret nextToken in several services - if (self.service, self.operation) in (('inspector', 'ListFindings'), ('logs', 'DescribeLogGroups')): - if response.get('nextToken'): - complete = False - del response['nextToken'] + #complete = apply_filters(self, response, complete) for key, value in response.items(): if not isinstance(value, list): @@ -380,3 +246,17 @@ def resources(self): # pylint:disable=too-many-branches response['truncated'] = [True] return response + + +class ResultListing(object): + """Represents a listing result summary acquired from the function acquire_listing""" + def __init__(self, input, result_type, details): + self.input = input + self.result_type = result_type + self.details = details + + @property + def to_tuple(self): + """Return a tiple of strings describing the result of an executed query""" + return (self.result_type, self.input.service, self.input.region, + self.input.operation, self.input.profile, self.details) diff --git a/aws_list_all/query.py b/aws_list_all/query.py index 337906a..0f882a7 100644 --- a/aws_list_all/query.py +++ b/aws_list_all/query.py @@ -1,6 +1,8 @@ from __future__ import print_function +import codecs import json +import os import sys import contextlib from collections import defaultdict @@ -12,13 +14,19 @@ from traceback import print_exc from .introspection import get_listing_operations, get_regions_for_service -from .listing import Listing +from .listing import RawListing, FilteredListing, ResultListing +from os.path import dirname; RESULT_NOTHING = '---' RESULT_SOMETHING = '+++' RESULT_ERROR = '!!!' RESULT_NO_ACCESS = '>:|' +DEPENDENT_OPERATIONS = { + 'ListKeys' : 'ListAliases', + 'DescribeInternetGateways' : 'DescribeVpcs', +} + # List of requests with legitimate, persistent errors that indicate that no listable resources are present. # # If the request would never return listable resources, it should not be done and be listed in one of the lists @@ -195,10 +203,11 @@ NOT_AVAILABLE_STRINGS = NOT_AVAILABLE_FOR_REGION_STRINGS + NOT_AVAILABLE_FOR_ACCOUNT_STRINGS -def do_query(services, selected_regions=(), selected_operations=(), verbose=0, parallel=32, selected_profile=None): +def do_query(services, selected_regions=(), selected_operations=(), verbose=0, parallel=32, selected_profile=None, unfilter=()): """For the given services, execute all selected operations (default: all) in selected regions (default: all)""" to_run = [] + dependencies = {} print('Building set of queries to execute...') for service in services: for region in get_regions_for_service(service, selected_regions): @@ -206,45 +215,62 @@ def do_query(services, selected_regions=(), selected_operations=(), verbose=0, p if verbose > 0: region_name = region or 'n/a' print('Service: {: <28} | Region: {:<15} | Operation: {}'.format(service, region_name, operation)) + if operation in DEPENDENT_OPERATIONS: + dependencies[DEPENDENT_OPERATIONS[operation], region] = [service, region, DEPENDENT_OPERATIONS[operation], selected_profile, unfilter] + if operation in DEPENDENT_OPERATIONS.values(): + dependencies[operation, region] = [service, region, operation, selected_profile, unfilter] + continue - to_run.append([service, region, operation, selected_profile]) + to_run.append([service, region, operation, selected_profile, unfilter]) shuffle(to_run) # Distribute requests across endpoints results_by_type = defaultdict(list) print('...done. Executing queries...') + + results_by_type = execute_query(dependencies.values(), verbose, parallel, results_by_type) + results_by_type = execute_query(to_run, verbose, parallel, results_by_type) + print('...done') + for result_type in (RESULT_NOTHING, RESULT_SOMETHING, RESULT_NO_ACCESS, RESULT_ERROR): + for result in sorted(results_by_type[result_type]): + print(*result.to_tuple) + + +def execute_query(to_run, verbose, parallel, results_by_type): # the `with` block is a workaround for a bug: https://bugs.python.org/issue35629 with contextlib.closing(ThreadPool(parallel)) as pool: for result in pool.imap_unordered(partial(acquire_listing, verbose), to_run): - results_by_type[result[0]].append(result) + results_by_type[result.result_type].append(result) if verbose > 1: - print('ExecutedQueryResult: {}'.format(result)) + print('ExecutedQueryResult: {}'.format(result.to_tuple)) else: - print(result[0][-1], end='') + print(result.to_tuple[0][-1], end='') sys.stdout.flush() - print('...done') - for result_type in (RESULT_NOTHING, RESULT_SOMETHING, RESULT_NO_ACCESS, RESULT_ERROR): - for result in sorted(results_by_type[result_type]): - print(*result) + return results_by_type def acquire_listing(verbose, what): """Given a service, region and operation execute the operation, serialize and save the result and return a tuple of strings describing the result.""" - service, region, operation, profile = what + service, region, operation, profile, unfilter = what start_time = time() try: if verbose > 1: print(what, 'starting request...') - listing = Listing.acquire(service, region, operation, profile) + listing = RawListing.acquire(service, region, operation, profile) + listingFile = FilteredListing(listing, './', unfilter) duration = time() - start_time if verbose > 1: print(what, '...request successful') print("timing [success]:", duration, what) - if listing.resource_total_count > 0: - with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: + with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: json.dump(listing.to_json(), jsonfile, default=datetime.isoformat) - return (RESULT_SOMETHING, service, region, operation, profile, ', '.join(listing.resource_types)) + + resource_count = listingFile.resource_total_count + if listingFile.input.error == RESULT_ERROR: + return ResultListing(listing, RESULT_ERROR, 'Error(Error during processing of resources)') + if resource_count > 0: + return ResultListing(listing, RESULT_SOMETHING, ', '.join(listingFile.resource_types)) else: - return (RESULT_NOTHING, service, region, operation, profile, ', '.join(listing.resource_types)) + return ResultListing(listing, RESULT_NOTHING, ', '.join(listingFile.resource_types)) except Exception as exc: # pylint:disable=broad-except duration = time() - start_time if verbose > 1: @@ -266,19 +292,41 @@ def acquire_listing(verbose, what): if not_available_string in str(exc): result_type = RESULT_NOTHING - return (result_type, service, region, operation, profile, repr(exc)) + listing = RawListing(service, region, operation, {}, profile, result_type) + with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile: + json.dump(listing.to_json(), jsonfile, default=datetime.isoformat) + return ResultListing(listing, result_type, repr(exc)) -def do_list_files(filenames, verbose=0): +def do_list_files(filenames, verbose=0, not_found=False, errors=False, denied=False, unfilter=()): """Print out a rudimentary summary of the Listing objects contained in the given files""" + dir = dirname(filenames[0]) for listing_filename in filenames: - listing = Listing.from_json(json.load(open(listing_filename, 'rb'))) - resources = listing.resources + listing = RawListing.from_json(json.load(open(listing_filename, 'rb'))) + listing_entry = FilteredListing(listing, dir, unfilter) + resources = listing_entry.resources + truncated = False + was_denied = False if 'truncated' in resources: truncated = resources['truncated'] del resources['truncated'] + if listing.error == RESULT_NO_ACCESS: + was_denied = True + if not resources and denied: + print(listing.service, listing.region, listing.operation, 'MISSING PERMISSION', '0') + if listing.error == RESULT_ERROR and errors: + print(listing.service, listing.region, listing.operation, 'ERROR', '0') + if listing.error == RESULT_ERROR and listing_entry.resource_total_count > 0: + continue + for resource_type, value in resources.items(): + if not not_found and len(value) == 0 and not was_denied: + continue + if not denied and was_denied: + continue + if was_denied: + resource_type = 'MISSING PERMISSION' len_string = '> {}'.format(len(value)) if truncated else str(len(value)) print(listing.service, listing.region, listing.operation, resource_type, len_string) if verbose > 0: @@ -313,3 +361,4 @@ def do_list_files(filenames, verbose=0): print(' - ', item) if truncated: print(' - ... (more items, query truncated)') + diff --git a/aws_list_all/resource_filter.py b/aws_list_all/resource_filter.py new file mode 100644 index 0000000..a687d37 --- /dev/null +++ b/aws_list_all/resource_filter.py @@ -0,0 +1,285 @@ +import json +import pprint + +import boto3 + +from .client import get_client + +class ResourceFilter: + def execute(self, listing, response): + pass + +class CloudfrontFilter: + def execute(self, listing, response): + # Transmogrify strange cloudfront results into standard AWS format + if listing.service == 'cloudfront': + assert len(response.keys()) == 1, 'Unexpected cloudfront response: {}'.format(response) + key = list(response.keys())[0][:-len('List')] + response = list(response.values())[0] + response[key] = response.get('Items', []) + +class MedialiveFilter: + def execute(self, listing, response): + # medialive List* things sends a next token; remove if no channels/lists + if listing.service == 'medialive': + if listing.operation == 'ListChannels' and not response['Channels']: + if 'Channels' in response: + del response['Channels'] + if 'NextToken' in response: + del response['NextToken'] + if listing.operation == 'ListInputs' and not response['Inputs']: + if 'Inputs' in response: + del response['Inputs'] + if 'NextToken' in response: + del response['NextToken'] + +class SSMListCommandsFilter: + def execute(self, listing, response): + # ssm ListCommands sends a next token; remove if no channels + if listing.service == 'ssm' and listing.operation == 'ListCommands': + if 'NextToken' in response and not response['Commands']: + del response['NextToken'] + +class SNSListSubscriptionsFilter: + def execute(self, listing, response): + # SNS ListSubscriptions always sends a next token... + if listing.service == 'sns' and listing.operation == 'ListSubscriptions': + del response['NextToken'] + +class AthenaWorkGroupsFilter: + def execute(self, listing, response): + # Athena has a "primary" work group that is always present + if listing.service == 'athena' and listing.operation == 'ListWorkGroups': + response['WorkGroups'] = [wg for wg in response.get('WorkGroups', []) if wg['Name'] != 'primary'] + +class ListEventBusesFilter: + def execute(self, listing, response): + # Remove default event buses + if listing.service == 'events' and listing.operation == 'ListEventBuses': + response['EventBuses'] = [wg for wg in response.get('EventBuses', []) if wg['Name'] != 'default'] + +class XRayGroupsFilter: + def execute(self, listing, response): + # XRay has a "Default" group that is always present + if listing.service == 'xray' and listing.operation == 'GetGroups': + response['Groups'] = [wg for wg in response.get('Groups', []) if wg['GroupName'] != 'Default'] + +class Route53ResolverFilter: + def execute(self, listing, response): + if listing.service == 'route53resolver': + if listing.operation == 'ListResolverRules': + response['ResolverRules'] = [ + rule for rule in response.get('ResolverRules', []) + if rule['Id'] != 'rslvr-autodefined-rr-internet-resolver' + ] + if listing.operation == 'ListResolverRuleAssociations': + response['ResolverRuleAssociations'] = [ + rule for rule in response.get('ResolverRuleAssociations', []) + if rule['ResolverRuleId'] != 'rslvr-autodefined-rr-internet-resolver' + ] + +class KMSListAliasesFilter: + def execute(self, listing, response): + # Special handling for Aliases in kms, there are some reserved AWS-managed aliases. + if listing.service == 'kms' and listing.operation == 'ListAliases': + response['Aliases'] = [ + alias for alias in response.get('Aliases', []) + if not alias.get('AliasName').lower().startswith('alias/aws') + ] + +class KMSListKeysFilter: + def __init__(self, directory): + self.directory = directory + + def execute(self, listing, response): + # Special handling for service-level kms keys; derived from alias name. + if listing.service == 'kms' and listing.operation == 'ListKeys': + #list_aliases = run_raw_listing_operation(self.service, self.region, 'ListAliases', self.profile) + aliases_file = '{}_{}_{}_{}.json'.format(listing.service, 'ListAliases', listing.region, listing.profile) + aliases_file = self.directory + aliases_file + aliases_listing = Listing.from_json(json.load(open(aliases_file, 'rb'))) + list_aliases = aliases_listing.response + service_key_ids = [ + k.get('TargetKeyId') for k in list_aliases.get('Aliases', []) + if k.get('AliasName').lower().startswith('alias/aws') + ] + response['Keys'] = [k for k in response.get('Keys', []) if k.get('KeyId') not in service_key_ids] + +class AppstreamImagesFilter: + def execute(self, listing, response): + # Filter PUBLIC images from appstream + if listing.service == 'appstream' and listing.operation == 'DescribeImages': + response['Images'] = [ + image for image in response.get('Images', []) if not image.get('Visibility', 'PRIVATE') == 'PUBLIC' + ] + +class CloudsearchFilter: + def execute(self, listing, response): + # This API returns a dict instead of a list + if listing.service == 'cloudsearch' and listing.operation == 'ListDomainNames': + response['DomainNames'] = list(response['DomainNames'].items()) + +class CloudTrailFilter: + def execute(self, listing, response): + # Only list CloudTrail trails in own/Home Region + if listing.service == 'cloudtrail' and listing.operation == 'DescribeTrails': + response['trailList'] = [ + trail for trail in response['trailList'] + if trail.get('HomeRegion') == self.region or not trail.get('IsMultiRegionTrail') + ] + +class CloudWatchFilter: + def execute(self, listing, response): + # Remove AWS-default cloudwatch metrics + if listing.service == 'cloudwatch' and listing.operation == 'ListMetrics': + response['Metrics'] = [ + metric for metric in response['Metrics'] if not metric.get('Namespace').startswith('AWS/') + ] + +class IAMPoliciesFilter: + def execute(self, listing, response): + # Remove AWS supplied policies + if listing.service == 'iam' and listing.operation == 'ListPolicies': + response['Policies'] = [ + policy for policy in response['Policies'] if not policy['Arn'].startswith('arn:aws:iam::aws:') + ] + +class S3OwnerFilter: + def execute(self, listing, response): + # Owner Info is not necessary + if listing.service == 's3' and listing.operation == 'ListBuckets': + del response['Owner'] + +class ECSClustersFailureFilter: + def execute(self, listing, response): + # Remove failures from ecs/DescribeClusters + if listing.service == 'ecs' and listing.operation == 'DescribeClusters': + if 'failures' in response: + del response['failures'] + +class PinpointGetAppsFilter: + def execute(self, listing, response): + # This API returns a dict instead of a list + if listing.service == 'pinpoint' and listing.operation == 'GetApps': + response['ApplicationsResponse'] = response.get('ApplicationsResponse', {}).get('Items', []) + +class SSMBaselinesFilter: + def execute(self, listing, response): + # Remove AWS-defined Baselines + if listing.service == 'ssm' and listing.operation == 'DescribePatchBaselines': + response['BaselineIdentities'] = [ + line for line in response['BaselineIdentities'] if not line['BaselineName'].startswith('AWS-') + ] + +class DBSecurityGroupsFilter: + def execute(self, listing, response): + # Remove default DB Security Group + if listing.service in 'rds' and listing.operation == 'DescribeDBSecurityGroups': + response['DBSecurityGroups'] = [ + group for group in response['DBSecurityGroups'] if group['DBSecurityGroupName'] != 'default' + ] + +class DBParameterGroupsFilter: + def execute(self, listing, response): + # Remove default DB Parameter Groups + if listing.service in ('rds', 'neptune', 'docdb') and listing.operation in 'DescribeDBParameterGroups': + response['DBParameterGroups'] = [ + group for group in response['DBParameterGroups'] + if not group['DBParameterGroupName'].startswith('default.') + ] + +class DBClusterParameterGroupsFilter: + def execute(self, listing, response): + # Remove default DB Cluster Parameter Groups + if listing.service in ('rds', 'neptune', 'docdb') and listing.operation in 'DescribeDBClusterParameterGroups': + response['DBClusterParameterGroups'] = [ + group for group in response['DBClusterParameterGroups'] + if not group['DBClusterParameterGroupName'].startswith('default.') + ] + +class DBOptionGroupsFilter: + def execute(self, listing, response): + # Remove default DB Option Groups + if listing.service == 'rds' and listing.operation == 'DescribeOptionGroups': + response['OptionGroupsList'] = [ + group for group in response['OptionGroupsList'] if not group['OptionGroupName'].startswith('default:') + ] + +class EC2VPCFilter: + def execute(self, listing, response): + # Filter default VPCs + if listing.service == 'ec2' and listing.operation == 'DescribeVpcs': + response['Vpcs'] = [vpc for vpc in response['Vpcs'] if not vpc['IsDefault']] + +class EC2SubnetsFilter: + def execute(self, listing, response): + # Filter default Subnets + if listing.service == 'ec2' and listing.operation == 'DescribeSubnets': + response['Subnets'] = [net for net in response['Subnets'] if not net['DefaultForAz']] + +class EC2SecurityGroupsFilter: + def execute(self, listing, response): + # Filter default SGs + if listing.service == 'ec2' and listing.operation == 'DescribeSecurityGroups': + response['SecurityGroups'] = [sg for sg in response['SecurityGroups'] if sg['GroupName'] != 'default'] + +class EC2RouteTablesFilter: + def execute(self, listing, response): + # Filter main route tables + if listing.service == 'ec2' and listing.operation == 'DescribeRouteTables': + response['RouteTables'] = [ + rt for rt in response['RouteTables'] if not any(x['Main'] for x in rt['Associations']) + ] + +class EC2NetworkAclsFilter: + def execute(self, listing, response): + # Filter default Network ACLs + if listing.service == 'ec2' and listing.operation == 'DescribeNetworkAcls': + response['NetworkAcls'] = [nacl for nacl in response['NetworkAcls'] if not nacl['IsDefault']] + +class EC2InternetGatewaysFilter: + def __init__(self, directory): + self.directory = directory + + def execute(self, listing, response): + # Filter default Internet Gateways + if listing.service == 'ec2' and listing.operation == 'DescribeInternetGateways': + #describe_vpcs = run_raw_listing_operation(self.service, self.region, 'DescribeVpcs', self.profile) + vpcs_file = '{}_{}_{}_{}.json'.format(listing.service, 'DescribeVpcs', listing.region, listing.profile) + vpcs_file = self.directory + vpcs_file + vpcs_listing = Listing.from_json(json.load(open(vpcs_file, 'rb'))) + describe_vpcs = vpcs_listing.response + vpcs = {v['VpcId']: v for v in describe_vpcs.get('Vpcs', [])} + internet_gateways = [] + # print(self.response) + # print(response) + for ig in response['InternetGateways']: + attachments = ig.get('Attachments', []) + # more than one, it cannot be default. + if len(attachments) != 1: + continue + vpc = attachments[0].get('VpcId') + if not vpcs.get(vpc, {}).get('IsDefault', False): + internet_gateways.append(ig) + response['InternetGateways'] = internet_gateways + +class EC2FpgaImagesFilter: + def execute(self, listing, response): + # Filter Public images from ec2.fpga images + if listing.service == 'ec2' and listing.operation == 'DescribeFpgaImages': + response['FpgaImages'] = [image for image in response.get('FpgaImages', []) if not image.get('Public')] + +class WorkmailDeletedOrganizationsFilter: + def execute(self, listing, response): + # Remove deleted Organizations + if listing.service == 'workmail' and listing.operation == 'ListOrganizations': + response['OrganizationSummaries'] = [ + s for s in response.get('OrganizationSummaries', []) if not s.get('State') == 'Deleted' + ] + +class ElasticacheSubnetGroupsFilter: + def execute(self, listing, response): + if listing.service == 'elasticache' and listing.operation == 'DescribeCacheSubnetGroups': + response['CacheSubnetGroups'] = [ + g for g in response.get('CacheSubnetGroups', []) if g.get('CacheSubnetGroupName') != 'default' + ] \ No newline at end of file