Improve visualization branch + Feature comparing

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 python:
   - "2.7"
-  - "3.6"
+  - "3.9"
 install:
   - pip install flake8 yapf pytest
   - python setup.py install

aws_list_all/__main__.py

@@ -2,14 +2,16 @@
 from __future__ import print_function
 
 import os
+import sys
+import webbrowser
 from resource import getrlimit, setrlimit, RLIMIT_NOFILE
 from argparse import ArgumentParser
 from sys import exit, stderr
 
 from .introspection import (
     get_listing_operations, get_services, get_verbs, introspect_regions_for_service, recreate_caches
 )
-from .query import do_list_files, do_query
+from .query import show_list_files, do_query, do_consecutive
 
 
 def increase_limit_nofiles():
@@ -72,6 +74,12 @@ def main():
         action='append',
         help='Restrict querying to the given operation (can be specified multiple times)'
     )
+    query.add_argument(
+        '-u',
+        '--unfilter',
+        action='append',
+        help='Exclude given default-value filter from being applied (can be specified multiple times)'
+    )
     query.add_argument('-p', '--parallel', default=32, type=int, help='Number of request to do in parallel')
     query.add_argument('-d', '--directory', default='.', help='Directory to save result listings to')
     query.add_argument('-v', '--verbose', action='count', help='Print detailed info during run')
@@ -83,6 +91,17 @@ def main():
     )
     show.add_argument('listingfile', nargs='*', help='listing file(s) to load and print')
     show.add_argument('-v', '--verbose', action='count', help='print given listing files with detailed info')
+    show.add_argument('-n', '--not_found', default=False, action='store_true', help='additionally print listing files of resources not found')
+    show.add_argument('-e', '--errors', default=False, action='store_true', help='additionally print listing files of resources where queries resulted in errors')
+    show.add_argument('-b', '--denied', default=False, action='store_true', help='additionally print listing files of resources with "missing permission" errors')
+    show.add_argument('-w', '--html', default='', help='Print and display the results in HTML-file with given name')
+    show.add_argument('-c', '--cmp', nargs='*', default='.', help='Compare target directory to this and display the results in HTML-file named cmp.html')
+    show.add_argument(
+        '-u',
+        '--unfilter',
+        action='append',
+        help='Exclude given default-value filter from being applied (can be specified multiple times)'
+    )
 
     # Introspection debugging is not the main function. So we put it all into a subcommand.
     introspect = subparsers.add_parser(
@@ -119,6 +138,45 @@ def main():
     )
     introspecters.add_parser('debug', description='Debug information', help='Debug information')
 
+    # Combine the features of query and show and optionally print the findings
+    # from query to an HTML file to be viewed in a browser
+    view = subparsers.add_parser(
+        'view', description='Print a listing to an HTML file to be viewed in a browser',
+        help='Create browser view of listing'
+    )
+    view.add_argument(
+        '-s',
+        '--service',
+        action='append',
+        help='Restrict querying to the given service (can be specified multiple times)'
+    )
+    view.add_argument(
+        '-r',
+        '--region',
+        action='append',
+        help='Restrict querying to the given region (can be specified multiple times)'
+    )
+    view.add_argument(
+        '-o',
+        '--operation',
+        action='append',
+        help='Restrict querying to the given operation (can be specified multiple times)'
+    )
+    view.add_argument(
+        '-u',
+        '--unfilter',
+        action='append',
+        help='Exclude given default-value filter from being applied (can be specified multiple times)'
+    )
+    view.add_argument('-p', '--parallel', default=32, type=int, help='Number of request to do in parallel')
+    view.add_argument('-d', '--directory', default='.', help='Directory to save result listings to')
+    view.add_argument('-v', '--verbose', action='count', help='Print detailed info during run')
+    view.add_argument('-c', '--profile', help='Use a specific .aws/credentials profile.')
+    view.add_argument('-n', '--not_found', default=False, action='store_true', help='additionally print listing files of resources not found')
+    view.add_argument('-e', '--errors', default=False, action='store_true', help='additionally print listing files of resources where queries resulted in errors')
+    view.add_argument('-b', '--denied', default=False, action='store_true', help='additionally print listing files of resources with "missing permission" errors')
+    view.add_argument('-w', '--html', default='', help='Print and display the query results inside HTML-file with given name')
+
     # Finally, refreshing the service/region caches comes last.
     caches = subparsers.add_parser(
         'recreate-caches',
@@ -156,12 +214,24 @@ def main():
             args.operation,
             verbose=args.verbose or 0,
             parallel=args.parallel,
-            selected_profile=args.profile
+            selected_profile=args.profile,
+            unfilter=args.unfilter
         )
     elif args.command == 'show':
+        orig_dir = os.getcwd()
         if args.listingfile:
             increase_limit_nofiles()
-            do_list_files(args.listingfile, verbose=args.verbose or 0)
+            show_list_files(
+                args.listingfile,
+                orig_dir,
+                args.cmp,
+                args.html,
+                verbose=args.verbose or 0,
+                not_found=args.not_found,
+                errors=args.errors,
+                denied=args.denied,
+                unfilter=args.unfilter
+            )
         else:
             show.print_help()
             return 1
@@ -183,6 +253,31 @@ def main():
         else:
             introspect.print_help()
             return 1
+    elif args.command == 'view':
+        orig_dir = os.getcwd()
+        if args.directory:
+            try:
+                os.makedirs(args.directory)
+            except OSError:
+                pass
+            os.chdir(args.directory)
+        increase_limit_nofiles()
+        services = args.service or get_services()
+        do_consecutive(
+            services,
+            orig_dir,
+            args.directory,
+            args.html,
+            args.region,
+            args.operation,
+            verbose=args.verbose or 0,
+            parallel=args.parallel,
+            selected_profile=args.profile,
+            unfilter=args.unfilter,
+            not_found=args.not_found,
+            errors=args.errors,
+            denied=args.denied
+        )
     elif args.command == 'recreate-caches':
         increase_limit_nofiles()
         recreate_caches(args.update_packaged_values)

aws_list_all/apply_filter.py

@@ -0,0 +1,152 @@
+import json
+import os
+import sys
+
+from .fixing_filter import *
+from .resource_filter import *
+
+def apply_filters(listing, unfilterList, response, complete):
+    """Apply filters for operations to be handled in a special way or 
+        to remove default resources from the response"""
+    apply_complete = complete
+
+    if not('cloudfront' in unfilterList):
+        filter = CloudfrontFilter()
+        filter.execute(listing, response)
+
+    if not('medialive' in unfilterList):
+        filter = MedialiveFilter()
+        filter.execute(listing, response)
+
+    if not('ssmListCommands' in unfilterList):
+        filter = SSMListCommandsFilter()
+        filter.execute(listing, response)
+
+    if not('snsListSubscriptions' in unfilterList):
+        filter = SNSListSubscriptionsFilter()
+        filter.execute(listing, response)
+
+    if not('athenaWorkGroups' in unfilterList):
+        filter = AthenaWorkGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('listEventBuses' in unfilterList):
+        filter = ListEventBusesFilter()
+        filter.execute(listing, response)
+
+    if not('xRayGroups' in unfilterList):
+        filter = XRayGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('route53Resolver' in unfilterList):
+        filter = Route53ResolverFilter()
+        filter.execute(listing, response)
+
+    filter = CountFilter(apply_complete)
+    filter.execute(listing, response)
+    apply_complete = filter.complete
+
+    filter = QuantityFilter(apply_complete)
+    filter.execute(listing, response)
+    apply_complete = filter.complete
+
+    filter = NeutralThingFilter()
+    filter.execute(listing, response)
+
+    filter = BadThingFilter(apply_complete)
+    filter.execute(listing, response)
+    apply_complete = filter.complete
+
+    if not('kmsListAliases' in unfilterList):
+        filter = KMSListAliasesFilter()
+        filter.execute(listing, response)
+
+    if not('appstreamImages' in unfilterList):
+        filter = AppstreamImagesFilter()
+        filter.execute(listing, response)
+
+    if not('cloudsearch' in unfilterList):
+        filter = CloudsearchFilter()
+        filter.execute(listing, response)
+
+    if not('cloudTrail' in unfilterList):
+        filter = CloudTrailFilter()
+        filter.execute(listing, response)
+
+    if not('cloudWatch' in unfilterList):
+        filter = CloudWatchFilter()
+        filter.execute(listing, response)
+
+    if not('iamPolicies' in unfilterList):
+        filter = IAMPoliciesFilter()
+        filter.execute(listing, response)
+
+    if not('s3Owner' in unfilterList):
+        filter = S3OwnerFilter()
+        filter.execute(listing, response)
+
+    if not('ecsClustersFailure' in unfilterList):
+        filter = ECSClustersFailureFilter()
+        filter.execute(listing, response)
+
+    if not('pinpointGetApps' in unfilterList):
+        filter = PinpointGetAppsFilter()
+        filter.execute(listing, response)
+
+    if not('ssmBaselines' in unfilterList):
+        filter = SSMBaselinesFilter()
+        filter.execute(listing, response)
+
+    if not('dbSecurityGroups' in unfilterList):
+        filter = DBSecurityGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('dbParameterGroups' in unfilterList):
+        filter = DBParameterGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('dbClusterParameterGroups' in unfilterList):
+        filter = DBClusterParameterGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('dbOptionGroups' in unfilterList):
+        filter = DBOptionGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('ec2VPC' in unfilterList):
+        filter = EC2VPCFilter()
+        filter.execute(listing, response)
+
+    if not('ec2Subnets' in unfilterList):
+        filter = EC2SubnetsFilter()
+        filter.execute(listing, response)
+
+    if not('ec2SecurityGroups' in unfilterList):
+        filter = EC2SecurityGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('ec2RouteTables' in unfilterList):
+        filter = EC2RouteTablesFilter()
+        filter.execute(listing, response)
+
+    if not('ec2NetworkAcls' in unfilterList):
+        filter = EC2NetworkAclsFilter()
+        filter.execute(listing, response) 
+
+    if not('ec2FpgaImages' in unfilterList):
+        filter = EC2FpgaImagesFilter()
+        filter.execute(listing, response) 
+
+    if not('workmailDeletedOrganizations' in unfilterList):
+        filter = WorkmailDeletedOrganizationsFilter()
+        filter.execute(listing, response) 
+
+    if not('elasticacheSubnetGroups' in unfilterList):
+        filter = ElasticacheSubnetGroupsFilter()
+        filter.execute(listing, response)    
+
+    filter = NextTokenFilter(apply_complete)
+    filter.execute(listing, response)
+    apply_complete = getattr(filter, 'complete')
+
+    return apply_complete