Improve visualization branch + Feature comparing

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 python:
   - "2.7"
-  - "3.6"
+  - "3.9"
 install:
   - pip install flake8 yapf pytest
   - python setup.py install

aws_list_all/__main__.py

@@ -2,14 +2,17 @@
 from __future__ import print_function
 
 import os
+import sys
+import webbrowser
 from resource import getrlimit, setrlimit, RLIMIT_NOFILE
 from argparse import ArgumentParser
 from sys import exit, stderr
 
+from .generate_html import  before_content, after_content
 from .introspection import (
     get_listing_operations, get_services, get_verbs, introspect_regions_for_service, recreate_caches
 )
-from .query import do_list_files, do_query
+from .query import do_list_files, do_query, print_query, compare_list_files
 
 
 def increase_limit_nofiles():
@@ -72,6 +75,12 @@ def main():
         action='append',
         help='Restrict querying to the given operation (can be specified multiple times)'
     )
+    query.add_argument(
+        '-u',
+        '--unfilter',
+        action='append',
+        help='Exclude given default-value filter from being applied (can be specified multiple times)'
+    )
     query.add_argument('-p', '--parallel', default=32, type=int, help='Number of request to do in parallel')
     query.add_argument('-d', '--directory', default='.', help='Directory to save result listings to')
     query.add_argument('-v', '--verbose', action='count', help='Print detailed info during run')
@@ -83,6 +92,15 @@ def main():
     )
     show.add_argument('listingfile', nargs='*', help='listing file(s) to load and print')
     show.add_argument('-v', '--verbose', action='count', help='print given listing files with detailed info')
+    show.add_argument('-n', '--not_found', default=False, action='store_true', help='additionally print listing files of resources not found')
+    show.add_argument('-e', '--errors', default=False, action='store_true', help='additionally print listing files of resources where queries resulted in errors')
+    show.add_argument('-d', '--denied', default=False, action='store_true', help='additionally print listing files of resources with "missing permission" errors')
+    show.add_argument(
+        '-u',
+        '--unfilter',
+        action='append',
+        help='Exclude given default-value filter from being applied (can be specified multiple times)'
+    )
 
     # Introspection debugging is not the main function. So we put it all into a subcommand.
     introspect = subparsers.add_parser(
@@ -119,6 +137,47 @@ def main():
     )
     introspecters.add_parser('debug', description='Debug information', help='Debug information')
 
+    # Print the findings from query to an HTML file to be viewed in a browser
+    viewhtml = subparsers.add_parser(
+        'print-html', description='Print a listing to an HTML file to be viewed in a browser',
+        help='Create browser view of listing'
+    )
+    viewhtml.add_argument(
+        '-s',
+        '--service',
+        action='append',
+        help='Restrict querying to the given service (can be specified multiple times)'
+    )
+    viewhtml.add_argument(
+        '-r',
+        '--region',
+        action='append',
+        help='Restrict querying to the given region (can be specified multiple times)'
+    )
+    viewhtml.add_argument(
+        '-o',
+        '--operation',
+        action='append',
+        help='Restrict querying to the given operation (can be specified multiple times)'
+    )
+    viewhtml.add_argument(
+        '-u',
+        '--unfilter',
+        action='append',
+        help='Exclude given default-value filter from being applied (can be specified multiple times)'
+    )
+    viewhtml.add_argument('-p', '--parallel', default=32, type=int, help='Number of request to do in parallel')
+    viewhtml.add_argument('-d', '--directory', default='.', help='Directory to save result listings to')
+    viewhtml.add_argument('-v', '--verbose', action='count', help='Print detailed info during run')
+    viewhtml.add_argument('-c', '--profile', help='Use a specific .aws/credentials profile.')
+
+    # Visually compare and display the differences in listings from two directories
+    compare = subparsers.add_parser(
+        'compare', description='Compare listings from two different directories', help='Compare saved listings'
+    )
+    compare.add_argument('-b', '--basedir', nargs='*', default='.', help='Listing file(s) used as comparison base')
+    compare.add_argument('-m', '--moddir', nargs='*', default='.', help='Listing file(s) to be compared')
+
     # Finally, refreshing the service/region caches comes last.
     caches = subparsers.add_parser(
         'recreate-caches',
@@ -156,12 +215,20 @@ def main():
             args.operation,
             verbose=args.verbose or 0,
             parallel=args.parallel,
-            selected_profile=args.profile
+            selected_profile=args.profile,
+            unfilter=args.unfilter
         )
     elif args.command == 'show':
         if args.listingfile:
             increase_limit_nofiles()
-            do_list_files(args.listingfile, verbose=args.verbose or 0)
+            do_list_files(
+                args.listingfile, 
+                verbose=args.verbose or 0,
+                not_found=args.not_found,
+                errors=args.errors,
+                denied=args.denied,
+                unfilter=args.unfilter
+            )
         else:
             show.print_help()
             return 1
@@ -183,6 +250,45 @@ def main():
         else:
             introspect.print_help()
             return 1
+    elif args.command == 'print-html':
+        origout = before_content('test.html')
+        if args.directory:
+            try:
+                os.makedirs(args.directory)
+            except OSError:
+                pass
+            os.chdir(args.directory)
+        increase_limit_nofiles()
+        services = args.service or get_services()
+        print_query(
+            services,
+            args.region,
+            args.operation,
+            verbose=args.verbose or 0,
+            parallel=args.parallel,
+            selected_profile=args.profile,
+            unfilter=args.unfilter
+        )
+        after_content(origout)
+
+        new = 2
+        path_up = os.path.dirname(os.getcwd())
+        os.chdir(path_up)
+        url = os.getcwd() + "/test.html"
+        webbrowser.open(url,new=new)
+    elif args.command == 'compare':
+        origout = before_content('compare.html')
+        if args.basedir and args.moddir:
+            increase_limit_nofiles()
+            compare_list_files(
+                args.basedir,
+                args.moddir
+            )
+        after_content(origout)
+
+        new = 2
+        url = os.getcwd() + "/compare.html"
+        webbrowser.open(url,new=new)
     elif args.command == 'recreate-caches':
         increase_limit_nofiles()
         recreate_caches(args.update_packaged_values)

aws_list_all/apply_filter.py

@@ -0,0 +1,152 @@
+import json
+import os
+import sys
+
+from .fixing_filter import *
+from .resource_filter import *
+
+def apply_filters(listing, unfilterList, response, complete):
+    """Apply filters for operations to be handled in a special way or 
+        to remove default resources from the response"""
+    apply_complete = complete
+
+    if not('cloudfront' in unfilterList):
+        filter = CloudfrontFilter()
+        filter.execute(listing, response)
+
+    if not('medialive' in unfilterList):
+        filter = MedialiveFilter()
+        filter.execute(listing, response)
+
+    if not('ssmListCommands' in unfilterList):
+        filter = SSMListCommandsFilter()
+        filter.execute(listing, response)
+
+    if not('snsListSubscriptions' in unfilterList):
+        filter = SNSListSubscriptionsFilter()
+        filter.execute(listing, response)
+
+    if not('athenaWorkGroups' in unfilterList):
+        filter = AthenaWorkGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('listEventBuses' in unfilterList):
+        filter = ListEventBusesFilter()
+        filter.execute(listing, response)
+
+    if not('xRayGroups' in unfilterList):
+        filter = XRayGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('route53Resolver' in unfilterList):
+        filter = Route53ResolverFilter()
+        filter.execute(listing, response)
+
+    filter = CountFilter(apply_complete)
+    filter.execute(listing, response)
+    apply_complete = filter.complete
+
+    filter = QuantityFilter(apply_complete)
+    filter.execute(listing, response)
+    apply_complete = filter.complete
+
+    filter = NeutralThingFilter()
+    filter.execute(listing, response)
+
+    filter = BadThingFilter(apply_complete)
+    filter.execute(listing, response)
+    apply_complete = filter.complete
+
+    if not('kmsListAliases' in unfilterList):
+        filter = KMSListAliasesFilter()
+        filter.execute(listing, response)
+
+    if not('appstreamImages' in unfilterList):
+        filter = AppstreamImagesFilter()
+        filter.execute(listing, response)
+
+    if not('cloudsearch' in unfilterList):
+        filter = CloudsearchFilter()
+        filter.execute(listing, response)
+
+    if not('cloudTrail' in unfilterList):
+        filter = CloudTrailFilter()
+        filter.execute(listing, response)
+
+    if not('cloudWatch' in unfilterList):
+        filter = CloudWatchFilter()
+        filter.execute(listing, response)
+
+    if not('iamPolicies' in unfilterList):
+        filter = IAMPoliciesFilter()
+        filter.execute(listing, response)
+
+    if not('s3Owner' in unfilterList):
+        filter = S3OwnerFilter()
+        filter.execute(listing, response)
+
+    if not('ecsClustersFailure' in unfilterList):
+        filter = ECSClustersFailureFilter()
+        filter.execute(listing, response)
+
+    if not('pinpointGetApps' in unfilterList):
+        filter = PinpointGetAppsFilter()
+        filter.execute(listing, response)
+
+    if not('ssmBaselines' in unfilterList):
+        filter = SSMBaselinesFilter()
+        filter.execute(listing, response)
+
+    if not('dbSecurityGroups' in unfilterList):
+        filter = DBSecurityGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('dbParameterGroups' in unfilterList):
+        filter = DBParameterGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('dbClusterParameterGroups' in unfilterList):
+        filter = DBClusterParameterGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('dbOptionGroups' in unfilterList):
+        filter = DBOptionGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('ec2VPC' in unfilterList):
+        filter = EC2VPCFilter()
+        filter.execute(listing, response)
+
+    if not('ec2Subnets' in unfilterList):
+        filter = EC2SubnetsFilter()
+        filter.execute(listing, response)
+
+    if not('ec2SecurityGroups' in unfilterList):
+        filter = EC2SecurityGroupsFilter()
+        filter.execute(listing, response)
+
+    if not('ec2RouteTables' in unfilterList):
+        filter = EC2RouteTablesFilter()
+        filter.execute(listing, response)
+
+    if not('ec2NetworkAcls' in unfilterList):
+        filter = EC2NetworkAclsFilter()
+        filter.execute(listing, response) 
+
+    if not('ec2FpgaImages' in unfilterList):
+        filter = EC2FpgaImagesFilter()
+        filter.execute(listing, response) 
+
+    if not('workmailDeletedOrganizations' in unfilterList):
+        filter = WorkmailDeletedOrganizationsFilter()
+        filter.execute(listing, response) 
+
+    if not('elasticacheSubnetGroups' in unfilterList):
+        filter = ElasticacheSubnetGroupsFilter()
+        filter.execute(listing, response)    
+
+    filter = NextTokenFilter(apply_complete)
+    filter.execute(listing, response)
+    apply_complete = getattr(filter, 'complete')
+
+    return apply_complete