official release of v3.4.0 (#1292)

* bumping version to 3.4.0

* updating to terraform v0.13.0 (#1290)

* updating terraform version to 0.13.0 and aws provider to 3.3.0

* misc updates to terraform code

* fixes

* updating for vpc flow logs

* patching out writing of vars in tests

* cloudtrail to cwl fix

* fix

Vagrantfile

@@ -59,7 +59,7 @@ def configure_streamalert(machine)
                    privileged: false
 end
 
-TERRAFORM_VERSION = ENV.fetch('SA_TERRAFORM_VERSION', '0.12.9')
+TERRAFORM_VERSION = ENV.fetch('SA_TERRAFORM_VERSION', '0.13.0')
 def configure_terraform(machine)
   # Install terraform with the specified version.
   machine.vm.provision :shell,

docs/source/getting-started.rst

@@ -10,17 +10,17 @@ Install Dependencies
 ********************
 
 1. Install Python 3.7 and `pip <https://pip.pypa.io/en/stable/installing/>`_
-2. Install `Terraform <https://www.terraform.io/intro/getting-started/install.html>`_ >= v0.12.9:
+2. Install `Terraform <https://www.terraform.io/intro/getting-started/install.html>`_ >= v0.13.0:
 
 .. code-block:: bash
 
   brew install terraform  # MacOS Homebrew
-  terraform --version     # Must be >= v0.12.9
+  terraform --version     # Must be >= v0.13.0
 
 .. note::
 
-  Terraform versions lower than 0.12 are not supported. We recommend installing Terraform
-  version 0.12.9 or greater.
+  Terraform versions lower than 0.13 are not supported. We recommend installing Terraform
+  version 0.13.0 or greater.
 
 
 3. If you are using Linux, you may need to install the Python development libraries:

streamalert/__init__.py

@@ -1,2 +1,2 @@
 """StreamAlert version."""
-__version__ = '3.3.0'
+__version__ = '3.4.0'

streamalert_cli/_infrastructure/_include.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = "~> 0.13.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 3.3.0"
+    }
+  }
+}

streamalert_cli/_infrastructure/providers.tf → streamalert_cli/_infrastructure/_providers.tf

@@ -3,6 +3,14 @@
 // define its own provider block.
 // See here for additional reading:
 //   https://www.terraform.io/docs/configuration/modules.html#providers-within-modules
+
+// Default provider
+//  TODO: this should be updated to use required_providers:
+//    https://www.terraform.io/docs/configuration/provider-requirements.html#requiring-providers
+provider "aws" {
+  region = var.region
+}
+
 provider "aws" {
   alias  = "ap-northeast-1"
   region = "ap-northeast-1"

streamalert_cli/_infrastructure/_variables.tf

@@ -0,0 +1,3 @@
+variable "region" {
+  type = string
+}

streamalert_cli/_infrastructure/modules/tf_artifact_extractor/iam.tf

@@ -66,9 +66,9 @@ data "aws_iam_policy_document" "firehose_s3" {
 // IAM Policy: Interact with the Glue Catalog
 resource "aws_iam_role_policy" "streamalert_firehose_glue" {
   name = "FirehoseReadGlueCatalog"
-  role = "${aws_iam_role.streamalert_kinesis_firehose.id}"
+  role = aws_iam_role.streamalert_kinesis_firehose.id
 
-  policy = "${data.aws_iam_policy_document.firehose_glue_catalog.json}"
+  policy = data.aws_iam_policy_document.firehose_glue_catalog.json
 }
 
 // IAM Policy Document: Interact with the Glue Catalog
@@ -77,7 +77,7 @@ data "aws_iam_policy_document" "firehose_glue_catalog" {
     effect = "Allow"
 
     actions = [
-      "glue:GetTableVersions"
+      "glue:GetTable*"
     ]
 
     resources = ["*"]

streamalert_cli/_infrastructure/modules/tf_cloudtrail/modules/tf_cloudtrail_cloudwatch/main.tf

@@ -52,14 +52,14 @@ data "aws_iam_policy_document" "cloudtrail_to_cloudwatch_create_logs" {
     sid       = "AWSCloudTrailCreateLogStream"
     effect    = "Allow"
     actions   = ["logs:CreateLogStream"]
-    resources = [aws_cloudwatch_log_group.cloudtrail_logging.arn]
+    resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}:log-stream:*"]
   }
 
   statement {
     sid       = "AWSCloudTrailPutLogEvents"
     effect    = "Allow"
     actions   = ["logs:PutLogEvents"]
-    resources = [aws_cloudwatch_log_group.cloudtrail_logging.arn]
+    resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}:log-stream:*"]
   }
 }
 

streamalert_cli/_infrastructure/modules/tf_cloudtrail/modules/tf_cloudtrail_cloudwatch/output.tf

@@ -2,6 +2,7 @@ output "cloudtrail_to_cloudwatch_logs_role" {
   value = aws_iam_role.cloudtrail_to_cloudwatch_role.arn
 }
 
+// CloudTrail requires the log stream wildcard here
 output "cloudwatch_logs_group_arn" {
-  value = aws_cloudwatch_log_group.cloudtrail_logging.arn
+  value = "${aws_cloudwatch_log_group.cloudtrail_logging.arn}:*"
 }

streamalert_cli/_infrastructure/modules/tf_flow_logs/main.tf

@@ -1,25 +1,28 @@
 resource "aws_flow_log" "vpc_flow_log" {
-  count          = length(var.vpcs)
-  vpc_id         = element(var.vpcs, count.index)
-  log_group_name = aws_cloudwatch_log_group.flow_log_group.name
-  iam_role_arn   = aws_iam_role.flow_log_role.arn
-  traffic_type   = "ALL"
+  count                = length(var.vpcs)
+  vpc_id               = element(var.vpcs, count.index)
+  log_destination      = aws_cloudwatch_log_group.flow_log_group.arn
+  log_destination_type = "cloud-watch-logs"
+  iam_role_arn         = aws_iam_role.flow_log_role.arn
+  traffic_type         = "ALL"
 }
 
 resource "aws_flow_log" "subnet_flow_log" {
-  count          = length(var.subnets)
-  subnet_id      = element(var.subnets, count.index)
-  log_group_name = aws_cloudwatch_log_group.flow_log_group.name
-  iam_role_arn   = aws_iam_role.flow_log_role.arn
-  traffic_type   = "ALL"
+  count                = length(var.subnets)
+  subnet_id            = element(var.subnets, count.index)
+  log_destination      = aws_cloudwatch_log_group.flow_log_group.arn
+  log_destination_type = "cloud-watch-logs"
+  iam_role_arn         = aws_iam_role.flow_log_role.arn
+  traffic_type         = "ALL"
 }
 
 resource "aws_flow_log" "eni_flow_log" {
-  count          = length(var.enis)
-  eni_id         = element(var.enis, count.index)
-  log_group_name = aws_cloudwatch_log_group.flow_log_group.name
-  iam_role_arn   = aws_iam_role.flow_log_role.arn
-  traffic_type   = "ALL"
+  count                = length(var.enis)
+  eni_id               = element(var.enis, count.index)
+  log_destination      = aws_cloudwatch_log_group.flow_log_group.arn
+  log_destination_type = "cloud-watch-logs"
+  iam_role_arn         = aws_iam_role.flow_log_role.arn
+  traffic_type         = "ALL"
 }
 
 resource "aws_cloudwatch_log_group" "flow_log_group" {
@@ -34,8 +37,7 @@ resource "aws_cloudwatch_log_group" "flow_log_group" {
 
 resource "aws_cloudwatch_log_subscription_filter" "flow_logs" {
   name            = "${aws_cloudwatch_log_group.flow_log_group.name}_to_kinesis"
-  log_group_name  = "${aws_cloudwatch_log_group.flow_log_group.name}"
-  filter_pattern  = "${var.flow_log_filter}"
-  destination_arn = "${var.cloudwatch_logs_destination_arn}"
+  log_group_name  = aws_cloudwatch_log_group.flow_log_group.name
+  filter_pattern  = var.flow_log_filter
+  destination_arn = var.cloudwatch_logs_destination_arn
 }
-

streamalert_cli/_infrastructure/modules/tf_globals/alerts_firehose/iam.tf

@@ -54,9 +54,9 @@ data "aws_iam_policy_document" "firehose_s3" {
 // IAM Policy: Interact with the Glue Catalog
 resource "aws_iam_role_policy" "stream_alert_firehose_glue" {
   name = "streamalert_firehose_read_glue_catalog"
-  role = "${aws_iam_role.firehose.id}"
+  role = aws_iam_role.firehose.id
 
-  policy = "${data.aws_iam_policy_document.firehose_glue_catalog.json}"
+  policy = data.aws_iam_policy_document.firehose_glue_catalog.json
 }
 
 // IAM Policy Document: Interact with the Glue Catalog

streamalert_cli/_infrastructure/modules/tf_kinesis_firehose_setup/iam.tf

@@ -88,9 +88,9 @@ data "aws_iam_policy_document" "firehose_cloudwatch" {
 // IAM Policy: Interact with the Glue Catalog
 resource "aws_iam_role_policy" "streamalert_firehose_glue" {
   name = "streamalert_firehose_read_glue_catalog"
-  role = "${aws_iam_role.streamalert_kinesis_firehose.id}"
+  role = aws_iam_role.streamalert_kinesis_firehose.id
 
-  policy = "${data.aws_iam_policy_document.firehose_glue_catalog.json}"
+  policy = data.aws_iam_policy_document.firehose_glue_catalog.json
 }
 
 // IAM Policy Document: Interact with the Glue Catalog
@@ -99,7 +99,7 @@ data "aws_iam_policy_document" "firehose_glue_catalog" {
     effect = "Allow"
 
     actions = [
-      "glue:GetTableVersions"
+      "glue:GetTable*"
     ]
 
     resources = ["*"]

streamalert_cli/terraform/generate.py

@@ -63,24 +63,20 @@
 from streamalert_cli.utils import CLICommand
 
 RESTRICTED_CLUSTER_NAMES = ('main', 'athena')
-TERRAFORM_VERSION = '~> 0.12.9'
-TERRAFORM_PROVIDER_VERSION = '~> 2.48.0'
 
 LOGGER = get_logger(__name__)
 
 
-def _terraform_defaults(region):
-    return infinitedict({
-        'terraform': {
-            'required_version': TERRAFORM_VERSION,
-        },
-        'provider': {
-            'aws': {
-                'region': region,
-                'version': TERRAFORM_PROVIDER_VERSION,
-            },
-        },
-    })
+def write_vars(config, **kwargs):
+    """Write root variables to a terraform.tfvars.json file
+
+    Keyword Args:
+        region (string): AWS region where infrastructure will be built
+    """
+    _create_terraform_module_file(
+        kwargs,
+        os.path.join(config.build_directory, 'terraform.tfvars.json')
+    )
 
 
 def generate_s3_bucket(bucket, logging, **kwargs):
@@ -164,7 +160,9 @@ def generate_main(config, init=False):
     Returns:
         dict: main.tf.json Terraform dict
     """
-    main_dict = _terraform_defaults(config['global']['account']['region'])
+    write_vars(config, region=config['global']['account']['region'])
+
+    main_dict = infinitedict()
 
     logging_bucket, create_logging_bucket = s3_access_logging_bucket(config)
 

tests/unit/streamalert_cli/terraform/test_generate.py

@@ -13,7 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from mock import ANY, patch
+from mock import ANY, Mock, patch
 
 from nose.tools import (
     assert_equal,
@@ -35,6 +35,7 @@
 )
 
 
+@patch('streamalert_cli.terraform.generate.write_vars', Mock())
 class TestTerraformGenerate:
     """Test class for the Terraform Cluster Generating"""
     # pylint: disable=no-self-use,attribute-defined-outside-init
@@ -44,8 +45,7 @@ def setup(self):
         self.cluster_dict = common.infinitedict()
         self.config = CLIConfig(config_path='tests/unit/conf')
 
-    @staticmethod
-    def test_generate_s3_bucket():
+    def test_generate_s3_bucket(self):
         """CLI - Terraform Generate S3 Bucket """
         bucket = generate.generate_s3_bucket(
             bucket='unit.test.bucket',
@@ -67,8 +67,7 @@ def test_generate_s3_bucket():
         assert_equal(bucket['bucket'], 'unit.test.bucket')
         assert_equal(set(bucket.keys()), required_keys)
 
-    @staticmethod
-    def test_generate_s3_bucket_lifecycle():
+    def test_generate_s3_bucket_lifecycle(self):
         """CLI - Terraform Generate S3 Bucket with Lifecycle"""
         bucket = generate.generate_s3_bucket(
             bucket='unit.test.bucket',
@@ -91,14 +90,7 @@ def test_generate_main(self):
         tf_main = generate.generate_main(config=self.config, init=False)
 
         tf_main_expected = {
-            'provider': {
-                'aws': {
-                    'version': '~> 2.48.0',  # Changes to this should require unit test update
-                    'region': 'us-west-1'
-                }
-            },
             'terraform': {
-                'required_version': '~> 0.12.9', # Changes to this should require unit test update
                 'backend': {
                     's3': {
                         'bucket': 'unit-test-streamalert-terraform-state',
@@ -229,7 +221,6 @@ def test_generate_main(self):
             }
         }
 
-        assert_dict_equal(tf_main['provider'], tf_main_expected['provider'])
         assert_dict_equal(tf_main['terraform'], tf_main_expected['terraform'])
         assert_dict_equal(tf_main['resource'], tf_main_expected['resource'])
 
@@ -389,7 +380,7 @@ def test_generate_cloudtrail_with_s3_events(self):
         """CLI - Terraform Generate CloudTrail Module, With S3 Events"""
         cluster_name = 'advanced'
         self.config['clusters']['advanced']['modules']['cloudtrail'] = {
-            's3_settings':{
+            's3_settings': {
                 'bucket_name': 'unit-test-bucket',
                 'cross_account_ids': ['456789012345'],
                 'enable_events': True,