From 27b86691c5cfa8d86e98b18b91a07a36786385a9 Mon Sep 17 00:00:00 2001 From: ryandeivert Date: Mon, 5 Oct 2020 14:15:11 -0700 Subject: [PATCH] official release of v3.4.1 (#1297) * bumping version to 3.4.1 * adding fix for #1294 to only copy terraform files once during init (#1295) * adding fix for #1294 * pylint --- conf/lambda.json | 2 +- docs/source/getting-started.rst | 4 -- streamalert/__init__.py | 2 +- streamalert_cli/config.py | 26 ++++++++--- streamalert_cli/terraform/generate.py | 43 ------------------- .../manage_lambda/test_package.py | 3 +- .../terraform/test_firehose.py | 3 +- .../terraform/test_generate.py | 26 ----------- tests/unit/streamalert_cli/test_cli_config.py | 3 +- 9 files changed, 28 insertions(+), 84 deletions(-) diff --git a/conf/lambda.json b/conf/lambda.json index 844802aea..c8953a909 100644 --- a/conf/lambda.json +++ b/conf/lambda.json @@ -54,7 +54,7 @@ "concurrency_limit": 10, "memory": 128, "timeout": 300, - "file_format": null, + "file_format": "parquet", "log_level": "info" }, "classifier_config": {}, diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index 1a6d25c86..3398af153 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -103,10 +103,6 @@ Deploy python manage.py configure aws_account_id 111111111111 # Replace with your 12-digit AWS account ID python manage.py configure prefix # Choose a unique name prefix (alphanumeric characters only) -.. note:: - - * Update the ``file_format`` value in ``conf/lambda.json``. Valid options are ``parquet`` or ``json``. The default value will be parquet in a future release, but this must be manually configured at this time. - .. code-block:: bash "athena_partitioner_config": { diff --git a/streamalert/__init__.py b/streamalert/__init__.py index 461eff87c..ad9304418 100644 --- a/streamalert/__init__.py +++ b/streamalert/__init__.py @@ -1,2 +1,2 @@ """StreamAlert version.""" -__version__ = '3.4.0' +__version__ = '3.4.1' diff --git a/streamalert_cli/config.py b/streamalert_cli/config.py index 12d732489..b40cdf5cb 100644 --- a/streamalert_cli/config.py +++ b/streamalert_cli/config.py @@ -23,11 +23,12 @@ from streamalert.apps import StreamAlertApp from streamalert.shared import CLUSTERED_FUNCTIONS, config, metrics from streamalert.shared.logger import get_logger -from streamalert_cli.helpers import continue_prompt +from streamalert_cli.terraform import TERRAFORM_FILES_PATH from streamalert_cli.apps.helpers import save_app_auth_info +from streamalert_cli.helpers import continue_prompt -DEFAULT_CONFIG_PATH = 'conf' +DEFAULT_CONFIG_PATH = 'conf' LOGGER = get_logger(__name__) @@ -69,8 +70,21 @@ def terraform_files(self): self.config['global']['general'].get('terraform_files', []) ) - @staticmethod - def _setup_build_directory(directory): + def _copy_terraform_files(self, directory): + """Copy all packaged terraform files and terraform files provided by the user to temp + + Args: + config (CLIConfig): Loaded StreamAlert config + """ + shutil.copytree(TERRAFORM_FILES_PATH, directory) + + # Copy any additional user provided terraform files to temp + for item in self.terraform_files: + shutil.copy2(item, directory) + + LOGGER.info('Copied Terraform configuration to \'%s\'', directory) + + def _setup_build_directory(self, directory): """Create the directory to be used for building infrastructure Args: @@ -87,7 +101,9 @@ def _setup_build_directory(directory): temp_dir.cleanup() if os.path.exists(directory): - shutil.rmtree(directory) + shutil.rmtree(directory) # shutil.copytree in python3.7 cannot handle existing dir + + self._copy_terraform_files(directory) return directory diff --git a/streamalert_cli/terraform/generate.py b/streamalert_cli/terraform/generate.py index 9923edb2b..7d929fe5c 100644 --- a/streamalert_cli/terraform/generate.py +++ b/streamalert_cli/terraform/generate.py @@ -15,14 +15,12 @@ """ import json import os -import shutil from streamalert.shared.config import ConfigError, firehose_alerts_bucket from streamalert.shared.logger import get_logger from streamalert.shared.utils import get_database_name, get_data_file_format from streamalert_cli.athena.helpers import generate_alerts_table_schema from streamalert_cli.helpers import check_credentials -from streamalert_cli.terraform import TERRAFORM_FILES_PATH from streamalert_cli.terraform.common import ( InvalidClusterName, infinitedict, @@ -384,28 +382,6 @@ def handler(cls, options, config): return terraform_generate_handler(config, check_creds=False) -def _copy_terraform_files(config): - """Copy all packaged terraform files and terraform files provided by the user to temp - - Args: - config (CLIConfig): Loaded StreamAlert config - """ - # Copy the packaged terraform files to temp - # Currently this ignores *.tf.json, in the instance that these - # exist in current deployments. This can be removed in a future release. - shutil.copytree( - TERRAFORM_FILES_PATH, - config.build_directory, - ignore=shutil.ignore_patterns('*.tf.json') # TODO: remove this eventually - ) - - # Copy any additional user provided terraform files to temp - for item in config.terraform_files: - shutil.copy2(item, config.build_directory) - - LOGGER.info('Copied Terraform configuration to \'%s\'', config.build_directory) - - def terraform_generate_handler(config, init=False, check_tf=True, check_creds=True): """Generate all Terraform plans for the configured clusters. @@ -424,8 +400,6 @@ def terraform_generate_handler(config, init=False, check_tf=True, check_creds=Tr if check_tf and not terraform_check(): return False - _copy_terraform_files(config) - # Setup the main.tf.json file LOGGER.debug('Generating cluster file: main.tf.json') _create_terraform_module_file( @@ -634,23 +608,6 @@ def generate_global_lambda_settings( tf_tmp_file (str): filename of terraform file, generated by CLI. message (str): Message will be logged by LOGGER. """ - if conf_name == 'athena_partitioner_config': - # Raise ConfigError when user doesn't explicitly set `file_format` - # in `athena_partitioner_config` in conf/lambda.json when upgrade to v3.1.0. - file_format = get_data_file_format(config) - - if not file_format or file_format not in ('parquet', 'json'): - message = ( - '[WARNING] ' - 'It is required to explicitly set "file_format" for ' - 'athena_partitioner_config in "conf/lambda.json" when upgrading to v3.1.0. ' - 'Available values are "parquet" and "json". For more information, refer to ' - 'https://github.com/airbnb/streamalert/issues/1143. ' - 'In the future release, the default value of "file_format" will ' - 'be changed to "parquet".' - ) - raise ConfigError(message) - tf_tmp_file = os.path.join(config.build_directory, '{}.tf.json'.format(tf_tmp_file_name)) if required and conf_name not in config['lambda']: diff --git a/tests/unit/streamalert_cli/manage_lambda/test_package.py b/tests/unit/streamalert_cli/manage_lambda/test_package.py index d1d1bf2b3..eadd79100 100644 --- a/tests/unit/streamalert_cli/manage_lambda/test_package.py +++ b/tests/unit/streamalert_cli/manage_lambda/test_package.py @@ -16,7 +16,7 @@ # pylint: disable=no-self-use,protected-access import os -from mock import patch +from mock import patch, Mock from pyfakefs import fake_filesystem_unittest from streamalert_cli.config import CLIConfig @@ -28,6 +28,7 @@ class PackageTest(fake_filesystem_unittest.TestCase): TEST_CONFIG_PATH = 'tests/unit/conf' MOCK_TEMP_PATH = '/tmp/test_packaging' + @patch('streamalert_cli.config.CLIConfig._copy_terraform_files', Mock()) def setUp(self): self.setUpPyfakefs() self.fs.add_real_directory(self.TEST_CONFIG_PATH) diff --git a/tests/unit/streamalert_cli/terraform/test_firehose.py b/tests/unit/streamalert_cli/terraform/test_firehose.py index 4c35a9c74..8be250adb 100644 --- a/tests/unit/streamalert_cli/terraform/test_firehose.py +++ b/tests/unit/streamalert_cli/terraform/test_firehose.py @@ -218,7 +218,6 @@ def test_firehose_enabled_log_json(self): 'json:embedded': {} } - self.config = CLIConfig(config_path='tests/unit/conf_athena') firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config) expected_result = { @@ -228,7 +227,7 @@ def test_firehose_enabled_log_json(self): 'source': './modules/tf_kinesis_firehose_delivery_stream', 'buffer_size': 128, 'buffer_interval': 900, - 'file_format': 'json', + 'file_format': 'parquet', 'stream_name': 'unit_test_streamalert_json_embedded', 'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}', 's3_bucket_name': 'unit-test-streamalert-data', diff --git a/tests/unit/streamalert_cli/terraform/test_generate.py b/tests/unit/streamalert_cli/terraform/test_generate.py index 1a3da5656..0d4bb8de8 100644 --- a/tests/unit/streamalert_cli/terraform/test_generate.py +++ b/tests/unit/streamalert_cli/terraform/test_generate.py @@ -854,32 +854,6 @@ def test_generate_main_with_sqs_url_false(self): assert_equal(result['module']['globals']['source'], './modules/tf_globals') assert_false(result['module']['globals']['sqs_use_prefix']) - def test_generate_athena_lambda_format_unspecified(self): - "CLI - Terraform Generate Global Lambda Settings, Unspecified Athena file_format" - self.config['lambda']['athena_partitioner_config']['file_format'] = None - - assert_raises( - ConfigError, - generate.generate_global_lambda_settings, - config=self.config, - conf_name='athena_partitioner_config', - generate_func='test_func', - tf_tmp_file_name='test_tf_tmp_file_path', - ) - - def test_generate_athena_lambda_format_invalid(self): - "CLI - Terraform Generate Global Lambda Settings, Invalid Athena file_format" - self.config['lambda']['athena_partitioner_config']['file_format'] = 'Parquet' - - assert_raises( - ConfigError, - generate.generate_global_lambda_settings, - config=self.config, - conf_name='athena_partitioner_config', - generate_func='test_func', - tf_tmp_file_name='test_tf_tmp_file_path', - ) - def test_generate_required_lambda_invalid_config(self): "CLI - Terraform Generate Global Lambda Settings, Invalid Config" diff --git a/tests/unit/streamalert_cli/test_cli_config.py b/tests/unit/streamalert_cli/test_cli_config.py index aa9662711..3ff7c16c9 100644 --- a/tests/unit/streamalert_cli/test_cli_config.py +++ b/tests/unit/streamalert_cli/test_cli_config.py @@ -16,7 +16,7 @@ # pylint: disable=protected-access import json -from mock import patch +from mock import patch, Mock from nose.tools import assert_equal, assert_true, assert_false from pyfakefs import fake_filesystem_unittest @@ -31,6 +31,7 @@ def __init__(self): self.config = None self.fs_patcher = None + @patch('streamalert_cli.config.CLIConfig._copy_terraform_files', Mock()) def setup(self): """Setup before each method""" config_data = basic_streamalert_config()