Merge branch 'dev/marian-anderson' into kconvey-copy-job

CHANGELOG.md

@@ -11,12 +11,14 @@
 
 ### Features
 - Add a BigQuery adapter macro to enable usage of CopyJobs ([#2709](https://github.com/fishtown-analytics/dbt/pull/2709))
+- Support TTL for BigQuery tables([#2711](https://github.com/fishtown-analytics/dbt/pull/2711))
 - Add better retry support when using the BigQuery adapter ([#2694](https://github.com/fishtown-analytics/dbt/pull/2694), follow-up to [#1963](https://github.com/fishtown-analytics/dbt/pull/1963))
 - Added a `dispatch` method to the context adapter and deprecated `adapter_macro`. ([#2302](https://github.com/fishtown-analytics/dbt/issues/2302), [#2679](https://github.com/fishtown-analytics/dbt/pull/2679))
 - The built-in schema tests now use `adapter.dispatch`, so they can be overridden for adapter plugins ([#2415](https://github.com/fishtown-analytics/dbt/issues/2415), [#2684](https://github.com/fishtown-analytics/dbt/pull/2684))
 - Add support for impersonating a service account using `impersonate_service_account` in the BigQuery profile configuration ([#2677](https://github.com/fishtown-analytics/dbt/issues/2677)) ([docs](https://docs.getdbt.com/reference/warehouse-profiles/bigquery-profile#service-account-impersonation))
 - Macros in the current project can override internal dbt macros that are called through `execute_macros`. ([#2301](https://github.com/fishtown-analytics/dbt/issues/2301), [#2686](https://github.com/fishtown-analytics/dbt/pull/2686))
 - Add state:modified and state:new selectors ([#2641](https://github.com/fishtown-analytics/dbt/issues/2641), [#2695](https://github.com/fishtown-analytics/dbt/pull/2695))
+- Add two new flags `--use-colors` and `--no-use-colors` to `dbt run` command to enable or disable log colorization from the command line ([#2708](https://github.com/fishtown-analytics/dbt/pull/2708))
 
 ### Fixes
 - Fix Redshift table size estimation; e.g. 44 GB tables are no longer reported as 44 KB. [#2702](https://github.com/fishtown-analytics/dbt/issues/2702)
@@ -29,9 +31,11 @@
 
 Contributors:
 - [@bbhoss](https://github.com/bbhoss) ([#2677](https://github.com/fishtown-analytics/dbt/pull/2677))
-- [@kconvey](https://github.com/kconvey) ([#2694](https://github.com/fishtown-analytics/dbt/pull/2694), [#2709](https://github.com/fishtown-analytics/dbt/pull/2709))
+- [@kconvey](https://github.com/kconvey) ([#2694](https://github.com/fishtown-analytics/dbt/pull/2694), [#2709](https://github.com/fishtown-analytics/dbt/pull/2709)), [#2711](https://github.com/fishtown-analytics/dbt/pull/2711))
 - [@vogt4nick](https://github.com/vogt4nick) ([#2702](https://github.com/fishtown-analytics/dbt/issues/2702))
 - [@stephen8chang](https://github.com/stephen8chang) ([docs#106](https://github.com/fishtown-analytics/dbt-docs/pull/106), [docs#108](https://github.com/fishtown-analytics/dbt-docs/pull/108), [docs#113](https://github.com/fishtown-analytics/dbt-docs/pull/113))
+- [@rsenseman](https://github.com/rsenseman) ([#2708](https://github.com/fishtown-analytics/dbt/pull/2708))
+
 
 ## dbt 0.18.0b2 (July 30, 2020)
 

core/dbt/contracts/connection.py

@@ -160,7 +160,7 @@ def to_dict(self, omit_none=True, validate=False, *, with_aliases=False):
 
 class UserConfigContract(Protocol):
     send_anonymous_usage_stats: bool
-    use_colors: bool
+    use_colors: Optional[bool]
     partial_parse: Optional[bool]
     printer_width: Optional[int]
 

core/dbt/contracts/project.py

@@ -14,7 +14,6 @@
 
 PIN_PACKAGE_URL = 'https://docs.getdbt.com/docs/package-management#section-specifying-package-versions' # noqa
 DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True
-DEFAULT_USE_COLORS = True
 
 
 Name = NewType('Name', str)
@@ -259,7 +258,7 @@ def parse_project_config(
 @dataclass
 class UserConfig(ExtensibleJsonSchemaMixin, Replaceable, UserConfigContract):
     send_anonymous_usage_stats: bool = DEFAULT_SEND_ANONYMOUS_USAGE_STATS
-    use_colors: bool = DEFAULT_USE_COLORS
+    use_colors: Optional[bool] = None
     partial_parse: Optional[bool] = None
     printer_width: Optional[int] = None
 
@@ -269,8 +268,8 @@ def set_values(self, cookie_dir):
         else:
             tracking.do_not_track()
 
-        if self.use_colors:
-            ui.use_colors()
+        if self.use_colors is not None:
+            ui.use_colors(self.use_colors)
 
         if self.printer_width:
             ui.printer_width(self.printer_width)

core/dbt/flags.py

@@ -2,6 +2,7 @@
 import multiprocessing
 from pathlib import Path
 from typing import Optional
+
 # initially all flags are set to None, the on-load call of reset() will set
 # them for their first time.
 STRICT_MODE = None
@@ -11,6 +12,7 @@
 TEST_NEW_PARSER = None
 WRITE_JSON = None
 PARTIAL_PARSE = None
+USE_COLORS = None
 
 
 def env_set_truthy(key: str) -> Optional[str]:
@@ -48,7 +50,7 @@ def _get_context():
 
 def reset():
     global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
-        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT
+        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS
 
     STRICT_MODE = False
     FULL_REFRESH = False
@@ -58,11 +60,12 @@ def reset():
     WRITE_JSON = True
     PARTIAL_PARSE = False
     MP_CONTEXT = _get_context()
+    USE_COLORS = True
 
 
 def set_from_args(args):
     global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
-        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT
+        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS
 
     USE_CACHE = getattr(args, 'use_cache', USE_CACHE)
 
@@ -78,6 +81,13 @@ def set_from_args(args):
     PARTIAL_PARSE = getattr(args, 'partial_parse', None)
     MP_CONTEXT = _get_context()
 
+    # The use_colors attribute will always have a value because it is assigned
+    # None by default from the add_mutually_exclusive_group function
+    use_colors_override = getattr(args, 'use_colors')
+
+    if use_colors_override is not None:
+        USE_COLORS = use_colors_override
+
 
 # initialize everything to the defaults on module load
 reset()

core/dbt/main.py

@@ -902,6 +902,30 @@ def parse_args(args, cls=DBTArgumentParser):
         If set, skip writing the manifest and run_results.json files to disk
         '''
     )
+    colors_flag = p.add_mutually_exclusive_group()
+    colors_flag.add_argument(
+        '--use-colors',
+        action='store_const',
+        const=True,
+        dest='use_colors',
+        help='''
+        Colorize the output DBT prints to the terminal. Output is colorized by
+        default and may also be set in a profile or at the command line.
+        Mutually exclusive with --no-use-colors
+        '''
+    )
+    colors_flag.add_argument(
+        '--no-use-colors',
+        action='store_const',
+        const=False,
+        dest='use_colors',
+        help='''
+        Do not colorize the output DBT prints to the terminal. Output is
+        colorized by default and may also be set in a profile or at the
+        command line.
+        Mutually exclusive with --use-colors
+        '''
+    )
 
     p.add_argument(
         '-S',

core/dbt/ui.py

@@ -1,3 +1,4 @@
+import dbt.flags as flags
 import textwrap
 from typing import Dict
 
@@ -11,8 +12,6 @@
 }
 
 
-USE_COLORS = False
-
 COLOR_FG_RED = COLORS['red']
 COLOR_FG_GREEN = COLORS['green']
 COLOR_FG_YELLOW = COLORS['yellow']
@@ -21,9 +20,8 @@
 PRINTER_WIDTH = 80
 
 
-def use_colors():
-    global USE_COLORS
-    USE_COLORS = True
+def use_colors(use_colors_val=True):
+    flags.USE_COLORS = use_colors_val
 
 
 def printer_width(printer_width):
@@ -32,7 +30,7 @@ def printer_width(printer_width):
 
 
 def color(text: str, color_code: str):
-    if USE_COLORS:
+    if flags.USE_COLORS:
         return "{}{}{}".format(color_code, text, COLOR_RESET_ALL)
     else:
         return text

plugins/bigquery/dbt/adapters/bigquery/impl.py

@@ -108,6 +108,7 @@ class BigqueryConfig(AdapterConfig):
     labels: Optional[Dict[str, str]] = None
     partitions: Optional[List[str]] = None
     grant_access_to: Optional[List[Dict[str, str]]] = None
+    hours_to_expiration: Optional[int] = None
 
 
 class BigQueryAdapter(BaseAdapter):
@@ -766,6 +767,12 @@ def get_table_options(
             expiration = 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)'
             opts['expiration_timestamp'] = expiration
 
+        if (config.get('hours_to_expiration') is not None) and (not temporary):
+            expiration = (
+                'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL '
+                '{} hour)').format(config.get('hours_to_expiration'))
+            opts['expiration_timestamp'] = expiration
+
         if config.persist_relation_docs() and 'description' in node:
             description = sql_escape(node['description'])
             opts['description'] = '"""{}"""'.format(description)

test/integration/022_bigquery_test/adapter-specific-models/expiring_table.sql

@@ -0,0 +1 @@
+select 1 as id

test/integration/022_bigquery_test/test_bigquery_adapter_specific.py

@@ -0,0 +1,38 @@
+""""Test adapter specific config options."""
+from test.integration.base import DBTIntegrationTest, use_profile
+import textwrap
+import yaml
+
+
+class TestBigqueryAdapterSpecific(DBTIntegrationTest):
+
+    @property
+    def schema(self):
+        return "bigquery_test_022"
+
+    @property
+    def models(self):
+        return "adapter-specific-models"
+
+    @property
+    def profile_config(self):
+        return self.bigquery_profile()
+
+    @property
+    def project_config(self):
+        return yaml.safe_load(textwrap.dedent('''\
+        config-version: 2
+        models:
+            test:
+                materialized: table
+                expiring_table:
+                    hours_to_expiration: 4    
+        '''))
+
+    @use_profile('bigquery')
+    def test_bigquery_hours_to_expiration(self):
+        _, stdout = self.run_dbt_and_capture(['--debug', 'run'])
+
+        self.assertIn(
+            'expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL '
+            '4 hour)', stdout)

test/integration/061_use_colors_tests/models/do_nothing_then_fail.sql

@@ -0,0 +1 @@
+select 1,

test/integration/061_use_colors_tests/test_no_use_colors.py

@@ -0,0 +1,29 @@
+
+from test.integration.base import DBTIntegrationTest, use_profile
+import logging
+import re
+import sys
+
+class TestNoUseColors(DBTIntegrationTest):
+
+    @property
+    def project_config(self):
+        return {'config-version': 2}
+
+    @property
+    def schema(self):
+        return "use_colors_tests_061"
+
+    @property
+    def models(self):
+        return "models"
+
+    @use_profile('postgres')
+    def test_postgres_no_use_colors(self):
+        # pattern to match formatted log output
+        pattern = re.compile(r'\[31m.*|\[33m.*')
+
+        results, stdout = self.run_dbt_and_capture(args=['--no-use-colors', 'run'], expect_pass=False)
+
+        stdout_contains_formatting_characters = bool(pattern.search(stdout))
+        self.assertFalse(stdout_contains_formatting_characters)

test/integration/061_use_colors_tests/test_use_colors.py

@@ -0,0 +1,29 @@
+
+from test.integration.base import DBTIntegrationTest, use_profile
+import logging
+import re
+import sys
+
+class TestUseColors(DBTIntegrationTest):
+
+    @property
+    def project_config(self):
+        return {'config-version': 2}
+
+    @property
+    def schema(self):
+        return "use_colors_tests_061"
+
+    @property
+    def models(self):
+        return "models"
+
+    @use_profile('postgres')
+    def test_postgres_use_colors(self):
+        # pattern to match formatted log output
+        pattern = re.compile(r'\[31m.*|\[33m.*')
+
+        results, stdout = self.run_dbt_and_capture(args=['--use-colors', 'run'], expect_pass=False)
+
+        stdout_contains_formatting_characters = bool(pattern.search(stdout))
+        self.assertTrue(stdout_contains_formatting_characters)

test/unit/test_bigquery_adapter.py

@@ -4,7 +4,7 @@
 import unittest
 from contextlib import contextmanager
 from requests.exceptions import ConnectionError
-from unittest.mock import patch, MagicMock, Mock, ANY
+from unittest.mock import patch, MagicMock, Mock, create_autospec, ANY
 
 import hologram
 
@@ -629,6 +629,35 @@ def test_parse_partition_by(self):
             }
         )
 
+    def test_hours_to_expiration(self):
+        adapter = self.get_adapter('oauth')
+        mock_config = create_autospec(
+            dbt.context.providers.RuntimeConfigObject)
+        config = {'hours_to_expiration': 4}
+        mock_config.get.side_effect = lambda name: config.get(name)
+
+        expected = {
+            'expiration_timestamp': 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 4 hour)',
+        }
+        actual = adapter.get_table_options(mock_config, node={}, temporary=False)
+        self.assertEqual(expected, actual)
+
+
+    def test_hours_to_expiration_temporary(self):
+        adapter = self.get_adapter('oauth')
+        mock_config = create_autospec(
+            dbt.context.providers.RuntimeConfigObject)
+        config={'hours_to_expiration': 4}
+        mock_config.get.side_effect = lambda name: config.get(name)
+
+        expected = {
+            'expiration_timestamp': (
+                'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)'),
+        }
+        actual = adapter.get_table_options(mock_config, node={}, temporary=True)
+        self.assertEqual(expected, actual)
+
+
 
 class TestBigQueryFilterCatalog(unittest.TestCase):
     def test__catalog_filter_table(self):