Fetch singer-io tap codebase

.circleci/config.yml

@@ -1,68 +1,240 @@
 version: 2.1
+
 orbs:
   slack: circleci/slack@3.4.2
 
+executors:
+  docker-executor:
+    docker:
+      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester
+
 jobs:
   build:
-    docker:
-      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:tap-tester-v4
+    executor: docker-executor
+    steps:
+      - run: echo 'CI done'
+  ensure_env:
+    executor: docker-executor
     steps:
       - checkout
       - run:
-          name: 'Setup virtual env'
+          name: 'Setup Virtual Env'
           command: |
-            python3 -mvenv /usr/local/share/virtualenvs/tap-facebook
+            python3 -m venv /usr/local/share/virtualenvs/tap-facebook
             source /usr/local/share/virtualenvs/tap-facebook/bin/activate
             pip install -U pip setuptools
             pip install .[dev]
+      - slack/notify-on-failure:
+          only_for_branches: master
+      - persist_to_workspace:
+          root: /usr/local/share/virtualenvs
+          paths:
+            - tap-facebook
+  run_pylint:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
-          name: 'pylint'
+          name: 'Run pylint'
           command: |
             source /usr/local/share/virtualenvs/tap-facebook/bin/activate
             pylint tap_facebook -d C,R,W
+      - slack/notify-on-failure:
+          only_for_branches: master
+  run_unit_tests:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
-          when: always
-          name: 'Unit Tests'
+          name: 'Run Unit Tests'
           command: |
             source /usr/local/share/virtualenvs/tap-facebook/bin/activate
             nosetests tests/unittests
-      - add_ssh_keys
+      - slack/notify-on-failure:
+          only_for_branches: master
+  run_integration_tests:
+    parameters:
+      test_name:
+        type: string
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
-          name: 'Integration Tests'
+          name: 'Run Integration Tests'
           command: |
             aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh
             source dev_env.sh
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
-            run-test --tap=tap-facebook \
-                     --target=target-stitch \
-                     --orchestrator=stitch-orchestrator \
-                     --email=harrison+sandboxtest@stitchdata.com \
-                     --password=$SANDBOX_PASSWORD \
-                     --client-id=50 \
-                     tests
+
+            run-test --tap=tap-facebook tests/<< parameters.test_name >>
+      - slack/notify-on-failure:
+          only_for_branches: master
+  deploy:
+    executor: docker-executor
+    steps:
+      - checkout
+      - add_ssh_keys:
+          fingerprints:
+            - '18:3e:7f:b6:39:58:d3:0f:70:54:32:d6:3e:d7:13:4e'
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
+      - run: source /usr/local/share/virtualenvs/tap-facebook/bin/activate
       - run:
-          name: 'pylint tests'
+          name: 'Create git tag'
           command: |
-            source /usr/local/share/virtualenvs/tap-tester/bin/activate
-            pip install pylint
-            pylint tests/*.py -d 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,duplicate-code,no-name-in-module,attribute-defined-outside-init,too-many-statements,cell-var-from-loop,too-many-public-methods,missing-docstring,use-a-generator'
+            version=$(python setup.py --version)
+            git config user.email "stitchintegrationdev@talend.com"
+            git config user.name "${CIRCLE_USERNAME}"
+            git tag -a v"${version}" -m "version ${version}"
+            git push --tags
+      - run:
+          name: 'Build and Upload'
+          command: |
+            pip install twine
+            python setup.py sdist --dist-dir dist
+            twine upload --username $PYPI_USERNAME --password $PYPI_PASSWORD ./dist/*
       - slack/notify-on-failure:
           only_for_branches: master
+      - slack/notify:
+          message: ':green_circle: A new version of the tap has been tagged and released'
 
 workflows:
   version: 2
   commit:
     jobs:
+      - ensure_env:
+          context: circleci-user
+      - run_pylint:
+          context: circleci-user
+          requires:
+            - ensure_env
+      - run_unit_tests:
+          context: circleci-user
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_start_date.py"
+          test_name: "test_facebook_start_date.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_field_selection.py"
+          test_name: "test_facebook_field_selection.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_discovery.py"
+          test_name: "test_facebook_discovery.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_bookmarks.py"
+          test_name: "test_facebook_bookmarks.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_automatic_fields.py"
+          test_name: "test_facebook_automatic_fields.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_tests_run.py"
+          test_name: "test_facebook_tests_run.py"
+          requires:
+            - ensure_env
       - build:
           context: circleci-user
+          requires:
+            - run_pylint
+            - run_unit_tests
+            - test_facebook_start_date.py
+            - test_facebook_field_selection.py
+            - test_facebook_discovery.py
+            - test_facebook_bookmarks.py
+            - test_facebook_automatic_fields.py
+            - test_facebook_tests_run.py
+      - deploy:
+          context: circleci-user
+          requires:
+            - build
+          filters:
+            branches:
+              only: master
   build_daily:
+    jobs:
+      - ensure_env:
+          context: circleci-user
+      - run_pylint:
+          context: circleci-user
+          requires:
+            - ensure_env
+      - run_unit_tests:
+          context: circleci-user
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_start_date.py"
+          test_name: "test_facebook_start_date.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_field_selection.py"
+          test_name: "test_facebook_field_selection.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_discovery.py"
+          test_name: "test_facebook_discovery.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_bookmarks.py"
+          test_name: "test_facebook_bookmarks.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_automatic_fields.py"
+          test_name: "test_facebook_automatic_fields.py"
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context: circleci-user
+          name: "test_facebook_tests_run.py"
+          test_name: "test_facebook_tests_run.py"
+          requires:
+            - ensure_env
+      - build:
+          context: circleci-user
+          requires:
+            - run_pylint
+            - run_unit_tests
+            - test_facebook_start_date.py
+            - test_facebook_field_selection.py
+            - test_facebook_discovery.py
+            - test_facebook_bookmarks.py
+            - test_facebook_automatic_fields.py
+            - test_facebook_tests_run.py
     triggers:
       - schedule:
           cron: "0 6 * * *"
           filters:
             branches:
               only:
                 - master
-    jobs:
-      - build:
-          context: circleci-user

CHANGELOG.md

@@ -1,20 +1,29 @@
 # Changelog
 
-# 1.14.0
+## 1.16.0
+  * Bump tap dependency, `facebook_business`, from `10.0.0` to `12.0.0` [#164](https://github.com/singer-io/tap-facebook/pull/164)
+
+## 1.15.1
+  * Bump tap dependency, `attrs`, from `16.3.0` to `17.3.0` [#161](https://github.com/singer-io/tap-facebook/pull/161)
+
+## 1.15.0
+  * Add `country` to `ad_insights_country`'s composite primary key [#154](https://github.com/singer-io/tap-facebook/pull/154)
+
+## 1.14.0
   * Add an Ads Insight Stream, broken down by `hourly_stats_aggregated_by_advertiser_time_zone` [#151](https://github.com/singer-io/tap-facebook/pull/151)
 
-# 1.13.0
+## 1.13.0
   * Bump API version from `v9` to `v10` [#146](https://github.com/singer-io/tap-facebook/pull/146)
   * Add feature for AdsInsights stream: The tap will shift the start date to 37 months ago in order to fetch data from this API
     * More info [here](https://www.facebook.com/business/help/1695754927158071?id=354406972049255)
 
-# 1.12.1
+## 1.12.1
   * Increased insights job timeout to 300 seconds [#148](https://github.com/singer-io/tap-facebook/pull/148)
 
-# 1.12.0
+## 1.12.0
   * Added leads stream [#143](https://github.com/singer-io/tap-facebook/pull/143)
 
-# 1.11.2
+## 1.11.2
   * Added unique_outbound_clicks to several streams [#138](https://github.com/singer-io/tap-facebook/pull/138)
 
 ## 1.11.1

README.md

@@ -3,15 +3,18 @@ This is a [Singer](https://singer.io) tap that produces JSON-formatted data foll
 
 This tap:
 - Pulls raw data from the [Facebook Marketing API](https://developers.facebook.com/docs/marketing-apis)
-- Extracts the following resources from Facebook for a one Ad account:
+- Extracts the following resources from Facebook for one Ad account:
   - Ad Creatives
   - Ads
   - Ad Sets
   - Campaigns
+  - Leads
   - Ads Insights
     - Breakdown by age and gender
     - Breakdown by country
     - Breakdown by placement and device
+    - Breakdown by region
+    - Breakdown by the hour for advertisers
 - Outputs the schema for each resource
 - Incrementally pulls data based on the input state
 

setup.py

@@ -3,17 +3,17 @@
 from setuptools import setup
 
 setup(name='tap-facebook',
-      version='1.14.0',
+      version='1.16.0',
       description='Singer.io tap for extracting data from the Facebook Ads API',
       author='Stitch',
       url='https://singer.io',
       classifiers=['Programming Language :: Python :: 3 :: Only'],
       py_modules=['tap_facebook'],
       install_requires=[
-          'attrs==16.3.0',
+          'attrs==17.3.0',
           'backoff==1.8.0',
           'pendulum==1.2.0',
-          'facebook_business==10.0.0',
+          'facebook_business==12.0.0',
           'requests==2.20.0',
           'singer-python==5.10.0',
       ],

tap_facebook/__init__.py

@@ -685,7 +685,8 @@ def __iter__(self):
     'ads_insights': {"breakdowns": []},
     'ads_insights_age_and_gender': {"breakdowns": ['age', 'gender'],
                                     "primary-keys": ['age', 'gender']},
-    'ads_insights_country': {"breakdowns": ['country']},
+    'ads_insights_country': {"breakdowns": ['country'],
+                             "primary-keys": ['country']},
     'ads_insights_platform_and_device': {"breakdowns": ['publisher_platform',
                                                         'platform_position', 'impression_device'],
                                          "primary-keys": ['publisher_platform',

tests/base.py

@@ -96,8 +96,8 @@ def expected_metadata(self):
                 self.REPLICATION_METHOD: self.INCREMENTAL,
                 self.REPLICATION_KEYS: {"date_start"}
             },
-            "ads_insights_country": {  # TODO | add country | https://stitchdata.atlassian.net/browse/SRCE-2555
-                self.PRIMARY_KEYS: {"campaign_id", "adset_id", "ad_id", "date_start"}, # , "country"},
+            "ads_insights_country": {
+                self.PRIMARY_KEYS: {"campaign_id", "adset_id", "ad_id", "date_start", "country"},
                 self.REPLICATION_METHOD: self.INCREMENTAL,
                 self.REPLICATION_KEYS: {"date_start"}
             },

tests/test_facebook_field_selection.py

@@ -55,7 +55,7 @@ def expected_pks():
             "campaigns" :                       {"id"},
             "ads_insights" :                    {"campaign_id", "adset_id", "ad_id", "date_start"},
             "ads_insights_age_and_gender" :     {"campaign_id", "adset_id", "ad_id", "date_start", "age", "gender"},
-            "ads_insights_country" :            {"campaign_id", "adset_id", "ad_id", "date_start"},
+            "ads_insights_country" :            {"campaign_id", "adset_id", "ad_id", "date_start", "country"},
             "ads_insights_platform_and_device": {"campaign_id", "adset_id", "ad_id", "date_start", "publisher_platform", "platform_position", "impression_device"},
             "ads_insights_region" :             {"campaign_id", "adset_id", "ad_id", "date_start"},
             "ads_insights_dma" :                {"campaign_id", "adset_id", "ad_id", "date_start"},