Fetch singer-io tap codebase

.circleci/config.yml

@@ -1,68 +1,166 @@
 version: 2.1
+
 orbs:
   slack: circleci/slack@3.4.2
 
+executors:
+  docker-executor:
+    docker:
+      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester
+
 jobs:
   build:
-    docker:
-      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:tap-tester-v4
+    executor: docker-executor
+    steps:
+      - run: echo 'CI done'
+  ensure_env:
+    executor: docker-executor
     steps:
       - checkout
       - run:
-          name: 'Setup virtual env'
+          name: 'Setup Virtual Env'
           command: |
-            python3 -mvenv /usr/local/share/virtualenvs/tap-facebook
+            python3 -m venv /usr/local/share/virtualenvs/tap-facebook
             source /usr/local/share/virtualenvs/tap-facebook/bin/activate
-            pip install -U pip setuptools
+            pip install 'pip==23.3.2'
+            pip install 'setuptools==56.0.0'
             pip install .[dev]
+      - slack/notify-on-failure:
+          only_for_branches: master
+      - persist_to_workspace:
+          root: /usr/local/share/virtualenvs
+          paths:
+            - tap-facebook
+  # Regression Job Definitions
+  run_pylint:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
-          name: 'pylint'
+          name: 'Run pylint'
           command: |
             source /usr/local/share/virtualenvs/tap-facebook/bin/activate
             pylint tap_facebook -d C,R,W
+      - slack/notify-on-failure:
+          only_for_branches: master
+  run_unit_tests:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
-          when: always
-          name: 'Unit Tests'
+          name: 'Run Unit Tests'
           command: |
             source /usr/local/share/virtualenvs/tap-facebook/bin/activate
-            nosetests tests/unittests
-      - add_ssh_keys
+            pip install nose2
+            nose2 -v -s tests/unittests
+      - store_test_results:
+          path: test_output/report.xml
+      - store_artifacts:
+          path: htmlcov
+      - slack/notify-on-failure:
+          only_for_branches: master
+  run_integration_tests:
+    executor: docker-executor
+    parallelism: 9
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
-          name: 'Integration Tests'
+          name: 'Run Integration Tests'
           command: |
             aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh
             source dev_env.sh
+            mkdir /tmp/${CIRCLE_PROJECT_REPONAME}
+            export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME}
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
-            run-test --tap=tap-facebook \
-                     --target=target-stitch \
-                     --orchestrator=stitch-orchestrator \
-                     --email=harrison+sandboxtest@stitchdata.com \
-                     --password=$SANDBOX_PASSWORD \
-                     --client-id=50 \
-                     tests
-      - run:
-          name: 'pylint tests'
-          command: |
-            source /usr/local/share/virtualenvs/tap-tester/bin/activate
-            pip install pylint
-            pylint tests/*.py -d 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,duplicate-code,no-name-in-module,attribute-defined-outside-init,too-many-statements,cell-var-from-loop,too-many-public-methods,missing-docstring,use-a-generator'
+            circleci tests glob "tests/*.py" | circleci tests split > ./tests-to-run
+            if [ -s ./tests-to-run ]; then
+              for test_file in $(cat ./tests-to-run)
+              do
+                run-test --tap=${CIRCLE_PROJECT_REPONAME} $test_file
+              done
+            fi
       - slack/notify-on-failure:
           only_for_branches: master
+      - store_artifacts:
+          path: /tmp/tap-facebook
 
 workflows:
   version: 2
   commit:
     jobs:
+      - ensure_env:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+      - run_pylint:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - run_unit_tests:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
       - build:
-          context: circleci-user
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - run_pylint
+            - run_unit_tests
+            - run_integration_tests
+
   build_daily:
+    jobs:
+      - ensure_env:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+      - run_pylint:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - run_unit_tests:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - build:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - run_pylint
+            - run_unit_tests
+            - run_integration_tests
     triggers:
       - schedule:
-          cron: "0 6 * * *"
+          cron: "0 1 * * *"
           filters:
             branches:
               only:
                 - master
-    jobs:
-      - build:
-          context: circleci-user

.github/pull_request_template.md

@@ -9,3 +9,7 @@
 
 # Rollback steps
  - revert this branch
+
+#### AI generated code
+https://internal.qlik.dev/general/ways-of-working/code-reviews/#guidelines-for-ai-generated-code
+- [ ] this PR has been written with the help of GitHub Copilot or another generative AI tool

CHANGELOG.md

@@ -1,20 +1,66 @@
 # Changelog
 
-# 1.14.0
+## 1.20.2
+  * Bump facebook_business SDK to v19.0.2 [#238](https://github.com/singer-io/tap-facebook/pull/239)
+
+## 1.20.1
+  * Bump facebook_business SDK to v19.0.0 [#238](https://github.com/singer-io/tap-facebook/pull/238)
+
+## 1.20.0
+  * Run on python 3.11.7 [#237](https://github.com/singer-io/tap-facebook/pull/237)
+
+## 1.19.1
+  * Add retry logic for status code - 503 [#226](https://github.com/singer-io/tap-facebook/pull/226)
+
+## 1.19.0
+  * Add conversions to insights streams [#204](https://github.com/singer-io/tap-facebook/pull/204)
+
+## 1.18.6
+  * Bump facebook_business SDK to v17.0.2 for token param bug fix [#219](https://github.com/singer-io/tap-facebook/pull/219)
+
+## 1.18.5
+  * Bump facebook_business SDK to v16.0.2 [#213](https://github.com/singer-io/tap-facebook/pull/213)
+
+## 1.18.4
+  * Facebook business API to v14.0 [#201](https://github.com/singer-io/tap-facebook/pull/201)
+
+## 1.18.3
+  * Facebook business API to V13.0 [#191] (https://github.com/singer-io/tap-facebook/pull/191)
+## 1.18.2
+  * Implemented Request Timeout [#173](https://github.com/singer-io/tap-facebook/pull/173)
+
+## 1.18.1
+  * Forced Replication Method implemented for couple of streams and replication keys [167](https://github.com/singer-io/tap-facebook/pull/167)
+  * Added Tap-tester test cases [168](https://github.com/singer-io/tap-facebook/pull/168)
+  * Updated schema file of ads_insights_age_and_gender and ads_insights_hourly_advertiser and added "format": "date-time" [#172](https://github.com/singer-io/tap-facebook/pull/172)
+
+## 1.17.0
+  * Added retry to AdsInsights job polling to resolve race-condition errors [#174](https://github.com/singer-io/tap-facebook/pull/174)
+
+## 1.16.0
+  * Bump tap dependency, `facebook_business`, from `10.0.0` to `12.0.0` [#164](https://github.com/singer-io/tap-facebook/pull/164)
+
+## 1.15.1
+  * Bump tap dependency, `attrs`, from `16.3.0` to `17.3.0` [#161](https://github.com/singer-io/tap-facebook/pull/161)
+
+## 1.15.0
+  * Add `country` to `ad_insights_country`'s composite primary key [#154](https://github.com/singer-io/tap-facebook/pull/154)
+
+## 1.14.0
   * Add an Ads Insight Stream, broken down by `hourly_stats_aggregated_by_advertiser_time_zone` [#151](https://github.com/singer-io/tap-facebook/pull/151)
 
-# 1.13.0
+## 1.13.0
   * Bump API version from `v9` to `v10` [#146](https://github.com/singer-io/tap-facebook/pull/146)
   * Add feature for AdsInsights stream: The tap will shift the start date to 37 months ago in order to fetch data from this API
     * More info [here](https://www.facebook.com/business/help/1695754927158071?id=354406972049255)
 
-# 1.12.1
+## 1.12.1
   * Increased insights job timeout to 300 seconds [#148](https://github.com/singer-io/tap-facebook/pull/148)
 
-# 1.12.0
+## 1.12.0
   * Added leads stream [#143](https://github.com/singer-io/tap-facebook/pull/143)
 
-# 1.11.2
+## 1.11.2
   * Added unique_outbound_clicks to several streams [#138](https://github.com/singer-io/tap-facebook/pull/138)
 
 ## 1.11.1

README.md

@@ -3,15 +3,18 @@ This is a [Singer](https://singer.io) tap that produces JSON-formatted data foll
 
 This tap:
 - Pulls raw data from the [Facebook Marketing API](https://developers.facebook.com/docs/marketing-apis)
-- Extracts the following resources from Facebook for a one Ad account:
+- Extracts the following resources from Facebook for one Ad account:
   - Ad Creatives
   - Ads
   - Ad Sets
   - Campaigns
+  - Leads
   - Ads Insights
     - Breakdown by age and gender
     - Breakdown by country
     - Breakdown by placement and device
+    - Breakdown by region
+    - Breakdown by the hour for advertisers
 - Outputs the schema for each resource
 - Incrementally pulls data based on the input state
 

setup.py

@@ -3,19 +3,19 @@
 from setuptools import setup
 
 setup(name='tap-facebook',
-      version='1.14.0',
+      version='1.20.2',
       description='Singer.io tap for extracting data from the Facebook Ads API',
       author='Stitch',
       url='https://singer.io',
       classifiers=['Programming Language :: Python :: 3 :: Only'],
       py_modules=['tap_facebook'],
       install_requires=[
-          'attrs==16.3.0',
-          'backoff==1.8.0',
+          'attrs==17.3.0',
+          'backoff==2.2.1',
+          'facebook_business==19.0.2',
           'pendulum==1.2.0',
-          'facebook_business==10.0.0',
           'requests==2.20.0',
-          'singer-python==5.10.0',
+          'singer-python==6.0.0',
       ],
       extras_require={
           'dev': [