scrape_posts.yaml

name: Scrape Latest Posts

on:
  schedule:
    - cron: "0 */2 * * *"
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: "pip"
      - run: pip install -r requirements.txt
      - run: python main.py
        env:
          BRIGHTDATA_API_KEY: ${{ secrets.BRIGHTDATA_API_KEY }}
          BROWSER_WSS_URL: ${{ secrets.BROWSER_WSS_URL }}
          DEV_MODE: ${{ secrets.DEV_MODE }}
          BRIGHTDATA_USERNAME: ${{ secrets.BRIGHTDATA_USERNAME }}
          BRIGHTDATA_PASSWORD: ${{ secrets.BRIGHTDATA_PASSWORD }}
          BRIGHTDATA_REMOTE_SERVER_ADDR: ${{ secrets.BRIGHTDATA_REMOTE_SERVER_ADDR }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_ENDPOINT_URL_S3: ${{ secrets.AWS_ENDPOINT_URL_S3 }}
          AWS_ENDPOINT_URL_IAM: ${{ secrets.AWS_ENDPOINT_URL_IAM }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          GENAI_API_KEY: ${{ secrets.GENAI_API_KEY }}
          DATABASE_URL: ${{ secrets.DATABASE_URL }}