Skip to content

crawl current data #4164

crawl current data

crawl current data #4164

Workflow file for this run

name: crawl current data
# 6시간마다 실행되게 해두었습니다.
# 총 1000번이므로 250일 운영 가능합니다.
# 한국어와 영문 크롤러로 변경할 필요 있음
on:
schedule:
- cron: "0 */6 * * *"
# on: [push]
jobs:
runCrawler:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
persist-credentials: false
- name: Set up Python 3.x
uses: actions/setup-python@v1
with:
python-version: "3.x"
architecture: "x64"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Install Chromedriver
run: |
CHROME_DRIVER=chromedriver
if [ -f "$CHROME_DRIVER" ]; then
echo "$FILE already exists."
else
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add
sudo apt-get install google-chrome-stable
wget https://chromedriver.storage.googleapis.com/2.40/chromedriver_linux64.zip
unzip ./chromedriver_linux64.zip
fi
- name: Run crawler
run: |
python "crawler/total_news_crawler.py"
- name: Commit files
run: |
git config --local user.email "paullabkorea@gmail.com"
git config --local user.name "paullabkorea"
git add .
git commit --allow-empty -m "Run crawler and update current data"
- name: Push changes
uses: ad-m/github-push-action@master
with:
# branch: "main"
github_token: ${{ secrets.GITHUB_TOKEN }}