crawl current data #4167
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: crawl current data | |
# 6시간마다 실행되게 해두었습니다. | |
# 총 1000번이므로 250일 운영 가능합니다. | |
# 한국어와 영문 크롤러로 변경할 필요 있음 | |
on: | |
schedule: | |
- cron: "0 */6 * * *" | |
# on: [push] | |
jobs: | |
runCrawler: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
persist-credentials: false | |
- name: Set up Python 3.x | |
uses: actions/setup-python@v1 | |
with: | |
python-version: "3.x" | |
architecture: "x64" | |
- name: Install dependencies | |
run: | | |
python -m pip install --upgrade pip | |
pip install -r requirements.txt | |
- name: Install Chromedriver | |
run: | | |
CHROME_DRIVER=chromedriver | |
if [ -f "$CHROME_DRIVER" ]; then | |
echo "$FILE already exists." | |
else | |
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add | |
sudo apt-get install google-chrome-stable | |
wget https://chromedriver.storage.googleapis.com/2.40/chromedriver_linux64.zip | |
unzip ./chromedriver_linux64.zip | |
fi | |
- name: Run crawler | |
run: | | |
python "crawler/total_news_crawler.py" | |
- name: Commit files | |
run: | | |
git config --local user.email "paullabkorea@gmail.com" | |
git config --local user.name "paullabkorea" | |
git add . | |
git commit --allow-empty -m "Run crawler and update current data" | |
- name: Push changes | |
uses: ad-m/github-push-action@master | |
with: | |
# branch: "main" | |
github_token: ${{ secrets.GITHUB_TOKEN }} |