- install dependencies
pip install -r requirements.txt
npm install
source .env # you can copy .env_sample to .env with your proxy server
- start proxy server
node proxy.js
- start se-scraper
docker run --name se-scraper -m 2gb --restart always -p 3000:3000 tschachn/se-scraper:latest
- start mysql DB
docker run --name bf-mysql -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -d mysql:latest
-
create database
bf
-
execute sql in
bf.sql
-
execute
python excel2db.py
to migrate data from excel to database -
execute
remove_duplicate.sql
to remove duplicate keyword in table
- start main function
python main.py
- restart docker
se-scraper
container every 2 hours
python task.py