const { spider } = require('webster');

class MySpider extends spider {
    get defUserAgent() {
        return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36';
    }
    get defDeviceType() {
        return 'pc';
    }
    async parseHtml(html) {
        return true;
    }
}

(async () => {
    const spider = new MySpider({
        actions: [
            {
                type: 'waitForSelector',
                selector: 'div.js-details-container',
            }
        ],
        targets: [
            {
                selector: 'div.Box-row[role=row]',
                type: 'text',
                field: 'sugs'
            }
        ],
    });
    const url = `https://github.com/zhuyingda/webster`;
    let crawlResult = await spider.startRequest(url);
    console.log(crawlResult);
})();

// producer
const Webster = require('webster');
const Producer = Webster.producer;
const Task = Webster.task;

let task = new Task({
    spiderType: 'browser',
    engineType: 'playwright',
    browserType: 'chromium',
    url: 'http://quotes.toscrape.com/tag/humor/',
    targets: [
        {
            selector: 'span.text',
            type: 'text',
            field: 'quote'
        },
        {
            selector: 'li.next > a',
            type: 'attr',
            attrName: 'href',
            field: 'link'
        }
    ],
    actions: [
        {
            type: 'waitAfterPageLoading',
            value: 500
        }
    ],
    referInfo: {
        para1: 'this is a refer field 1',
        para2: 'this is a refer field 2'
    }
});

let myProducer = new Producer({
    channel: 'demo_channel1',
    dbConf: {
        redis: {
            host: 'redis-12419.c44.us-east-1-2.ec2.cloud.redislabs.com',
            port: 12419,
            password: 'X2AcjziaOOYPppWFOPiP4rmzZ9RFLViv'
        }
    }
});
myProducer.generateTask(task).then(() => {
    console.log('done');
    process.exit();
});

// consumer
const Webster = require('webster');
const Consumer = Webster.consumer;

class MyConsumer extends Consumer {
    constructor(option) {
        super(option);
    }
    afterCrawlRequest(result) {
        console.log('your scrape result:', result);
    }
}

let myConsumer = new MyConsumer({
    channel: 'demo_channel1',
    sleepTime: 5000,
    deviceType: 'pc',
    dbConf: {
        redis: {
            host: 'redis-12419.c44.us-east-1-2.ec2.cloud.redislabs.com',
            port: 12419,
            password: 'X2AcjziaOOYPppWFOPiP4rmzZ9RFLViv'
        }
    }
});
myConsumer.startConsume();