Google keyword search and find target website
+ 查看更多

Here is the sample code to show how to input a keyword and search in google with puppeteer API. You can change it as you want. By click run button, you can see whole proccess in the monitor and result in the console.

Keyword:

Target Words:

The website domain you want to find in the google result.

Source code:

//This is an example code for how to google search using puppeteer docker.
var cheerio = require('cheerio');
var request = require('request');
var argv = require('minimist')(process.argv.slice(2));
var puppeteer = require('puppeteer-extra');
var StealthPlugin = require('puppeteer-extra-plugin-stealth');
let browser_ags = [
    '--no-sandbox',
    "--hide-scrollbars",
    '--ignore-certifcate-errors',
    '--ignore-certifcate-errors-spki-list',
    '--disable-setuid-sandbox',
    '--disable-dev-shm-usage',
    '--disable-accelerated-2d-canvas',
    '--disable-notifications',
    '--disable-infobars',
    '--disable-gpu',
    '--disable-web-security',
];

(async () => {

$('#center_col .g').each((i, link) => {
            results.results.push({
                link: $(link).find('a').attr('href'),
                title: $(link).find('a h3').text(),
                snippet: '',
                visible_link: $(link).find('cite').text(),
            })
        });

results.no_results = (results.results.length === 0);
        results.num_results = results.results.length;

// clean some results
        results.results = clean_results(results.results, ['title', 'link']);

results.time = (new Date()).toUTCString();
        return results;
    }

//get your param
    let results = [];
    results["rank"]  = 0;
    results["page_num"]  = 0;
    let page_num = 1;
    let result_rank = 1;
    let max_page_num = argv["u_max_num"]?argv["u_max_num"]:10;
    let keyword = argv["u_keyword"]?argv["u_keyword"]:"sohu";
    let target_words = argv["u_targetword"]?argv["u_targetword"]:"sohu";
    target_words = target_words.split(";");
    if(!keyword || !target_words){
        console.log("empty");
        return;
    }
    //
    puppeteer.use(StealthPlugin());
    //launch
    var browser = await puppeteer.launch({
        headless: true,
        args: browser_ags
    });

const page = await browser.newPage();

await page.setViewport({
        width: 1440,
        height: 1080,
    });

let startUrl = 'https://www.google.com/search?gl=us&hl=en&start=0&num=10';

try{
        await page.goto(startUrl, {waitUntil:"networkidle2", timeout:30000});
        await page.waitForSelector('input[name="q"]', { timeout: 10000 });
    }catch (e) {
        console.log(e.message);
        browser.close();
        return;
    }
    // input keyword
    const input = await page.$('input[name="q"]');
    await page.evaluate((value, selector) => {
        return document.querySelector(selector).value = value;
    }, keyword, `input[name="q"]`);
    await sleep(50);
    await input.focus();
    await page.keyboard.press("Enter");

while(true){
        console.log("Search keyword "+keyword+" on page "+page_num);
        try {
            await page.waitForSelector('#fbar', {timeout: 10000});
        }catch (e){
            console.log(e.message);
            break;
        }
        let html = await page.content();
        let parsed = parse(html);
        console.log("page "+page_num+" item:"+parsed.results.length);
        console.log(parsed);
        let find = false;
        for (var i=0; i < parsed.results.length; i++) {
            let res = parsed.results[i];
            for (var j=0; j < target_words.length; j++){
                let word = target_words[j];
                if (res.visible_link && res.visible_link.includes(word)) {
                    console.log("["+word+"] find in page:"+page_num +",rank:"+res.rank);
                    results["rank"] = res.rank;
                    results["page_num"] = page_num;
                    find = true;
                    break;
                }
            }

if(find){ break;}
        }

if(find){
            break;
        }

//next page
        if (page_num <= max_page_num) {
            let next_page_loaded = await next_page();
            if (next_page_loaded === false) {
                console.log("None next paget");
                break;
            }
        }else{
            break;
        }
        page_num++;
    }
    console.log( "Rank:"+results["rank"]+",Page:"+results["page_num"]);

console.log(results);

// const formData = results;
    //
    // request.post({url:argv["u_callback"], formData: formData}, function optionalCallback(err, httpResponse, body) {
    //     if (err) {
    //         return console.error('post failed:', err);
    //     }
    // });

browser.close();
})();

Our API can process your code zip package

Real time preview:

Console:

Google keyword search and find target website+ 查看更多

Google keyword search and find target website
+ 查看更多