【爬虫】爬取简书某ID所有文章并保存为pdf

npm i puppeteer 

const browser = await puppeteer.launch({
        // 设置false可以看到页面的执行步骤
        headless: false,
    });

BASE_URL =  https://www.jianshu.com ;

//文章列表地址
HOME_URL = `${BASE_URL}/u/f46becd1ed83`;

const viewport_size = {
        width: 0,
        height: 0,
    };

const page = await browser.newPage();

//设置浏览器的宽、高
page.setViewport(viewport_size);

//打开文章主页
await page.goto(HOME_URL);

function autoScroll(page) {
    return page.evaluate(() => {
        return new Promise((resolve, reject) => {
            var totalHeight = 0;
            var distance = 100;
            var timer = setInterval(() => {
                console.log( 执行间断函数 );
                var scrollHeight = document.body.scrollHeight;
                window.scrollBy(0, distance);
                totalHeight += distance;

                if (totalHeight >= scrollHeight) {
                    console.log( 滑动到底 );
                    clearInterval(timer);
                    resolve();
                }
            }, 100);
        })
    });
}

const articles = await page.$eval( .note-list , articles_element => {
        const article_elements = articles_element.querySelectorAll( li );
        const articleElementArray = Array.prototype.slice.call(article_elements);

        return articleElementArray.map(item => {
            const a_element = item.querySelector( .title );
            return {
                href: a_element.getAttribute( href ),
                title: a_element.innerHTML.trim(),
         };
     });
});

for (let article of articles) {
        const articlePage = await browser.newPage();
        articlePage.setViewport(viewport_size);
        articlePage.goto(`${BASE_URL}${article.href}`, {
            waitUntil:  networkidle2
        });

        articlePage.waitForSelector( .post );
        console.log( 文章详情页面加载完成 );
}

await articlePage.$eval( body , body => {
            body.querySelector( .navbar ).style.display =  none ;
            body.querySelector( #note-fixed-ad-container ).style.display =  none ;
            body.querySelector( .note-bottom ).style.display =  none ;
            body.querySelector( .side-tool ).style.display =  none ;
            // body.querySelector( .author ).style.display =  none ;
            body.querySelector( .meta-bottom ).style.display =  none ;
            body.querySelector( #web-note-ad-1 ).style.display =  none ;
            body.querySelector( #comment-list ).style.display =  none ;
            body.querySelector( .follow-detail ).style.display =  none ;
            body.querySelector( .show-foot ).style.display =  none ;

            Promise.resolve();
        });

await page.emulateMedia( screen );
        await articlePage.pdf({
            path: fileFullPath,
            format:  A4
 });

const browser = await puppeteer.launch({
        headless: true,
    });

node jian_shu.js