#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Nov  4 17:15:06 2021
@author: huyi
"""
 
from selenium import webdriver
import time
 
# =============================================================================
# 爬取动态下滑加载网页
# =============================================================================
def pa(url):
    driver = webdriver.Chrome('/usr/local/bin/chromedriver')
    driver.get(url)
    js = '''
                let height = 0
        let interval = setInterval(() => {
            window.scrollTo({
                top: height,
                behavior: "smooth"
            });
            height += 500
        }, 500);
        setTimeout(() => {
            clearInterval(interval)
        }, 20000);
    '''
    driver.execute_script(js)
    time.sleep(20)
    source = driver.page_source
    driver.close()
    return source

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Nov  4 21:53:22 2021
@author: huyi
"""
 
 
import jieba.analyse
 
 
def get_key_word(sentence):
    result_dic = {}
    words_lis = jieba.analyse.extract_tags(
        sentence, topK=3, withWeight=True, allowPOS=())
    for word, flag in words_lis:
        if word in result_dic:
            result_dic[word] += 1
        else:
            result_dic[word] = 1
    return result_dic

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Nov  4 14:01:38 2021
@author: huyi
"""
from lxml import etree
from tools.dynamic_page import pa
from tools.analyse_word import get_key_word
 
 
csdn_url = 'https://blog.csdn.net/rank/list'
source = etree.HTML(pa(csdn_url))
 
titles = source.xpath("//div[@class='hosetitem-title']/a/text()")
key_word_dic = {}
for x in titles:
    if x:
        for k, v in get_key_word(x).items():
            if k.lower() in key_word_dic:
                key_word_dic[k.lower()] += v
            else:
                key_word_dic[k.lower()] = v
 
word_count_sort = sorted(key_word_dic.items(),
                         key=lambda x: x[1], reverse=True)
 
with open('result.txt', mode='w', encoding='utf-8') as f:
    for y in word_count_sort:
        f.write('{},{}\n'.format(y[0], y[1]))