from datetime import datetime, timedelta

from selenium import webdriver

from tqdm import tqdm


def get_chrome_driver():
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--no-sandbox')
    chrome_options.add_argument('--disable-dev-shm-usage')

    chrome_driver = webdriver.Chrome('./chromedriver', options=chrome_options)

    return chrome_driver


_SECTIONS = {
    '100': '정치',
    '101': '경제',
    '102': '사회',
    '103': '생활/문화',
    '104': '세계',
    '105': 'IT/과학'
}


def get_popular_day_news_contents(date, section_id):
    url = f'https://news.naver.com/main/ranking/popularDay.nhn' \
        f'?rankingType=popular_day' \
        f'&sectionId={section_id}&date={date}'
    driver.get(url)
    driver.implicitly_wait(1)

    ranking = driver.find_elements_by_class_name('ranking')

    if ranking:
        headlines = ranking[0].find_elements_by_class_name('ranking_headline')
        ledes = ranking[0].find_elements_by_class_name('ranking_lede')
        offices = ranking[0].find_elements_by_class_name('ranking_office')
        views = ranking[0].find_elements_by_class_name('ranking_view')

        return zip(headlines, ledes, offices, views)
    else:
        return None


def get_popular_day_news(driver, start_date, end_date):
    fp = open(f'./popularDay-{start_date}-{end_date}.tsv', 'w')
    columns = ['date', 'section', 'office', 'view', 'headline', 'lede', 'link']
    fp.write('\t'.join(columns) + '\n')

    target_dates = get_date_strings(start_date, end_date)

    for date_str in tqdm(target_dates):
        for section_id, section_name in _SECTIONS.items():
            contents = get_popular_day_news_contents(date_str, section_id)
            if contents:
                for headline, lede, office, view in contents:
                    headline_a_tag = headline.find_element_by_tag_name('a')
                    contents = '\t'.join([
                        date_str,
                        section_name,
                        office.text,
                        view.text,
                        headline_a_tag.get_attribute('title'),
                        lede.text,
                        headline_a_tag.get_attribute('href')
                    ])
                    fp.write(contents + '\n')

    driver.quit()


def get_date_strings(start_date, end_date):
    date_strings = []
    curr = datetime.strptime(start_date, '%Y%m%d')
    end = datetime.strptime(end_date, '%Y%m%d')

    while True:
        if curr > end:
            break
        else:
            date_strings.append(
                curr.strftime('%Y%m%d')
            )
            curr = curr + timedelta(days=1)
    return date_strings


if __name__ == '__main__':
    driver = get_chrome_driver()
    start_date = '20200101'
    end_date = '20200131'

    date_strings = get_date_strings(start_date, end_date)
    print(date_strings)
    get_popular_day_news(driver, start_date, end_date)

import warnings
warnings.filterwarnings(action='ignore')

import pandas as pd
import soynlp
import matplotlib.pyplot as plt
import re
import pyecharts
import matplotlib.font_manager as fm

plt.rc('font', family='NanumGothic')
print(plt.rcParams['font.family'])

dataset_dir = '~/private/news-crawler'
filenames = [
    'popularDay-20200101-20200131.tsv',
    'popularDay-20200201-20200229.tsv',
    'popularDay-20200301-20200331.tsv',
    'popularDay-20200401-20200426.tsv'
]
data = pd.concat([
    pd.read_csv(f'{dataset_dir}/{f}', sep='\t', error_bad_lines = False, warn_bad_lines=False) 
    for f in filenames
])
data['datetime'] = pd.to_datetime(data['date'], format='%Y%m%d')
data.head()

from soynlp.utils import DoublespaceLineCorpus
from soynlp.noun import LRNounExtractor_v2
_office_list = list(data.office) + list(o[:-1] for o in data.office)

def prep(sent: str):
    prep = re.sub('\W+', ' ', sent)
    prep = re.sub(r'\d+일', '', prep)
    prep = re.sub(r'\d+년', '', prep)
    prep = re.sub(r'\d+대', '', prep)
    return prep


def is_office(word):
    if word in _office_list:
        return True
    else:
        return False


def get_global_top_keywords(data):
    sents = list(data['headline']) + list(data['lede'])
    prep_sents = [prep(sent) for sent in sents]
    
    noun_extractor = LRNounExtractor_v2(verbose=True, extract_compound=True)
    noun_extractor.train(prep_sents)
    nouns = noun_extractor.extract()
    top_keywords = []
    for n, k in sorted(nouns.items(), key=lambda x: x[1].frequency, reverse=True):
        if len(n) >= 3 and k.frequency >= 30 and not is_office(n):
            top_keywords.append((n, k.frequency))
    return top_keywords

top_keywords = get_global_top_keywords(data)
top_keywords[:50]

def get_keywords_from_news(headline, lede, keywords):
    news_keywords = list(set([
        keyword[0] for keyword in keywords 
        if keyword[0] in headline or keyword[0] in lede
    ]))
    return ','.join(news_keywords)

data['keywords'] = data.apply(lambda x: get_keywords_from_news(x['headline'], x['lede'], top_keywords), axis=1)
data.head(10)

top_keywords1 = get_global_top_keywords(
    data[(data['datetime'] >= '20200101') & (data['datetime'] < '20200201')]
)
top_keywords2 = get_global_top_keywords(
    data[(data['datetime'] >= '20200201') & (data['datetime'] < '20200301')]
)
top_keywords3 = get_global_top_keywords(
    data[(data['datetime'] >= '20200301') & (data['datetime'] < '20200401')]
)
top_keywords4 = get_global_top_keywords(
    data[(data['datetime'] >= '20200401') & (data['datetime'] < '20200501')]
)
tops = list(set([
    keyword for keyword, _ in 
    top_keywords1[:20] + top_keywords2[:20] + top_keywords3[:20] + top_keywords4[:20]
]))

tops

def get_keyword_summ(unique_date, data, keyword):
    tmp = data.copy()
    tmp[f'has_{keyword}'] = tmp.keywords.str.contains(keyword)

    keyword_summ = tmp[['date', f'has_{keyword}']][tmp[f'has_{keyword}']].groupby(
        ['date']
    ).count().reset_index()
    
    keyword_data = pd.merge(unique_date, keyword_summ, on='date', how='left')
    keyword_data = keyword_data.fillna(0)
    return keyword_data


unique_date_df = pd.DataFrame(data['date'].unique(), columns=['date'])
plt.figure(figsize=(20, 15))
for keyword in tops:
    keyword_data = get_keyword_summ(unique_date_df, data, keyword)
    plt.plot(
        pd.to_datetime(keyword_data['date'], format='%Y%m%d'),
        keyword_data[f'has_{keyword}'],
        label=keyword
    )
plt.legend(fontsize=10)

from pyecharts import Line
unique_dates = [str(date) for date in list(keyword_data['date'])]

line = Line()

for keyword in ['코로나', '우한폐렴', '바이러스', '마스크', '신천지', '재난지원금']:
    keyword_data = get_keyword_summ(unique_date_df, data, keyword)

    line.add(
        name=keyword,
        x_axis=unique_dates,
        y_axis=[int(item) for item in list(keyword_data[f'has_{keyword}'])],
        mark_point=['max']
    )
# line - notebook 에서 로딩 안됨 ㅠ 아래는 그냥 이미지를 넣음

line = Line()
for keyword in ['기생충', '봉준호']:
    keyword_data = get_keyword_summ(unique_date_df, data, keyword)

    line.add(
        name=keyword,
        x_axis=unique_dates,
        y_axis=[int(item) for item in list(keyword_data[f'has_{keyword}'])],
        mark_point=['max']
    )

# line - notebook 에서 로딩 안됨 ㅠ 아래는 그냥 이미지를 넣음

line = Line()
for keyword in ['텔레그램', '조주빈', 'n번방']:
    keyword_data = get_keyword_summ(unique_date_df, data, keyword)

    line.add(
        name=keyword,
        x_axis=unique_dates,
        y_axis=[int(item) for item in list(keyword_data[f'has_{keyword}'])],
        mark_point=['max']
    )
# line - notebook 에서 로딩 안됨 ㅠ 아래는 그냥 이미지를 넣음

line = Line()
for keyword in ['김정은']:
    keyword_data = get_keyword_summ(unique_date_df, data, keyword)

    line.add(
        name=keyword,
        x_axis=unique_dates,
        y_axis=[int(item) for item in list(keyword_data[f'has_{keyword}'])],
        mark_point=['max']
    )

# line - notebook 에서 로딩 안됨 ㅠ 아래는 그냥 이미지를 넣음

line = Line()
for keyword in ['트럼프']:
    keyword_data = get_keyword_summ(unique_date_df, data, keyword)

    line.add(
        name=keyword,
        x_axis=unique_dates,
        y_axis=[int(item) for item in list(keyword_data[f'has_{keyword}'])],
        mark_point=['max']
    )

# line - notebook 에서 로딩 안됨 ㅠ 아래는 그냥 이미지를 넣음