from sklearn.feature_extraction.text import TfidfVectorizer,  CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline

%matplotlib inline
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd
import scipy as sp
import pickle
import time
from selenium import webdriver

def making_date(start, end):
    
    """
    [입력형식]
    ㅇ start, end = '년도-월-일'
    
    ex) 
    '2018-11-01'
    '2018-12-14'
    
    [return 값]
    ㅇ date : start ~ end까지 날짜리스트
    
    """
    
    date_list = []
    
    if start > end :
        print("error(start > end) try again")
        return None
    
    elif (start or end) is (int or float):
        print("error try again")
        return None
    
    elif start == end:
        making_date = start
        temp = str(making_date)[:4] + str(making_date)[5:7] + str(making_date)[8:10]
        date_list.append(temp)
        return date_list
            
    else :
        making_date = pd.date_range(start, end)
        for count in range(len(making_date)):
            temp = str(making_date[count])[:4] + str(making_date[count])[5:7] + str(making_date[count])[8:10]
            date_list.append(temp)
            
    return date_list

date_list = making_date("2018-11-01","2018-12-02")
len(date_list)

def crawler(date_list,headless=False):

    '''
    [category]
    1 : 축구
    2 : 야구
    3 : 농구
    4 : 배구
    5 : 골프

    '''

    result = []
    category_list = ["kfootball","wfootball","kbaseball","wbaseball","basketball","volleyball","golf"]
    count = 0
    category_count = 0

    for category_data in category_list:

        if category_data == "wfootball" :
            pass
        elif category_data == "wbaseball" :
            pass
        else :
            category_count += 1

        for date in date_list:

            for page_number in range(1,3):

                url = "https://sports.news.naver.com/{}/news/index.nhn?date={}&page={}&isphoto=N".format(category_data,date,page_number)
                
                if headless:
                    options = webdriver.ChromeOptions()
                    options.add_argument('headless')
                    driver = webdriver.Chrome(options=options)
                else :
                    driver = webdriver.Chrome()

                driver.get(url)

                articles = driver.find_elements_by_css_selector("#_newsList ul li")

                for article in articles:
                    title = article.find_element_by_css_selector('div .title span').text
                    count += 1

                    data = {
                            "ID" : count,
                            "category" : category_count,
                            "title" : title,
                        }

                    result.append(data)

                driver.quit()

    df = pd.DataFrame(result)

    return df

df = crawler(date_list,headless=False)
df

df.to_csv("naver.csv", mode='w')

df = pd.read_csv("naver.csv")
df[:10]

from sklearn.feature_extraction.text import TfidfVectorizer,  CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline

x_train, x_test, y_train, y_test = train_test_split(df.title, df.category,
                                                   test_size = 0.1, random_state = 1)
len(x_train),len(x_test), len(y_train), len(y_test)

clf = Pipeline([
    ('vect',TfidfVectorizer()),
    ('clf',MultinomialNB(alpha = 0.01))
])

model = clf.fit(x_train, y_train)

y_pred = model.predict(x_test)

from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

confusion_matrix(y_test, y_pred)

print(classification_report(y_test, y_pred))

test_str = "‘정지석 22득점’ 대한항공, 한국전력 꺾고 2연승 질주"
model.predict_proba([test_str])

test_str2 = "두산베어스 한국시리즈 준우승"
model.predict_proba([test_str2])