%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
import requests
import pandas as pd
from selenium import webdriver
from fake_useragent import UserAgent
import nltk
from wordcloud import WordCloud
from nltk import FreqDist
from nltk.tag import pos_tag
from nltk import Text
from nltk.tokenize import RegexpTokenizer

result = []
df_bbc = []
df_newyork = []
df_fox = []
df_abc = []

url_list = ['https://twitter.com/bbcworld',
            'https://twitter.com/nytimes',
            'https://twitter.com/CNN',
            'https://twitter.com/FoxNews',
            'https://twitter.com/ABC'
           ]

for member in url_list:

    options = webdriver.ChromeOptions() 
    options.add_argument("user-agent={}".format(UserAgent().chrome))

    driver = webdriver.Chrome()
    driver.get(member)

    for count in range(10):
        
        time.sleep(1)

        script = "window.scrollTo(0, 100000);"
        driver.execute_script(script)

    items = driver.find_elements_by_css_selector("#stream-items-id .content")
    print("{} : ".format(member[20:]), len(items))

    for idx, item in enumerate(items):
        
        try:
            time_before = item.find_element_by_css_selector("div:nth-child(1) > small > a > span").text
        except:
            time_before = None
        try:
            news_comment = item.find_element_by_css_selector(".js-tweet-text-container p").text   
        except:
            news_comment = None
        try:
            news_url = item.find_element_by_css_selector("div:nth-child(1) > small > a").get_attribute("href")   
        except:
            news_url =  None
        try:
            reaction = item.find_element_by_css_selector("div:nth-child(4) > div:nth-child(2) > div:nth-child(1) .ProfileTweet-actionCountForPresentation").text   
        except:
            reaction = 0
        try:
            retweet = item.find_element_by_css_selector("div:nth-child(4) > div:nth-child(2) > div:nth-child(2) .ProfileTweet-actionCountForPresentation").text
        except:
            retweet = 0
        try:
            like = item.find_element_by_css_selector("div:nth-child(4) > div:nth-child(2) > div:nth-child(3) .ProfileTweet-actionCountForPresentation").text
        except:
            like = 0
        
        data = {
            "time(before)" : time_before,
            "news_comment" : news_comment,
            "news_url" : news_url,
            "reaction" : reaction,
            "retweet" : retweet,
            "like" : like,
            "brand" : member[20:]
        }

        result.append(data)

        df = pd.DataFrame(result)
        df = df[["brand","time(before)","reaction","retweet","like","news_comment","news_url"]]
            
    driver.quit()

df

comment_text = ''

for idx in range(len(df['news_comment'])):
    comment_text += df['news_comment'][idx] + ' '

stopwords = ["A",'US','CNN','S','News','New','Sessions','south']

retokenize = RegexpTokenizer("[\w]+")
tokens = pos_tag(retokenize.tokenize(comment_text))
names_list = [data[0] for data in tokens if data[1] == "NNP" and data[0] not in stopwords]
fd_names = FreqDist(names_list)

fd_names.most_common(20)

wc = WordCloud(width=1000, height=600, background_color="white", random_state=0)
plt.imshow(wc.generate_from_frequencies(fd_names))
plt.axis("off")
plt.show()