# Before we start coding, we first import all packages that we need.
import numpy as np
import pandas as pd
import tensorflow as tf

import re
import string

from tensorflow.keras import layers
from tensorflow.keras import losses
from tensorflow import keras

from tensorflow.keras.layers.experimental.preprocessing import TextVectorization
from tensorflow.keras.layers.experimental.preprocessing import StringLookup

from matplotlib import pyplot as plt

from sklearn.decomposition import PCA
import plotly.express as px 

train_url = "https://github.com/PhilChodrow/PIC16b/blob/master/datasets/fake_news_train.csv?raw=true"
df = pd.read_csv(train_url)
df

# import the stopwords library
from sklearn.feature_extraction import text
stop = text.ENGLISH_STOP_WORDS
# now "stop" contains the list of stopwords that we wish to remove from the data.
# we thus remove all stopwords by using lambda expression and .apply method

df['text'] = df['text'].apply(lambda x: ' '.join([word for word in x.split() if word.lower() not in (stop)]))
df['title'] = df['title'].apply(lambda x: ' '.join([word for word in x.split() if word.lower() not in (stop)]))
df

 data = tf.data.Dataset.from_tensor_slices(
            ({
                    "title" : df[["title"]],
                    "text" : df[["text"]]            
                }, {
                    "fake" : df[["fake"]]
                }))
    

def make_dataset(df):
    """
    A function that creates a fakenews tf.data.Dataset from a dataframe after cleaning all stopwords
    It will also add a batch configue to the returned dataset, the size of batches being 100
    
    input: df, pd.DataFrame object
    output: tf.data.Dataset with batch_size 100
    """
    # Excluding the stopwords from the dataframe
    stop = text.ENGLISH_STOP_WORDS
    df['text'] = df['text'].apply(lambda x: ' '.join([word for word in x.split() if word.lower() not in (stop)]))
    df['title'] = df['title'].apply(lambda x: ' '.join([word for word in x.split() if word.lower() not in (stop)]))
    
    # Convert the dataframe into the Dataset object in tensorflow
    # Here, we need to tell tensorflow which columns are the inputs ("text", "title") and which column is the desired out ("fake")
    data = tf.data.Dataset.from_tensor_slices(
            ({
                    "title" : df[["title"]],
                    "text" : df[["text"]]            
                }, {
                    "fake" : df[["fake"]]
                }))
    
    # We set the dataset batch size to 100
    # This means when we train our model, it will take one batch each time
    # And will not loop over the entire 20000+ rows every time
    data = data.batch(100)
    return data

# We create the dataset using the above defined function
data = make_dataset(df)

train_size = int(0.8*len(data))
val_size   = int(0.2*len(data))

# The take(n method allows us to take the first n 'rows'
train = data.take(train_size)
# And the skip method allows us to ignore the first n 'rows' and take the rest
val   = data.skip(train_size).take(val_size)

# Check the length of the train and test dataset
len(train), len(val)

# We first specify the maximum number of different words we want to consider
# We are only going to look at the 2000 words which appear most frequently in the texts/titles
size_vocabulary = 2000

# The first step in vectorization is to standardize the texts/titles
# We are going to make all words appear in lower cases, and we will drop all the punctuations
# After we define the function, tensorflow will automatically apply it when necessary
def standardization(input_data):
    """
    A standardization function that converts all letters to lower cases and drop punctuations
    
    input: tf.data.Dataset object
    ouput: the same dataset in lower case with no punctuations
    """
    lowercase = tf.strings.lower(input_data)
    no_punctuation = tf.strings.regex_replace(lowercase,
                        '[%s]' % re.escape(string.punctuation),'')
    return no_punctuation 


# Then we can use vectorization layer from tensorflow to vectorize the standardized data
# Pass the size_vocabulary and the standardization function to it
def make_vectorization_layer():
    vectorize_layer = TextVectorization(
    standardize=standardization, # automatically apply the standardization function
    max_tokens=size_vocabulary, # only consider this amount of words
    output_mode='int',
    output_sequence_length=500)
    return vectorize_layer

# Then we are ready to build the models

vectorize_layer1 = make_vectorization_layer()
vectorize_layer1.adapt(train.map(lambda x, y: x["title"]))

title_input = keras.Input(
    shape = (1,), 
    name = "title",
    dtype = "string"
)

# First, let's include the vectorizaion layer we definied above
title_features = vectorize_layer1(title_input)
# Then an embedding layer. We use dimension = 10 for the model
title_features = layers.Embedding(size_vocabulary, 10, name = "embedding_title")(title_features)
# Drop 20% of the indicators to avoid overfitting
title_features = layers.Dropout(0.2)(title_features)
# Consolidate the features using GlobalAveragePooling
title_features = layers.GlobalAveragePooling1D()(title_features)
# Drop again to avoid overfitting
title_features = layers.Dropout(0.2)(title_features)
# Pass it to a Dense Layer to do feature identification
title_features = layers.Dense(32, activation='relu')(title_features)
# Output the result, because we are dealing with 2 cases: true and false
# thus, the output whould have exactly two units
title_features = layers.Dense(2, name = "fake")(title_features)

# Then we make a model with the specified input and output
model1 = keras.Model(
    inputs = [title_input],
    outputs = title_features
)
# we can check the model structure
model1.summary()

model1.compile(optimizer = "adam",
              loss = losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy']
)

history = model1.fit(train,
            validation_data = val,
            epochs = 15, # we train the model 15 times using 15 different batches from the dataset
            verbose = True)

plt.plot(history.history["accuracy"], label = "Training")
plt.plot(history.history["val_accuracy"], label = "Validations")
plt.savefig("model1.jpg")
plt.legend()

# Second vectorization layer only with the text part
vectorize_layer2 = make_vectorization_layer()
vectorize_layer2.adapt(train.map(lambda x, y: x["text"]))

# A single input using only the texts
text_input = keras.Input(
    shape = (1,), 
    name = "text",
    dtype = "string"
)

# First, let's include the vectorizaion layer we definied above
text_features = vectorize_layer2(text_input)
# First, let's include the vectorizaion layer we definied above
text_features = layers.Embedding(size_vocabulary, 10, name = "embedding_text")(text_features)
# Drop 20% of the indicators to avoid overfitting
text_features = layers.Dropout(0.2)(text_features)
# Let's consolidate the features using GlobalAveragePooling
text_features = layers.GlobalAveragePooling1D()(text_features)
# Drop again to avoid overfitting
text_features = layers.Dropout(0.2)(text_features)
# We pass it to a Dense Layer to do feature identification
text_features = layers.Dense(32, activation='relu')(text_features)
# Output the result, because we are dealing with 2 cases: true and false
text_features = layers.Dense(2, name = "fake")(text_features)

# Declare the model
model2 = keras.Model(
    inputs = text_input,
    outputs = text_features
)

# Take a look at the model structure
model2.summary()

# Compile our model2
model2.compile(optimizer = "adam",
              loss = losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy']
)

# Train the model
history = model2.fit(train,
            validation_data = val,
            epochs = 15, 
            verbose = True)

plt.plot(history.history["accuracy"],label = "Training")
plt.plot(history.history["val_accuracy"],label = "Validations")
plt.legend()
plt.savefig("model2.jpg")

# our 3rd vectorization layer, and this time feed it with both titles and texts
vectorize_layer3 = make_vectorization_layer()
vectorize_layer3.adapt(train.map(lambda x, y: x["title"]))
vectorize_layer3.adapt(train.map(lambda x, y: x["text"]))

embedding_layer = layers.Embedding(size_vocabulary, 10, name = "embedding")

text_features = vectorize_layer3(text_input)
text_features = embedding_layer(text_features)
text_features = layers.Dropout(0.2)(text_features)
text_features = layers.GlobalAveragePooling1D()(text_features)
text_features = layers.Dropout(0.2)(text_features)
text_features = layers.Dense(32, activation='relu')(text_features)
title_features = vectorize_layer3(title_input)
title_features = embedding_layer(title_features)
title_features = layers.Dropout(0.2)(title_features)
title_features = layers.GlobalAveragePooling1D()(title_features)
title_features = layers.Dropout(0.2)(title_features)
title_features = layers.Dense(32, activation='relu')(title_features)

main = layers.concatenate([text_features, title_features], axis = 1)

main = layers.Dropout(0.2)(main)
main = layers.Dense(32, activation='relu')(main)
main = layers.Dropout(0.2)(main)
output = layers.Dense(2, name = "fake")(main)

model3 = keras.Model(
    inputs = [text_input, title_input],
    outputs = output
)

model3.compile(optimizer = "adam",
              loss = losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy']
)
history = model3.fit(train,
          epochs = 15,
          validation_data = val, 
          verbose = True)
plt.plot(history.history["accuracy"],label = "Training")
plt.plot(history.history["val_accuracy"],label = "Validations")
plt.legend()
plt.savefig("model3.jpg")

#The test dataset can be accessed directly from Professor Chodrow’s github page.
test_url = "https://github.com/PhilChodrow/PIC16b/blob/master/datasets/fake_news_test.csv?raw=true"
de = pd.read_csv(test_url)

# In order to feed data to the models, we need to make it a dataset using the earlier makedata_set function.
test = make_dataset(de)

model2.evaluate(test)

model3.evaluate(test)

weights = model3.get_layer('embedding').get_weights()[0] # get the weights from the embedding layer
vocab = vectorize_layer3.get_vocabulary() # get the vocabulary 
# Apply PCA on the high-dimensional weights
pca = PCA(n_components=2)
weights = pca.fit_transform(weights)

# Create a dataframe accordingly
# Using the first weight as x value
# And the second weight as y value
embedding_df = pd.DataFrame({
    'word' : vocab, 
    'x0'   : weights[:,0],
    'x1'   : weights[:,1]
})
# Plot our word embedding
fig = px.scatter(embedding_df, 
          x = "x0", 
          y = "x1", 
          size = list(np.ones(len(embedding_df))),
          size_max = 2,
          hover_name = "word")

from plotly.io import write_html
write_html(fig, "PCA.html")
fig.show()