base line

import pandas as pd
pd.set_option('display.max_colwidth', None)
from sklearn import preprocessing as pp
from sklearn.model_selection import train_test_split
import scipy.sparse as sp
import numpy as np
import random
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import time
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm

ratings = pd.read_csv('initial_Train.csv')
ratings['interactions'] = 1

df = ratings.sort_values(by = ['timestamp'])
print("Rating Distribution")
# df.groupby(['rating'])['rating'].count()
names = ['userId', 'movieId', 'rating', 'timestamp', 'interactions']
train_, test_ = train_test_split(ratings.values, test_size=0.3, shuffle=False)
train_ = pd.DataFrame(train_, columns = names )
# train_.to_csv('initial_train_data.csv')
test_ = pd.DataFrame(test_, columns = names)

import numpy
from numpy.compat import pickle 
le_user = pp.LabelEncoder()
le_item = pp.LabelEncoder()
# le_user.classes_ = le_user.classes_.astype(np.float64)
# le_item.classes_ = le_item.classes_.astype(np.float64)
train_['user_id_idx'] = le_user.fit_transform(train_['userId'].values)
train_['item_id_idx'] = le_item.fit_transform(train_['movieId'].values)



train_user_ids = train_['userId'].unique()
train_item_ids = train_['movieId'].unique()
test_ = test_[(test_['userId'].isin(train_user_ids))&(test_['movieId'].isin(train_item_ids))]
test_['user_id_idx'] = le_user.transform(test_['userId'].values)
test_['item_id_idx'] = le_item.transform(test_['movieId'].values)

rating_matrix = train_.pivot(index='user_id_idx', columns='item_id_idx', values='interactions')
n_users, n_movies = rating_matrix.shape

rating_matrix[rating_matrix.isnull()] = 0
rating_matrix = rating_matrix.astype(float)
rating_matrix = torch.FloatTensor(rating_matrix.values)

latent_vectors = 64
user_features = torch.randn(n_users, latent_vectors, requires_grad=True)
user_features.data.mul_(0.01)
movie_features = torch.randn(n_movies, latent_vectors, requires_grad=True)
movie_features.data.mul_(0.01)

class PMFLoss(torch.nn.Module):
    def __init__(self, lam_u=0.3, lam_v=0.3):
        super().__init__()
        self.lam_u = lam_u
        self.lam_v = lam_v
    
    def forward(self, matrix, u_features, v_features):
        non_zero_mask = (matrix != -1).type(torch.FloatTensor)
        predicted = torch.sigmoid(torch.mm(u_features, v_features.t()))
        
        diff = (matrix - predicted)**2
        prediction_error = torch.sum(diff*non_zero_mask)

        u_regularization = self.lam_u * torch.sum(u_features.norm(dim=1))
        v_regularization = self.lam_v * torch.sum(v_features.norm(dim=1))
        
        return prediction_error + u_regularization + v_regularization

criterion = PMFLoss()
loss = criterion(rating_matrix, user_features, movie_features)

latent_vectors = 64
user_features = torch.randn(n_users, latent_vectors, requires_grad=True)
user_features.data.mul_(0.01)
movie_features = torch.randn(n_movies, latent_vectors, requires_grad=True)
movie_features.data.mul_(0.01)
loss_ = []
step_ = []

pmferror = PMFLoss(lam_u=0.05, lam_v=0.05)
optimizer = torch.optim.Adam([user_features, movie_features], lr=0.01)
for step, epoch in enumerate(range(3000)):
    optimizer.zero_grad()
    loss = pmferror(rating_matrix, user_features, movie_features)
    loss.backward()
    optimizer.step()
    if step % 50 == 0:
        loss_.append(loss)
        step_.append(step)
        print(f"Step {step}, {loss:.3f}")

def convert_to_sparse_tensor(dok_mtrx):
    
    dok_mtrx_coo = dok_mtrx.tocoo().astype(np.float32)
    values = dok_mtrx_coo.data
    indices = np.vstack((dok_mtrx_coo.row, dok_mtrx_coo.col))

    i = torch.LongTensor(indices)
    v = torch.FloatTensor(values)
    shape = dok_mtrx_coo.shape

    dok_mtrx_sparse_tensor = torch.sparse.FloatTensor(i, v, torch.Size(shape))

    return dok_mtrx_sparse_tensor

from sklearn import metrics
from sklearn.preprocessing import label_binarize
def get_metrics(user_Embed_wts, item_Embed_wts, n_users, n_items, train_data, test_data, K):
    test = sp.dok_matrix((n_users, n_items), dtype = np.float32)
    test[test_data['user_id_idx'], test_data['item_id_idx']] = 1.0
    test = test.toarray()
    user_Embedding = nn.Embedding(user_Embed_wts.size()[0], user_Embed_wts.size()[1], _weight = user_Embed_wts)
    item_Embedding = nn.Embedding(item_Embed_wts.size()[0], item_Embed_wts.size()[1], _weight = item_Embed_wts)

    test_user_ids = torch.LongTensor(test_data['user_id_idx'].unique())

    relevance_score = torch.matmul(user_Embed_wts, torch.transpose(item_Embed_wts,0, 1))
    itms = test_data['item_id_idx'].unique()
    usrs = test_data['user_id_idx'].unique()


    
    R = sp.dok_matrix((n_users, n_items), dtype = np.float32)
    R[train_data['user_id_idx'], train_data['item_id_idx']] = 1.0
    
    R_tensor = convert_to_sparse_tensor(R)
    R_tensor_dense = R_tensor.to_dense()

    R_tensor_dense = R_tensor_dense*(-np.inf)
    R_tensor_dense = torch.nan_to_num(R_tensor_dense, nan=0.0)
    

    relevance_score = relevance_score + R_tensor_dense
    # print(torch.topk(relevance_score, K))
    topk_relevance_score = torch.topk(relevance_score, K).values
    # print(len(topk_relevance_score))
    topk_relevance_indices = torch.topk(relevance_score, K).indices
    # print(topk_relevance_indices)
    #covert topk_relevance_indices to cpu
    topk_relevance_indices = topk_relevance_indices.
    topk_relevance_indices_df = pd.DataFrame(topk_relevance_indices.numpy(),columns =['top_indx_'+str(x+1) for x in range(K)])
    # print(topk_relevance_indices_df)
    topk_relevance_indices_df['user_ID'] = topk_relevance_indices_df.index
    #
    topk_relevance_indices_df['top_rlvnt_itm'] = topk_relevance_indices_df[['top_indx_'+str(x+1) for x in range(K)]].values.tolist()
    topk_relevance_indices_df = topk_relevance_indices_df[['user_ID','top_rlvnt_itm']]
    # print(topk_relevance_indices_df)
    test_interacted_items = test_data.groupby('user_id_idx')['item_id_idx'].apply(list).reset_index()
    
    metrics_df = pd.merge(test_interacted_items,topk_relevance_indices_df, how= 'left', left_on = 'user_id_idx',right_on = ['user_ID'])
    # print(metrics_df)
    metrics_df['intrsctn_itm'] = [list(set(a).intersection(b)) for a, b in zip(metrics_df.item_id_idx, metrics_df.top_rlvnt_itm)]
   

    metrics_df['recall'] = metrics_df.apply(lambda x : len(x['intrsctn_itm'])/len(x['item_id_idx']), axis = 1) 
    metrics_df['precision'] = metrics_df.apply(lambda x : len(x['intrsctn_itm'])/K, axis = 1)
    # print(metrics_df)
    # metrics_df['auc'] = metrics_df.apply(lambda x : metrics.roc_auc_score([1]*len(x['intrsctn_itm']) + [0]*(K-len(x['intrsctn_itm'])), x['intrsctn_itm'] + [0]*(K-len(x['intrsctn_itm']))), axis = 1)


    def get_hit_list(item_id_idx, top_rlvnt_itm):
        return [1 if x in set(item_id_idx) else 0 for x in top_rlvnt_itm ]

    metrics_df['hit_list'] = metrics_df.apply(lambda x : get_hit_list(x['item_id_idx'], x['top_rlvnt_itm']), axis = 1)


    usert = test_data['user_id_idx'].unique()
    # print(intrsctn_itm)
    def get_auc(test, relevance_score, usert):
        relevance_score = relevance_score.cpu()
        auc = []
        for i in usert:
            a = metrics.roc_auc_score(test[i], relevance_score[i])
            auc.append(a)
        return auc
            
            
    def get_dcg_idcg(item_id_idx, hit_list):
        idcg  = sum([1 / np.log1p(idx+1) for idx in range(min(len(item_id_idx),len(hit_list)))])
        dcg =  sum([hit / np.log1p(idx+1) for idx, hit in enumerate(hit_list)])
        return dcg/idcg

    def get_cumsum(hit_list):
        return np.cumsum(hit_list)

    def get_map(item_id_idx, hit_list, hit_list_cumsum):
        return sum([hit_cumsum*hit/(idx+1) for idx, (hit, hit_cumsum) in enumerate(zip(hit_list, hit_list_cumsum))])/len(item_id_idx)
  

    metrics_df['ndcg'] = metrics_df.apply(lambda x : get_dcg_idcg(x['item_id_idx'], x['hit_list']), axis = 1)
    metrics_df['hit_list_cumsum'] = metrics_df.apply(lambda x : get_cumsum(x['hit_list']), axis = 1)
    
    metrics_df['map'] = metrics_df.apply(lambda x : get_map(x['item_id_idx'], x['hit_list'], x['hit_list_cumsum']), axis = 1)
    a_m = get_auc(test, relevance_score, usert)

    return  a_m, metrics_df['recall'], metrics_df['precision'], metrics_df['ndcg'], metrics_df['map'] 

train_ = train_.sort_values(by = ['user_id_idx'])
train_ = train_.reset_index(drop = True)
user_id_idx_end = train_.groupby('user_id_idx').tail(1).index
test_ = test_.sort_values(by = ['user_id_idx'])

user_id_idx_dict = dict(zip(train_['userId'], train_['user_id_idx']))

import json
import math
import copy


begin = 0 #index of the first user
number_of_users = 1000
end = number_of_users - 1
auc_ = []
ndcg_ = []
map_ = []
recall_ = []
precision_ = []
while(begin != len(train_)):



    temp = user_id_idx_end[end]
    print('\n')
    print('begin: ', begin)
    print('\n')
    print('temp: ', temp)
    print('\n')
    print('number_of_users: ', number_of_users)     
    train = copy.deepcopy(train_.iloc[begin:temp+1])
    train.iloc[:]['user_id_idx'] -= train.iloc[0]['user_id_idx']
    test_user_dic = dict(zip(train['userId'].unique(), train['user_id_idx'].unique()))

    test = copy.deepcopy(test_[test_['userId'].isin(test_user_dic.keys())])
    test['user_id_idx'] = test.iloc[:]['userId'].map(test_user_dic)
    

#-----------------------------------------------
    b = train.iloc[0]['userId']
    b = user_id_idx_dict[b]
    e = train.iloc[-1]['userId']
    e = user_id_idx_dict[e]
    auc ,recall,  precision, ndcg, map = get_metrics(user_features[b:e+1], movie_features, number_of_users, n_movies, train, test, K=5)
    print(b)
    print(e)
    auc_.extend(auc)
    ndcg_.extend(ndcg)
    precision_.extend(precision)
    map_.extend(map)
    recall_.extend(recall)

#=----------------------------------------------
    begin = temp + 1
    
    if ((end + number_of_users) <= len(user_id_idx_end)-1):
        temp = user_id_idx_end[end]#index of the last user
        end += number_of_users
    else:
        print('-------------------------------------------')
        number_of_users = len(user_id_idx_end) - end - 1
        temp = user_id_idx_end[-1]   
        end = len(user_id_idx_end)- 1

    # user_final_embedding =torch.flip(user_final_embedding, [0])

print("auc: ", numpy.array(auc_).mean())
print("test_topK_recall: ", numpy.array(recall_).mean())
print("test_topK_precision: ", numpy.array(precision_).mean())
print("test_topK_ndcg: ", numpy.array(ndcg_).mean())
print("test_topK_map: ", numpy.array(map_).mean())

user_features = user_features.detach()
movie_features = movie_features.detach()
auc ,test_topK_recall,  test_topK_precision, test_topK_ndcg, test_topK_map  = get_metrics(user_features, movie_features, n_users, n_movies, train_, test_, K=5)
print("auc: ", auc)
print("test_topK_recall: ", test_topK_recall)
print("test_topK_precision: ", test_topK_precision)
print("test_topK_ndcg: ", test_topK_ndcg)
print("test_topK_map: ", test_topK_map)

test_topK_recall,  test_topK_precision, test_topK_ndcg, test_topK_map  = get_metrics(user_e.to('cpu'), item_e.to('cpu'), n_users, n_items, train_, test_, K=5)
print("test_topK_recall: ", test_topK_recall)
print("test_topK_precision: ", test_topK_precision)
print("test_topK_ndcg: ", test_topK_ndcg)
print("test_topK_map: ", test_topK_map)

plt.plot(step_, loss_, label='Total Training Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()y