refinend the algo1, starting to test algo2
Luca Lombardo
2 years ago
parent
990d59e3ab
commit
aa7364c261
@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Importing the libraries
|
||||
import os
|
||||
import wget
|
||||
import gzip
|
||||
import time
|
||||
import warnings
|
||||
import scipy as sp
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import networkx as nx
|
||||
import plotly.graph_objs as go
|
||||
from scipy.sparse import *
|
||||
from scipy.sparse.linalg import norm
|
||||
from os.path import exists
|
||||
|
||||
|
||||
warnings.simplefilter(action='ignore', category=FutureWarning)
|
||||
# some stupid pandas function that doesn't work
|
||||
|
||||
class utilities:
|
||||
# Importing the dataset
|
||||
def load_data():
|
||||
# Loading the dataset
|
||||
dataset = int(input("Choose the dataset:\n [1] web-Stanford\n [2] web-BerkStan: \nEnter an option: "))
|
||||
|
||||
if dataset == 1:
|
||||
if exists('../data/web-Stanford.txt'):
|
||||
dataset = '../data/web-Stanford.txt'
|
||||
else:
|
||||
print("\nThe file doesn't exist, download it from https://snap.stanford.edu/data/web-Stanford.html")
|
||||
|
||||
# if there is no folder data, create it
|
||||
if not exists('../data'):
|
||||
os.makedirs('../data')
|
||||
|
||||
# Downloading the dataset
|
||||
url = 'https://snap.stanford.edu/data/web-Stanford.txt.gz'
|
||||
wget.download(url, '../data/web-Stanford.txt.gz')
|
||||
# Unzipping the dataset
|
||||
with gzip.open('../data/web-Stanford.txt.gz', 'rb') as f_in:
|
||||
with open('../data/web-Stanford.txt', 'wb') as f_out:
|
||||
f_out.write(f_in.read())
|
||||
|
||||
dataset = '../data/web-Stanford.txt'
|
||||
print("\nDataset downloaded\n")
|
||||
|
||||
elif dataset == 2:
|
||||
if exists('../data/web-BerkStan.txt'):
|
||||
dataset = '../data/web-BerkStan.txt'
|
||||
else:
|
||||
print("\nThe file doesn't exist, download it from https://snap.stanford.edu/data/web-BerkStan.html")
|
||||
|
||||
# if there is no folder data, create it
|
||||
if not exists('../data'):
|
||||
os.makedirs('../data')
|
||||
|
||||
# Downloading the dataset
|
||||
url = 'https://snap.stanford.edu/data/web-BerkStan.txt.gz'
|
||||
wget.download(url, '../data/web-BerkStan.txt.gz')
|
||||
# Unzipping the dataset
|
||||
with gzip.open('../data/web-BerkStan.txt.gz', 'rb') as f_in:
|
||||
with open('../data/web-BerkStan.txt', 'wb') as f_out:
|
||||
f_out.write(f_in.read())
|
||||
|
||||
dataset = '../data/web-BerkStan.txt'
|
||||
print("\nDataset downloaded\n")
|
||||
|
||||
return dataset
|
||||
|
||||
# Creating the graph from the dataset
|
||||
def create_graph(dataset):
|
||||
print("\nCreating the graph...")
|
||||
G = nx.read_edgelist(dataset, create_using=nx.DiGraph(), nodetype=int)
|
||||
n = G.number_of_nodes()
|
||||
print("Graph created based on the dataset\n")
|
||||
return G, n
|
||||
|
||||
# # Creating the transition probability matrix
|
||||
# The matrix is filled with zeros and the (i,j) element is x if the node i is connected to the node j. Where x is 1/(number of nodes connected to i).
|
||||
def create_matrix(G):
|
||||
print("Creating the transition probability matrix...")
|
||||
P = sp.sparse.lil_matrix((n,n))
|
||||
for i in G.nodes():
|
||||
for j in G[i]: #G[i] is the list of nodes connected to i, it's neighbors
|
||||
P[i-1,j-1] = 1/len(G[i])
|
||||
print("Transition probability matrix created\n")
|
||||
return P
|
||||
|
||||
# The vector is filled with d(i) = 1 if the i row of the matrix P is filled with zeros, other wise is 0
|
||||
def dangling_nodes(P,n):
|
||||
print("Creating the list of dangling nodes...")
|
||||
d = sp.sparse.lil_matrix((n,1))
|
||||
for i in range(n):
|
||||
if P[i].sum() == 0:
|
||||
d[i] = 1
|
||||
print("List of dangling nodes created\n")
|
||||
return d
|
||||
|
||||
def probability_vector(n):
|
||||
print("Creating the probability vector...")
|
||||
v = sp.sparse.lil_matrix((n,1))
|
||||
for i in range(n):
|
||||
v[i] = 1/n
|
||||
print("Probability vector created\n")
|
||||
return v
|
||||
|
||||
def transition_matrix(P, v, d):
|
||||
print("Creating the transition matrix...")
|
||||
Pt = P + v.dot(d.T)
|
||||
print("Transition matrix created\n")
|
||||
return Pt
|
||||
|
||||
def alpha():
|
||||
a = []
|
||||
for i in range(85,100):
|
||||
a.append(i/100)
|
||||
return a
|
||||
|
||||
class Plotting:
|
||||
def tau_over_iterations(dataframe):
|
||||
dataframe = df
|
||||
x = df['tau'][::-1].tolist()
|
||||
y = df['iterations'].tolist()
|
||||
|
||||
fig = go.Figure(data=go.Scatter(x=x, y=y, mode='lines+markers'),
|
||||
layout=go.Layout(title='Iterations needed for the convergence', xaxis_title='tau', yaxis_title='iterations'))
|
||||
|
||||
# save the figure as a html file
|
||||
fig.write_html("../data/results/algo1/taus_over_iterations.html")
|
||||
print("The plot has been saved in the folder data/results/algo1")
|
||||
|
||||
def tau_over_time(df):
|
||||
x1 = df['tau'][::-1].tolist()
|
||||
y1 = df['time'].tolist()
|
||||
|
||||
fig = go.Figure(data=go.Scatter(x=x1, y=y1, mode='lines+markers'),
|
||||
layout=go.Layout(title='Time needed for the convergence', xaxis_title='tau', yaxis_title='time (seconds)'))
|
||||
|
||||
# save the plot in a html file
|
||||
fig.write_html("../data/results/algo1/taus_over_time.html")
|
||||
print("The plot has been saved in the folder data/results/algo1")
|
||||
|
||||
class Algorithms:
|
||||
def algo1(Pt, v, tau, max_mv, a: list):
|
||||
start_time = time.time()
|
||||
|
||||
print("STARTING ALGORITHM 1...")
|
||||
u = Pt.dot(v) - v
|
||||
mv = 1 # number of iteration
|
||||
r = sp.sparse.lil_matrix((n,1))
|
||||
Res = sp.sparse.lil_matrix((len(a),1))
|
||||
x = sp.sparse.lil_matrix((n,1))
|
||||
|
||||
for i in range(len(a)):
|
||||
r = a[i]*(u)
|
||||
normed_r = norm(r)
|
||||
Res[i] = normed_r
|
||||
|
||||
if Res[i] > tau:
|
||||
x = r + v
|
||||
|
||||
while max(Res) > tau and mv < max_mv:
|
||||
u = Pt*u # should it be the same u of the beginning?
|
||||
mv += 1
|
||||
|
||||
for i in range(len(a)):
|
||||
if Res[i] >= tau:
|
||||
r = (a[i]**(mv+1))*(u)
|
||||
Res[i] = norm(r)
|
||||
|
||||
if Res[i] > tau:
|
||||
x = r + x
|
||||
|
||||
if mv == max_mv:
|
||||
print("The algorithm didn't converge in ", max_mv, " iterations")
|
||||
else:
|
||||
print("The algorithm converged in ", mv, " iterations")
|
||||
|
||||
total_time = time.time() - start_time
|
||||
total_time = round(total_time, 2)
|
||||
|
||||
print("The algorithm took ", total_time, " seconds to run\n")
|
||||
|
||||
return mv, x, r, total_time
|
||||
|
||||
# pandas dataframe to store the results
|
||||
df = pd.DataFrame(columns=['alpha', 'iterations', 'tau', 'time'])
|
||||
|
||||
# Main
|
||||
if __name__ == "__main__":
|
||||
dataset = utilities.load_data()
|
||||
# maximum number of iterations, asked to the user
|
||||
max_mv = int(input("Insert the maximum number of iterations: "))
|
||||
|
||||
G, n = utilities.create_graph(dataset)
|
||||
P = utilities.create_matrix(G)
|
||||
d = utilities.dangling_nodes(P,n)
|
||||
v = utilities.probability_vector(n)
|
||||
Pt = utilities.transition_matrix(P, v, d)
|
||||
a = utilities.alpha()
|
||||
|
||||
# run the algorithm for different values of tau from 10^-5 to 10^-9 with step 10^-1
|
||||
for i in range(5,10):
|
||||
tau = 10**(-i)
|
||||
print("\ntau = ", tau)
|
||||
mv, x, r, total_time = Algorithms.algo1(Pt, v, tau, max_mv, a)
|
||||
|
||||
# store the results in the dataframe
|
||||
df = df.append({'alpha': a, 'iterations': mv, 'tau': tau, 'time': total_time}, ignore_index=True)
|
||||
|
||||
# save the results in a csv file
|
||||
df.to_csv('../data/results/algo1/different_tau.csv', index=False)
|
||||
|
||||
# plot the results
|
||||
Plotting.tau_over_iterations(df)
|
||||
Plotting.tau_over_time(df)
|
||||
|
||||
# print in the terminal the columns of the dataframe iterations, tau and time
|
||||
print("\n", df[['iterations', 'tau', 'time']])
|
||||
Loading…
Reference in New Issue