Consider only actors with many films

main
Luca Lombardo 3 years ago
parent e67fa9dac6
commit 599f49d8a4

@ -1,9 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import requests import requests
import pandas as pd import pandas as pd
import numpy as np
import os import os
import csv import csv
MIN_MOVIES = 42 # Only keep relations for actors that have made more than this many movies
def download_url(url): def download_url(url):
print("Downloading:", url) print("Downloading:", url)
file_name_start_pos = url.rfind("/") + 1 file_name_start_pos = url.rfind("/") + 1
@ -35,8 +38,6 @@ df_attori = pd.read_csv(
dtype={'primaryName': 'U', 'primaryProfession': 'U'}, dtype={'primaryName': 'U', 'primaryProfession': 'U'},
converters={'nconst': lambda x: int(x.lstrip("nm0"))}) converters={'nconst': lambda x: int(x.lstrip("nm0"))})
df_attori.query('primaryProfession.str.contains("actor") or primaryProfession.str.contains("actress")', inplace=True) df_attori.query('primaryProfession.str.contains("actor") or primaryProfession.str.contains("actress")', inplace=True)
df_attori.to_csv('data/Attori.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['nconst', 'primaryName'], header=False, index=False)
del df_attori # Free memory
print("Filtering films...") print("Filtering films...")
df_film = pd.read_csv( df_film = pd.read_csv(
@ -46,9 +47,7 @@ df_film = pd.read_csv(
converters={'tconst': lambda x: int(x.lstrip("t0")), 'isAdult': lambda x: x != "0"}) converters={'tconst': lambda x: int(x.lstrip("t0")), 'isAdult': lambda x: x != "0"})
df_film.query('not isAdult and titleType in ["movie", "tvSeries", "tvMovie", "tvMiniSeries"]', df_film.query('not isAdult and titleType in ["movie", "tvSeries", "tvMovie", "tvMiniSeries"]',
inplace=True) inplace=True)
df_film.to_csv('data/FilmFiltrati.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['tconst', 'primaryTitle'], header=False, index=False)
filtered_tconsts = df_film["tconst"].to_list() filtered_tconsts = df_film["tconst"].to_list()
del df_film # Free memory
print("Filtering relations...") print("Filtering relations...")
df_relazioni = pd.read_csv( df_relazioni = pd.read_csv(
@ -57,4 +56,22 @@ df_relazioni = pd.read_csv(
dtype={'category': 'U'}, dtype={'category': 'U'},
converters={'nconst': lambda x: int(x.lstrip("nm0")), 'tconst': lambda x: int(x.lstrip("t0"))}) converters={'nconst': lambda x: int(x.lstrip("nm0")), 'tconst': lambda x: int(x.lstrip("t0"))})
df_relazioni.query('(category == "actor" or category == "actress") and tconst in @filtered_tconsts', inplace=True) df_relazioni.query('(category == "actor" or category == "actress") and tconst in @filtered_tconsts', inplace=True)
# Returns an array of unique actor ids (nconsts) and an array of how many times they appear (counts) => the number of movies they appear in
nconsts, counts = np.unique(df_relazioni["nconst"].to_numpy(), return_counts=True)
filtered_nconsts = nconsts[counts>=MIN_MOVIES]
df_relazioni.query("nconst in @filtered_nconsts", inplace=True)
# Now select only films and actors that have at lest a relation
print("Re-filtering actors...")
nconsts_with_relations = df_relazioni["nconst"].unique()
df_attori.query("nconst in @nconsts_with_relations", inplace=True)
print("Re-filtering films...")
tconsts_with_relations = df_relazioni["tconst"].unique()
df_film.query("tconst in @tconsts_with_relations", inplace=True)
# Write the filtered files
df_attori.to_csv('data/Attori.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['nconst', 'primaryName'], header=False, index=False)
df_film.to_csv('data/FilmFiltrati.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['tconst', 'primaryTitle'], header=False, index=False)
df_relazioni.to_csv('data/Relazioni.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['tconst', 'nconst'], header=False, index=False) df_relazioni.to_csv('data/Relazioni.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['tconst', 'nconst'], header=False, index=False)
# Takes about 1 min 30 s

@ -1,4 +1,4 @@
// g++ -Wall -pedantic -std=c++17 kenobi.cpp -o kenobi // g++ -Wall -pedantic -std=c++17 -Ofast kenobi.cpp -o kenobi
#include <iostream> #include <iostream>
#include <iomanip> #include <iomanip>
#include <vector> #include <vector>

Loading…
Cancel
Save