Consider only actors with many films

filtro.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
 import requests
 import pandas as pd
+import numpy as np
 import os
 import csv
 
+MIN_MOVIES = 42  # Only keep relations for actors that have made more than this many movies
+
 def download_url(url):
   print("Downloading:", url)
   file_name_start_pos = url.rfind("/") + 1
@@ -35,8 +38,6 @@ df_attori = pd.read_csv(
   dtype={'primaryName': 'U', 'primaryProfession': 'U'},
   converters={'nconst': lambda x: int(x.lstrip("nm0"))})
 df_attori.query('primaryProfession.str.contains("actor") or primaryProfession.str.contains("actress")', inplace=True)
-df_attori.to_csv('data/Attori.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['nconst', 'primaryName'], header=False, index=False)
-del df_attori  # Free memory
 
 print("Filtering films...")
 df_film = pd.read_csv(
@@ -46,9 +47,7 @@ df_film = pd.read_csv(
   converters={'tconst': lambda x: int(x.lstrip("t0")), 'isAdult': lambda x: x != "0"})
 df_film.query('not isAdult and titleType in ["movie", "tvSeries", "tvMovie", "tvMiniSeries"]',
               inplace=True)
-df_film.to_csv('data/FilmFiltrati.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['tconst', 'primaryTitle'], header=False, index=False)
 filtered_tconsts = df_film["tconst"].to_list()
-del df_film  # Free memory
 
 print("Filtering relations...")
 df_relazioni = pd.read_csv(
@@ -57,4 +56,22 @@ df_relazioni = pd.read_csv(
   dtype={'category': 'U'},
   converters={'nconst': lambda x: int(x.lstrip("nm0")), 'tconst': lambda x: int(x.lstrip("t0"))})
 df_relazioni.query('(category == "actor" or category == "actress") and tconst in @filtered_tconsts', inplace=True)
+# Returns an array of unique actor ids (nconsts) and an array of how many times they appear (counts) => the number of movies they appear in
+nconsts, counts = np.unique(df_relazioni["nconst"].to_numpy(), return_counts=True)
+filtered_nconsts = nconsts[counts>=MIN_MOVIES]
+df_relazioni.query("nconst in @filtered_nconsts", inplace=True)
+
+# Now select only films and actors that have at lest a relation
+print("Re-filtering actors...")
+nconsts_with_relations = df_relazioni["nconst"].unique()
+df_attori.query("nconst in @nconsts_with_relations", inplace=True)
+print("Re-filtering films...")
+tconsts_with_relations = df_relazioni["tconst"].unique()
+df_film.query("tconst in @tconsts_with_relations", inplace=True)
+
+# Write the filtered files
+df_attori.to_csv('data/Attori.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['nconst', 'primaryName'], header=False, index=False)
+df_film.to_csv('data/FilmFiltrati.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['tconst', 'primaryTitle'], header=False, index=False)
 df_relazioni.to_csv('data/Relazioni.txt', sep='\t', quoting=csv.QUOTE_NONE, escapechar='\\', columns=['tconst', 'nconst'], header=False, index=False)
+
+# Takes about 1 min 30 s

kenobi.cpp

@@ -1,4 +1,4 @@
-// g++ -Wall -pedantic -std=c++17 kenobi.cpp -o kenobi
+// g++ -Wall -pedantic -std=c++17 -Ofast kenobi.cpp -o kenobi
 #include <iostream>
 #include <iomanip>
 #include <vector>