!pip install --upgrade colorcet dash gensim datashader "holoviews[recommended]" jupyter_bokeh bokeh python-louvain -q

# Import libraries
import pandas as pd
import collections
import networkx as nx
import colorcet as cc
import plotly.graph_objects as go
from dash import Dash, dcc, html, Input, Output
import random
from IPython.display import IFrame
from holoviews.operation.datashader import (datashade, aggregate, dynspread, bundle_graph, split_dataframe,  regrid)
from holoviews.element.graphs import layout_nodes
from datashader.layout import forceatlas2_layout, random_layout, circular_layout
import holoviews as hv
import datashader as ds
import datashader.transfer_functions as tf
from datashader.bundling import connect_edges, hammer_bundle
hv.extension("bokeh")
import numpy as np
#from google.colab import drive
#drive.mount('/content/drive')
import time
from holoviews import opts
import math
from scipy.interpolate import splprep, splev
from itertools import combinations
import string
translator = str.maketrans('', '', string.punctuation)
import community as louvain_community
from bokeh.io import output_notebook
output_notebook()
from matplotlib.colors import ListedColormap
import panel as pn
pn.extension()
import colorsys
import matplotlib.colors as mcolors
from typing import List
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from PIL import Image as PILImage
import gc
from IPython.display import display, Image
import io

# Functions to work with datashader
cvsopts = dict(plot_height = 1000, plot_width = 1000)

def nodesplot(nodes, name = None, canvas = None, cat = None):

    if canvas is None:

        eps = 1e-6
        xr = (float(nodes.x.min()) - eps, float(nodes.x.max()) + eps)
        yr = (float(nodes.y.min()) - eps, float(nodes.y.max()) + eps)
        canvas = ds.Canvas(x_range = xr, y_range = yr, **cvsopts)

    # Strip cats before mapping
    if cat:

        nodes[cat] = nodes[cat].cat.remove_unused_categories()

    aggregator = None if cat is None else ds.count_cat(cat)
    agg = canvas.points(nodes,'x','y',aggregator)

    if cat:

        cats = list(nodes[cat].cat.categories)
        color_key = dict(zip(cats, make_hex_palette(len(cats))))

        return tf.dynspread(tf.shade(agg, color_key = color_key, name = name))

    return tf.dynspread(tf.shade(agg, cmap = ["#FF3333"], name = name))

def edgesplot(edges, name = None, canvas = None):

    if canvas is None:

        eps = 1e-6
        xr = (float(edges.x.min()) - eps, float(edges.x.max()) + eps)
        yr = (float(edges.y.min()) - eps, float(edges.y.max()) + eps)
        canvas = ds.Canvas(x_range = xr, y_range = yr, **cvsopts)

    return tf.shade(canvas.line(edges, 'x','y', agg = ds.count()), name = name)

def graphplot(nodes, edges, name = "", canvas = None, cat = None, pad = 0.025):

    if canvas is None:

        xmin, xmax = nodes.x.min(), nodes.x.max()
        ymin, ymax = nodes.y.min(), nodes.y.max()

        xpad = (xmax - xmin) * pad
        ypad = (ymax - ymin) * pad

        eps = 1e-6
        xr = (float(xmin - xpad) - eps, float(xmax + xpad) + eps)
        yr = (float(ymin - ypad) - eps, float(ymax + ypad) + eps)

        canvas = ds.Canvas(x_range = xr, y_range = yr, **cvsopts)

    nodeplot = nodesplot(nodes, name + " nodes", canvas, cat)
    edgeplot = edgesplot(edges, name + " edges", canvas)

    return tf.stack(edgeplot, nodeplot, how = "over", name = name)

# Function to create a colormap
def make_hex_palette(n):

    """High-contrast palette optimised for dark backgrounds."""
    result = []

    for i in range(n):

        h = i / n

        # lightness=0.65, saturation=0.95 — bright, vivid, dark-bg friendly
        r, g, b = colorsys.hls_to_rgb(h, 0.65, 0.95)
        result.append('#{:02x}{:02x}{:02x}'.format(int(r * 255), int(g * 255), int(b * 255)))

    return result

# Function to visualize colormap
def plot_colortable(hex_colors: List[str]):

    """Creates a colorbar using custom hex colors."""
    cmap = mcolors.ListedColormap(hex_colors)

    plt.figure(figsize = (8, 2), dpi = 150)
    plt.imshow([list(range(len(hex_colors)))], cmap = cmap, aspect = 'auto')
    plt.axis('off')
    plt.show()

def show_ds_images(images, titles, ncols = 2, fig_width = 18, cell_px = 1000, bg = 'black', title_color = 'white', title_size = 11):

    """
    Display a list of datashader images with styled titles and background.

    Parameters
    ----------
    images : list of datashader Image objects
    titles : list of str
    ncols  : number of columns in the grid
    fig_width : total figure width in inches
    cell_px : resolution to render each image (square)
    bg     : hex background colour applied to each image AND the figure
    """

    nrows = -(-len(images) // ncols)
    cell_in = fig_width / ncols
    fig, axes = plt.subplots(nrows, ncols, figsize = (fig_width, cell_in * nrows), facecolor = bg, dpi = 150)
    axes = list(np.atleast_1d(axes).flat)

    for ax, img, title in zip(axes, images, titles):

        # Apply background colour, then upscale with Lanczos for sharpness
        styled = tf.set_background(img, bg)
        pil_img = styled.to_pil()
        pil_img = pil_img.resize((cell_px, cell_px), PILImage.LANCZOS)
        ax.imshow(pil_img, interpolation = 'lanczos')
        ax.set_title(title, color = title_color, fontsize = title_size, fontweight = 'bold', pad = 12, loc = 'center')
        ax.set_facecolor(bg)
        ax.axis('off')

    # Hide any unused axes
    for ax in list(axes)[len(images):]:

        ax.set_visible(False)

    plt.subplots_adjust(wspace = 0.04, hspace = 0.12)
    buf = io.BytesIO()
    plt.savefig(buf, format = 'png', bbox_inches = 'tight', facecolor = bg)
    plt.close()
    buf.seek(0)
    display(Image(data = buf.read()))
    buf.close()

def normalise_layout(df, margin = 0.05):

    """Rescale x, y to [margin, 1-margin] regardless of FA2 coordinate explosion."""
    df['x'] = (df['x'] - df['x'].min()) / (df['x'].max() - df['x'].min())
    df['y'] = (df['y'] - df['y'].min()) / (df['y'].max() - df['y'].min())
    df['x'] = df['x'] * (1 - 2*margin) + margin
    df['y'] = df['y'] * (1 - 2*margin) + margin

    return df

# Holoviews
kwargs = dict(width = 1000, height = 1000, xaxis = None, yaxis = None)
opts.defaults(opts.Nodes(**kwargs), opts.Graph(**kwargs))
colors = ['#000000'] + hv.Cycle('Category20').values

np.random.seed(100)
n = 100
m = 20000

nodes = pd.DataFrame(["node"+str(i) for i in range(n)], columns = ['name'])
print(f"Nodes data:\n {nodes.tail()}\n")

edges = pd.DataFrame(np.random.randint(0, len(nodes), size = (m, 2)), columns = ['source', 'target'])
print(f'Edges data:\n {edges.tail()}')

# Create the layout
circular  = circular_layout(nodes, uniform = False)
randomloc = random_layout(nodes)
randomloc.tail()

forcedirected = forceatlas2_layout(nodes, edges)

show_ds_images(images = [nodesplot(randomloc, "Random layout"), nodesplot(circular, "Circular layout"), nodesplot(forcedirected, "ForceAtlas2 layout")],
               titles = ['Layout- Random', 'Layout- Circular', 'Layout- Force Directed'],
               ncols = 3)

cd = circular
fd = forcedirected

show_ds_images(images = [graphplot(cd, connect_edges(cd, edges), "Circular layout"),
                         graphplot(fd, connect_edges(fd, edges), "Force-directed"),
                         graphplot(cd, hammer_bundle(cd, edges), "Circular layout, bundled"),
                         graphplot(fd, hammer_bundle(fd, edges), "Force-directed, bundled")],
               titles = ['Circular Layout', 'Force-Directed', 'Circular Layout, Bundled', 'Force-Directed Layout, Bundled'], ncols = 2)

# Read the file using pandas
df = pd.read_csv('/content/drive/MyDrive/netflix_titles.csv')

# Split the cells to extract data
df['directors'] = df['director'].apply(lambda l: [] if pd.isna(l) else [i.strip() for i in l.split(",")])
df['categories'] = df['listed_in'].apply(lambda l: [] if pd.isna(l) else [i.strip() for i in l.split(",")])
df['actors'] = df['cast'].apply(lambda l: [] if pd.isna(l) else [i.strip() for i in l.split(",")])
df['countries'] = df['country'].apply(lambda l: [] if pd.isna(l) else [i.strip() for i in l.split(",")])

df.head()

df2 = df[df['directors'].map(len) > 0]
df2 = df2[df2['actors'].map(len) > 0][['title', 'directors', 'actors']]

"Matt Duffer, Ross Duffer"  →  ["Matt Duffer", "Ross Duffer"]

# Redesign the edges as a people network
people_edges = []

for idx, row in df2.iterrows():

    combos = [(item1, item2) for item1 in row['directors'] for item2 in row['actors'] if item1 is not None]
    pairs = list(combinations(row['actors'], 2))

    for combo in combos:

        people_edges.append(tuple(sorted(combo)))

    for pair in pairs:

        people_edges.append(tuple(sorted(pair)))

people_edges = pd.DataFrame(people_edges, columns = ['source', 'target'])

weighted_people_edges = people_edges.groupby(['source', 'target']).size().reset_index().rename({0: 'weight'}, axis = 1)
weighted_people_edges.sort_values('weight', ascending = False).head()

weighted_people_edges['source'] = weighted_people_edges['source'].astype(str).apply(lambda x: x.translate(translator))
weighted_people_edges['target'] = weighted_people_edges['target'].apply(lambda x: x.translate(translator))

people_nodes = pd.DataFrame(pd.unique(weighted_people_edges[['source', 'target']].values.ravel('K')), columns = ['name'])

{"Tom Hanks": 0, "Steven Spielberg": 1, "Cate Blanchett": 2, …}

# Convert the nodes to indices for mapping
node_to_idx = {node: i for i, node in enumerate(people_nodes['name'])}
weighted_people_edges["source"] = weighted_people_edges["source"].map(node_to_idx)
weighted_people_edges["target"] = weighted_people_edges["target"].map(node_to_idx)

weighted_people_edges.head()

# Filter the graph for meaningful connections
filtered_people_edges = (weighted_people_edges[weighted_people_edges.weight >= 2]).copy()

# Keep nodes that appear in filtered edges
remaining_nodes = pd.unique(filtered_people_edges[['source','target']].values.ravel())

filtered_people_nodes = people_nodes[people_nodes.index.isin(remaining_nodes)].copy()

print('Filtering graph based on edge weights..\n')
print(f"Initial Configuration: \nNodes- {people_nodes.shape[0]}\nEdges- {weighted_people_edges.shape[0]}\n")
print(f"Filtered Result: \nNodes- {filtered_people_nodes.shape[0]}\nEdges- {filtered_people_edges.shape[0]}\n")

# Get position information
circularloc  = circular_layout(filtered_people_nodes, uniform = False)
forcedirectedloc = forceatlas2_layout(filtered_people_nodes, filtered_people_edges)

# Compile layouts
cd = circularloc
fd = forcedirectedloc

show_ds_images(images = [graphplot(cd, connect_edges(cd, filtered_people_edges), "Filtered Netflix Social Network: Circular Layout"),
                         graphplot(fd, connect_edges(fd, filtered_people_edges), "Filtered Netflix Social Network: Force-Directed Layout"),
                         graphplot(cd, hammer_bundle(cd, filtered_people_edges), "Filtered Netflix Social Network: Circular Layout, Bundled"),
                         graphplot(fd, hammer_bundle(fd, filtered_people_edges), "Filtered Netflix Social Network: Force-Directed Layout, Bundled")],
               titles = ["Filtered Netflix Social Network: Circular Layout",
                         "Filtered Netflix Social Network: Force-Directed Layout",
                         "Filtered Netflix Social Network: Circular Layout, Bundled",
                         "Filtered Netflix Social Network: Force-Directed Layout, Bundled"],
               ncols = 2)

print('Analyzing the Netflix neighborhood structure..')

graph = nx.from_pandas_edgelist(filtered_people_edges, source = 'source', target = 'target', edge_attr = 'weight')

print(f"Netflix Graph-\nNodes: {graph.number_of_nodes()}, Edges: {graph.number_of_edges()}")

# Find communities
print('Finding communities..')
partition = louvain_community.best_partition(graph, weight = 'weight', random_state = 100)

print('Communities detected..\nUpdating params..')

# Map community id onto the nodes DataFrame (which holds x, y positions)
filtered_people_nodes['community'] = filtered_people_nodes.index.map(partition).fillna(-1).astype(int)
filtered_people_nodes['community'] = pd.Categorical(filtered_people_nodes['community'])

# Merge community onto the layout DataFrames
community_categories = sorted(filtered_people_nodes['community'].unique().astype(str))

# Assign as proper pd.Categorical — this is what datashader requires
forcedirectedloc['community'] = pd.Categorical(forcedirectedloc.index.map(filtered_people_nodes['community']).astype(str), categories = community_categories)
circularloc['community'] = pd.Categorical(circularloc.index.map(filtered_people_nodes['community']).astype(str), categories = community_categories)

print(f"Process complete..\nNumber of communities: {filtered_people_nodes['community'].nunique()}")
print(filtered_people_nodes['community'].value_counts().head(10))

# Update function to show communities
def nodesplot(nodes, name = None, canvas = None, cat = None):

    canvas = ds.Canvas(**cvsopts) if canvas is None else canvas
    aggregator = None if cat is None else ds.count_cat(cat)
    agg = canvas.points(nodes, 'x', 'y', aggregator)

    # Use a categorical colormap when a category column is provided
    if cat:

        # Remove_unused_categories ensures cats and colors always match
        cats = list(nodes[cat].cat.remove_unused_categories().cat.categories)
        color_key = dict(zip(cats, make_hex_palette(len(cats))))

        return tf.spread(tf.shade(agg, color_key = color_key), px = 3, name = name)

    else:

        return tf.spread(tf.shade(agg, cmap = ["#FF3333"]), px = 3, name = name)

cats = list(filtered_people_nodes['community'].cat.categories)
hex_colors = make_hex_palette(len(cats))
plot_colortable(hex_colors)

show_ds_images(images = [graphplot(forcedirectedloc, hammer_bundle(forcedirectedloc, filtered_people_edges), "Filtered Netflix: Communities, Force-Directed Bundled", cat = 'community')],
               titles = ['Filtered Netflix Social Network, Weighted and Bundled'],
               ncols = 1,
               fig_width = 10)

# Build the cmap again
cats = list(filtered_people_nodes['community'].cat.categories)
hex_colors = make_hex_palette(len(cats))
color_key = dict(zip(cats, hex_colors))
n_communities = filtered_people_nodes['community'].nunique()
plot_colortable(hex_colors)

# Filter the graph again to be more informative
degree = pd.Series(dict(graph.degree()), name = 'degree')
weighted_degree = pd.Series(dict(graph.degree(weight = 'weight')), name = 'weighted_degree')

# Combine into a single stats DataFrame
node_stats = pd.DataFrame({'degree': degree, 'weighted_degree': weighted_degree})
node_stats = node_stats.join(filtered_people_nodes[['name', 'community']])
node_stats = node_stats[['name', 'community', 'degree', 'weighted_degree']]

print(node_stats.sort_values('weighted_degree', ascending = False).head(20))

# Filter the graph for top communities and nodes
comm_sizes = node_stats['community'].value_counts()
top_communities = comm_sizes.head(25).index
node_stats_filtered = node_stats[node_stats['community'].isin(top_communities)]

# Within each community keep only the most connected individuals ────────────
top_nodes = (node_stats_filtered.groupby('community', group_keys = False).apply(lambda g: g.nlargest(50, 'weighted_degree')))
top_node_ids = set(top_nodes.index)
print(f"Keeping {len(top_node_ids)} nodes from 25 communities")

# Filter edge list for these nodes
filtered_edges = weighted_people_edges[weighted_people_edges['source'].isin(top_node_ids) & weighted_people_edges['target'].isin(top_node_ids)].copy()

# Flter by minimum edge weight to remove weak links
filtered_edges = filtered_edges[filtered_edges['weight'] >= 2]

print(f"Edges before filtering: {len(weighted_people_edges)}")
print(f"Edges after filtering: {len(filtered_edges)}")

filtered_edges.head()

remaining_nodes = pd.unique(filtered_edges[['source', 'target']].values.ravel())
print(f"# of Remaining Nodes- {len(remaining_nodes)}")

new_idx = {old: new for new, old in enumerate(remaining_nodes)}

filtered_edges['source'] = filtered_edges['source'].map(new_idx)
filtered_edges['target'] = filtered_edges['target'].map(new_idx)

# Rebuild nodes DataFrame with name + community
filtered_nodes = (top_nodes.loc[remaining_nodes].reset_index().rename(columns = {'index': 'old_idx'}))
filtered_nodes.index = filtered_nodes['old_idx'].map(new_idx)
filtered_nodes = filtered_nodes.sort_index()

# Assign categorical community
community_cats = sorted(filtered_nodes['community'].unique().astype(str))
filtered_nodes['community'] = pd.Categorical(filtered_nodes['community'].astype(str), categories = community_cats)

print(filtered_nodes.head())

# Recompute the layouts
layout_nodes_df = filtered_nodes[[]].copy()
layout_nodes_df['name'] = filtered_nodes.index.astype(str)

fd_filtered = forceatlas2_layout(layout_nodes_df, filtered_edges)

# Normalize the edges before drawing
xmin, xmax = fd_filtered.x.min(), fd_filtered.x.max()
ymin, ymax = fd_filtered.y.min(), fd_filtered.y.max()
margin = 0.05

fd_filtered['x'] = (fd_filtered.x - xmin) / (xmax - xmin) * (1 - 2 * margin) + margin
fd_filtered['y'] = (fd_filtered.y - ymin) / (ymax - ymin) * (1 - 2 * margin) + margin

# ← always strip here, immediately after assignment
fd_filtered_xy = fd_filtered[['x', 'y']]
fd_filtered['community'] = pd.Categorical(filtered_nodes['community'].values, categories = community_cats)
fd_filtered['community'] = fd_filtered['community'].cat.remove_unused_categories()
filtered_nodes['community'] = filtered_nodes['community'].cat.remove_unused_categories()

show_ds_images(images = [graphplot(fd_filtered, hammer_bundle(fd_filtered_xy, filtered_edges), "Netflix Universe: Top Individuals by Community", cat = 'community')], titles = ['Filtered Edges w/ Important Communities'], ncols = 1, fig_width = 10)

# Build a new nodes df for hv
node_data = pd.DataFrame({'x': fd_filtered['x'].values,
                          'y': fd_filtered['y'].values,
                          'index': np.arange(len(filtered_nodes)),
                          'name': filtered_nodes['name'].values,
                          'community': filtered_nodes['community'].astype(str).values,
                          'weighted_degree': filtered_nodes['weighted_degree'].values})

hv_nodes = hv.Nodes(node_data, kdims = ['x', 'y', 'index'], vdims = ['name', 'community', 'weighted_degree'])

# Sanity check — x/y must NOT be all zeros
print("Node position sample:")
print(node_data[['name', 'x', 'y', 'community', 'weighted_degree']].head())
print(f"\nX range: {node_data.x.min():.3f} → {node_data.x.max():.3f}")
print(f"Y range: {node_data.y.min():.3f} → {node_data.y.max():.3f}")

# Build the graph
hv_graph = hv.Graph((filtered_edges[['source', 'target', 'weight']], hv_nodes), kdims = ['source', 'target'], vdims = ['weight'])

print("Initial Graph:", hv_graph)
print("Nodes in graph:", len(hv_graph.nodes))

bundled = bundle_graph(hv_graph)
print("Edge bundling complete ✓")

display(bundled)

# Add interactivity with datashader and bokeh
n_cats = filtered_nodes['community'].nunique()
hex_colors = make_hex_palette(n_cats)
shaded_edges = datashade(bundled, normalization = 'linear', width = 1000, height = 1000)
interactive_nodes = bundled.nodes.opts(opts.Nodes(color = 'community',
                                                  cmap = hex_colors,
                                                  size = 8,
                                                  tools = ['hover'],
                                                  width = 1000,
                                                  height = 1000,
                                                  xaxis = None,
                                                  yaxis = None,
                                                  legend_position = 'right',
                                                  padding = 0.06))

interactive_plot = (shaded_edges * interactive_nodes)

# pn.panel is the reliable display path for datashaded overlays in Colab
pn.panel(interactive_plot)

x_range = node_data.x.min(), node_data.x.max()
y_range = node_data.y.min(), node_data.y.max()

# Compute label positions
centroids = (node_data.groupby('community').agg(x = ('x', 'mean'), y = ('y', 'mean'), size = ('name', 'count')).reset_index())
centroids['label'] = 'Community #' + centroids['community'].astype(str)

print(centroids.sort_values('size', ascending = False).head(10))

community_labels = hv.Labels(centroids[['x', 'y', 'label']], kdims = ['x', 'y'], vdims = ['label']).opts(xlim = x_range,
                                                                                                         ylim = y_range,
                                                                                                         text_font_size = '8pt',
                                                                                                         text_color = 'black',
                                                                                                         text_alpha = 0.85,
                                                                                                         xaxis = None,
                                                                                                         yaxis = None)

labelled_plot = (shaded_edges * interactive_nodes * community_labels)

pn.panel(labelled_plot)