Unlock SEO Insights with Data Science: Visualising Your Website’s Internal Link Graph

Subscriber Content

import pandas as pd
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import re
import networkx as nx
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from networkx.algorithms import community

# Step 1: Data Collection
def collect_data(base_url):
    to_visit = [base_url]
    visited = set()
    pages = []

    while to_visit:
        url = to_visit.pop(0)
        if url in visited:
            continue
        visited.add(url)

        response = requests.get(url)
        soup = BeautifulSoup(response.text, 'html.parser')

        # Exclude non-HTML pages
        if 'text/html' not in response.headers.get('Content-Type', ''):
            continue

        # Extract page title
        title = soup.title.string if soup.title else 'No Title'

        # Extract internal links excluding header/footer
        body = soup.find('body')
        links = set()
        for a_tag in body.find_all('a', href=True):
            href = a_tag['href']
            if not href.startswith('#') and not re.match(r'^https?://', href):
                full_url = urljoin(base_url, href)
                if urlparse(full_url).netloc == urlparse(base_url).netloc:
                    links.add(full_url)

        pages.append({'url': url, 'title': title, 'links': links})
        to_visit.extend(links - visited)

    return pages

base_url = 'https://www.python.org/'
data = collect_data(base_url)

# Step 2: Graph Creation
def create_graph(data):
    g = nx.Graph()  # Create an undirected graph

    # Add nodes and edges
    url_to_id = {}
    for page in data:
        if page['url'] not in url_to_id:
            url_to_id[page['url']] = len(url_to_id)
            g.add_node(page['url'], title=page['title'])

    for page in data:
        for link in page['links']:
            if link in url_to_id:
                g.add_edge(page['url'], link)

    return g

graph = create_graph(data)

# Step 3: Community Detection
def detect_related_topics(graph):
    return list(community.greedy_modularity_communities(graph))

related_topics = detect_related_topics(graph)

# Print community information
community_info = []
for i, cluster in enumerate(related_topics):
    for vertex in cluster:
        community_info.append({
            'Community': i,
            'Page Title': graph.nodes[vertex]['title'],
            'URL': vertex,
            'Degree': graph.degree(vertex)
        })

# Create a DataFrame for the community information
df_community_info = pd.DataFrame(community_info)
df_community_info.to_csv('community_info.csv', index=False)
print("Community information saved to community_info.csv")

# Step 5: Visualization
def visualize_graph(graph, related_topics):
    pos = nx.spring_layout(graph)
    fig, ax = plt.subplots(figsize=(20, 20))

    # Plot nodes
    colors = cm.rainbow([i / len(related_topics) for i in range(len(related_topics))])
    for i, cluster in enumerate(related_topics):
        color = colors[i]
        for node in cluster:
            x, y = pos[node]
            ax.plot(x, y, 'o', color=color, markersize=5)
            ax.text(x, y, graph.nodes[node]['title'], fontsize=8, color=color)

    # Plot edges
    for edge in graph.edges():
        x1, y1 = pos[edge[0]]
        x2, y2 = pos[edge[1]]
        ax.plot([x1, x2], [y1, y2], 'k-', alpha=0.5)

    plt.show()

visualize_graph(graph, related_topics)

Add content here that will only be visible to your subscribers.