Predicting NBA lineup point differentials with ML

import time
from functools import reduce

import pandas as pd
from bs4 import BeautifulSoup
import selenium
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys


def select_season(browser, season):
    "Navigate browser to the given season."
    url = f'https://www.basketball-reference.com/leagues/NBA_{season}_per_poss.html'
    browser.get(url)

def select_stat_table(browser, stat_table):
    "Navigate browser to the given statistic table. Current season must be 1995-96 onward."
    stat_table_dict = {'totals': 1,
                      'per_game': 2,
                      'per_minute': 3,
                      'per_poss': 4,
                      'advanced': 5,
                      'pbp': 6,
                      'shooting': 7,
                      'adj_shooting': 8}
    n = stat_table_dict[stat_table]
    button_xpath = f'/html/body/div[2]/div[5]/div[2]/div[{n}]/a'
    browser.find_element_by_xpath(button_xpath).click()

def read_stat_table(soup):
    "Parse the html table on the bs4 object's current page and return stats DataFrame."
    table = soup.find(class_="table_outer_container")
    tbody = table.find('tbody')
    tbody_tr = tbody.find_all('tr')
    stat_cols, all_stats, all_ids = [], [], []

    for row in tbody_tr:
        # skip partial rows i.e. player entries with multiple teams
        if row['class'] != ["full_table"]: continue
        cols = row.find_all('td')
        if stat_cols == []:  # extract stat names
            stat_cols = [col.get('data-stat') for col in cols]
        player_stats = [col.text for col in cols]  # extract player stat values
        all_stats.append(player_stats)
        player_id = cols[0].get('data-append-csv')  # extract unique player id
        all_ids.append(player_id)

    df = pd.DataFrame(all_stats, columns=stat_cols).set_index('player')
    df['player_id'] = all_ids

    return df

def process_scraped_table(df, season):
    "Drop redundant columns, add year column, update index. Return DataFrame."
    nunique = df.apply(pd.Series.nunique)
    drop_cols = nunique[nunique == 1].index
    df.drop(drop_cols, axis=1, inplace=True)
    df['year'] = [season] * len(df)
    df.index += f' {str(season)}'
    return df

def combine_season_stats(dfs):
    "Merge the stat tables from the same season on player ids."
    dfs = [df.reset_index().set_index('player_id') for df in dfs]
    season_df = reduce(lambda left, right: pd.merge(left, right[right.columns.difference(left.columns)], left_index=True, right_index=True), dfs)
    return season_df.reset_index().set_index('player')

def scrape(seasons, stat_types, data_dir):
    """
    Scrape player season stats from basketball reference for the given stat types
    across the given seasons and write out csv to given directory. Return DataFrame.
    """
    options = webdriver.firefox.options.Options()
    options.headless = True
    browser = webdriver.Firefox(executable_path="scraping/drivers/geckodriver", options=options)
    dfs = []

    for season in seasons:
        print('scraping:', season)
        season_dfs = []
        select_season(browser, season); time.sleep(3)

        for stat_type in stat_types:
            select_stat_table(browser, stat_type); time.sleep(3)
            page_source = browser.page_source
            soup = BeautifulSoup(page_source, 'html.parser')
            df = read_stat_table(soup)
            df = process_scraped_table(df, season)
            filename = data_dir + str(season) + '_' + str(stat_type).replace(' ', '_') + '.csv'
            df.to_csv(filename)
            season_dfs.append(df)

        combo_season_df = combine_season_stats(season_dfs)
        filename = data_dir + str(season) + '_' + '_'.join(stat_types) + '.csv'
        combo_season_df.to_csv(filename)
        dfs.append(combo_season_df)

    if len(seasons) < 2:
        return dfs[0]

    master_df = pd.concat(dfs)
    filename = data_dir + str(seasons[0]) + '_to_' + str(seasons[-1]) + '_' + '_'.join(stat_types) + '.csv'
    master_df.to_csv(filename)

    return master_df

from bball_ref_scraping_funcs import scrape

data_dir = 'nba_data/'
seasons = [year for year in range(1997, 2021)]
stat_types = ['advanced', 'shooting', 'pbp', 'per_poss']

if __name__ == '__main__':
    scrape(seasons, stat_types, data_dir)

def select_team_and_year_lineups(browser, team, year):
    "Navigate browser to given franchise page and year."
    url = f'https://www.basketball-reference.com/teams/{team}/{year}/lineups/'
    browser.get(url)

def read_lineup_table(soup, n_man_lineup):
    """
    Parse the html lineup stat table for the lineup type given by n_man_lineup
    (i.e. 4-player or 5-player lineups) on the current bs4 object page.
    Return DataFrame.
    """
    all_stats, all_lineup_ids, stat_cols = [], [], []
    table = soup.find('table', {'id': f'lineups_{n_man_lineup}-man_'})
    tbody = table.find('tbody')

    for row in tbody.find_all('tr')[0:-1]:
        cols = row.find_all('td')
        stat_cols = [col.get('data-stat') for col in cols]

        lineup_ids = [cols[0].get('csk')]
        all_lineup_ids.append(lineup_ids)

        lineup_stats = [col.text for col in cols]
        all_stats.append(lineup_stats)

    df = pd.DataFrame(all_stats, columns=stat_cols).set_index('lineup')
    df['player_ids'] = all_lineup_ids
    df['number_players'] = [n_man_lineup] * len(df)

    return df

def process_lineup_df(df, team, year):
    """
    Process the extracted lineup stat table:
      - Convert minutes played to numeric datatype
      - Fill blanks with NaNs
      - Convert differential stats to floats
      - Add year and team columns
      - Update index to reflect players, team, and year

    Return DataFrame.
    """
    df.loc[:, 'mp'] = df['mp'].apply(lambda x: float(x.split(':')[0]) + float(x.split(':')[1])/60)
    df.loc[:, 'player_ids'] = df['player_ids'].apply(lambda x: x[0].replace(':', ', '))
    df.replace('', np.nan)
    conv_to_float_cols = df.columns[df.columns.str.contains('diff')]
    df.loc[:, conv_to_float_cols] = df[conv_to_float_cols].apply(lambda series: pd.to_numeric(series, errors='ignore'))
    df['year'] = [year] * len(df)
    df['team'] = [team] * len(df)
    df.index = df.index + f' {team} {year}'

    return df

def scrape_lineup_data(teams, seasons, n_man_lineup, data_dir):
    """
    Scrape lineup data from basketball reference for given teams and seasons
    for the given lineup type (i.e. 4-player or 5-player lineups). Write out team
    lineup DataFrames and combined DataFrame to the given data directory.
    Return combined DataFrame.
    """
    options = webdriver.firefox.options.Options()
    options.headless = True
    browser = webdriver.Firefox(executable_path="scraping/drivers/geckodriver", options=options)
    start = datetime.now()
    all_dfs = []
    for team in teams:
        team_dfs = []
        print('scraping:', team, '   |   ', datetime.now()-start)
        for season in seasons:
            # handle team name switches etc.
            if team == 'NJN' and season > 2012: team = 'BRK'; print('Season:', season, '-->', team)
            if team == 'NOH' and season < 2003: continue # not a team prior to 2002-03
            if team == 'NOH' and season == 2006: team = 'NOK'; print('Season:', season, '-->', team)
            if team == 'NOK' and season == 2008: team = 'NOH'; print('Season:', season, '-->', team)
            if team == 'NOH' and season == 2014: team = 'NOP'; print('Season:', season, '-->', team)
            if team == 'SEA' and season > 2008: team = 'OKC'; print('Season:', season, '-->', team)
            if team == 'WSB' and season > 1997: team = 'WAS'; print('Season:', season, '-->', team)
            if team == 'CHH' and season > 2002: team = 'CHA'; print('Season:', season, '-->', team)
            if team == 'CHA' and season in [2002, 2003, 2004]: continue  # no charlotte team 2002-2004
            if team == 'CHA' and season > 2014: team = 'CHO'; print('Season:', season, '-->', team)
            if team == 'VAN' and season > 2001 : team = 'MEM'; print('Season:', season, '-->', team)

            select_team_and_year_lineups(browser, team, season)
            page_source = browser.page_source
            soup = BeautifulSoup(page_source, 'html.parser')
            df = read_lineup_table(soup, n_man_lineup)
            df = process_lineup_df(df, team, season)
            team_dfs.append(df)

        combined_team_df = pd.concat(team_dfs)
        filename = data_dir + f'{team}/{team}_{n_man_lineup}_man_lineups_{seasons[0]}_{seasons[-1]}.csv'
        os.makedirs(os.path.dirname(filename), exist_ok=True)
        combined_team_df.to_csv(filename)
        all_dfs.append(combined_team_df)

    df = pd.concat(all_dfs)
    filename = f'bballref_data/raw_all_teams_{n_man_lineup}_man_lineups_{seasons[0]}_{seasons[-1]}.csv'
    df.to_csv(filename)
    return df

from bballref_lineup_scraping_funcs import scrape_lineup_data

data_dir = 'bballref_data/lineups/'
n_man_lineup = 5
seasons = [year for year in range(1997, 2021)]
teams = [
        'ATL',
        'NJN',  # switches to BKN
        'BOS',
        'CHH',  # switches to CHA then CHO
        'CHI',
        'CLE',
        'DAL',
        'DEN',
        'DET',
        'GSW',
        'HOU',
        'IND',
        'LAC',
        'LAL',
        'VAN',  # switches to MEM
        'MIA',
        'MIL',
        'MIN',
        'NOH',  # switches to NOP
        'NYK',
        'SEA',  # switches to OKC
        'ORL',
        'PHI',
        'PHO',
        'POR',
        'SAC',
        'SAS',
        'TOR',
        'UTA',
        'WSB', # switches to WAS
        ]

if __name__ == '__main__':
    scrape_lineup_data(teams, seasons, n_man_lineup, data_dir)

def verify_players(lineup, df_players):
    """
    Return a boolean indicating whether all players in given lineup are
    present in the provided player dataset.
    """
    player_ids = lineup['player_ids'].split(', ')
    year = lineup['year']

    for pid in player_ids:
        if df_players[(df_players['year']==year) &
            (df_players['player_id']==pid)].empty:
            return False
    return True

def get_lineup_cluster_profile(lineup, df_players):
    """
    Build the plain lineup cluster profile and bpm-weighted lineup cluster
    profiles by summing the individual player cluster labels across all players
    in the given lineup.

    Return a (1x24) np array of 12 bpm-weighted cluster labels and 12 plain
    cluster labels.
    """
    player_ids = lineup['player_ids'].split(', ')
    year = lineup['year']
    cluster_bpm_cols = [f'cluster_{n}_bpm' for n in range(12)]
    cluster_nonbpm_cols = [f'cluster_{n}' for n in range(12)]

    lineup_bpm_clusters = np.zeros(12)
    lineup_nonbpm_clusters = np.zeros(12)

    for pid in player_ids:
        player = df_players[(df_players['year'] == year) & (df_players['player_id'] == pid)]
        lineup_bpm_clusters += player[cluster_bpm_cols].to_numpy().reshape(-1)
        lineup_nonbpm_clusters += player[cluster_nonbpm_cols].to_numpy().reshape(-1)

    return np.append(lineup_bpm_clusters, lineup_nonbpm_clusters)

## Building 4man lineups (same process for 5-man lineups)
df_players = pd.read_csv('datasets/master_player_stats_bio_bpm_clusters.csv', index_col=0)
df_all_4 = pd.read_csv('datasets/lineups/4man_lineups_300min_1997_2020.csv', index_col=0)
df_all_4 = df_all_4[df_all_4['year'] >=2000]

## Drop invalid lineups with players who are not in the player dataset
indices = df_all_4.apply(lambda lineup: verify_players(lineup, df_players), axis=1)
df4 = df_all_4.loc[indices[indices==True].index, :]

## Construct pd.Series with lineup cluster profiles as values and same index as df4
lineup_cluster_profile_series = df4.apply(lambda lineup: get_lineup_cluster_profile(lineup, df_players), axis=1)
lineup_cluster_profiles = lineup_cluster_profile_series.values
lineup_cluster_profiles = [[val for val in lineup_cluster_profiles[row]] for
                            row in range(len(lineup_cluster_profiles))]

## Add the cluster profiles to the lineup dataset, write out
cluster_bpm_cols = [f'cluster_{n}_bpm' for n in range(12)]
cluster_nonbpm_cols = [f'cluster_{n}' for n in range(12)]
cols = cluster_bpm_cols + cluster_nonbpm_cols
df4[cols] = lineup_cluster_profiles
df4.to_csv('datasets/master_4man_lineup_clusters_2000_2020.csv')

def get_lineup_salary(lineup, df_players):
    player_ids = lineup['player_ids'].split(', ')
    year = lineup['year']
    total_salary = 0
    for pid in player_ids:
        player = df_players[(df_players['year']==year) & (df_players['player_id']==pid)]
        total_salary += player['salary'].item()
    return total_salary

def get_salary_zscore(lineup, yearly_salary_mu_sigma):
    year = lineup['year']
    mu, sigma = yearly_salary_mu_sigma.loc[year].values
    return (lineup['total_salary'] - mu) / sigma

# df5 is the 5-player lineup dataset
yearly_mu_sigma = df5.groupby('year')['total_salary'].agg(['mean', 'std'])
df5['total_salary'] = df5.apply(lambda lineup: get_lineup_salary(lineup, df_players), axis=1)
df5['salary_zscore'] = df5.apply(lambda lineup: get_salary_zscore(lineup, yearly_mu_sigma), axis=1)

from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn


def build_MLP(sizes, activation, output_activation=nn.Identity):
    layers = []
    for i in range(len(sizes)-1):
        active_fn = activation if i < len(sizes) - 2 else output_activation
        layers += [nn.Linear(in_features=sizes[i], out_features=sizes[i+1]), active_fn()]
    mlp = nn.Sequential(*layers)
    return mlp

def torch_train_test_split(X, y, test_size, random_state=3):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size, random_state)
    datasets = []
    for dataset in [X_train, X_test, y_train, y_test]:
        datasets.append(torch.tensor(dataset.to_numpy(), dtype=torch.float32))
    return datasets

### Load data
df5 = pd.read_csv('../datasets/master_5man_lineup_clustes_2000_2020.csv', index_col=0)
cluster_bpm_cols = [f'cluster_{n}_bpm' for n in range(12)]
cluster_nonbpm_cols = [f'cluster_{n}' for n in range(12)]

X5 = df5[cluster_cols]
y5 = df5['diff_pts']
X_bpm5 = df5[bpm_cluster_cols]
y_bpm5 = df5['diff_pts']

### Setup network, learning rate, inputs, labels,  etc.
input_dim = [12]
hidden_sizes = [24, 24]
output_dim = [1]
sizes = input_dim + hidden_sizes + output_dim
activation = nn.ReLU
mlp = build_MLP(sizes, activation)
mlp_learning_rate = 0.0001
loss_fn = nn.MSELoss()
optimizer = torch.optim.Adam(params=mlp.parameters(), lr=mlp_learning_rate)
epochs = 5000
test_interval = 50
X_train_tensor, X_test_tensor, y_train_tensor, y_test_tensor = torch_train_test_split(X5, y5, 0.2, seed)
training_losses, test_losses, test_epochs = [], [], []

### Training loop
for epoch in range(epochs):
    optimizer.zero_grad()
    y_hat = mlp(X_train_tensor)
    train_loss = loss_fn(y_hat.reshape(-1), y_train_tensor)
    training_losses.append(train_loss.item())
    train_loss.backward()
    optimizer.step()

    ### Evaluate on test set
    if epoch % test_interval == 0:
        test_epochs.append(epoch)
        with torch.no_grad():
            y_hat_test = mlp(X_test_tensor)
            test_loss = loss_fn(y_hat_test.reshape(-1), y_test_tensor)
            test_losses.append(test_loss.item())

### Plot training results:
fig, ax = plt.subplots()
train_plot = sns.lineplot(x=[i for i in range(epochs)], y=training_losses, label='Training Loss', ax=ax)
test_plot = sns.lineplot(x=test_epochs, y=test_losses, label='Test Loss', ax=ax)
ax.set_title('Loss by epoch: 5-player, non-bpm, 2 hidden layers', fontsize=15)
ax.set_ylabel('MSE Loss')
ax.set_xlabel('Epoch')
plt.tight_layout()
plt.legend();