Forecasting at Scale: Time-Series with Prophet

# Import libraries
import pandas as pd
import time
from multiprocessing import Pool, cpu_count
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import matplotlib.pyplot as plt
%matplotlib inline
import zipfile
import itertools
from prophet import Prophet
import os
import sys
import logging
import random 

# List contents
for dirname, _, filenames in os.walk('/kaggle/input'):
    
    for filename in filenames:
        
        print(os.path.join(dirname, filename))

# Load data
train = pd.read_csv('/kaggle/input/store-sales-time-series-forecasting/train.csv')
test = pd.read_csv('/kaggle/input/store-sales-time-series-forecasting/test.csv')
holidays = pd.read_csv('/kaggle/input/store-sales-time-series-forecasting/holidays_events.csv').rename({'date': 'ds', 'description': 'holiday'}, axis = 1)
oil = pd.read_csv('/kaggle/input/store-sales-time-series-forecasting/oil.csv').rename({'date': 'ds', 'dcoilwtico': 'oil'}, axis = 1).fillna(method = 'bfill')

# Inspect data
print(f"Unique Item Families:    {train['family'].nunique()}") 
print(f"# Stores:                {train['store_nbr'].nunique()}")
print(f"Dataset start date:      {min(train['date'])}")
print(f"Dataset end date:        {max(train['date'])}")
print(f"Test set start date:     {min(test['date'])}")
print(f"Test set end date:       {max(test['date'])}")

interval = pd.date_range(min(train['date']), max(train['date']), freq = 'd')

print(f"Num Days:               {len(interval)}")
print(f"Train Shape:            {train['date'].nunique()}")

oil.head()

# Fix dates
train['ds'] = pd.to_datetime(train['date'])
test['ds'] = pd.to_datetime(test['date'])
oil['ds'] = pd.to_datetime(oil['ds'])

# Create a master label for training 
train['item'] = train['store_nbr'].astype(str) + '_' + train['family']
test['item'] = test['store_nbr'].astype(str) + '_' + test['family']

# Merge data
test1 = test.merge(oil, how = 'left', on = 'ds').drop('date', axis = 1)
train1 = train.merge(oil, how = 'left', on = 'ds').drop('date', axis = 1)

test.shape

test1.shape

test1.head()

train1.head()

# Create an index for each product
df = train1.pivot(index = 'ds', columns = 'item', values = 'sales').reset_index() #.rename({'date': 'ds'}, axis = 1)
promos = train1.pivot(index = 'ds', columns = 'item', values = 'onpromotion').reset_index() #.rename({'date': 'ds'}, axis = 1)
test_promos = test1.pivot(index = 'ds', columns = 'item', values = 'onpromotion').reset_index() #.rename({'date': 'ds'}, axis = 1)

# Instantiate processing power for parallelism
num_cpus = cpu_count()

# Get an index of all products
items = df.drop('ds', axis = 1).columns

#!kaggle competitions submit -c store-sales-time-series-forecasting -f submission.csv -m "prophet_forecasting_at_scale"

test.isnull().sum()

model = Prophet(holidays = holidays[['ds', 'holiday']])
model.add_regressor('onpromotion') #, standardize = False)
model.add_regressor('oil' )#, standardize = False)

# Compile data
trainingdata = df[['ds', items[0]]].rename({items[0]: 'y'}, axis = 1)
trainingdata['y'] = trainingdata['y'].astype(float)

# cONVERT DATES
trainingdata['ds'] = pd.to_datetime(trainingdata['ds'])
promos['ds'] = pd.to_datetime(promos['ds'])

# Add regressors
trainingdata = trainingdata.merge(promos[['ds', items[0]]]).rename({items[0]: 'onpromotion'}, axis = 1).merge(oil, how = 'left', on = 'ds')

trainingdata['oil'] = trainingdata['oil'].fillna(method = 'bfill')

trainingdata.head()

model.fit(trainingdata)

fut = model.make_future_dataframe(periods = 16)
merge_content = test_promos[['ds', items[0]]].rename({items[0]: 'onpromotion'}, axis = 1).merge(oil, how = 'left', on = 'ds')
fut = fut.merge(merge_content)
fut['oil'] = fut['oil'].fillna(method = 'bfill')

fut

fut['ds'] = pd.to_datetime(fut['ds'])
preds = model.predict(fut)

preds[['ds', 'onpromotion', 'oil', 'yhat']]

out = test[test['item'] == items[0]].drop('item', axis = 1)
out['ds'] = pd.to_datetime(out['ds'])
out.head()

out.tail()

# cONVERT DATES
df['ds'] = pd.to_datetime(df['ds'])
promos['ds'] = pd.to_datetime(promos['ds'])
test['ds'] = pd.to_datetime(test['ds'])

# Create a function to perform modeling
def prophet_model(item):
    
    # Init model
    model = Prophet(holidays = holidays[['ds', 'holiday']])
    model.add_regressor('onpromotion') #, standardize = False)
    model.add_regressor('oil' )#, standardize = False)

    # Compile data
    trainingdata = df[['ds', item]].rename({item: 'y'}, axis = 1)
    trainingdata['y'] = trainingdata['y'].astype(float)

    # Add regressors
    trainingdata = trainingdata.merge(promos[['ds', item]], how = 'left').rename({item: 'onpromotion'}, axis = 1).merge(oil, how = 'left', on = 'ds')
    trainingdata['onpromotion'] = trainingdata['onpromotion'].fillna(0)
    trainingdata['oil'] = trainingdata['oil'].fillna(method = 'bfill')
    
    # Train
    model.fit(trainingdata)
    
    # Init predictions
    fut = model.make_future_dataframe(periods = 16)
    merge_content = test_promos[['ds', item]].rename({item: 'onpromotion'}, axis = 1).merge(oil, how = 'left', on = 'ds')
    fut = fut.merge(merge_content)
    fut['oil'] = fut['oil'].fillna(method = 'bfill')
    
    # Model
    preds = model.predict(fut)[['ds', 'yhat']] 
    preds['ds'] = pd.to_datetime(preds['ds'])
    
    # Output
    out = test[test['item'] == item].drop('item', axis = 1)    
    
    return out.merge(preds, how = 'left', on = 'ds').rename({'yhat': 'sales'}, axis = 1) #.drop(['ds', 'oil', 'onpromotion'], axis = 1)

random.choice(items)

prophet_model(random.choice(items))

# Forecast
#start = time.time()

# Disable outputs
#logging.getLogger("cmdstanpy").disabled = True 

# Parallelization
#p = Pool(num_cpus)
#result = p.map(prophet_model, items)

#end = time.time()

#print(f'Elapsed Modeling Time:  {round((start - end) / 60, 2)} minutes..')

# Compile output
#res = pd.concat([i for i in result])
#out = res[['id', 'sales']]

#out.shape

test.shape

# Write file
out.to_csv('submission.csv', index = False)

# Visualize outputs
sample = random.choice(items)

sample

# Init model
model = Prophet(holidays = holidays[['ds', 'holiday']])
model.add_regressor('onpromotion') #, standardize = False)
model.add_regressor('oil' )#, standardize = False)

# Compile data
trainingdata = df[['ds', sample]].rename({sample: 'y'}, axis = 1)
trainingdata['y'] = trainingdata['y'].astype(float)

# Add regressors
trainingdata = trainingdata.merge(promos[['ds', sample]], how = 'left').rename({sample: 'onpromotion'}, axis = 1).merge(oil, how = 'left', on = 'ds')
trainingdata['onpromotion'] = trainingdata['onpromotion'].fillna(0)
trainingdata['oil'] = trainingdata['oil'].fillna(method = 'bfill')
    
# Train
model.fit(trainingdata)
    
# Init predictions
fut = model.make_future_dataframe(periods = 16)
merge_content = test_promos[['ds', sample]].rename({sample: 'onpromotion'}, axis = 1).merge(oil, how = 'left', on = 'ds')
fut = fut.merge(merge_content)
fut['oil'] = fut['oil'].fillna(method = 'bfill')
    
# Model
preds = model.predict(fut) #[['ds', 'yhat']] 
preds['ds'] = pd.to_datetime(preds['ds'])
    
# Output
out = test[test['item'] == sample].drop('item', axis = 1)    
out = out.merge(preds, how = 'left', on = 'ds').rename({'yhat': 'sales'}, axis = 1) #.drop(['ds', 'oil', 'onpromotion'], axis = 1)

print(f"Examining Outputs for {sample}")
    
model.plot(preds)

model.plot_components(preds)

print('Showing Time Series for Model: ')

out.head()

out.shape

fig, ax = plt.subplots()
ax.scatter(trainingdata['ds'], trainingdata['y'], color = 'dodgerblue', s = .1)
ax.scatter(out['ds'], out['sales'], color = 'red', s = .2)
plt.ylabel('Count of Item')
plt.xlabel('Date')