#Installing the libraries with the specified versions
#!pip install numpy==2.0.2 pandas==2.2.2 scikit-learn==1.6.1 matplotlib==3.10.0 seaborn==0.13.2 joblib==1.4.2 xgboost==2.1.4 requests==2.32.3 huggingface_hub==0.30.1 -q
!pip install numpy pandas scikit-learn matplotlib seaborn joblib xgboost requests huggingface_hub -q

import warnings
warnings.filterwarnings("ignore")

# Libraries to help with reading and manipulating data
import numpy as np
import pandas as pd
from typing import Tuple, Dict

# For splitting the dataset
from sklearn.model_selection import train_test_split, RandomizedSearchCV, KFold, cross_val_score

# Libaries to help with data visualization
import matplotlib.pyplot as plt
import seaborn as sns

# Removes the limit for the number of displayed columns
pd.set_option("display.max_columns", None)
# Sets the limit for the number of displayed rows
pd.set_option("display.max_rows", 100)


# Libraries different ensemble classifiers
from sklearn.ensemble import (
    BaggingRegressor,
    RandomForestRegressor,
    AdaBoostRegressor,
    GradientBoostingRegressor,
)
from xgboost import XGBRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.impute import SimpleImputer
import xgboost as xgb

# Libraries to get different metric scores
from sklearn.metrics import (
    confusion_matrix,
    accuracy_score,
    precision_score,
    recall_score,
    f1_score,
    mean_squared_error,
    mean_absolute_error,
    r2_score,
    mean_absolute_percentage_error
)

# To create the pipeline
from sklearn.compose import make_column_transformer, ColumnTransformer
from sklearn.pipeline import make_pipeline,Pipeline

# To tune different models and standardize
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler,OneHotEncoder

# To serialize the model
import joblib

# os related functionalities
import os

# API request
import requests

# For hyperparameter tuning
import itertools

# for hugging face space authentication to upload files
from huggingface_hub import login, HfApi

# Loading the dataset
dataset=pd.read_csv("SuperKart.csv")

# Making a copy of the original dataset
data=dataset.copy()

### View the first 5 rows
data.head()

# view the last 5 rows
data.tail()

# Checking the shape of the dataset
print(f"There are {data.shape[0]} rows and {data.shape[1]} columns.")

There are 8763 rows and 12 columns.

# Checking the data types and non-null values
data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 8763 entries, 0 to 8762
Data columns (total 12 columns):
 #   Column                     Non-Null Count  Dtype  
---  ------                     --------------  -----  
 0   Product_Id                 8763 non-null   object 
 1   Product_Weight             8763 non-null   float64
 2   Product_Sugar_Content      8763 non-null   object 
 3   Product_Allocated_Area     8763 non-null   float64
 4   Product_Type               8763 non-null   object 
 5   Product_MRP                8763 non-null   float64
 6   Store_Id                   8763 non-null   object 
 7   Store_Establishment_Year   8763 non-null   int64  
 8   Store_Size                 8763 non-null   object 
 9   Store_Location_City_Type   8763 non-null   object 
 10  Store_Type                 8763 non-null   object 
 11  Product_Store_Sales_Total  8763 non-null   float64
dtypes: float64(4), int64(1), object(7)
memory usage: 821.7+ KB

# Getting a statistical summary of the dataset
data.describe(include="all").T

# Checking for missing values
data.duplicated().sum()

np.int64(0)

# Checking for missing values
data.isnull().sum()

Product_Id                   0
Product_Weight               0
Product_Sugar_Content        0
Product_Allocated_Area       0
Product_Type                 0
Product_MRP                  0
Store_Id                     0
Store_Establishment_Year     0
Store_Size                   0
Store_Location_City_Type     0
Store_Type                   0
Product_Store_Sales_Total    0
dtype: int64

# Function to plot combined histogram and boxplot for a numerical feature

def histogram_boxplot(
    data, feature, figsize=(12, 7), kde=True, bins=30,
    color="royalblue", title=None, show_mean=True, show_median=True, grid=True
):
    """
    Combined boxplot and histogram for a numerical feature.

    Parameters
    ----------
    data : pandas.DataFrame
        Dataset containing the feature to visualize.
    feature : str
        Column name of the numerical feature to plot.
    figsize : tuple, optional
        Figure size, default is (12, 7).
    kde : bool, optional
        Whether to show a Kernel Density Estimate on the histogram. Default is True.
    bins : int, optional
        Number of histogram bins. Default is 30.
    color : str, optional
        Base color for plots. Default is 'royalblue'.
    title : str, optional
        Title for the combined plot. If None, uses the feature name.
    show_mean : bool, optional
        Whether to display a vertical line for the mean. Default is True.
    show_median : bool, optional
        Whether to display a vertical line for the median. Default is True.
    grid : bool, optional
        Whether to show grid lines on the histogram. Default is True.

    Returns
    -------
    matplotlib.figure.Figure
        The generated Matplotlib figure.
    """

    # Create subplots: boxplot on top, histogram below
    fig, (ax_box, ax_hist) = plt.subplots(
        nrows=2,
        sharex=True,
        gridspec_kw={"height_ratios": (0.25, 0.75)},
        figsize=figsize
    )

    # --- Boxplot ---
    sns.boxplot(
        data=data,
        x=feature,
        ax=ax_box,
        color=sns.color_palette("coolwarm", 8)[2],
        showmeans=True,
        meanprops={"marker": "o", "markerfacecolor": "white", "markeredgecolor": "black"}
    )
    ax_box.set(xlabel=None)
    ax_box.set_title(
        title if title else f"Distribution of {feature}",
        fontsize=14,
        fontweight="bold"
    )

    # --- Histogram ---
    sns.histplot(
        data=data,
        x=feature,
        bins=bins,
        kde=kde,
        ax=ax_hist,
        color=color,
        edgecolor="black",
        alpha=0.75
    )

    # Add mean and median lines
    mean_val = data[feature].mean()
    median_val = data[feature].median()
    if show_mean:
        ax_hist.axvline(mean_val, color="red", linestyle="--", linewidth=1.5, label=f"Mean: {mean_val:.2f}")
    if show_median:
        ax_hist.axvline(median_val, color="black", linestyle="-", linewidth=1.2, label=f"Median: {median_val:.2f}")

    # Final touches
    if grid:
        ax_hist.grid(alpha=0.3)
    ax_hist.legend()
    plt.tight_layout()

    return fig

# Identify numerical features (excluding the target if desired)
numeric_features = data.select_dtypes(include=["int64", "float64"]).columns.tolist()

print("Numeric features:", numeric_features)

Numeric features: ['Product_Weight', 'Product_Allocated_Area', 'Product_MRP', 'Store_Establishment_Year', 'Product_Store_Sales_Total']

# Plot distributions for all numeric variables
for col in numeric_features:
    if col != "Store_Establishment_Year":  # exclude Store_Establishment_Year
        histogram_boxplot(data, col, kde=True, bins=30, color="steelblue")

# Function to create a labeled bar plot for categorical features

def labeled_barplot(
    data: pd.DataFrame,
    feature: str,
    top: int | None = None,          # Show only top N categories
    perc: bool = False,              # Display percentages instead of counts
    sort: str = "desc",              # Sorting order: 'desc', 'asc', or 'alpha'
    orient: str = "v",               # Orientation: 'v' (vertical) or 'h' (horizontal)
    color: str | None = None,        # Single color for bars
    palette: str = "Paired",         # Palette if no single color is set
    min_pct: float | None = None,    # Group categories below this percentage as 'Other' (e.g. 0.01)
    dropna: bool = True,             # Exclude NaN values
    title: str | None = None,        # Custom title for the plot
    figsize: tuple | None = None,    # Automatically adjusts if not provided
    value_fmt: str | None = None,    # Format for values (e.g. "{:,}" for comma separators)
    font_size: int = 11              # Font size for the bar labels
):
    """
    Create a labeled bar plot with optional percentage annotations and category filtering.

    Parameters
    ----------
    data : pandas.DataFrame
        The input dataset containing the feature.
    feature : str
        The categorical column name to visualize.
    top : int, optional
        Display only the top N categories. If None, show all.
    perc : bool, optional
        Display percentages instead of absolute counts. Default is False.
    sort : str, optional
        Sort order: 'desc' (default), 'asc', or 'alpha' (alphabetical).
    orient : str, optional
        Orientation of bars: 'v' for vertical or 'h' for horizontal.
    color : str, optional
        Fixed color for all bars. If None, uses the palette argument.
    palette : str, optional
        Seaborn color palette when color is not provided.
    min_pct : float, optional
        Combine categories below this fraction of total into 'Other'. Example: 0.02 groups <2%.
    dropna : bool, optional
        Whether to exclude NaN values from the plot. Default is True.
    title : str, optional
        Title for the plot. If None, a default descriptive title is created.
    figsize : tuple, optional
        Custom figure size. If None, automatically adjusted based on number of categories.
    value_fmt : str, optional
        Format string for numerical labels. Example: "{:,}" to show thousands separators.
    font_size : int, optional
        Font size for the numeric labels above/beside bars.

    Returns
    -------
    fig, ax : tuple
        Matplotlib Figure and Axes objects for further customization.
    """

    # --- Prepare the series for plotting ---
    s = data[feature].dropna() if dropna else data[feature]
    total = len(s)

    # Group rare categories as "Other"
    if min_pct is not None and min_pct > 0:
        freqs = s.value_counts(normalize=True)
        rare = freqs[freqs < min_pct].index
        s = s.mask(s.isin(rare), "Other")

    # Compute category counts
    counts = s.value_counts(dropna=not dropna)

    # Keep only the top-N categories
    if top is not None and top > 0:
        counts = counts.head(top)

    # Apply sorting
    if sort == "asc":
        counts = counts.sort_values(ascending=True)
    elif sort == "alpha":
        counts = counts.sort_index()
    else:  # Default: descending
        counts = counts.sort_values(ascending=False)

    cats = counts.index.tolist()
    vals = counts.values
    n_cats = len(cats)

    # Dynamically determine figure size if not provided
    if figsize is None:
        if orient == "v":
            figsize = (max(6, min(0.6 * n_cats + 2, 18)), 5.5)
        else:
            figsize = (8.5, max(5, min(0.4 * n_cats + 2, 16)))

    fig, ax = plt.subplots(figsize=figsize)

    # Choose color or palette
    bar_kwargs = {"color": color} if color else {"palette": palette}

    # Create the bar plot
    if orient == "v":
        sns.barplot(x=cats, y=vals, ax=ax, **bar_kwargs)
        ax.set_xlabel(feature)
        ax.set_ylabel("Percentage" if perc else "Count")
        ax.tick_params(axis="x", rotation=45)
    else:
        sns.barplot(x=vals, y=cats, ax=ax, **bar_kwargs)
        ax.set_ylabel(feature)
        ax.set_xlabel("Percentage" if perc else "Count")

    # Annotate each bar with value or percentage
    for p in ax.patches:
        val = p.get_height() if orient == "v" else p.get_width()
        label = f"{val/total:.1%}" if perc else (value_fmt.format(val) if value_fmt else f"{int(val):,}")

        if orient == "v":
            ax.annotate(
                label,
                (p.get_x() + p.get_width() / 2, val),
                ha="center", va="bottom",
                xytext=(0, 3), textcoords="offset points",
                fontsize=font_size
            )
        else:
            ax.annotate(
                label,
                (val, p.get_y() + p.get_height() / 2),
                ha="left", va="center",
                xytext=(3, 0), textcoords="offset points",
                fontsize=font_size
            )

    # Set the plot title
    if title is None:
        title = f"{feature} — Top {top}" if top else f"{feature} — Distribution"
        if min_pct:
            title += f" (rare < {min_pct:.0%} → 'Other')"
    ax.set_title(title, fontsize=13, fontweight="bold")

    sns.despine()
    plt.tight_layout()
    return fig, ax

# Identify categorical variables in the dataset
categorical_features = data.select_dtypes(include=["object"]).columns.tolist()

print("Categorical features:")
for col in categorical_features:
    print(f" - {col}")

Categorical features:
 - Product_Id
 - Product_Sugar_Content
 - Product_Type
 - Store_Id
 - Store_Size
 - Store_Location_City_Type
 - Store_Type

# Plot labeled barplots for each categorical variable
for col in categorical_features:
    # Skip IDs if they are not meaningful for visualization
    if col not in ["Product_Id" ]:
        labeled_barplot(
            data,
            feature=col,
            perc=True,           # Display percentages
            top=None,            # Show all categories
            orient="v",          # Vertical bars
            min_pct=0.01,        # Group rare categories under "Other"
            font_size=11
        )

# Set a consistent theme for all plots
sns.set_theme(style="whitegrid")

# -----------------------------
# Numeric ↔ Numeric
# -----------------------------
def scatter_with_trend(
    data, x, y, hue=None,
    sample=5000, alpha=0.3,
    trend="auto",           # 'auto' | 'lowess' | 'poly' | 'none'
    poly_degree=2,          # grado para la tendencia polinómica
    color_trend="black",
    linewidth=2.0
):
    """
    Scatter plot with an optional trend line.

    - If trend='lowess', it tries to use statsmodels LOWESS.
    - If trend='poly', it fits a polynomial regression of given degree.
    - If trend='auto', it uses LOWESS if statsmodels is installed, otherwise polynomial.
    - If trend='none', no trend line is drawn.
    """

    # Subset & sample
    cols = [x, y] + ([hue] if hue else [])
    df = data[cols].dropna()
    if sample and len(df) > sample:
        df = df.sample(sample, random_state=42)

    plt.figure(figsize=(7.5, 5.5))
    ax = sns.scatterplot(data=df, x=x, y=y, hue=hue, alpha=alpha, edgecolor=None)

    # Decide trend mode
    mode = trend
    if trend == "auto":
        try:
            import statsmodels.api as sm  # noqa
            mode = "lowess"
        except Exception:
            mode = "poly"

    # Plot LOWESS if available
    if mode == "lowess":
        # use statsmodels lowess explicitly (works even if seaborn doesn't find it)
        from statsmodels.nonparametric.smoothers_lowess import lowess
        xx = df[x].to_numpy()
        yy = df[y].to_numpy()
        # sort x for a clean line
        order = np.argsort(xx)
        smoothed = lowess(yy[order], xx[order], frac=0.25, it=1, return_sorted=True)
        ax.plot(smoothed[:, 0], smoothed[:, 1], color=color_trend, linewidth=linewidth, label="LOWESS")
    elif mode == "poly":
        # simple polynomial fit as fallback
        xx = df[x].to_numpy()
        yy = df[y].to_numpy()
        order = np.argsort(xx)
        coeffs = np.polyfit(xx, yy, deg=poly_degree)
        y_hat = np.polyval(coeffs, xx[order])
        ax.plot(xx[order], y_hat, color=color_trend, linewidth=linewidth, label=f"Poly (deg={poly_degree})")
    else:
        # 'none' -> no trend line
        pass

    ax.set_title(f"{y} vs {x}" + ("" if mode == "none" else f" (trend: {mode})"))
    if hue is None and mode in {"lowess", "poly"}:
        ax.legend(loc="best")
    plt.tight_layout()
    return ax

def hexbin_density(data, x, y, gridsize=30):
    """
    Hexbin density plot (fast for large datasets).
    """
    df = data[[x, y]].dropna()
    plt.figure(figsize=(7.5, 5.5))
    hb = plt.hexbin(df[x], df[y], gridsize=gridsize, cmap="viridis", mincnt=1)
    plt.colorbar(hb, label="count")
    plt.xlabel(x); plt.ylabel(y)
    plt.title(f"Hexbin density: {y} vs {x}")
    plt.tight_layout()

def corr_heatmap(data, numeric_cols):
    """
    Correlation heatmap for a list of numeric columns.
    """
    corr = data[numeric_cols].corr(numeric_only=True)
    plt.figure(figsize=(8, 6))
    ax = sns.heatmap(corr, annot=True, fmt=".2f", cmap="vlag", linewidths=.5)
    ax.set_title("Correlation Heatmap (numeric variables)")
    plt.tight_layout()
    return ax

# -----------------------------
# Categorical ↔ Numeric
# -----------------------------
def box_violin_numeric_by_category(data, cat, num, kind="box", show_points=False):
    """
    Boxplot/Violinplot of a numeric variable conditioned on a category.
    Optionally overlays jitter points for distribution detail.
    """
    df = data[[cat, num]].dropna()
    plt.figure(figsize=(9, 5.5))
    if kind == "violin":
        ax = sns.violinplot(data=df, x=cat, y=num, inner="quartile", cut=0)
    else:
        ax = sns.boxplot(data=df, x=cat, y=num, showmeans=True,
                         meanprops={"marker":"o","markerfacecolor":"white","markeredgecolor":"black"})
    if show_points:
        sns.stripplot(data=df, x=cat, y=num, color="black", alpha=0.25, dodge=True)
    ax.set_title(f"{num} by {cat}")
    ax.tick_params(axis="x", rotation=30)
    plt.tight_layout()
    return ax

def mean_ci_bar(data, cat, num, estimator="mean", ci=95, order=None, top=None):
    """
    Barplot of an aggregate (mean by default) with confidence intervals.
    You can optionally plot only the top-N categories by sample size.
    """
    df = data[[cat, num]].dropna()
    if top:
        top_levels = df[cat].value_counts().head(top).index
        df = df[df[cat].isin(top_levels)]
    plt.figure(figsize=(9, 5.5))
    ax = sns.barplot(data=df, x=cat, y=num, estimator=getattr(np, estimator), ci=ci, order=order)
    ax.set_title(f"{num} {estimator} by {cat} (±{ci}% CI)")
    ax.tick_params(axis="x", rotation=30)
    plt.tight_layout()
    return ax

# -----------------------------
# Categorical ↔ Categorical
# -----------------------------
def stacked_ratio_bar(data, cat1, cat2, normalize="index"):
    """
    Stacked ratio bars between two categorical variables.
    normalize='index' -> % row-wise; 'columns' -> % col-wise; None -> counts.
    """
    ctab = pd.crosstab(data[cat1], data[cat2], normalize=normalize)
    ctab.plot(kind="bar", stacked=True, figsize=(9, 5.5), colormap="tab20")
    plt.title(f"{cat1} vs {cat2} ({'ratio' if normalize else 'count'})")
    plt.ylabel("Ratio" if normalize else "Count")
    plt.tight_layout()

def cat_heatmap(data, cat1, cat2, normalize=None):
    """
    Heatmap using a crosstab between two categorical variables.
    normalize=None -> counts; 'index' or 'columns' for row/col proportions.
    """
    ctab = pd.crosstab(data[cat1], data[cat2], normalize=normalize)
    plt.figure(figsize=(8, 6))
    ax = sns.heatmap(ctab, annot=True, fmt=".2f" if normalize else "d", cmap="Blues")
    ax.set_title(f"Heatmap: {cat1} x {cat2} ({'ratio' if normalize else 'count'})")
    plt.tight_layout()
    return ax

# -----------------------------
# 1) Identify variable types
# -----------------------------
# 'data' is our main DataFrame
target = "Product_Store_Sales_Total"  

numeric_cols = data.select_dtypes(include=["int64", "float64"]).columns.tolist()
categorical_cols = data.select_dtypes(include=["object"]).columns.tolist()

# skip high-cardinality IDs when plotting categorical vars
id_like = {"Product_Id"}  # keep them if you want to review bias
cat_for_plots = [c for c in categorical_cols if c not in id_like]

print("Numeric:", numeric_cols)
print("Categorical:", categorical_cols)

# -----------------------------
# 2) Enumerate bivariate combos
# -----------------------------
from itertools import combinations

# (a) Numeric–Numeric: all pairwise combos among numeric variables 
num_num_pairs = list(combinations([c for c in numeric_cols if c != target], 2))

# (b) Categorical–Numeric: each categorical vs target, and optionally vs other numeric vars
cat_num_pairs_target = [(c, target) for c in categorical_cols]
cat_num_pairs_all = [(c, n) for c in categorical_cols for n in numeric_cols]  

# (c) Categorical–Categorical: pairs among categorical variables
cat_cat_pairs = list(combinations(categorical_cols, 2))

print(f"# Num–Num pairs: {len(num_num_pairs)}")
print(f"# Cat–Num (vs target) pairs: {len(cat_num_pairs_target)}")
print(f"# Cat–Cat pairs: {len(cat_cat_pairs)}")

# -----------------------------
# 3) Recommended pairs for SuperKart
# -----------------------------
# Focus on the most interpretable/useful combos for this dataset:

recommended = {
    "num_num": [
        ("Product_Weight", "Product_MRP"),
        ("Product_Weight", "Product_Allocated_Area"),
        ("Product_MRP", "Product_Allocated_Area"),
        ("Store_Establishment_Year", "Product_Allocated_Area"),
        ("Store_Establishment_Year", "Product_MRP"),
        ("Store_Establishment_Year", "Product_Weight"),
    ],
    "cat_num_target": [
        ("Product_Type", target),
        ("Product_Sugar_Content", target),
        ("Store_Size", target),
        ("Store_Location_City_Type", target),
        ("Store_Type", target),
        # keep Store_Id if you want to reveal store-level bias:
        ("Store_Id", target),
    ],
    "cat_cat": [
        ("Store_Size", "Store_Location_City_Type"),
        ("Store_Type", "Store_Location_City_Type"),
        ("Store_Type", "Store_Size"),
        ("Product_Type", "Product_Sugar_Content"),
        ("Store_Id", "Store_Type"),
    ]
}

print("Recommended Num–Num:", recommended["num_num"])
print("Recommended Cat–Num (target):", recommended["cat_num_target"])
print("Recommended Cat–Cat:", recommended["cat_cat"])

# -----------------------------
# 4) Plot helpers by pair type
# -----------------------------
# (a) Numeric–Numeric: scatter + LOWESS; if relation is dense, try hexbin
for x, y in recommended["num_num"]:
    scatter_with_trend(
        data, x, y,
        trend="auto",      
        poly_degree=2,     
        alpha=0.35
    )
    plt.show()
    # hexbin_density(data, x, y); plt.show()   

# (b) Categorical–Numeric (vs target): box/violin and mean+CI
for cat, num in recommended["cat_num_target"]:
    # Boxplot (use violin for shape)
    box_violin_numeric_by_category(data, cat=cat, num=num, kind="box", show_points=False)
    plt.show()
    # Mean with 95% CI (top levels if high cardinality)
    mean_ci_bar(data, cat=cat, num=num, estimator="mean", ci=95, top=12)
    plt.show()

# (c) Categorical–Categorical: stacked ratio and heatmap
for c1, c2 in recommended["cat_cat"]:
    stacked_ratio_bar(data, c1, c2, normalize="index")
    plt.show()
    cat_heatmap(data, c1, c2, normalize="index")
    plt.show()

Numeric: ['Product_Weight', 'Product_Allocated_Area', 'Product_MRP', 'Store_Establishment_Year', 'Product_Store_Sales_Total']
Categorical: ['Product_Id', 'Product_Sugar_Content', 'Product_Type', 'Store_Id', 'Store_Size', 'Store_Location_City_Type', 'Store_Type']
# Num–Num pairs: 6
# Cat–Num (vs target) pairs: 7
# Cat–Cat pairs: 21
Recommended Num–Num: [('Product_Weight', 'Product_MRP'), ('Product_Weight', 'Product_Allocated_Area'), ('Product_MRP', 'Product_Allocated_Area'), ('Store_Establishment_Year', 'Product_Allocated_Area'), ('Store_Establishment_Year', 'Product_MRP'), ('Store_Establishment_Year', 'Product_Weight')]
Recommended Cat–Num (target): [('Product_Type', 'Product_Store_Sales_Total'), ('Product_Sugar_Content', 'Product_Store_Sales_Total'), ('Store_Size', 'Product_Store_Sales_Total'), ('Store_Location_City_Type', 'Product_Store_Sales_Total'), ('Store_Type', 'Product_Store_Sales_Total'), ('Store_Id', 'Product_Store_Sales_Total')]
Recommended Cat–Cat: [('Store_Size', 'Store_Location_City_Type'), ('Store_Type', 'Store_Location_City_Type'), ('Store_Type', 'Store_Size'), ('Product_Type', 'Product_Sugar_Content'), ('Store_Id', 'Store_Type')]

# Fix inconsistent category labels
data.Product_Sugar_Content.replace(to_replace=["reg"], value=["Regular"], inplace=True)

# Check the value counts to confirm the fix
data.Product_Sugar_Content.value_counts()

Product_Sugar_Content
Low Sugar    4885
Regular      2359
No Sugar     1519
Name: count, dtype: int64

# Create a new feature extracting the first two characters from Product_Id
data["Product_ID_Code"] = data["Product_Id"].str[:2]
data.head()

# Check the unique values of the new feature
data["Product_ID_Code"].unique()

array(['FD', 'NC', 'DR'], dtype=object)

# Check the unique values of Product_Type for a specific Product_ID_Code
data.loc[data.Product_ID_Code=="FD","Product_Type"].unique()

array(['Frozen Foods', 'Dairy', 'Canned', 'Baking Goods', 'Snack Foods',
       'Meat', 'Fruits and Vegetables', 'Breads', 'Breakfast',
       'Starchy Foods', 'Seafood'], dtype=object)

# Check the unique values of Product_Type for another specific Product_ID_Code
data.loc[data.Product_ID_Code=="NC","Product_Type"].unique()

array(['Health and Hygiene', 'Household', 'Others'], dtype=object)

# Check the unique values of Product_Type for another specific Product_ID_Code
data.loc[data.Product_ID_Code=="DR","Product_Type"].unique()

array(['Hard Drinks', 'Soft Drinks'], dtype=object)

# Create a new feature representing the age of the store in years
data["Store_Age_Years"] = 2025 - data.Store_Establishment_Year

# List of perishable product types
perishables = [
    "Dairy",
    "Meat",
    "Fruits and Vegetables",
    "Breakfast",
    "Breads",
    "Seafood",
]

# Function to categorize products as Perishables or Non Perishables
def change(x):
    if x in perishables:
        return "Perishables"
    else:
        return "Non Perishables"

# Create a new feature categorizing Product_Type into Perishables and Non Perishables
data['Product_Type_Category'] = data['Product_Type'].apply(change)

# Display the first few rows of the updated DataFrame
data.head()

# Visualize boxplots for all numeric variables to check for outliers
numeric_columns = data.select_dtypes(include=np.number).columns.tolist()
numeric_columns.remove("Store_Establishment_Year")
numeric_columns.remove("Store_Age_Years")


plt.figure(figsize=(15, 12))

for i, variable in enumerate(numeric_columns):
    plt.subplot(4, 4, i + 1)
    plt.boxplot(data[variable], whis=1.5)
    plt.tight_layout()
    plt.title(variable)

plt.show()

# Drop unnecessary columns that won't be used in modeling
data = data.drop(["Product_Id","Product_Type","Store_Id","Store_Establishment_Year"], axis=1)

# Display the shape of the updated DataFrame
data.shape

(8763, 11)

# Display the first few rows of the updated DataFrame
data.head()

# Separating features and target variable
X = data.drop("Product_Store_Sales_Total", axis=1)
y = data["Product_Store_Sales_Total"]

# Splitting the dataset into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)

# Displaying the shapes of the training and testing sets
X_train.shape, X_test.shape, y_train.shape, y_test.shape

((6134, 10), (2629, 10), (6134,), (2629,))

# --- Robust dtype detection ---
# Categorical = object or pandas 'category'
categorical_features = X.select_dtypes(include=["object", "category"]).columns.tolist()

# Numeric = anything recognized by pandas as a number (int/float/nullable)
numeric_features = X.select_dtypes(include=["number"]).columns.tolist()

print("Categorical features:", categorical_features)
print("Numeric features:", numeric_features)

Categorical features: ['Product_Sugar_Content', 'Store_Size', 'Store_Location_City_Type', 'Store_Type', 'Product_ID_Code', 'Product_Type_Category']
Numeric features: ['Product_Weight', 'Product_Allocated_Area', 'Product_MRP', 'Store_Age_Years']

# --- OneHotEncoder: keep dense output regardless of sklearn version ---
try:
    ohe = OneHotEncoder(handle_unknown="ignore", sparse_output=False)
except TypeError:
    # fallback for older scikit-learn where 'sparse_output' doesn't exist
    ohe = OneHotEncoder(handle_unknown="ignore", sparse=False)

# --- Pipelines for each data type ---
numeric_pipeline = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="median")),
        ("scaler", StandardScaler())
    ]
)

categorical_pipeline = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("encoder", ohe)
    ]
)

# --- Option A (clear & explicit): ColumnTransformer ---
preprocessor = ColumnTransformer(
    transformers=[
        ("num", numeric_pipeline, numeric_features),
        ("cat", categorical_pipeline, categorical_features),
    ],
    remainder="drop"   # drop anything not listed above (safer for reproducibility)
)

# (Alternative) Option B using make_column_transformer (equivalent):
# preprocessor = make_column_transformer(
#     (numeric_pipeline, numeric_features),
#     (categorical_pipeline, categorical_features),
#     remainder="drop"
# )

preprocessor = ColumnTransformer(
    transformers=[
        ("num", numeric_pipeline, numeric_features),
        ("cat", categorical_pipeline, categorical_features),
    ],
    remainder="drop"
)

# Adjusted R² calculation
def adj_r2_score_from_prediction(n: int, k: int, r2: float) -> float:
    """
    Adjusted R² given:
    - n: number of samples used for evaluation
    - k: number of predictors (AFTER preprocessing/encoding)
    - r2: plain R²
    """
    # Guard against division by zero when n - k - 1 <= 0
    denom = max(n - k - 1, 1)
    return 1.0 - ((1.0 - r2) * (n - 1) / denom)

# Effective k calculation for pipelines
def _effective_k_from_pipeline(model, X) -> int:
    """
    Compute effective number of predictors (k) for a fitted Pipeline.
    If a 'preprocess' step exists, we use its transformed shape.
    Otherwise we fallback to X.shape[1].
    """
    try:
        pre = getattr(model, "named_steps", {}).get("preprocess", None)
        if pre is not None:
            # model must be already fitted; transform uses train-fit params
            Xt = pre.transform(X)
            return Xt.shape[1]
    except Exception:
        pass
    # Fallback (no preprocess or cannot transform)
    return X.shape[1]

# Model performance function for regression
def model_performance_regression(model, predictors, target) -> pd.DataFrame:
    """
    Compute regression metrics for a (fitted) model or pipeline:
    - RMSE, MAE, R², Adjusted R², MAPE

    model: fitted regressor or sklearn Pipeline (e.g., preprocess + model)
    predictors: X matrix BEFORE preprocessing (pipeline will handle it)
    target: y vector/series
    """
    # Predict with the pipeline/model
    pred = model.predict(predictors)

    r2 = r2_score(target, pred)
    rmse = float(np.sqrt(mean_squared_error(target, pred)))
    mae = float(mean_absolute_error(target, pred))
    mape = float(mean_absolute_percentage_error(target, pred))

    # Adjusted R² with k derived after preprocessing
    n = predictors.shape[0]
    k = _effective_k_from_pipeline(model, predictors)
    adjr2 = adj_r2_score_from_prediction(n=n, k=k, r2=r2)

    df_perf = pd.DataFrame(
        {
            "RMSE": [rmse],
            "MAE": [mae],
            "R-squared": [float(r2)],
            "Adj. R-squared": [float(adjr2)],
            "MAPE": [mape],
            "n_samples": [int(n)],
            "k_effective": [int(k)],
        }
    )
    return df_perf

# Check if xgboost is installed
try:
    HAS_XGB = True
except Exception:
    HAS_XGB = False


def evaluate_models_with_custom_metrics(
    X_train, y_train, X_test, y_test, preprocessor,
    random_state: int = 42, cv_splits: int = 5, n_jobs: int = -1
) -> Tuple[pd.DataFrame, Pipeline]:
    """
    Train and evaluate Decision Tree, Bagging, Random Forest, AdaBoost,
    Gradient Boosting, and XGBoost (if available) using a unified preprocessing pipeline.

    Returns:
    - results_df: leaderboard sorted by Test RMSE with CV RMSE mean/std and
                  Test RMSE/MAE/R²/AdjR²/MAPE + n,k.
    - best_pipe:  fitted pipeline of the best model by Test RMSE.
    """

    models: Dict[str, object] = {
        "DecisionTree": DecisionTreeRegressor(random_state=random_state),
        "Bagging": BaggingRegressor(n_estimators=200, random_state=random_state, n_jobs=n_jobs),
        "RandomForest": RandomForestRegressor(n_estimators=300, random_state=random_state, n_jobs=n_jobs),
        "AdaBoost": AdaBoostRegressor(n_estimators=300, learning_rate=0.05, random_state=random_state),
        "GradientBoosting": GradientBoostingRegressor(n_estimators=300, learning_rate=0.05, max_depth=3, random_state=random_state),
    }
    if HAS_XGB:
        models["XGBoost"] = XGBRegressor(
            objective="reg:squarederror",
            n_estimators=600, learning_rate=0.05, max_depth=6,
            subsample=0.8, colsample_bytree=0.8, reg_lambda=1.0,
            n_jobs=n_jobs, random_state=random_state, tree_method="hist"
        )

    results = []
    best_name, best_rmse, best_pipe = None, np.inf, None
    kf = KFold(n_splits=cv_splits, shuffle=True, random_state=random_state)

    for name, est in models.items():
        pipe = Pipeline([("preprocess", preprocessor), ("model", est)])

        # CV RMSE on training set
        cv_scores = cross_val_score(
            pipe, X_train, y_train,
            scoring="neg_root_mean_squared_error",
            cv=kf, n_jobs=n_jobs
        )
        cv_rmse_mean = float(-cv_scores.mean())
        cv_rmse_std  = float(cv_scores.std())

        # Fit and evaluate on test with full metric suite
        pipe.fit(X_train, y_train)
        perf = model_performance_regression(pipe, X_test, y_test).iloc[0].to_dict()

        row = {
            "model": name,
            "cv_rmse_mean": cv_rmse_mean,
            "cv_rmse_std": cv_rmse_std,
            "test_rmse": perf["RMSE"],
            "test_mae": perf["MAE"],
            "test_r2": perf["R-squared"],
            "test_adj_r2": perf["Adj. R-squared"],
            "test_mape": perf["MAPE"],
            "n_test": int(perf["n_samples"]),
            "k_effective": int(perf["k_effective"]),
        }
        results.append(row)

        if perf["RMSE"] < best_rmse:
            best_rmse = perf["RMSE"]
            best_name = name
            best_pipe = pipe

    results_df = pd.DataFrame(results).sort_values("test_rmse").reset_index(drop=True)
    print(f"Best model by Test RMSE: {best_name} (RMSE = {best_rmse:.3f})")
    if not HAS_XGB:
        print("Note: XGBoost was skipped because 'xgboost' is not installed.")
    return results_df, best_pipe

# Evaluate models and get the leaderboard
leaderboard, best_model = evaluate_models_with_custom_metrics(
    X_train, y_train, X_test, y_test, preprocessor,
    random_state=42, cv_splits=5, n_jobs=-1
)

leaderboard  # Display the leaderboard

Best model by Test RMSE: RandomForest (RMSE = 282.701)

# pick top-2 by test_rmse
top2 = leaderboard.nsmallest(2, "test_rmse")["model"].tolist()
top2

['RandomForest', 'Bagging']

# Hyperparameter tuning for Random Forest using RandomizedSearchCV

cv = KFold(n_splits=5, shuffle=True, random_state=42)

rf_pipe = Pipeline([
    ("preprocess", preprocessor),
    ("model", RandomForestRegressor(random_state=42, n_jobs=-1))
])

rf_param_dist = {
    "model__n_estimators": [300, 500, 800, 1000],
    "model__max_depth": [None, 10, 15, 20, 30],
    "model__min_samples_split": [2, 5, 10, 20],
    "model__min_samples_leaf": [1, 2, 4, 8],
    "model__max_features": ["sqrt", "log2", 0.5, 0.7, 0.9],
    "model__bootstrap": [True],   # False can lead to overfitting
}

rf_search = RandomizedSearchCV(
    rf_pipe,
    param_distributions=rf_param_dist,
    n_iter=50,  # number of random combinations to try
    scoring="neg_root_mean_squared_error",
    cv=cv,
    n_jobs=-1,
    random_state=42,
    verbose=1
)

rf_search.fit(X_train, y_train)

print("Best RF params:", rf_search.best_params_)
print("Best RF CV RMSE:", -rf_search.best_score_)

rf_tuned_perf = model_performance_regression(rf_search.best_estimator_, X_test, y_test)
rf_tuned_perf

Fitting 5 folds for each of 50 candidates, totalling 250 fits
Best RF params: {'model__n_estimators': 500, 'model__min_samples_split': 5, 'model__min_samples_leaf': 4, 'model__max_features': 0.9, 'model__max_depth': 20, 'model__bootstrap': True}
Best RF CV RMSE: 287.3779027980666

# Hyperparameter tuning for Bagging using RandomizedSearchCV
base_tree = DecisionTreeRegressor(random_state=42)

bag_pipe = Pipeline([
    ("preprocess", preprocessor),
    ("model", BaggingRegressor(
        # base_estimator was renamed to estimator in sklearn 1.2+
        **({"estimator": base_tree} if "estimator" in BaggingRegressor().__dict__ else {"base_estimator": base_tree}),
        n_estimators=300,
        random_state=42,
        n_jobs=-1
    ))
])

bag_param_dist = {
    "model__n_estimators": [200, 300, 500, 800],
    # base tree-level
    "model__estimator__max_depth": [None, 8, 12, 16, 20] if "estimator" in BaggingRegressor().__dict__ else
                                   "model__base_estimator__max_depth",
    "model__estimator__min_samples_split": [2, 5, 10, 20] if "estimator" in BaggingRegressor().__dict__ else
                                           "model__base_estimator__min_samples_split",
    "model__estimator__min_samples_leaf": [1, 2, 4, 8] if "estimator" in BaggingRegressor().__dict__ else
                                          "model__base_estimator__min_samples_leaf",
    # bagging-level
    "model__max_samples": [0.6, 0.7, 0.8, 1.0],
    "model__max_features": [0.6, 0.8, 1.0],
    "model__bootstrap": [True],
    "model__bootstrap_features": [False, True],
    "model__oob_score": [True, False],
}

# Adjust param_dist keys 
if "estimator" not in BaggingRegressor().__dict__:
    bag_param_dist = {
        "model__n_estimators": [200, 300, 500, 800],
        "model__base_estimator__max_depth": [None, 8, 12, 16, 20],
        "model__base_estimator__min_samples_split": [2, 5, 10, 20],
        "model__base_estimator__min_samples_leaf": [1, 2, 4, 8],
        "model__max_samples": [0.6, 0.7, 0.8, 1.0],
        "model__max_features": [0.6, 0.8, 1.0],
        "model__bootstrap": [True],
        "model__bootstrap_features": [False, True],
        "model__oob_score": [True, False],
    }

bag_search = RandomizedSearchCV(
    bag_pipe,
    param_distributions=bag_param_dist,
    n_iter=50,
    scoring="neg_root_mean_squared_error",
    cv=cv,
    n_jobs=-1,
    random_state=42,
    verbose=1
)

bag_search.fit(X_train, y_train)

print("Best Bagging params:", bag_search.best_params_)
print("Best Bagging CV RMSE:", -bag_search.best_score_)

bag_tuned_perf = model_performance_regression(bag_search.best_estimator_, X_test, y_test)
bag_tuned_perf

Fitting 5 folds for each of 50 candidates, totalling 250 fits
Best Bagging params: {'model__oob_score': True, 'model__n_estimators': 300, 'model__max_samples': 0.8, 'model__max_features': 1.0, 'model__estimator__min_samples_split': 2, 'model__estimator__min_samples_leaf': 2, 'model__estimator__max_depth': None, 'model__bootstrap_features': False, 'model__bootstrap': True}
Best Bagging CV RMSE: 286.9880052862924

# --- Helper: convert tuned metrics DF -> leaderboard schema ---
def tuned_row_from_search(name: str, search, X_test, y_test) -> pd.DataFrame:
    """
    Build a leaderboard-style row for a tuned model using a fitted
    RandomizedSearchCV/GridSearchCV object.
    - Obtains CV mean/std at best_index_ (convert from negative scores to +RMSE)
    - Computes test metrics with model_performance_regression
    """
    # CV stats at the best configuration
    i = search.best_index_
    mean_cv = search.cv_results_["mean_test_score"][i]   # negative RMSE
    std_cv  = search.cv_results_["std_test_score"][i]    # std of negative RMSE (same magnitude)

    cv_rmse_mean = float(-mean_cv)
    cv_rmse_std  = float(abs(std_cv))

    # Test metrics 
    perf = model_performance_regression(search.best_estimator_, X_test, y_test).iloc[0]

    row = {
        "model": name,
        "cv_rmse_mean": cv_rmse_mean,
        "cv_rmse_std":  cv_rmse_std,
        "test_rmse": float(perf["RMSE"]),
        "test_mae":  float(perf["MAE"]),
        "test_r2":   float(perf["R-squared"]),
        "test_adj_r2": float(perf["Adj. R-squared"]),
        "test_mape": float(perf["MAPE"]),
        "n_test":    int(perf["n_samples"]),
        "k_effective": int(perf["k_effective"]),
    }
    return pd.DataFrame([row])

# Build tuned rows for RF and Bagging
rf_tuned_row = tuned_row_from_search("RF_tuned", rf_search, X_test, y_test)
bag_tuned_row = tuned_row_from_search("Bagging_tuned", bag_search, X_test, y_test)

tuned_df = pd.concat([rf_tuned_row, bag_tuned_row], ignore_index=True)

# Reorder columns to match leaderboard
cols_order = [
    "model", "cv_rmse_mean", "cv_rmse_std",
    "test_rmse", "test_mae", "test_r2", "test_adj_r2", "test_mape",
    "n_test", "k_effective",
]
tuned_df = tuned_df[cols_order]

# Combine original leaderboard with tuned results
leaderboard_plus = pd.concat([leaderboard[cols_order], tuned_df], ignore_index=True)
leaderboard_plus = leaderboard_plus.sort_values("test_rmse").reset_index(drop=True)
leaderboard_plus

# --- Model Serialization ---

# Create a directory for model artifacts if it doesn't exist
os.makedirs("backend_files", exist_ok=True)

# The fitted model pipeline from RandomizedSearchCV
final_model = rf_search.best_estimator_

# Save the model pipeline (includes preprocessing + model)
model_path = "backend_files/rf_tuned_pipeline.joblib"
joblib.dump(final_model, model_path)

print(f"✅ Model successfully saved at: {model_path}")

✅ Model successfully saved at: backend_files/rf_tuned_pipeline.joblib

# --- Model Loading and Testing ---

# Load the serialized model
loaded_model = joblib.load(model_path)

# Verify it works with the test set
y_true = np.ravel(y_test)
y_pred_flat = np.ravel(loaded_model.predict(X_test))

rmse_loaded = np.sqrt(mean_squared_error(y_true, y_pred_flat))
r2_loaded = r2_score(y_true, y_pred_flat)

print(f"Loaded model RMSE: {rmse_loaded}")
print(f"Loaded model R²: {r2_loaded}")

Loaded model RMSE: 276.8045875615
Loaded model R²: 0.9325281945169243

%%writefile backend_files/app.py
# Import necessary libraries
import os
import numpy as np
import joblib  # For loading the serialized model
import pandas as pd  # For data manipulation
from flask import Flask, request, jsonify  # For creating the Flask API

# Initialize Flask app
superkart_api = Flask("SuperKart_Sales_API")

# Load the trained sales prediction pipeline (preprocess + RF tuned)
model = joblib.load("rf_tuned_pipeline.joblib")

# Define a route for the home page
@superkart_api.get("/")
def home():
    return "SuperKart Sales Forecast API is up. POST to /v1/predict"

# Define an endpoint to predict sales for a single product-store entry
@superkart_api.post("/v1/predict")
def predict_sales():
    try:
        # Get JSON data from the request
        data = request.get_json(force=True)

        # Build the sample in the SAME schema used for training
        sample = {
            "Product_Weight": data["Product_Weight"],
            "Product_Sugar_Content": data["Product_Sugar_Content"],
            "Product_Allocated_Area": data["Product_Allocated_Area"],
            "Product_MRP": data["Product_MRP"],
            "Store_Size": data["Store_Size"],
            "Store_Location_City_Type": data["Store_Location_City_Type"],
            "Store_Type": data["Store_Type"],
            "Product_ID_Code": data["Product_ID_Code"],
            "Store_Age_Years": data["Store_Age_Years"],
            "Product_Type_Category": data["Product_Type_Category"],
        }

        # Convert the extracted data into a DataFrame
        input_data = pd.DataFrame([sample])

        # Make a prediction using the trained pipeline
        prediction = float(model.predict(input_data).tolist()[0])

        # Return the prediction as a JSON response
        return jsonify({"Sales": prediction})

    except KeyError as e:
        return jsonify({"error": f"Missing required field: {str(e)}"}), 400
    except Exception as e:
        return jsonify({"error": str(e)}), 500

# Run the Flask app (use 0.0.0.0 for containers / HF Spaces)
if __name__ == "__main__":
    port = int(os.environ.get("PORT", 7860))
    superkart_api.run(host="0.0.0.0", port=port, debug=True)

Overwriting backend_files/app.py

import pkg_resources

packages = [
    "pandas", "numpy", "scikit-learn", "seaborn", "joblib", "xgboost",
    "Werkzeug", "flask", "gunicorn", "requests", "uvicorn", "streamlit"
]

lines = []
for pkg in packages:
    try:
        version = pkg_resources.get_distribution(pkg).version
        print(f"{pkg}=={version}")
        lines.append(f"{pkg}=={version}")
    except:
        lines.append(f"{pkg}")  # por si no está instalado

with open("backend_files/requirements.txt", "w") as f:
    f.write("\n".join(lines))

print("✅ requirements.txt created with installed versions.")

pandas==2.3.2
numpy==1.26.4
scikit-learn==1.7.1
seaborn==0.13.2
joblib==1.5.2
xgboost==3.0.5
Werkzeug==3.1.3
flask==3.1.2
gunicorn==23.0.0
requests==2.32.5
uvicorn==0.35.0
streamlit==1.48.1
✅ requirements.txt created with installed versions.

%%writefile backend_files/Dockerfile
FROM python:3.12-slim

# Install system dependencies for building packages
RUN apt-get update && apt-get install -y --no-install-recommends \
    build-essential \
    gcc \
    libgomp1 \
 && rm -rf /var/lib/apt/lists/*

# Set the working directory inside the container
WORKDIR /app

# Copy all files from the current directory to the container's working directory
COPY . .

# Install dependencies from the requirements file without using cache to reduce image size
RUN pip install --no-cache-dir --upgrade -r requirements.txt

# Expose the app port (útil para documentación; algunos PaaS lo ignoran y usan PORT env)
EXPOSE 7860

# Define the command to start the application using Gunicorn with 4 worker processes
# -w 4: 4 workers
# -b 0.0.0.0:7860: bind to all interfaces on port 7860
# app:superkart_api -> módulo app.py y objeto Flask 'superkart_api'
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:7860", "app:superkart_api"]

Writing backend_files/Dockerfile

# --- Login to Hugging Face Hub ---
from huggingface_hub import login

# Login 
login(token="hf_********REDACTED********")

# Create a new Space on Hugging Face Hub
from huggingface_hub import create_repo

try:
    create_repo(
        "superkart-backend",           # 👈 nombre de tu repo/space
        repo_type="space",             # tipo space
        space_sdk="docker",            # usa el modo Docker
        private=False                  # True si quieres mantenerlo privado
    )
    print("✅ Space created successfully!")

except Exception as e:
    if "RepositoryAlreadyExistsError" in str(e):
        print("⚠️ Repository already exists. Skipping creation.")
    else:
        print(f"❌ Error creating repository: {e}")

✅ Space created successfully!

# --- Upload backend files to the Hugging Face Space ---
from huggingface_hub import HfApi, login

# Token and repo
access_key = "hf_********REDACTED********"
repo_id = "josegzzv/superkart-backend"  # 👈 tu usuario + nombre del space

# Authenticate
login(token=access_key)

# Initialize API
api = HfApi()

# Upload folder
api.upload_folder(
    folder_path="backend_files",  # Backend files
    repo_id=repo_id,
    repo_type="space"
)

print("✅ Backend uploaded successfully to Hugging Face Space!")

✅ Backend uploaded successfully to Hugging Face Space!

import requests
from IPython.display import display, Markdown
import json

# === Backend URL (Hugging Face Space) ===
BACKEND_URL = "https://josegzzv-superkart-backend.hf.space/v1/predict"

# === Payload matching your cURL example ===
sample_payload = {
    "Product_Weight": 12.6,
    "Product_Sugar_Content": "Regular",
    "Product_Allocated_Area": 2.5,
    "Product_MRP": 149.99,
    "Store_Size": "Medium",
    "Store_Location_City_Type": "Metro",
    "Store_Type": "Supermarket",
    "Product_ID_Code": "FD",
    "Store_Age_Years": 6,
    "Product_Type_Category": "Beverages"
}

# === Test the backend prediction endpoint ===
try:
    # Send POST request to backend
    response = requests.post(
        BACKEND_URL,
        headers={"Content-Type": "application/json"},
        json=sample_payload,
        timeout=15
    )

    # Check the response
    if response.status_code == 200:
        display(Markdown("✅ **Backend is running and returned a prediction successfully:**"))
        print(json.dumps(response.json(), indent=2, ensure_ascii=False))
    else:
        display(Markdown(f"⚠️ **Backend responded with status code {response.status_code}:**"))
        print(response.text)

# Handle connection or timeout errors
except Exception as e:
    display(Markdown("❌ **Failed to connect to the backend.**"))
    print(f"Error: {e}")

{
  "Sales": 4117.598791108056
}

{"Sales": 4117.598791108056}
This confirms that the model pipeline (preprocessing → inference → serialization → API serving) is functioning correctly end-to-end.

# Create a folder for frontend UI
os.makedirs("frontend_files", exist_ok=True)

%%writefile frontend_files/app.py
import streamlit as st
import requests

st.set_page_config(page_title="SuperKart — Sales Forecast", page_icon="🛒")

# Read the backend URL from Streamlit secrets (defined in .streamlit/secrets.toml)
BACKEND_URL = st.secrets["BACKEND_URL"]

st.title("🛒 SuperKart — Sales Forecast (Frontend)")

st.markdown(
    """
    Enter the product and store information below to predict **Product_Store_Sales_Total**.
    """
)

# --------------------------
# Input fields for the UI
# --------------------------
col1, col2 = st.columns(2)

with col1:
    Product_Weight = st.number_input(
        "Product Weight (kg)",
        min_value=0.0, step=0.01, value=12.66
    )
    Product_Allocated_Area = st.number_input(
        "Product Allocated Area (0–1)",
        min_value=0.0, max_value=1.0, step=0.001, value=0.068
    )
    Product_MRP = st.number_input(
        "Product MRP",
        min_value=0.0, step=0.01, value=147.03
    )
    Store_Age_Years = st.number_input(
        "Store Age (years)",
        min_value=0, step=1, value=20
    )
    Product_ID_Code = st.selectbox(
        "Product ID (prefix)",
        options=["FD", "NC", "DR"],  # based on EDA analysis
        index=0
    )

with col2:
    Product_Sugar_Content = st.selectbox(
        "Product Sugar Content",
        options=["Low Sugar", "Regular", "No Sugar"],
        index=0
    )
    Store_Size = st.selectbox(
        "Store Size",
        options=["Small", "Medium", "High"],
        index=1
    )
    Store_Location_City_Type = st.selectbox(
        "Store Location City Type",
        options=["Tier 1", "Tier 2", "Tier 3"],
        index=1
    )
    Store_Type = st.selectbox(
        "Store Type",
        options=["Departmental Store", "Supermarket Type1", "Supermarket Type2", "Food Mart"],
        index=2
    )
    Product_Type_Category = st.selectbox(
        "Product Type Category",
        options=["Perishables", "Non Perishables"],
        index=1
    )

# --------------------------
# Prepare the payload for the API
# --------------------------
product_data = {
    "Product_Weight": float(Product_Weight),
    "Product_Sugar_Content": Product_Sugar_Content,
    "Product_Allocated_Area": float(Product_Allocated_Area),
    "Product_MRP": float(Product_MRP),
    "Store_Size": Store_Size,
    "Store_Location_City_Type": Store_Location_City_Type,
    "Store_Type": Store_Type,
    "Product_ID_Code": Product_ID_Code,
    "Store_Age_Years": int(Store_Age_Years),
    "Product_Type_Category": Product_Type_Category,
}

st.markdown("---")
st.subheader("Prediction")

# Allow the user to manually edit or confirm backend URL
backend_url = st.text_input("Backend URL", BACKEND_URL).rstrip("/")
predict_endpoint = f"{backend_url}/v1/predict"

# --------------------------
# Button to make API call
# --------------------------
if st.button("Predict", type="primary"):
    if "<user>-<backend-space>" in backend_url:
        st.warning("⚠️ Please set your real BACKEND_URL (Hugging Face Space URL) before predicting.")
    else:
        try:
            resp = requests.post(predict_endpoint, json=product_data, timeout=30)
            if resp.status_code == 200:
                result = resp.json()
                y_hat = float(result.get("Sales"))
                st.success(f"✅ Predicted Product Store Sales Total: **{y_hat:,.2f}**")
                with st.expander("Show Payload Sent to API"):
                    st.json(product_data)
            else:
                st.error(f"API returned error ({resp.status_code}): {resp.text}")
        except Exception as e:
            st.exception(e)

st.caption("Powered by Streamlit • SuperKart • v1")

Overwriting frontend_files/app.py

import pkg_resources

packages = [
    "streamlit", "requests"
]

lines = []
for pkg in packages:
    try:
        version = pkg_resources.get_distribution(pkg).version
        print(f"{pkg}=={version}")
        lines.append(f"{pkg}=={version}")
    except:
        lines.append(f"{pkg}")  # por si no está instalado

with open("frontend_files/requirements.txt", "w") as f:
    f.write("\n".join(lines))

print("✅ requirements.txt created with installed versions.")

streamlit==1.48.1
requests==2.32.5
✅ requirements.txt created with installed versions.

%%writefile frontend_files/Dockerfile
# ---------- Base Image ----------
# frontend_files/Dockerfile
FROM python:3.12-slim

WORKDIR /app
COPY . .

RUN pip install --no-cache-dir --upgrade -r requirements.txt

# Hugging Face exposes the port via $PORT (defaults to 7860)
ENV HOME=/app \
    STREAMLIT_CONFIG_DIR=/app/.streamlit \
    STREAMLIT_BROWSER_GATHERUSAGESTATS=false \
    STREAMLIT_SERVER_PORT=7860 \
    PORT=7860

# Create the .streamlit directory and set permissions
RUN mkdir -p /app/.streamlit && chmod -R 777 /app/.streamlit

EXPOSE 7860

# Use a shell form so $PORT gets expanded
CMD ["streamlit", "run", "app.py", \
     "--server.port=7860", \
     "--server.address=0.0.0.0", \
     "--server.headless=true", \
     "--server.enableXsrfProtection=false", \
     "--server.enableCORS=false"]

Overwriting frontend_files/Dockerfile

# --- Upload frontend files to the Hugging Face Space ---
from huggingface_hub import HfApi, login

# Token and repository
access_key = "hf_********REDACTED********"
repo_id = "josegzzv/superkart-frontend"  

# Authenticate
login(token=access_key)

# Create a new Space on Hugging Face Hub
from huggingface_hub import create_repo

try:
    create_repo(
        "superkart-frontend",           # name of your repo/space
        repo_type="space",             # space type
        space_sdk="docker",            # Docker mode
        private=False                  # True if you want to keep it private
    )
    print("✅ Space created successfully!")

except Exception as e:
    if "RepositoryAlreadyExistsError" in str(e):
        print("⚠️ Repository already exists. Skipping creation.")
    else:
        print(f"❌ Error creating repository: {e}")

✅ Space created successfully!

# --- Upload frontend files to the Hugging Face Space ---
# Initialize API
api = HfApi()

# Upload folder
api.upload_folder(
    folder_path="frontend_files",  # Frontend files
    repo_id=repo_id,
    repo_type="space",
    ignore_patterns=None
)

print("✅ Frontend uploaded successfully to Hugging Face Space!")

✅ Frontend uploaded successfully to Hugging Face Space!

import re
import requests
from IPython.display import display, Markdown

FRONTEND_URL = "https://josegzzv-superkart-frontend.hf.space"

try:
    response = requests.get(FRONTEND_URL, timeout=20)
    status = response.status_code
    html = response.text or ""


    if 200 <= status < 300:
        display(Markdown("✅ **Frontend is reachable (HTTP 2xx).**"))
        display(Markdown(f"- **Status Code:** `{status}`"))

    else:
        display(Markdown(f"⚠️ **Frontend responded with HTTP {status}.**"))
        print(html[:1000])

except Exception as e:
    display(Markdown("❌ **Unable to reach the frontend.**"))
    print(f"{type(e).__name__}: {e}")

	Product_Id	Product_Weight	Product_Sugar_Content	Product_Allocated_Area	Product_Type	Product_MRP	Store_Id	Store_Establishment_Year	Store_Size	Store_Location_City_Type	Store_Type	Product_Store_Sales_Total
8758	NC7546	14.80	No Sugar	0.016	Health and Hygiene	140.53	OUT004	2009	Medium	Tier 2	Supermarket Type2	3806.53
8759	NC584	14.06	No Sugar	0.142	Household	144.51	OUT004	2009	Medium	Tier 2	Supermarket Type2	5020.74
8760	NC2471	13.48	No Sugar	0.017	Health and Hygiene	88.58	OUT001	1987	High	Tier 2	Supermarket Type1	2443.42
8761	NC7187	13.89	No Sugar	0.193	Household	168.44	OUT001	1987	High	Tier 2	Supermarket Type1	4171.82
8762	FD306	14.73	Low Sugar	0.177	Snack Foods	224.93	OUT002	1998	Small	Tier 3	Food Mart	2186.08

	model	cv_rmse_mean	cv_rmse_std	test_rmse	test_mae	test_r2	test_adj_r2	test_mape	n_test	k_effective
0	RandomForest	291.736994	20.845229	282.701362	106.843682	0.929623	0.929029	0.038883	2629	22
1	Bagging	292.721101	20.705200	283.157203	107.348400	0.929396	0.928800	0.039055	2629	22
2	XGBoost	304.322843	15.869530	293.363133	127.009633	0.924214	0.923575	0.045798	2629	22
3	GradientBoosting	314.160573	21.181424	301.247656	136.633763	0.920086	0.919411	0.051118	2629	22
4	DecisionTree	406.776303	24.409774	388.029106	135.941579	0.867412	0.866293	0.049736	2629	22
5	AdaBoost	483.141647	11.260480	480.116760	375.442619	0.797012	0.795299	0.132692	2629	22

	model	cv_rmse_mean	cv_rmse_std	test_rmse	test_mae	test_r2	test_adj_r2	test_mape	n_test	k_effective
0	RF_tuned	287.377903	18.298050	276.804588	104.289940	0.932528	0.931959	0.039303	2629	22
1	Bagging_tuned	286.988005	18.849473	277.332545	103.242706	0.932271	0.931699	0.038122	2629	22
2	RandomForest	291.736994	20.845229	282.701362	106.843682	0.929623	0.929029	0.038883	2629	22
3	Bagging	292.721101	20.705200	283.157203	107.348400	0.929396	0.928800	0.039055	2629	22
4	XGBoost	304.322843	15.869530	293.363133	127.009633	0.924214	0.923575	0.045798	2629	22
5	GradientBoosting	314.160573	21.181424	301.247656	136.633763	0.920086	0.919411	0.051118	2629	22
6	DecisionTree	406.776303	24.409774	388.029106	135.941579	0.867412	0.866293	0.049736	2629	22
7	AdaBoost	483.141647	11.260480	480.116760	375.442619	0.797012	0.795299	0.132692	2629	22

Model	CV RMSE (mean ± std)	Test RMSE	Test R²	MAPE
Random Forest	291.7 ± 20.8	282.7	0.9296	3.9%
Bagging	292.7 ± 20.7	283.1	0.9294	3.9%
XGBoost	304.3 ± 15.9	293.4	0.9242	4.6%
Gradient Boosting	314.2 ± 21.2	301.2	0.9201	5.1%
Decision Tree	406.8 ± 24.4	388.0	0.8674	4.9%
AdaBoost	483.1 ± 11.3	480.1	0.7970	13.3%

Model	CV RMSE Mean	Test RMSE	Test MAE	R²	Adj. R²	MAPE
RF_tuned	287.38	276.80	104.29	0.9325	0.9319	3.93%
Bagging_tuned	286.99	277.33	103.24	0.9323	0.9317	3.81%
RandomForest	291.74	282.70	106.84	0.9296	0.9290	3.89%
Bagging	292.72	283.16	107.35	0.9294	0.9288	3.91%
XGBoost	304.32	293.36	127.01	0.9242	0.9236	4.58%
GradientBoosting	314.16	301.25	136.63	0.9201	0.9194	5.11%

	Product_Id	Product_Weight	Product_Sugar_Content	Product_Allocated_Area	Product_Type	Product_MRP	Store_Id	Store_Establishment_Year	Store_Size	Store_Location_City_Type	Store_Type	Product_Store_Sales_Total
0	FD6114	12.66	Low Sugar	0.027	Frozen Foods	117.08	OUT004	2009	Medium	Tier 2	Supermarket Type2	2842.40
1	FD7839	16.54	Low Sugar	0.144	Dairy	171.43	OUT003	1999	Medium	Tier 1	Departmental Store	4830.02
2	FD5075	14.28	Regular	0.031	Canned	162.08	OUT001	1987	High	Tier 2	Supermarket Type1	4130.16
3	FD8233	12.10	Low Sugar	0.112	Baking Goods	186.31	OUT001	1987	High	Tier 2	Supermarket Type1	4132.18
4	NC1180	9.57	No Sugar	0.010	Health and Hygiene	123.67	OUT002	1998	Small	Tier 3	Food Mart	2279.36

	count	unique	top	freq	mean	std	min	25%	50%	75%	max
Product_Id	8763	8763	FD6114	1	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Product_Weight	8763.0	NaN	NaN	NaN	12.653792	2.21732	4.0	11.15	12.66	14.18	22.0
Product_Sugar_Content	8763	4	Low Sugar	4885	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Product_Allocated_Area	8763.0	NaN	NaN	NaN	0.068786	0.048204	0.004	0.031	0.056	0.096	0.298
Product_Type	8763	16	Fruits and Vegetables	1249	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Product_MRP	8763.0	NaN	NaN	NaN	147.032539	30.69411	31.0	126.16	146.74	167.585	266.0
Store_Id	8763	4	OUT004	4676	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Store_Establishment_Year	8763.0	NaN	NaN	NaN	2002.032751	8.388381	1987.0	1998.0	2009.0	2009.0	2009.0
Store_Size	8763	3	Medium	6025	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Store_Location_City_Type	8763	3	Tier 2	6262	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Store_Type	8763	4	Supermarket Type2	4676	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Product_Store_Sales_Total	8763.0	NaN	NaN	NaN	3464.00364	1065.630494	33.0	2761.715	3452.34	4145.165	8000.0

Problem Statement¶

Business Context¶

Objective¶

Data Description¶

Installing and Importing the necessary libraries¶

Loading the dataset¶

Data Overview¶

View the first 5 and last 5 rows of the dataset¶

Checking the shape of the dataset¶

Checking the data types and non-null values¶

Getting a statistical summary of the dataset¶

Checking for missing values¶

Checking for missing values¶

🧭 Data Overview — Observations and Insights¶

General Structure¶

Data Completeness¶

Data Types¶

Numerical Summary¶

Categorical Insights¶

Key Takeaways¶

Exploratory Data Analysis (EDA)¶

Univariate Analysis¶

🔹 Univariate Analysis — Numerical Variables¶

🔹 Univariate Analysis — Categorical Variables¶

🧭 Univariate Analysis — Observations and Insights¶

🔹 Numerical Variables¶

Product_Weight¶

Product_Allocated_Area¶

Product_MRP (Maximum Retail Price)¶

Product_Store_Sales_Total (Target Variable)¶

🔹 Categorical Variables¶

Product_Sugar_Content¶

Product_Type¶

Store_Id¶

Store_Size¶

Store_Location_City_Type¶

Store_Type¶

⚙️ Key Takeaways¶

Bivariate Analysis¶

🔍 Bivariate Analysis — Observations and Insights¶

1️⃣ Numeric vs Numeric Relationships¶

Product_Weight vs Product_MRP¶

Product_Weight vs Product_Allocated_Area¶

Product_MRP vs Product_Allocated_Area¶

Store_Establishment_Year vs Product Attributes¶

2️⃣ Categorical vs Numeric (Target: Product_Store_Sales_Total)¶

Product_Type vs Sales¶

Product_Sugar_Content vs Sales¶

Store_Size vs Sales¶

Store_Location_City_Type vs Sales¶

Store_Type vs Sales¶

Store_Id vs Sales¶

3️⃣ Categorical vs Categorical Relationships¶

Store_Size vs Store_Location_City_Type¶

Store_Type vs Store_Location_City_Type¶

Store_Type vs Store_Size¶

Product_Type vs Product_Sugar_Content¶

Store_Id vs Store_Type¶

⚙️ Key Takeaways¶

Data Preprocessing¶

🧩 Data Preprocessing — Observations and Insights¶

1️⃣ Overview¶

2️⃣ Data Cleaning and Feature Engineering¶

Category Label Fix¶

New Engineered Features¶

3️⃣ Outlier Review¶

4️⃣ Feature Transformation with ColumnTransformer¶

Numeric Pipeline¶

Categorical Pipeline¶

Unified Preprocessor¶

Model Building¶

Define functions for Model Evaluation¶

Model Performance Improvement - Hyperparameter Tuning¶

The best models where RandomForest and Bagging¶

Model Performance Comparison, Final Model Selection, and Serialization¶

🧠 Model Building — Observations & Insights¶

1. Model Evaluation Summary¶

2. Cross-Validation and Performance Trends¶

3. Hyperparameter Tuning Results¶

🔹 Random Forest (Tuned)¶