Evaluating a Prediction Model with MSE

Introduction to Evaluation in Predictive Modeling

Welcome to our lesson on the " Evaluation of a Prediction Model ". In this session, we delve into evaluating the accuracy of predictive modeling using the Mean Square Error (MSE), explaining residuals and MSE with real-world examples for illustration. Predictive models are pivotal in transforming raw data into actionable insights, but to ensure these models perform as expected, rigorous evaluation is paramount. Such evaluations are crucial for measuring how closely the predicted outcomes align with the actual values, serving as the cornerstone for refining and optimizing models. Today, we'll uncover the importance of meticulous model evaluation and proceed to break down the concepts of residuals and MSE as fundamental elements of this process.

Understanding Residuals and Mean Square Error (MSE)

MSE = \frac{1}{n} \sum_{i=1}^{n} (y_i - \hat{y}_i)^2

Mean Square Error Function

Before diving into the practical implementation of MSE in predictive modeling, it's beneficial to encapsulate the calculation in a Python function. This approach not only enhances readability but also facilitates reuse across different models and datasets. Python # MSE calculation function def calculate_mse(y_true, y_pred): return np.mean((y_true - y_pred)**2) # Deduction of MSE happens here # MSE calculation function def calculate_mse(y_true, y_pred): return np.mean((y_true - y_pred)**2) # Deduction of MSE happens here This function takes in the true values y_true and the predicted values y_pred as arguments, returning the calculated MSE. This computation forms the bedrock of evaluating and interpreting the performance of predictive models.

Join the 1M+ learners on CodeSignal

Be a part of our community of 1M+ users who develop and demonstrate their skills on CodeSignal

Python

# MSE calculation function
def calculate_mse(y_true, y_pred):
    return np.mean((y_true - y_pred)**2) # Deduction of MSE happens here

Python

from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
import numpy as np

# Fetching the dataset
housing = fetch_california_housing()
# Selecting the Median Income feature
X = housing.data[:, 0]  # Selecting the feature
X = X.reshape(-1, 1)  # Reshaping for sklearn compatibility
y = housing.target

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
import numpy as np

# Fetching the dataset
housing = fetch_california_housing()
# Selecting the Median Income feature
X = housing.data[:, 0]  # Selecting the feature
X = X.reshape(-1, 1)  # Reshaping for sklearn compatibility
y = housing.target

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

Python

# Creating and training the Linear Regression model
model = LinearRegression()
model.fit(X_train, y_train)

# Making predictions using the trained model on the test set
y_pred = model.predict(X_test)

# Calculate MSE using the function
mse = calculate_mse(y_test, y_pred)

# First 3 predicted values
print(f"Predicted Values: {y_pred[:3]}") # [1.14958917 1.50606882 1.90393718]
# First 3 true values
print(f"True Values: {y_test[:3]}") # [0.477   0.458   5.00001]
# MSE calculated for all values
print(f"Mean Square Error (MSE): {mse:.4f}") # 0.7091

# Creating and training the Linear Regression model
model = LinearRegression()
model.fit(X_train, y_train)

# Making predictions using the trained model on the test set
y_pred = model.predict(X_test)

# Calculate MSE using the function
mse = calculate_mse(y_test, y_pred)

# First 3 predicted values
print(f"Predicted Values: {y_pred[:3]}") # [1.14958917 1.50606882 1.90393718]
# First 3 true values
print(f"True Values: {y_test[:3]}") # [0.477   0.458   5.00001]
# MSE calculated for all values
print(f"Mean Square Error (MSE): {mse:.4f}") # 0.7091

Python

# Select 10 random data points for plotting
indexes = np.random.choice(range(len(y_test)), 10, replace=False)
X_test_selected = X_test[indexes]
y_test_selected = y_test[indexes]
y_pred_selected = y_pred[indexes]

# Select 10 random data points for plotting
indexes = np.random.choice(range(len(y_test)), 10, replace=False)
X_test_selected = X_test[indexes]
y_test_selected = y_test[indexes]
y_pred_selected = y_pred[indexes]

Python

import matplotlib.pyplot as plt
import numpy as np

# Plot the selected data points, regression line for these points, and residuals
plt.figure(figsize=(10, 6))
plt.scatter(X_test_selected, y_test_selected, color='blue', label='True Values')
plt.plot(X_test_selected, y_pred_selected, 'ro-', label='Predictions')  # 'ro-' for red dots connected by a line

# Visualizing residuals for the selected data points
for i in range(len(X_test_selected)):
    plt.plot([X_test_selected[i], X_test_selected[i]], [y_test_selected[i], y_pred_selected[i]], 'g--', label='Residual' if i == 0 else "")

plt.xlabel('Median Income')
plt.ylabel('Housing Value')
plt.title('Linear Regression: Selected True Values, Predictions, and Residuals')
plt.legend(loc='upper left')
plt.show()

import matplotlib.pyplot as plt
import numpy as np

# Plot the selected data points, regression line for these points, and residuals
plt.figure(figsize=(10, 6))
plt.scatter(X_test_selected, y_test_selected, color='blue', label='True Values')
plt.plot(X_test_selected, y_pred_selected, 'ro-', label='Predictions')  # 'ro-' for red dots connected by a line

# Visualizing residuals for the selected data points
for i in range(len(X_test_selected)):
    plt.plot([X_test_selected[i], X_test_selected[i]], [y_test_selected[i], y_pred_selected[i]], 'g--', label='Residual' if i == 0 else "")

plt.xlabel('Median Income')
plt.ylabel('Housing Value')
plt.title('Linear Regression: Selected True Values, Predictions, and Residuals')
plt.legend(loc='upper left')
plt.show()

Python

from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
import numpy as np

# Fetching the dataset
housing = fetch_california_housing()
# Selecting the Median Income feature
X = housing.data[:, 0]  # Selecting the feature
X = X.reshape(-1, 1)  # Reshaping for sklearn compatibility
y = housing.target

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

Python

# Creating and training the Linear Regression model
model = LinearRegression()
model.fit(X_train, y_train)

# Making predictions using the trained model on the test set
y_pred = model.predict(X_test)

# Calculate MSE using the function
mse = calculate_mse(y_test, y_pred)

# First 3 predicted values
print(f"Predicted Values: {y_pred[:3]}") # [1.14958917 1.50606882 1.90393718]
# First 3 true values
print(f"True Values: {y_test[:3]}") # [0.477   0.458   5.00001]
# MSE calculated for all values
print(f"Mean Square Error (MSE): {mse:.4f}") # 0.7091

Python

# Select 10 random data points for plotting
indexes = np.random.choice(range(len(y_test)), 10, replace=False)
X_test_selected = X_test[indexes]
y_test_selected = y_test[indexes]
y_pred_selected = y_pred[indexes]

Python

import matplotlib.pyplot as plt
import numpy as np

# Plot the selected data points, regression line for these points, and residuals
plt.figure(figsize=(10, 6))
plt.scatter(X_test_selected, y_test_selected, color='blue', label='True Values')
plt.plot(X_test_selected, y_pred_selected, 'ro-', label='Predictions')  # 'ro-' for red dots connected by a line

# Visualizing residuals for the selected data points
for i in range(len(X_test_selected)):
    plt.plot([X_test_selected[i], X_test_selected[i]], [y_test_selected[i], y_pred_selected[i]], 'g--', label='Residual' if i == 0 else "")

plt.xlabel('Median Income')
plt.ylabel('Housing Value')
plt.title('Linear Regression: Selected True Values, Predictions, and Residuals')
plt.legend(loc='upper left')
plt.show()

Introduction to Evaluation in Predictive Modeling

Understanding Residuals and Mean Square Error (MSE)

Mean Square Error Function

Join the 1M+ learners on CodeSignal

Implementing MSE: Setting Up Datasets for Training and Testing

Implementing MSE: Model Evaluation

Visualization of Predictions and Residuals

Lesson Summary