Lecture 1: How Machines See - Traditional Computer Vision

# Import necessary libraries
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
from PIL import Image
import requests
from io import BytesIO
from sklearn import datasets
from scipy import ndimage
from skimage import filters, feature, color
import warnings
warnings.filterwarnings('ignore')

# Set random seed for reproducibility
np.random.seed(42)

# Configure matplotlib for better-looking plots
plt.rcParams['figure.figsize'] = (12, 6)
plt.rcParams['font.size'] = 11

print("Libraries loaded successfully!")

# Load MNIST-like digits dataset
from sklearn.datasets import load_digits

# Load the dataset (8x8 images, simplified MNIST)
digits = load_digits()
images = digits.images
labels = digits.target

print(f"Dataset loaded: {len(images)} images")
print(f"Image shape: {images[0].shape}")
print(f"Labels: {np.unique(labels)}")

# Display a few examples
fig, axes = plt.subplots(2, 5, figsize=(12, 5))
for i, ax in enumerate(axes.flat):
    ax.imshow(images[i], cmap='gray')
    ax.set_title(f"Label: {labels[i]}")
    ax.axis('off')
plt.suptitle("Student Handwritten Digits (Sample)", fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()

# Select one image to examine
sample_idx = 0
sample_image = images[sample_idx]
sample_label = labels[sample_idx]

# Create progressive zoom visualization
fig, axes = plt.subplots(1, 4, figsize=(16, 4))

# Original image
axes[0].imshow(sample_image, cmap='gray', interpolation='nearest')
axes[0].set_title(f"Original (Label: {sample_label})")
axes[0].axis('off')

# Zoom level 1: Show grid
axes[1].imshow(sample_image, cmap='gray', interpolation='nearest')
axes[1].grid(True, color='red', linewidth=0.5, alpha=0.3)
axes[1].set_title("With Grid Overlay")
axes[1].set_xticks(np.arange(-.5, 8, 1), minor=True)
axes[1].set_yticks(np.arange(-.5, 8, 1), minor=True)
axes[1].grid(which='minor', color='red', linewidth=0.5)

# Zoom level 2: Crop to 4x4 region
crop_region = sample_image[2:6, 2:6]
axes[2].imshow(crop_region, cmap='gray', interpolation='nearest')
axes[2].set_title("Zoomed Region (4×4 pixels)")
for i in range(4):
    for j in range(4):
        axes[2].text(j, i, f"{int(crop_region[i,j])}", 
                    ha='center', va='center', color='red', fontsize=10, fontweight='bold')
axes[2].grid(True, color='red', linewidth=1)
axes[2].set_xticks(np.arange(-.5, 4, 1), minor=True)
axes[2].set_yticks(np.arange(-.5, 4, 1), minor=True)
axes[2].grid(which='minor', color='red', linewidth=1)

# Zoom level 3: Single pixel enlarged
single_pixel = np.full((10, 10), crop_region[1, 2])
axes[3].imshow(single_pixel, cmap='gray', vmin=0, vmax=16)
axes[3].set_title(f"Single Pixel\nValue: {int(crop_region[1,2])}")
axes[3].axis('off')

plt.tight_layout()
plt.show()

print("\n🔍 Key Insight: Images are just 2D arrays of numbers!")
print(f"Each pixel stores a value representing brightness.")

# Demonstrate grayscale values
grayscale_bar = np.linspace(0, 255, 256).reshape(1, -1).astype(np.uint8)
grayscale_bar = np.repeat(grayscale_bar, 50, axis=0)

plt.figure(figsize=(12, 2))
plt.imshow(grayscale_bar, cmap='gray', aspect='auto')
plt.title("Grayscale Values: 0 (Black) → 255 (White)", fontsize=12, fontweight='bold')
plt.xlabel("Pixel Value")
plt.xticks([0, 64, 128, 192, 255], ['0\n(Black)', '64', '128\n(Mid Gray)', '192', '255\n(White)'])
plt.yticks([])
plt.tight_layout()
plt.show()

print("\n💡 Binary Representation:")
print(f"  0 (Black):   {bin(0):>10s}")
print(f"  128 (Gray):  {bin(128):>10s}")
print(f"  255 (White): {bin(255):>10s}")

# Create a simple color image to demonstrate RGB
color_image = np.zeros((100, 300, 3), dtype=np.uint8)
color_image[:, :100, 0] = 255  # Red channel
color_image[:, 100:200, 1] = 255  # Green channel
color_image[:, 200:, 2] = 255  # Blue channel

fig, axes = plt.subplots(1, 4, figsize=(16, 3))

axes[0].imshow(color_image)
axes[0].set_title("RGB Color Image")
axes[0].axis('off')

axes[1].imshow(color_image[:,:,0], cmap='Reds')
axes[1].set_title("Red Channel")
axes[1].axis('off')

axes[2].imshow(color_image[:,:,1], cmap='Greens')
axes[2].set_title("Green Channel")
axes[2].axis('off')

axes[3].imshow(color_image[:,:,2], cmap='Blues')
axes[3].set_title("Blue Channel")
axes[3].axis('off')

plt.tight_layout()
plt.show()

print("\n🎨 Color mixing works like mixing light:")
print(f"  Red + Green = Yellow")
print(f"  Red + Blue = Magenta")
print(f"  Green + Blue = Cyan")
print(f"  Red + Green + Blue = White")

# Get examples of digit "1" and other digits
ones_indices = np.where(labels == 1)[0][:5]
others_indices = [np.where(labels == i)[0][0] for i in [0, 2, 3, 4]]

# Function to compute column sums
def column_sums(image):
    return np.sum(image, axis=0)

# Visualize column sums for "1" vs others
fig, axes = plt.subplots(3, 5, figsize=(15, 9))

for col, idx in enumerate(ones_indices):
    img = images[idx]
    sums = column_sums(img)
    
    axes[0, col].imshow(img, cmap='gray')
    axes[0, col].set_title(f"Digit: {labels[idx]}")
    axes[0, col].axis('off')
    
    axes[1, col].bar(range(len(sums)), sums, color='blue', alpha=0.7)
    axes[1, col].set_ylim([0, 120])
    axes[1, col].set_title(f"Max sum: {sums.max():.0f}")
    axes[1, col].set_xlabel("Column")
    if col == 0:
        axes[1, col].set_ylabel("Sum")

for col, idx in enumerate(others_indices):
    img = images[idx]
    sums = column_sums(img)
    
    axes[2, col].imshow(img, cmap='gray')
    axes[2, col].set_title(f"Digit: {labels[idx]}")
    axes[2, col].axis('off')
    
    # Plot column sums
    # (Reusing plot from row 1 structure)

# Add empty axis for alignment
axes[2, 4].axis('off')

plt.suptitle("Column Sum Analysis: Detecting Digit '1'", fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()

print("\n📊 Observation: Digit '1' tends to have a sharper, taller spike in column sums.")
print("   But is this enough to reliably distinguish it from other digits?")

def spike_detector(image, threshold=80):
    """
    Simple spike detector for digit '1'.
    Returns True if max column sum exceeds threshold.
    """
    sums = column_sums(image)
    return np.max(sums) > threshold

# Test on all digits
predictions = []
for img, true_label in zip(images[:100], labels[:100]):
    pred = spike_detector(img)
    predictions.append({
        'pred': 1 if pred else 0,
        'true': 1 if true_label == 1 else 0
    })

# Calculate accuracy
correct = sum(p['pred'] == p['true'] for p in predictions)
accuracy = correct / len(predictions)

print(f"\n✅ Spike Detector Accuracy: {accuracy*100:.1f}%")
print(f"   Correct: {correct}/{len(predictions)}")

# Show some failures
failures = [(i, p) for i, p in enumerate(predictions) if p['pred'] != p['true']]
if failures:
    print(f"\n❌ Number of failures: {len(failures)}")
    print("\nLet's examine some failure cases...")
    
    fig, axes = plt.subplots(1, min(5, len(failures)), figsize=(12, 3))
    if len(failures) == 1:
        axes = [axes]
    for ax, (idx, p) in zip(axes, failures[:5]):
        ax.imshow(images[idx], cmap='gray')
        ax.set_title(f"True: {labels[idx]}\nPred: {'1' if p['pred'] else 'Not 1'}")
        ax.axis('off')
    plt.suptitle("Failure Cases", fontsize=12, fontweight='bold')
    plt.tight_layout()
    plt.show()

print("\n🤔 Reflection: Hand-coded rules are fragile. Different handwriting breaks them.")

# Apply Sobel edge detection
sample_image = images[10]

# Compute gradients
edges_x = filters.sobel_h(sample_image)
edges_y = filters.sobel_v(sample_image)
edges = np.hypot(edges_x, edges_y)

# Visualize
fig, axes = plt.subplots(1, 4, figsize=(16, 4))

axes[0].imshow(sample_image, cmap='gray')
axes[0].set_title(f"Original (Digit: {labels[10]})")
axes[0].axis('off')

axes[1].imshow(edges_x, cmap='gray')
axes[1].set_title("Horizontal Edges (Sobel X)")
axes[1].axis('off')

axes[2].imshow(edges_y, cmap='gray')
axes[2].set_title("Vertical Edges (Sobel Y)")
axes[2].axis('off')

axes[3].imshow(edges, cmap='gray')
axes[3].set_title("Edge Magnitude")
axes[3].axis('off')

plt.tight_layout()
plt.show()

print("\n🔍 Edge detection highlights shape boundaries, removing uniform regions.")

# Normalize images to [0, 1]
X = images / 16.0  # Original range is [0, 16] for this dataset
y = labels

print(f"Dataset shape: {X.shape}")
print(f"Pixel value range: [{X.min():.2f}, {X.max():.2f}]")
print(f"Number of classes: {len(np.unique(y))}")

# Flatten images into feature vectors
X_flat = X.reshape(len(X), -1)

print(f"Original shape: {X.shape}")
print(f"Flattened shape: {X_flat.shape}")
print(f"\nEach image is now a {X_flat.shape[1]}-dimensional vector.")

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X_flat, y, test_size=0.3, random_state=42, stratify=y
)

print(f"Training set: {X_train.shape[0]} images")
print(f"Test set: {X_test.shape[0]} images")

from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report

# Train SVM
print("Training SVM...")
svm = SVC(kernel='rbf', gamma='scale', random_state=42)
svm.fit(X_train, y_train)

# Predict on test set
y_pred = svm.predict(X_test)

# Evaluate
accuracy = accuracy_score(y_test, y_pred)
print(f"\n✅ Test Accuracy: {accuracy*100:.2f}%")

# Classification report
print("\n📊 Classification Report:")
print(classification_report(y_test, y_pred))

# Compute confusion matrix
cm = confusion_matrix(y_test, y_pred)

# Visualize
fig, ax = plt.subplots(figsize=(10, 8))
im = ax.imshow(cm, cmap='Blues')

# Add labels
ax.set_xticks(np.arange(10))
ax.set_yticks(np.arange(10))
ax.set_xticklabels(np.arange(10))
ax.set_yticklabels(np.arange(10))
ax.set_xlabel('Predicted Label', fontsize=12)
ax.set_ylabel('True Label', fontsize=12)
ax.set_title('Confusion Matrix', fontsize=14, fontweight='bold')

# Add text annotations
for i in range(10):
    for j in range(10):
        text = ax.text(j, i, cm[i, j],
                      ha='center', va='center', 
                      color='white' if cm[i, j] > cm.max()/2 else 'black')

plt.colorbar(im, ax=ax)
plt.tight_layout()
plt.show()

print("\n🤔 Which digits are most often confused?")
# Find off-diagonal maximums
cm_nodiag = cm.copy()
np.fill_diagonal(cm_nodiag, 0)
max_confusion = np.unravel_index(cm_nodiag.argmax(), cm_nodiag.shape)
print(f"   Most common confusion: Digit {max_confusion[0]} predicted as {max_confusion[1]}")
print(f"   This happened {cm_nodiag[max_confusion]} times.")

# Find misclassified examples
misclassified_idx = np.where(y_pred != y_test)[0]

print(f"\n❌ Misclassified: {len(misclassified_idx)} out of {len(y_test)} ({len(misclassified_idx)/len(y_test)*100:.1f}%)")

# Show some failures
n_show = min(10, len(misclassified_idx))
fig, axes = plt.subplots(2, 5, figsize=(15, 6))

for i, ax in enumerate(axes.flat):
    if i < n_show:
        idx = misclassified_idx[i]
        # Reshape back to image
        img = X_test[idx].reshape(8, 8)
        true_label = y_test[idx]
        pred_label = y_pred[idx]
        
        ax.imshow(img, cmap='gray')
        ax.set_title(f"True: {true_label}, Pred: {pred_label}", color='red')
        ax.axis('off')
    else:
        ax.axis('off')

plt.suptitle("Misclassified Examples", fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()

print("\n💡 Reflection Questions:")
print("   - Are some of these ambiguous even for humans?")
print("   - What features would help distinguish these cases?")
print("   - Is 100% accuracy achievable? Should it be the goal?")