California Housing Regression

import * as math from 'mathjs';
import seedrandom from 'seedrandom';
import fs from 'fs';
import Papa from 'papaparse';
import { trainTestSplit, standardScaler } from './main.js'; // Adjust path as needed

// 1. Load the California Housing dataset from CSV
const csv = fs.readFileSync('data/california_housing.csv', 'utf8');
const parsed = Papa.parse(csv, {
    header: true,
    dynamicTyping: true,
    skipEmptyLines: true
});

// Extract feature names and target
const allColumns = Object.keys(parsed.data[0]);
const featureNames = allColumns.slice(0, -1);
const targetName = allColumns[allColumns.length - 1];

// --- SUBSAMPLE DATA FOR FASTER TRAINING ---
const SAMPLE_SIZE = 1000;
const totalRows = parsed.data.length;
const random = seedrandom('subsample'); // For reproducibility
const indices = Array.from({ length: totalRows }, (_, i) => i);
// Shuffle indices
for (let i = indices.length - 1; i > 0; i--) {
    const j = Math.floor(random() * (i + 1));
    [indices[i], indices[j]] = [indices[j], indices[i]];
}
const selectedIndices = indices.slice(0, SAMPLE_SIZE);

// Build X and y using only the selected indices
const X = [];
const y = [];
for (const idx of selectedIndices) {
    const row = parsed.data[idx];
    X.push(featureNames.map(f => row[f]));
    y.push([row[targetName]]);
}

// 2. Split data into training and testing sets
const { XTrain, XTest, yTrain, yTest } = trainTestSplit(X, y, 0.2, 42);

// 3. Apply feature scaling (Standardization)
const scalerX = standardScaler(XTrain);
const XTrainScaled = scalerX.data;
const XTestScaled = scalerX.transform(XTest);

const scalerY = standardScaler(yTrain);
const yTrainScaled = scalerY.data;
const yTestScaled = scalerY.transform(yTest);

const numFeatures = XTrainScaled[0].length; // Will be 8 for this dataset