Price Engine
three claims, one stopwatch.

def make_splits — the leakage rule

def make_splits(X, y, seed: int = SEED):
    """
    Three-way split, in this order, on purpose.

    LEAKAGE RULE: early stopping is a decision made from data. Whatever it
    watches, it has partly fit to. So it watches a VALIDATION slice that is
    carved OUT OF TRAIN — never the test set. If early stopping peeked at
    test, the tree count would be tuned to test, and the final MAE would be
    an optimistic number reported on data the model had already been shaped
    around. The test set is opened exactly once, at final scoring, and is
    never used to choose anything.
    """
    X_train_full, X_test, y_train_full, y_test = train_test_split(
        X, y, test_size=0.20, random_state=seed
    )
    # 15% of TRAIN (not of the whole dataset) becomes the validation slice.
    X_tr, X_val, y_tr, y_val = train_test_split(
        X_train_full, y_train_full, test_size=0.15, random_state=seed
    )
    return X_tr, y_tr, X_val, y_val, X_test, y_test

def train_lgbm — the model chooses its own size

def train_lgbm(X_tr, y_tr, X_val, y_val, n_estimators: int = N_ESTIMATORS, seed: int = SEED):
    """Fit LightGBM against a validation slice and let it choose its own size."""
    model = LGBMRegressor(
        n_estimators=n_estimators,
        learning_rate=LEARNING_RATE,
        num_leaves=NUM_LEAVES,
        random_state=seed,
        verbose=-1,
    )
    t0 = time.perf_counter()
    model.fit(
        X_tr,
        y_tr,
        eval_set=[(X_val, y_val)],          # validation only. never test.
        eval_metric="l2",
        callbacks=[lgb.early_stopping(EARLY_STOPPING_ROUNDS, verbose=False)],
    )
    return model, time.perf_counter() - t0

def race_sklearn — matched work, different split search

def race_sklearn(X_tr, y_tr, X_test, y_test, n_trees: int, seed: int = SEED):
    """
    Fair fight: sklearn is capped at exactly the tree count LightGBM's own
    early stopping settled on, gets the same learning rate, and trains on the
    identical rows. Same work, same data — the only difference left is HOW
    each library searches for splits.
    """
    sk = GradientBoostingRegressor(
        n_estimators=n_trees,
        learning_rate=LEARNING_RATE,
        max_depth=5,          # ~2^5 leaves, the closest match to num_leaves=31
        random_state=seed,
    )
    t0 = time.perf_counter()
    sk.fit(X_tr, y_tr)
    sk_train_s = time.perf_counter() - t0
    return sk, float(mean_absolute_error(y_test, sk.predict(X_test))), sk_train_s

the leakage tests

def test_test_set_appears_in_neither_train_nor_validation(small_run):
    test_rows = _rows(small_run["X_test"])
    assert not (test_rows & _rows(small_run["X_tr"]))
    assert not (test_rows & _rows(small_run["X_val"])), (
        "validation must be carved out of TRAIN — if it overlaps test, early "
        "stopping is tuning the tree count on the scoring set"
    )

def test_fit_is_only_ever_given_the_validation_slice():
    """Guards against someone 'helpfully' adding test to eval_set later."""
    src = (ROOT / "price_engine.py").read_text()
    eval_sets = re.findall(r"eval_set=\[([^\]]*)\]", src)
    assert eval_sets, "no eval_set found — did the fit call move?"
    for es in eval_sets:
        assert "X_val" in es and "y_val" in es
        assert "test" not in es

the verdict — both outcomes are printable

// Both outcomes are printable. If scikit-learn wins on some machine, the page
// says so in the same place, with the same emphasis — a benchmark that can
// only report one result is an advertisement.
const maeGap = R.sk.mae - R.lgbm.mae;
const accuracy = Math.abs(maeGap) < 0.01 * R.lgbm.mae
  ? `Accuracy is a tie — ${usd(Math.abs(maeGap))} apart on a typical error of ${usd(R.lgbm.mae)}, which is noise.`
  : (maeGap > 0
    ? `LightGBM was also ${usd(maeGap)} more accurate.`
    : `scikit-learn edged it on accuracy by ${usd(-maeGap)}.`);

set("verdict", faster
  ? `<b>${R.speedup.toFixed(1)}× faster</b> on identical data and an identical
     ${R.best_iteration}-tree budget. The gap is histogram-based split finding: LightGBM
     buckets each feature into ~255 bins once and scans histograms, while scikit-learn's
     exact greedy search sorts and sweeps every candidate threshold at every node.
     ${accuracy}`
  : `<b>scikit-learn won this run</b> — LightGBM took ${(1 / R.speedup).toFixed(1)}× longer.
     Histogram binning has fixed overhead, and on a small, narrow, all-numeric table it
     does not always pay for itself. Printed anyway: the benchmark reports what ran.
     ${accuracy}`);

extract — splits and honesty

## Splits and honesty

- **Validation carved out of TRAIN, never out of test** — early stopping is a decision made
  from data, so whatever it watches it has partly fitted to; letting it watch test would
  tune the tree count to the scoring set and make the reported MAE optimistic.
- **80/20 train/test, then 15% of train to validation** — standard proportions, and stating
  the order matters: the 15% is of train, not of the whole dataset.
- **The test set is opened exactly once, at final scoring** — no threshold, no tree count,
  no feature choice is made with it.
- **Two tests enforce the leakage rule mechanically, not by comment** — one asserts the test
  rows appear in neither train nor validation, one greps the source so a future edit that
  puts test into `eval_set` fails the suite.