Kross-validatsiya va qayta o'qitish

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier

X, y = load_breast_cancer(return_X_y=True)

for tasodif in [0, 1, 2, 3, 4]:
    X_o, X_s, y_o, y_s = train_test_split(
        X, y, test_size=0.2, random_state=tasodif
    )
    model = RandomForestClassifier(n_estimators=100, random_state=42)
    model.fit(X_o, y_o)
    print(f"random_state={tasodif}: aniqlik = {model.score(X_s, y_s):.3f}")

from sklearn.model_selection import cross_val_score

model = RandomForestClassifier(n_estimators=100, random_state=42)
ballar = cross_val_score(model, X, y, cv=5, scoring="accuracy")

print(f"Har bir bo'linish: {ballar.round(3)}")
print(f"O'rtacha:          {ballar.mean():.3f}")
print(f"Standart og'ish:   {ballar.std():.3f}")
print(f"Ishonch oralig'i:  {ballar.mean():.3f} ± {ballar.std() * 2:.3f}")

from sklearn.model_selection import (
    KFold, StratifiedKFold, TimeSeriesSplit,
    LeaveOneOut, GroupKFold, ShuffleSplit,
)

# Oddiy
kf = KFold(n_splits=5, shuffle=True, random_state=42)

# Sinf nisbatini saqlaydi - klassifikatsiya uchun standart
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Vaqt qatorlari uchun
tss = TimeSeriesSplit(n_splits=5)

# Har bir namuna alohida sinov - juda sekin
loo = LeaveOneOut()

# Guruhlar aralashmasin
gkf = GroupKFold(n_splits=5)

from sklearn.model_selection import cross_validate

natijalar = cross_validate(
    model, X, y, cv=5,
    scoring=["accuracy", "precision", "recall", "f1", "roc_auc"],
    return_train_score=True,
)

import pandas as pd
jadval = pd.DataFrame(natijalar)
print(jadval[[u for u in jadval.columns if u.startswith("test_")]].mean().round(3))

from sklearn.model_selection import learning_curve
import matplotlib.pyplot as plt

olchamlar, ball_oqitish, ball_validatsiya = learning_curve(
    RandomForestClassifier(n_estimators=100, random_state=42),
    X, y, cv=5,
    train_sizes=np.linspace(0.1, 1.0, 10),
    n_jobs=-1,
)

plt.figure(figsize=(9, 5))
plt.plot(olchamlar, ball_oqitish.mean(axis=1), "o-", label="O'qitish")
plt.plot(olchamlar, ball_validatsiya.mean(axis=1), "o-", label="Validatsiya")
plt.fill_between(olchamlar,
                 ball_validatsiya.mean(axis=1) - ball_validatsiya.std(axis=1),
                 ball_validatsiya.mean(axis=1) + ball_validatsiya.std(axis=1),
                 alpha=0.15)
plt.xlabel("O'qitish namunalari soni")
plt.ylabel("Aniqlik")
plt.legend()
plt.grid(alpha=0.3)
plt.show()

from sklearn.model_selection import validation_curve
from sklearn.tree import DecisionTreeClassifier

chuqurliklar = range(1, 21)

ball_o, ball_v = validation_curve(
    DecisionTreeClassifier(random_state=42),
    X, y,
    param_name="max_depth",
    param_range=chuqurliklar,
    cv=5,
)

for chuqurlik, o, v in zip(chuqurliklar, ball_o.mean(axis=1), ball_v.mean(axis=1)):
    if chuqurlik in [1, 2, 3, 5, 10, 20]:
        print(f"chuqurlik={chuqurlik:2d}  o'qitish={o:.3f}  validatsiya={v:.3f}")

# XATO
from sklearn.preprocessing import StandardScaler

X_masshtab = StandardScaler().fit_transform(X)
ballar = cross_val_score(model, X_masshtab, y, cv=5)
print(f"Sizib chiqish bilan: {ballar.mean():.3f}")

# TO'G'RI
from sklearn.pipeline import make_pipeline

quvur = make_pipeline(StandardScaler(), model)
ballar = cross_val_score(quvur, X, y, cv=5)
print(f"Pipeline bilan:      {ballar.mean():.3f}")

# Bemor kasalligini bashorat qilamiz
belgilar = [
    "yosh", "bosim", "xolesterin",
    "buyurilgan_dori",     # SHUBHALI!
]

# 1. Avval sinov to'plamini ajratamiz va UNGA TEGMAYMIZ
X_toliq, X_sinov, y_toliq, y_sinov = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# 2. Qolganini o'qitish va validatsiyaga bo'lamiz
X_o, X_val, y_o, y_val = train_test_split(
    X_toliq, y_toliq, test_size=0.25, random_state=42, stratify=y_toliq
)

print(f"O'qitish:    {len(X_o)}")
print(f"Validatsiya: {len(X_val)}")
print(f"Sinov:       {len(X_sinov)}")

from sklearn.model_selection import GridSearchCV

# 1. Sinov to'plamini ajratamiz
X_toliq, X_sinov, y_toliq, y_sinov = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# 2. Pipeline quramiz
quvur = make_pipeline(StandardScaler(), RandomForestClassifier(random_state=42))

# 3. Giperparametrlarni kross-validatsiya bilan tanlaymiz
tor = {
    "randomforestclassifier__n_estimators": [50, 100, 200],
    "randomforestclassifier__max_depth": [3, 5, 10, None],
}

qidiruv = GridSearchCV(quvur, tor, cv=5, n_jobs=-1, scoring="f1")
qidiruv.fit(X_toliq, y_toliq)

print(f"Eng yaxshi parametrlar: {qidiruv.best_params_}")
print(f"CV natijasi:            {qidiruv.best_score_:.3f}")

# 4. Faqat endi sinov to'plamini ochamiz
print(f"Yakuniy sinov:          {qidiruv.score(X_sinov, y_sinov):.3f}")