Amaliy loyiha - talaba muvaffaqiyatini bashorat qilish

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

np.random.seed(42)
sns.set_theme(style="whitegrid")

n = 1200

talabalar = pd.DataFrame({
    "yosh":            np.random.randint(17, 26, n),
    "oqish_soati":     np.random.gamma(3, 2.2, n).clip(0, 25),
    "davomat":         np.random.beta(6, 2, n) * 100,
    "oldingi_baho":    np.random.normal(3.6, 0.7, n).clip(2, 5),
    "uyqu_soati":      np.random.normal(7, 1.4, n).clip(3, 12),
    "topshiriqlar":    np.random.binomial(12, 0.72, n),
    "kutubxona":       np.random.poisson(4, n),
    "ishlaydi":        np.random.choice([0, 1], n, p=[0.65, 0.35]),
    "internet":        np.random.choice([0, 1], n, p=[0.15, 0.85]),
    "yonalish":        np.random.choice(
        ["Dasturlash", "Iqtisod", "Filologiya", "Muhandislik"], n),
    "turar_joy":       np.random.choice(
        ["Yotoqxona", "Uy", "Ijara"], n, p=[0.4, 0.45, 0.15]),
})

# Yashirin qonuniyat
ball = (
    talabalar["oqish_soati"] * 0.32
    + talabalar["davomat"] * 0.055
    + talabalar["oldingi_baho"] * 1.4
    + talabalar["topshiriqlar"] * 0.28
    + talabalar["kutubxona"] * 0.12
    - talabalar["ishlaydi"] * 0.9
    + np.random.normal(0, 1.3, n)
)

talabalar["otdi"] = (ball > np.percentile(ball, 28)).astype(int)

# Haqiqiy ma'lumotdagi kabi muammolar qo'shamiz
bosh_indeks = np.random.choice(n, 90, replace=False)
talabalar.loc[bosh_indeks[:50], "uyqu_soati"] = np.nan
talabalar.loc[bosh_indeks[50:], "kutubxona"] = np.nan
talabalar.loc[np.random.choice(n, 20, replace=False), "oqish_soati"] = 999

print(talabalar.shape)
print(talabalar["otdi"].value_counts(normalize=True).round(3))

print(talabalar.info())
print(talabalar.describe().round(2))

# Bo'sh qiymatlar
print((talabalar.isna().sum() / len(talabalar) * 100).round(1)[lambda s: s > 0])

# Chetdagi qiymatlar
print(f"999 qiymatlar soni: {(talabalar['oqish_soati'] == 999).sum()}")

sonli = talabalar.select_dtypes(include="number")

plt.figure(figsize=(10, 8))
sns.heatmap(sonli.corr(), annot=True, fmt=".2f", cmap="coolwarm", center=0)
plt.title("Belgilar korrelyatsiyasi")
plt.tight_layout()
plt.show()

print(sonli.corr()["otdi"].drop("otdi").sort_values(ascending=False).round(3))

# 999 - aniq xato, uni bo'sh deb belgilaymiz
talabalar.loc[talabalar["oqish_soati"] > 100, "oqish_soati"] = np.nan

# IQR bilan qolgan chetdagi qiymatlarni tekshiramiz
def chetdagilarni_sana(malumot, ustun):
    q1, q3 = malumot[ustun].quantile([0.25, 0.75])
    iqr = q3 - q1
    quyi, yuqori = q1 - 1.5 * iqr, q3 + 1.5 * iqr
    return ((malumot[ustun] < quyi) | (malumot[ustun] > yuqori)).sum()


for ustun in ["oqish_soati", "davomat", "uyqu_soati", "kutubxona"]:
    print(f"{ustun:14s}: {chetdagilarni_sana(talabalar, ustun)} ta chetdagi qiymat")

# Yangi belgilar
talabalar["topshiriq_foizi"] = talabalar["topshiriqlar"] / 12 * 100
talabalar["kunlik_oqish"] = talabalar["oqish_soati"] / 7
talabalar["faol_talaba"] = (
    (talabalar["davomat"] > 80) & (talabalar["topshiriqlar"] >= 10)
).astype(int)
talabalar["kam_uyqu"] = (talabalar["uyqu_soati"] < 6).astype(int)

from sklearn.model_selection import train_test_split

X = talabalar.drop(columns=["otdi"])
y = talabalar["otdi"]

assert "otdi" not in X.columns

X_toliq, X_sinov, y_toliq, y_sinov = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

print(f"O'qitish + validatsiya: {len(X_toliq)}")
print(f"Sinov (yopiq):          {len(X_sinov)}")
print(f"Sinf nisbati saqlandi:  {y_toliq.mean():.3f} / {y_sinov.mean():.3f}")

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer

sonli_ustunlar = [
    "yosh", "oqish_soati", "davomat", "oldingi_baho", "uyqu_soati",
    "topshiriqlar", "kutubxona", "topshiriq_foizi", "kunlik_oqish",
]
ikkilik_ustunlar = ["ishlaydi", "internet", "faol_talaba", "kam_uyqu"]
toifali_ustunlar = ["yonalish", "turar_joy"]

sonli_quvur = Pipeline([
    ("toldirish", SimpleImputer(strategy="median")),
    ("masshtab", StandardScaler()),
])

toifali_quvur = Pipeline([
    ("toldirish", SimpleImputer(strategy="most_frequent")),
    ("kodlash", OneHotEncoder(handle_unknown="ignore", drop="first")),
])

tayyorlovchi = ColumnTransformer([
    ("sonli", sonli_quvur, sonli_ustunlar),
    ("ikkilik", SimpleImputer(strategy="most_frequent"), ikkilik_ustunlar),
    ("toifali", toifali_quvur, toifali_ustunlar),
])

from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, HistGradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.dummy import DummyClassifier
from sklearn.model_selection import cross_validate, StratifiedKFold

bolinish = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

modellar = {
    "Asos (dummy)":  DummyClassifier(strategy="most_frequent"),
    "Logistik":      LogisticRegression(max_iter=1000, class_weight="balanced"),
    "Random Forest": RandomForestClassifier(
        n_estimators=200, class_weight="balanced", random_state=42, n_jobs=-1),
    "Boosting":      HistGradientBoostingClassifier(random_state=42),
    "SVM":           SVC(class_weight="balanced", probability=True, random_state=42),
}

natijalar = []

for nom, model in modellar.items():
    quvur = Pipeline([("tayyorlash", tayyorlovchi), ("model", model)])

    ballar = cross_validate(
        quvur, X_toliq, y_toliq, cv=bolinish,
        scoring=["accuracy", "precision", "recall", "f1", "roc_auc"],
    )

    natijalar.append({
        "Model":     nom,
        "Aniqlik":   ballar["test_accuracy"].mean(),
        "Precision": ballar["test_precision"].mean(),
        "Recall":    ballar["test_recall"].mean(),
        "F1":        ballar["test_f1"].mean(),
        "AUC":       ballar["test_roc_auc"].mean(),
    })

jadval = pd.DataFrame(natijalar).set_index("Model").round(3)
print(jadval)

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint, uniform

quvur = Pipeline([
    ("tayyorlash", tayyorlovchi),
    ("model", HistGradientBoostingClassifier(random_state=42)),
])

taqsimot = {
    "model__max_iter": randint(100, 400),
    "model__learning_rate": uniform(0.01, 0.25),
    "model__max_depth": [3, 5, 7, None],
    "model__min_samples_leaf": randint(5, 40),
    "model__l2_regularization": uniform(0, 2),
}

qidiruv = RandomizedSearchCV(
    quvur, taqsimot,
    n_iter=60,
    cv=bolinish,
    scoring="recall",
    n_jobs=-1,
    random_state=42,
)
qidiruv.fit(X_toliq, y_toliq)

print(f"Eng yaxshi CV recall: {qidiruv.best_score_:.3f}")
for kalit, qiymat in qidiruv.best_params_.items():
    print(f"  {kalit.replace('model__', ''):22s} = {qiymat}")

from sklearn.metrics import precision_recall_curve, f1_score

eng_yaxshi = qidiruv.best_estimator_
ehtimollik = eng_yaxshi.predict_proba(X_toliq)[:, 1]

print(f"{'Chegara':>8} {'Precision':>10} {'Recall':>8} {'F1':>7} {'Xavf ostida':>12}")

for chegara in [0.3, 0.4, 0.5, 0.6, 0.7]:
    bashorat = (ehtimollik >= chegara).astype(int)
    xavf_ostida = (bashorat == 0).sum()

    from sklearn.metrics import precision_score, recall_score
    print(f"{chegara:>8.1f} "
          f"{precision_score(y_toliq, bashorat):>10.3f} "
          f"{recall_score(y_toliq, bashorat):>8.3f} "
          f"{f1_score(y_toliq, bashorat):>7.3f} "
          f"{xavf_ostida:>12d}")

CHEGARA = 0.4

from sklearn.metrics import (
    classification_report, confusion_matrix,
    ConfusionMatrixDisplay, roc_auc_score,
)

ehtimollik_sinov = eng_yaxshi.predict_proba(X_sinov)[:, 1]
bashorat_sinov = (ehtimollik_sinov >= CHEGARA).astype(int)

print(classification_report(
    y_sinov, bashorat_sinov,
    target_names=["Xavf ostida", "O'tadi"],
))

print(f"AUC: {roc_auc_score(y_sinov, ehtimollik_sinov):.3f}")
print(confusion_matrix(y_sinov, bashorat_sinov))

from sklearn.inspection import permutation_importance

natija = permutation_importance(
    eng_yaxshi, X_sinov, y_sinov,
    n_repeats=20, random_state=42, scoring="recall", n_jobs=-1,
)

muhimlik = pd.Series(
    natija.importances_mean, index=X_sinov.columns
).sort_values(ascending=False)

print(muhimlik.head(8).round(4))

plt.figure(figsize=(9, 6))
muhimlik.head(10).sort_values().plot(kind="barh", color="steelblue")
plt.xlabel("Recall ga ta'siri")
plt.title("Belgilar muhimligi (permutatsiya usuli)")
plt.tight_layout()
plt.show()

def talabani_tushuntir(model, talaba, chegara=CHEGARA):
    ehtimol = model.predict_proba(talaba)[0, 1]
    holat = "XAVF OSTIDA" if ehtimol < chegara else "Xavf yo'q"

    sabablar = []
    q = talaba.iloc[0]

    if q["davomat"] < 70:
        sabablar.append(f"davomat past ({q['davomat']:.0f}%)")
    if q["topshiriqlar"] < 8:
        sabablar.append(f"topshiriqlar kam ({int(q['topshiriqlar'])}/12)")
    if q["oldingi_baho"] < 3.2:
        sabablar.append(f"oldingi baho past ({q['oldingi_baho']:.1f})")
    if q["oqish_soati"] < 4:
        sabablar.append(f"kam o'qiydi ({q['oqish_soati']:.1f} soat/hafta)")
    if q["ishlaydi"] == 1:
        sabablar.append("ishlaydi")

    return {
        "holat": holat,
        "otish_ehtimoli": round(float(ehtimol), 3),
        "sabablar": sabablar or ["aniq xavf omili topilmadi"],
    }


namuna = X_sinov.iloc[[3]]
natija = talabani_tushuntir(eng_yaxshi, namuna)

print(f"Holat:     {natija['holat']}")
print(f"Ehtimol:   {natija['otish_ehtimoli']}")
print(f"Sabablari: {', '.join(natija['sabablar'])}")

import joblib, json, sklearn
from datetime import datetime
from pathlib import Path

Path("modellar").mkdir(exist_ok=True)

joblib.dump(eng_yaxshi, "modellar/talaba_xavfi_v1.joblib", compress=3)

metama = {
    "nom": "talaba_xavfi",
    "versiya": "1.0.0",
    "yaratilgan": datetime.now().isoformat(),
    "sklearn": sklearn.__version__,
    "model_turi": "HistGradientBoostingClassifier",
    "chegara": CHEGARA,
    "belgilar": list(X.columns),
    "asosiy_metrika": "recall",
    "natijalar": {
        "cv_recall": round(float(qidiruv.best_score_), 4),
        "sinov_recall": round(float(recall_score(y_sinov, bashorat_sinov)), 4),
        "sinov_auc": round(float(roc_auc_score(y_sinov, ehtimollik_sinov)), 4),
    },
    "giperparametrlar": {
        k.replace("model__", ""): (float(v) if isinstance(v, float) else v)
        for k, v in qidiruv.best_params_.items()
    },
    "oqitish_namunalari": len(X_toliq),
    "cheklovlar": [
        "Model faqat 17-25 yoshli talabalar uchun tekshirilgan",
        "Sun'iy ma'lumotda o'qitilgan - haqiqiy ma'lumotda qayta sinash kerak",
        "Har semestr oxirida qayta o'qitilishi tavsiya etiladi",
    ],
}

with open("modellar/talaba_xavfi_v1.json", "w", encoding="utf-8") as fayl:
    json.dump(metama, fayl, ensure_ascii=False, indent=2)

print("Model saqlandi.")

# testlar/test_model.py
import pytest
import joblib
import pandas as pd


@pytest.fixture(scope="module")
def model():
    return joblib.load("modellar/talaba_xavfi_v1.joblib")


@pytest.fixture
def yaxshi_talaba():
    return pd.DataFrame([{
        "yosh": 20, "oqish_soati": 15, "davomat": 95, "oldingi_baho": 4.5,
        "uyqu_soati": 8, "topshiriqlar": 12, "kutubxona": 8,
        "ishlaydi": 0, "internet": 1,
        "yonalish": "Dasturlash", "turar_joy": "Yotoqxona",
        "topshiriq_foizi": 100, "kunlik_oqish": 15 / 7,
        "faol_talaba": 1, "kam_uyqu": 0,
    }])


def test_model_yuklanadi(model):
    assert hasattr(model, "predict_proba")


def test_yaxshi_talaba_otadi(model, yaxshi_talaba):
    ehtimol = model.predict_proba(yaxshi_talaba)[0, 1]
    assert ehtimol > 0.7


def test_yomon_talaba_xavf_ostida(model, yaxshi_talaba):
    yomon = yaxshi_talaba.copy()
    yomon.loc[0, ["oqish_soati", "davomat", "oldingi_baho",
                  "topshiriqlar", "topshiriq_foizi", "faol_talaba"]] = [
        1, 45, 2.3, 3, 25, 0
    ]
    ehtimol = model.predict_proba(yomon)[0, 1]
    assert ehtimol < 0.5


def test_davomat_ijobiy_tasir(model, yaxshi_talaba):
    past = yaxshi_talaba.copy()
    past.loc[0, "davomat"] = 50

    assert (model.predict_proba(yaxshi_talaba)[0, 1]
            > model.predict_proba(past)[0, 1])


def test_bosh_qiymat_bilan_ishlaydi(model, yaxshi_talaba):
    import numpy as np
    bosh = yaxshi_talaba.copy()
    bosh.loc[0, "uyqu_soati"] = np.nan

    natija = model.predict_proba(bosh)
    assert 0 <= natija[0, 1] <= 1

pytest testlar/ -v