Ma'lumotni tayyorlash

import pandas as pd
import numpy as np

np.random.seed(42)

uylar = pd.DataFrame({
    "maydon":   [65, 90, np.nan, 45, 78, 110, 55, 5000, 82, 95],
    "xonalar":  [2, 3, 4, 1, 3, 4, 2, 3, 3, 3],
    "tuman":    ["Chilonzor", "Sergeli", "Chilonzor", None, "Yunusobod",
                 "Sergeli", "Chilonzor", "Yunusobod", "Sergeli", "Chilonzor"],
    "holati":   ["yangi", "o'rta", "eski", "o'rta", "yangi",
                 "yangi", "eski", "o'rta", "yangi", "o'rta"],
    "yosh":     [12, 5, 30, 25, 3, 1, 40, 8, 4, 15],
    "narx":     [620, 780, 950, 380, 810, 1200, 470, 850, 820, 760],
})

print(uylar.isna().sum())
print(uylar.describe())

# Ustunlar bo'yicha foiz
bosh_foiz = (uylar.isna().sum() / len(uylar) * 100).round(1)
print(bosh_foiz[bosh_foiz > 0])

from sklearn.impute import SimpleImputer

# Sonli ustunlar - mediana bilan
sonli_toldiruvchi = SimpleImputer(strategy="median")
uylar[["maydon"]] = sonli_toldiruvchi.fit_transform(uylar[["maydon"]])

# Toifali ustunlar - eng ko'p uchraydigan bilan
toifali_toldiruvchi = SimpleImputer(strategy="most_frequent")
uylar[["tuman"]] = toifali_toldiruvchi.fit_transform(uylar[["tuman"]])

print(uylar.isna().sum().sum())

def chetdagilarni_top(malumot, ustun, koeffitsiyent=1.5):
    q1 = malumot[ustun].quantile(0.25)
    q3 = malumot[ustun].quantile(0.75)
    iqr = q3 - q1

    quyi = q1 - koeffitsiyent * iqr
    yuqori = q3 + koeffitsiyent * iqr

    return malumot[(malumot[ustun] < quyi) | (malumot[ustun] > yuqori)]


chetdagilar = chetdagilarni_top(uylar, "maydon")
print(chetdagilar[["maydon", "narx"]])

# Chegaralash (winsorization)
quyi = uylar["maydon"].quantile(0.01)
yuqori = uylar["maydon"].quantile(0.99)
uylar["maydon"] = uylar["maydon"].clip(quyi, yuqori)

# O'chirish
uylar = uylar[uylar["maydon"] < 500]

kodlangan = pd.get_dummies(uylar, columns=["tuman"], drop_first=True, dtype=int)
print(kodlangan.filter(like="tuman").head())

tartib = {"eski": 0, "o'rta": 1, "yangi": 2}
uylar["holati_kod"] = uylar["holati"].map(tartib)

print(uylar[["holati", "holati_kod"]].head())

# 500 ta noyob shahar bo'lsa - one-hot 500 ta ustun yaratadi
print(uylar["shahar"].nunique())

# Yechim 1: eng ko'p uchraydiganlarni qoldirish
eng_kop = uylar["shahar"].value_counts().nlargest(10).index
uylar["shahar"] = uylar["shahar"].where(
    uylar["shahar"].isin(eng_kop), "boshqa"
)

# Yechim 2: nishon bo'yicha kodlash
ortacha_narx = uylar.groupby("shahar")["narx"].mean()
uylar["shahar_kod"] = uylar["shahar"].map(ortacha_narx)

from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler

# Standartlashtirish: o'rtacha 0, standart og'ish 1
standart = StandardScaler()
X_standart = standart.fit_transform(X)

# Min-max: [0, 1] oralig'iga
minmax = MinMaxScaler()
X_minmax = minmax.fit_transform(X)

# Robust: mediana va IQR asosida - chetdagi qiymatlarga chidamli
robust = RobustScaler()
X_robust = robust.fit_transform(X)

from sklearn.model_selection import train_test_split

X = kodlangan.drop(columns=["narx"])
y = kodlangan["narx"]

X_oqitish, X_sinov, y_oqitish, y_sinov = train_test_split(
    X, y,
    test_size=0.2,        # 20% sinov uchun
    random_state=42,      # takrorlanuvchanlik
)

print(f"O'qitish: {X_oqitish.shape[0]} namuna")
print(f"Sinov:    {X_sinov.shape[0]} namuna")

# XATO: butun ma'lumotda masshtablash
X_masshtab = standart.fit_transform(X)
X_oqitish, X_sinov = train_test_split(X_masshtab, ...)

# TO'G'RI: avval bo'lish, keyin masshtablash
X_oqitish, X_sinov, y_oqitish, y_sinov = train_test_split(X, y, test_size=0.2)

standart = StandardScaler()
X_oqitish = standart.fit_transform(X_oqitish)   # fit + transform
X_sinov   = standart.transform(X_sinov)         # faqat transform

X_oqitish, X_sinov, y_oqitish, y_sinov = train_test_split(
    X, y, test_size=0.2, random_state=42,
    stratify=y,        # toifalar nisbati ikkalasida bir xil bo'ladi
)

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer

sonli_ustunlar  = ["maydon", "xonalar", "yosh"]
toifali_ustunlar = ["tuman", "holati"]

sonli_quvur = Pipeline([
    ("toldirish", SimpleImputer(strategy="median")),
    ("masshtab", StandardScaler()),
])

toifali_quvur = Pipeline([
    ("toldirish", SimpleImputer(strategy="most_frequent")),
    ("kodlash", OneHotEncoder(handle_unknown="ignore", drop="first")),
])

tayyorlovchi = ColumnTransformer([
    ("sonli", sonli_quvur, sonli_ustunlar),
    ("toifali", toifali_quvur, toifali_ustunlar),
])

X_oqitish_tayyor = tayyorlovchi.fit_transform(X_oqitish)
X_sinov_tayyor   = tayyorlovchi.transform(X_sinov)

uylar["narx_m2"]      = uylar["narx"] / uylar["maydon"]
uylar["xona_maydoni"] = uylar["maydon"] / uylar["xonalar"]
uylar["yangi_uy"]     = (uylar["yosh"] < 5).astype(int)

# Sana ustunidan
uylar["oy"]       = uylar["sana"].dt.month
uylar["hafta_kun"] = uylar["sana"].dt.dayofweek
uylar["dam_olish"] = (uylar["hafta_kun"] >= 5).astype(int)