Zaxira nusxa va falokatdan tiklanish

# Biznes ta'sirini tahlil qilish

| Jarayon | 1 soat | 4 soat | 1 kun | 1 hafta |
|---------|--------|--------|-------|---------|
| Onlayn savdo | 5 mln | 20 mln | 120 mln | 800 mln |
| Ombor boshqaruvi | 0 | 2 mln | 15 mln | 100 mln |
| Buxgalteriya | 0 | 0 | 5 mln | 40 mln |
| Marketing | 0 | 0 | 0 | 10 mln |

Bu jadval ustuvorlikni belgilaydi: onlayn savdo birinchi tiklanadi.

# Odatiy jadval
# Yakshanba: to'liq zaxira
# Dushanba-Shanba: inkremental

0 2 * * 0 /usr/local/bin/zaxira.sh --toliq
0 2 * * 1-6 /usr/local/bin/zaxira.sh --inkremental

#!/bin/bash
# db-zaxira.sh
set -euo pipefail

SANA=$(date +%F_%H%M)
ZAXIRA=/zaxira/db
[email protected]

mkdir -p "$ZAXIRA"

echo "==> Mantiqiy zaxira (mysqldump)"
mysqldump \
    --single-transaction \
    --routines --triggers --events \
    --default-character-set=utf8mb4 \
    --set-gtid-purged=OFF \
    dokon | gzip -9 > "$ZAXIRA/dokon_${SANA}.sql.gz"

echo "==> Shifrlash"
gpg --encrypt --recipient "$KALIT_ID" --trust-model always \
    "$ZAXIRA/dokon_${SANA}.sql.gz"
rm "$ZAXIRA/dokon_${SANA}.sql.gz"

echo "==> Tekshirish"
gpg --decrypt "$ZAXIRA/dokon_${SANA}.sql.gz.gpg" 2>/dev/null \
    | gunzip -t
echo "Arxiv butun"

echo "==> Xesh"
sha256sum "$ZAXIRA/dokon_${SANA}.sql.gz.gpg" \
    > "$ZAXIRA/dokon_${SANA}.sql.gz.gpg.sha256"

echo "==> Bulutga yuborish"
aws s3 cp "$ZAXIRA/dokon_${SANA}.sql.gz.gpg" \
    "s3://dokon-zaxira/db/" \
    --storage-class STANDARD_IA

echo "==> Eskilarini tozalash (30 kundan eski)"
find "$ZAXIRA" -name "*.gpg" -mtime +30 -delete
find "$ZAXIRA" -name "*.sha256" -mtime +30 -delete

echo "==> Tayyor: dokon_${SANA}.sql.gz.gpg"

# Nuqta-vaqtga tiklash uchun binlog
# my.cnf
[mysqld]
log_bin = /var/log/mysql/mysql-bin
binlog_format = ROW
expire_logs_days = 14

# Kechagi to'liq zaxira + bugungi binlog = istalgan vaqtga tiklash
mysql dokon < dokon_2026-09-15.sql

mysqlbinlog \
    --start-datetime="2026-09-15 02:00:00" \
    --stop-datetime="2026-09-16 14:22:00" \
    /var/log/mysql/mysql-bin.* | mysql dokon

# Restic - zamonaviy, shifrlangan, dedublikatsiyali
export RESTIC_REPOSITORY="s3:s3.amazonaws.com/dokon-zaxira"
export RESTIC_PASSWORD_FILE="/root/.restic-parol"

# Birinchi marta
restic init

# Zaxira
restic backup /var/www/dokon/storage \
    --exclude-file=/etc/restic/istisnolar.txt \
    --tag kunlik

# Ro'yxat
restic snapshots

# Tiklash
restic restore latest --target /tiklash

# Faqat bitta fayl
restic restore latest --target /tiklash --include /var/www/dokon/storage/muhim.pdf

# Eskilarini tozalash
restic forget \
    --keep-daily 7 \
    --keep-weekly 4 \
    --keep-monthly 12 \
    --keep-yearly 3 \
    --prune

# Yaxlitlikni tekshirish
restic check --read-data-subset=10%

# /etc/restic/istisnolar.txt
*.log
*.tmp
node_modules/
vendor/
.git/
cache/

# S3 Object Lock - o'chirib bo'lmaydigan saqlash
aws s3api create-bucket \
    --bucket dokon-zaxira-immutable \
    --object-lock-enabled-for-bucket

aws s3api put-object-lock-configuration \
    --bucket dokon-zaxira-immutable \
    --object-lock-configuration '{
        "ObjectLockEnabled": "Enabled",
        "Rule": {
            "DefaultRetention": {
                "Mode": "COMPLIANCE",
                "Days": 30
            }
        }
    }'

# Zaxira uchun alohida hisob va cheklangan huquq
# IAM siyosati: faqat yozish, o'chirish yo'q

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:PutObject",
                "s3:GetBucketLocation"
            ],
            "Resource": [
                "arn:aws:s3:::dokon-zaxira/*",
                "arn:aws:s3:::dokon-zaxira"
            ]
        }
    ]
}

#!/bin/bash
# tiklash-sinovi.sh - har oy avtomatik ishga tushadi
set -euo pipefail

SINOV_BAZA="dokon_tiklash_sinovi"
HISOBOT="/var/log/tiklash-sinovi-$(date +%F).log"

{
    echo "=== Tiklash sinovi: $(date -Iseconds) ==="

    BOSHLANDI=$(date +%s)

    echo "1. Oxirgi zaxirani yuklab olish"
    OXIRGI=$(aws s3 ls s3://dokon-zaxira/db/ | sort | tail -1 | awk '{print $4}')
    aws s3 cp "s3://dokon-zaxira/db/$OXIRGI" /tmp/

    echo "2. Xeshni tekshirish"
    aws s3 cp "s3://dokon-zaxira/db/${OXIRGI}.sha256" /tmp/
    (cd /tmp && sha256sum -c "${OXIRGI}.sha256")

    echo "3. Shifrni ochish"
    gpg --decrypt "/tmp/$OXIRGI" > /tmp/tiklash.sql.gz

    echo "4. Bazaga tiklash"
    mysql -e "DROP DATABASE IF EXISTS $SINOV_BAZA; CREATE DATABASE $SINOV_BAZA;"
    gunzip -c /tmp/tiklash.sql.gz | mysql "$SINOV_BAZA"

    TUGADI=$(date +%s)
    VAQT=$((TUGADI - BOSHLANDI))

    echo "5. Ma'lumotni tekshirish"
    for jadval in mijozlar buyurtmalar mahsulotlar; do
        SONI=$(mysql -N -e "SELECT COUNT(*) FROM $SINOV_BAZA.$jadval;")
        echo "   $jadval: $SONI qator"

        if [ "$SONI" -eq 0 ]; then
            echo "   XATO: $jadval bo'sh!"
            exit 1
        fi
    done

    echo "6. Oxirgi yozuv sanasi"
    mysql -N -e "SELECT MAX(yaratilgan) FROM $SINOV_BAZA.buyurtmalar;"

    echo "7. Tozalash"
    mysql -e "DROP DATABASE $SINOV_BAZA;"
    rm -f /tmp/tiklash.sql.gz "/tmp/$OXIRGI"*

    echo "=== MUVAFFAQIYATLI. Tiklash vaqti: ${VAQT} soniya ==="

} 2>&1 | tee "$HISOBOT"

# Natijani yuborish
mail -s "Tiklash sinovi: $(date +%F)" [email protected] < "$HISOBOT"

# MySQL replikatsiya (asosiy server)
[mysqld]
server-id = 1
log_bin = /var/log/mysql/mysql-bin
binlog_format = ROW
gtid_mode = ON
enforce_gtid_consistency = ON

-- Nusxa serverda
CHANGE REPLICATION SOURCE TO
    SOURCE_HOST = '10.0.2.10',
    SOURCE_USER = 'replikator',
    SOURCE_PASSWORD = '...',
    SOURCE_AUTO_POSITION = 1;

START REPLICA;
SHOW REPLICA STATUS\G

-- Kechikishni kuzatish
SELECT
    SERVICE_STATE,
    LAST_ERROR_MESSAGE,
    APPLYING_TRANSACTION
FROM performance_schema.replication_applier_status_by_worker;

# Falokatdan tiklanish rejasi

## 1. Ssenariylar

| Ssenariy | Ehtimol | Ta'sir | RTO |
|----------|---------|--------|-----|
| Bitta server qulashi | Yuqori | Past | 15 daq |
| Ma'lumotlar bazasi buzilishi | O'rta | Yuqori | 2 soat |
| Serverxona ishdan chiqishi | Past | Kritik | 8 soat |
| Ransomware | O'rta | Kritik | 24 soat |
| Provayder to'liq ishlamasligi | Past | Kritik | 12 soat |

## 2. Tiklash ustuvorligi

1. Ma'lumotlar bazasi
2. API xizmati
3. Veb sayt
4. Admin panel
5. Hisobot tizimi
6. Ichki vositalar

## 3. Kontaktlar

| Rol | Kim | Telefon | Zaxira |
|-----|-----|---------|--------|
| Tiklash rahbari | @husanboy | +998 90 xxx | @aziz |
| Baza | @aziz | +998 91 xxx | Tashqi maslahatchi |
| Tarmoq | @jasur | +998 93 xxx | Provayder |
| Bulut provayder | AWS Support | Case #xxx | — |

## 4. Tiklash qadamlari (baza buzilishi)

1. Muammoni tasdiqlash (5 daq)
2. Xizmatni texnik ish rejimiga o'tkazish (2 daq)
3. Oxirgi zaxirani aniqlash (5 daq)
4. Yangi server tayyorlash (20 daq)
5. Zaxirani tiklash (45 daq)
6. Binlog bilan nuqta-vaqtga tiklash (15 daq)
7. Ma'lumotni tekshirish (15 daq)
8. Ilovani ulash (5 daq)
9. Bosqichma-bosqich ishga tushirish (10 daq)

**Jami: ~2 soat**

## 5. Aloqa

- Status sahifasi: status.dokon.uz
- Mijozlarga: 30 daqiqada bir marta yangilanish
- Ichki: Slack #falokat kanali

# Status sahifasi (alohida infratuzilmada!)
# Asosiy sayt ishlamaganda ham u ishlashi kerak

# EBS snapshot (AWS)
aws ec2 create-snapshot \
    --volume-id vol-0123456789abcdef0 \
    --description "Kunlik zaxira $(date +%F)" \
    --tag-specifications 'ResourceType=snapshot,Tags=[{Key=Nomi,Value=dokon-kunlik}]'

# Avtomatik siyosat
aws dlm create-lifecycle-policy \
    --execution-role-arn arn:aws:iam::123456789012:role/AWSDataLifecycleManagerDefaultRole \
    --description "Kunlik snapshot" \
    --state ENABLED \
    --policy-details file://siyosat.json

{
    "ResourceTypes": ["VOLUME"],
    "TargetTags": [{"Key": "Zaxira", "Value": "kunlik"}],
    "Schedules": [{
        "Name": "Kunlik",
        "CreateRule": {"Interval": 24, "IntervalUnit": "HOURS", "Times": ["02:00"]},
        "RetainRule": {"Count": 7},
        "CopyTags": true
    }]
}

final class ZaxiraNazorati
{
    private const MAKS_YOSH_SOAT = 26;   // kunlik zaxira + zaxira vaqt
    private const MIN_HAJM_MB = 100;

    public function tekshiring(): array
    {
        $muammolar = [];

        $oxirgi = $this->zaxiralar->oxirgisi();

        if ($oxirgi === null) {
            return ['Zaxira umuman topilmadi'];
        }

        $yosh = (time() - $oxirgi->vaqt) / 3600;

        if ($yosh > self::MAKS_YOSH_SOAT) {
            $muammolar[] = sprintf('Oxirgi zaxira %.1f soat oldin', $yosh);
        }

        if ($oxirgi->hajmMb < self::MIN_HAJM_MB) {
            $muammolar[] = sprintf('Zaxira hajmi juda kichik: %d MB', $oxirgi->hajmMb);
        }

        // Hajm keskin o'zgardimi?
        $oldingi = $this->zaxiralar->oldingisi();

        if ($oldingi !== null) {
            $farq = abs($oxirgi->hajmMb - $oldingi->hajmMb) / $oldingi->hajmMb;

            if ($farq > 0.3) {
                $muammolar[] = sprintf('Hajm %.0f%% ga o\'zgardi', $farq * 100);
            }
        }

        // Oxirgi tiklash sinovi
        $sinov = $this->sinovlar->oxirgisi();

        if ($sinov === null || (time() - $sinov->vaqt) > 35 * 86400) {
            $muammolar[] = 'Tiklash sinovi 35 kundan beri o\'tkazilmagan';
        }

        return $muammolar;
    }
}