{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 实验4：Deformable DETR 多尺度检测\n",
    "\n",
    "流程：**读取完整 COCO → 构建 train/val 教学子集 → 数据检查 → 构建数据集 → 构建模型与训练器 → 完整训练 → 测试集评估 → 推理 → loss、学习率和 mAP 曲线**。配置和训练由 MMDetection 官方源码执行，本 Notebook 负责准备、审计和记录。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 参数与环境准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pathlib：跨平台处理项目路径；json：保存实验清单；os：读取环境变量；\n",
    "# sys：把项目源码加入导入路径；time：测速；platform：记录运行平台。\n",
    "from pathlib import Path\n",
    "import json, os, sys, time, platform\n",
    "\n",
    "ROOT = Path.cwd()\n",
    "while not (ROOT / \"pyproject.toml\").exists() and ROOT != ROOT.parent:\n",
    "    ROOT = ROOT.parent\n",
    "sys.path.insert(0, str(ROOT / \"src\"))\n",
    "from rsdet.notebook_utils import (\n",
    "    set_seed,\n",
    "    build_coco_teaching_subset,\n",
    "    build_yolo_teaching_subset,\n",
    "    build_dota_teaching_subset,\n",
    "    read_yolo_names,\n",
    "    coco_summary,\n",
    "    print_manifest,\n",
    "    find_first_image,\n",
    ")\n",
    "\n",
    "SEED = int(os.getenv(\"SEED\", \"42\"))\n",
    "set_seed(SEED)\n",
    "RUN_TRAINING = bool(int(os.getenv(\"RUN_TRAINING\", \"0\")))\n",
    "# Notebook 读取完整数据源，再写出可追溯的教学子集；原始目录不被覆盖。\n",
    "SUBSET_MAX_IMAGES = int(os.getenv(\"SUBSET_MAX_IMAGES\", \"3000\"))\n",
    "SUBSET_MAX_CLASSES = int(os.getenv(\"SUBSET_MAX_CLASSES\", \"6\"))\n",
    "DATA_ROOT = Path(os.getenv(\"DATA_ROOT\", str(ROOT / \"data\" / \"processed\" / \"dior\")))\n",
    "RUN_ROOT = ROOT / \"runs\" / \"notebook_outputs\"\n",
    "RUN_ROOT.mkdir(parents=True, exist_ok=True)\n",
    "ARTIFACTS = {\"seed\": SEED, \"platform\": platform.platform(), \"python\": platform.python_version()}\n",
    "print({\"root\": str(ROOT), \"run_training\": RUN_TRAINING, \"subset_max_images\": SUBSET_MAX_IMAGES, \"subset_max_classes\": SUBSET_MAX_CLASSES, \"seed\": SEED})\n",
    "\n",
    "def save_json(name, payload):\n",
    "    \"\"\"将阶段性结果写入 runs/notebook_outputs，方便报告引用和复核。\"\"\"\n",
    "    path = RUN_ROOT / name\n",
    "    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2, default=str), encoding=\"utf-8\")\n",
    "    ARTIFACTS[name] = str(path)\n",
    "    return path\n",
    "\n",
    "def plot_history(history, name=\"training_curves.png\"):\n",
    "    \"\"\"读取训练历史并绘制 loss、学习率曲线；无历史时只给出提示。\"\"\"\n",
    "    if not history:\n",
    "        print(\"暂无训练历史；设置 RUN_TRAINING=1 后会保存曲线。\")\n",
    "        return None\n",
    "    try:\n",
    "        import matplotlib.pyplot as plt\n",
    "    except ImportError:\n",
    "        print(\"matplotlib 未安装，跳过曲线绘制。\")\n",
    "        return None\n",
    "    epochs = [row.get(\"epoch\", i + 1) for i, row in enumerate(history)]\n",
    "    fig, axes = plt.subplots(1, 2, figsize=(10, 4))\n",
    "    axes[0].plot(epochs, [row.get(\"loss\", float(\"nan\")) for row in history], marker=\"o\")\n",
    "    axes[0].set_title(\"loss\"); axes[0].set_xlabel(\"epoch\"); axes[0].grid(alpha=.3)\n",
    "    axes[1].plot(epochs, [row.get(\"lr\", float(\"nan\")) for row in history], marker=\"o\", color=\"darkorange\")\n",
    "    axes[1].set_title(\"learning rate\"); axes[1].set_xlabel(\"epoch\"); axes[1].grid(alpha=.3)\n",
    "    fig.tight_layout()\n",
    "    path = RUN_ROOT / name; fig.savefig(path, dpi=150); plt.close(fig)\n",
    "    ARTIFACTS[name] = str(path)\n",
    "    print(\"曲线已保存:\", path)\n",
    "    return path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 DIOR 数据准备：DIOR → COCO 格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importlib.util 检查 MMDetection 依赖；subprocess 调用官方训练和测试脚本。\n",
    "import importlib.util, subprocess\n",
    "MMDET_ROOT = Path(os.getenv(\"MMDET_ROOT\", str(ROOT / \"third_party\" / \"mmdetection-v3.1.0\")))\n",
    "SOURCE_COCO_ROOT = Path(os.getenv(\"COCO_ROOT\", str(ROOT / \"data\" / \"processed\" / \"dior\")))\n",
    "COCO_ROOT = RUN_ROOT / \"deformable_detr_subset\"\n",
    "COCO_ROOT.mkdir(parents=True, exist_ok=True)\n",
    "source_train = SOURCE_COCO_ROOT / \"annotations\" / \"train.json\"\n",
    "source_val = SOURCE_COCO_ROOT / \"annotations\" / \"val.json\"\n",
    "train_subset = build_coco_teaching_subset(\n",
    "    source_train, COCO_ROOT / \"annotations\" / \"train.json\",\n",
    "    max_images=SUBSET_MAX_IMAGES, max_classes=SUBSET_MAX_CLASSES, seed=SEED,\n",
    "    source_image_dir=SOURCE_COCO_ROOT,\n",
    "    class_names=[\"airplane\", \"ship\", \"vehicle\", \"bridge\", \"storage-tank\", \"harbor\"],\n",
    ")\n",
    "val_subset = build_coco_teaching_subset(\n",
    "    source_val, COCO_ROOT / \"annotations\" / \"val.json\",\n",
    "    max_images=max(1, SUBSET_MAX_IMAGES // 5), max_classes=SUBSET_MAX_CLASSES, seed=SEED,\n",
    "    source_image_dir=SOURCE_COCO_ROOT,\n",
    "    class_names=[\"airplane\", \"ship\", \"vehicle\", \"bridge\", \"storage-tank\", \"harbor\"],\n",
    ")\n",
    "ARTIFACTS[\"subset_manifest_train\"] = str(train_subset[\"manifest\"])\n",
    "ARTIFACTS[\"subset_manifest_val\"] = str(val_subset[\"manifest\"])\n",
    "CFG = ROOT / \"configs\" / \"deformable_detr_remote_sensing.py\"\n",
    "TRAIN_SCRIPT, TEST_SCRIPT = MMDET_ROOT / \"tools\" / \"train.py\", MMDET_ROOT / \"tools\" / \"test.py\"\n",
    "print(\"完整 COCO 数据已处理为教学子集:\", {\"train\": train_subset, \"val\": val_subset}, {\"mmdetection_root\": str(MMDET_ROOT), \"data_root\": str(COCO_ROOT), \"config\": str(CFG)})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 数据检查与类别统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 先检查目录规模和文件组织，不改变完整数据集内容。\n",
    "print(\"COCO 目录应包含 train/、val/ 和 annotations/train.json、annotations/val.json。\")\n",
    "if COCO_ROOT.exists(): print(\"COCO_ROOT 已存在，文件数:\", sum(1 for _ in COCO_ROOT.rglob(\"*\")))\n",
    "else: print(\"未发现真实 COCO_ROOT；只执行配置审计。\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 构建数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COCO 摘要函数只读取元信息：图像数、标注数、类别数和类别名称。\n",
    "annotation_candidates = [COCO_ROOT / \"annotations\" / \"train.json\", COCO_ROOT / \"annotations\" / \"val.json\", COCO_ROOT / \"annotations.json\"]\n",
    "annotation_files = [p for p in annotation_candidates if p.exists()]\n",
    "dataset_summary = {}\n",
    "for annotation_file in annotation_files:\n",
    "    try:\n",
    "        dataset_summary[str(annotation_file)] = coco_summary(annotation_file)\n",
    "    except (ValueError, KeyError, json.JSONDecodeError) as exc:\n",
    "        dataset_summary[str(annotation_file)] = {\"error\": str(exc)}\n",
    "print(\"COCO 数据集摘要:\", dataset_summary or \"尚无 COCO 标注文件\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5 构建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 配置审计用于提前发现类别数、数据根目录和训练轮数缺失。\n",
    "cfg_text = CFG.read_text(encoding=\"utf-8\") if CFG.exists() else \"\"\n",
    "required = [\"_base_\", \"data_root\", \"num_classes\", \"max_epochs\"]\n",
    "print(\"配置检查:\", {token: token in cfg_text for token in required})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6 构建训练器与命令"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"MMDetection 依赖状态:\", {name: importlib.util.find_spec(name) is not None for name in [\"mmengine\", \"mmdet\"]})\n",
    "def run_mmdet(stage=\"train\"):\n",
    "    \"\"\"组装 MMDetection 官方 train.py/test.py 命令并返回子进程结果。\"\"\"\n",
    "    script = TRAIN_SCRIPT if stage == \"train\" else TEST_SCRIPT\n",
    "    if not script.exists(): raise FileNotFoundError(script)\n",
    "    work_dir = RUN_ROOT / \"deformable_detr\"; cmd = [sys.executable, str(script), str(CFG)]\n",
    "    if stage == \"train\": cmd += [\"--work-dir\", str(work_dir)]\n",
    "    else: cmd += [str(work_dir / \"latest.pth\")]\n",
    "    print(\"执行命令:\", \" \".join(map(str, cmd))); return subprocess.run(cmd, cwd=MMDET_ROOT, check=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7 开始完整训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_TRAINING:\n",
    "    if not TRAIN_SCRIPT.exists() or not COCO_ROOT.exists(): raise RuntimeError(\"MMDetection、配置或 COCO 数据未准备好\")\n",
    "    run_mmdet(\"train\")\n",
    "else: print(\"RUN_TRAINING=0：跳过外部训练命令。\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8 测试集评估与性能指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_TRAINING and TEST_SCRIPT.exists(): run_mmdet(\"test\")\n",
    "print(\"正式报告应记录 COCO mAP、AP_small、收敛 epoch、FPS、显存和与 YOLOv8 的对照。\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9 测试集推理与可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"推理结果应保存预测 JSON、带框图像和小目标错误样本。\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10 绘制 loss、学习率和 mAP 曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_file = RUN_ROOT / \"deformable_detr\" / \"vis_data\" / \"scalars.json\"\n",
    "if log_file.exists():\n",
    "    try:\n",
    "        import matplotlib.pyplot as plt\n",
    "        rows = [json.loads(line) for line in log_file.read_text(encoding=\"utf-8\").splitlines() if line.strip()]\n",
    "        steps = [r.get(\"step\") for r in rows if r.get(\"step\") is not None]\n",
    "        losses = [r.get(\"loss\") for r in rows if r.get(\"loss\") is not None]\n",
    "        if steps and losses:\n",
    "            fig, ax = plt.subplots(figsize=(7, 4)); ax.plot(steps, losses, linewidth=1.5)\n",
    "            ax.set(xlabel=\"step\", ylabel=\"loss\", title=\"Deformable DETR loss\"); ax.grid(alpha=.3)\n",
    "            curve = RUN_ROOT / \"deformable_detr_curves.png\"; fig.tight_layout(); fig.savefig(curve, dpi=150); plt.close(fig)\n",
    "            ARTIFACTS[\"curve\"] = str(curve); print(\"曲线已保存:\", curve)\n",
    "        else:\n",
    "            print(\"日志存在，但尚未提供可绘制的 step/loss 字段。\")\n",
    "    except (ImportError, json.JSONDecodeError) as exc:\n",
    "        print(\"日志曲线绘制跳过:\", exc)\n",
    "else:\n",
    "    print(\"尚无 MMDetection 日志；正式训练后读取 scalars.json 绘制 loss、学习率和 mAP 曲线。\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 11 实验结论与排错"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json(\"deformable_detr_audit.json\", {\"config\": str(CFG), \"data_root\": str(COCO_ROOT), \"mmdetection_root\": str(MMDET_ROOT)})\n",
    "print(\"优先排查 _base_ 路径、num_classes、ann_file、评估器路径和显存。\")\n",
    "print_manifest(RUN_ROOT, **ARTIFACTS)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.9+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}