12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455 |
"""演示25: 梯度灾难 — 梯度消失/爆炸 与 梯度裁剪。
1. 深层网络激活梯度随层数衰减/爆炸的示意曲线(理论: 梯度 × 权重谱半径 逐层乘)。
2. 梯度裁剪效果: 训练中出现梯度爆炸时, 裁剪 L2 范数到阈值可稳定训练。
"""
import os
import numpy as np
import matplotlib.pyplot as plt
from figure_style import figures_dir, configure
configure()
OUT = figures_dir()
os.makedirs(OUT, exist_ok=True)
layers = np.arange(1, 21)
def grad_norm(layer, spectral_radius):
"""梯度范数 ≈ λ^layer (λ = 权重谱半径)。"""
return spectral_radius ** layer
fig, axes = plt.subplots(1, 2, figsize=(12, 4.8))
# ① 梯度消失/爆炸
ax = axes[0]
for lam, name in [(0.8, "λ=0.8 (消失)"), (1.0, "λ=1.0 (稳定)"), (1.2, "λ=1.2 (爆炸)")]:
ax.plot(layers, grad_norm(layers, lam), lw=2, label=name)
ax.axhline(1.0, color="k", ls=":", lw=0.8)
ax.set_yscale("log")
ax.set_xlabel("层数(从输出往输入)"); ax.set_ylabel("梯度范数 (log)")
ax.set_title("梯度消失/爆炸: 逐层相乘导致指数衰减或放大")
ax.legend(); ax.grid(alpha=0.3)
# ② 梯度裁剪
ax = axes[1]
rng = np.random.default_rng(0)
steps = np.arange(200)
raw = np.cumsum(np.where(rng.random(200) < 0.02, rng.normal(0, 20, 200), rng.normal(0, 1, 200)))
def clip(g, threshold):
"""L2 范数裁剪到阈值内(标量情形用绝对值)。"""
norm = abs(g)
return np.clip(g, -threshold, threshold) if norm > threshold else g
clipped = [clip(g, 5.0) for g in raw]
ax.plot(raw, "tab:red", lw=1.2, alpha=0.7, label="无裁剪(有爆炸尖峰)")
ax.plot(np.array(clipped), "tab:blue", lw=1.8, label="L2 裁剪到 5.0")
ax.set_xlabel("训练步数"); ax.set_ylabel("梯度值")
ax.set_title("梯度裁剪: 限制单步更新幅度, 防爆炸")
ax.legend(); ax.grid(alpha=0.3)
fig.suptitle("梯度灾难: 消失/爆炸的诊断与梯度裁剪解法", fontsize=14)
fig.tight_layout(rect=(0, 0, 1, 0.93))
out = os.path.join(OUT, "fig_train_gradient.png")
fig.savefig(out, dpi=150)
print(f"-> {out}")
|
