123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051 |
"""演示20: 参数初始化策略 — Xavier vs He vs 零初始化。
在 10 层全连接网络上比较三种初始化:
- 零初始化: 梯度对称, 无法学习;
- Xavier: 适合 Sigmoid/Tanh, 保持前向方差;
- He: 适合 ReLU, 修正 ReLU 的方差折半问题。
观察 10 层后激活值分布(是否发散/消失)。
"""
import os
import numpy as np
import matplotlib.pyplot as plt
from figure_style import figures_dir, configure
configure()
OUT = figures_dir()
os.makedirs(OUT, exist_ok=True)
rng = np.random.default_rng(0)
n_in = 100
x = rng.normal(0, 1, (1000, n_in)) # 输入: 1000 样本
strategies = {
"Zero": lambda fan_in, fan_out: np.zeros((fan_in, fan_out)),
"Xavier": lambda fan_in, fan_out: rng.normal(0, np.sqrt(2 / (fan_in + fan_out)),
(fan_in, fan_out)),
"He": lambda fan_in, fan_out: rng.normal(0, np.sqrt(2 / fan_in), (fan_in, fan_out)),
}
fig, axes = plt.subplots(1, 3, figsize=(14, 4.5))
for (name, init), ax in zip(strategies.items(), axes):
h = x.copy()
for layer in range(10):
W = init(n_in, n_in)
h = np.maximum(0, h @ W) # ReLU 激活
if np.isnan(h).any() or np.abs(h).max() > 1e6:
ax.set_title(f"{name}: 第 {layer+1} 层后发散/消失 (|h|>1e6)")
break
ax.hist(h.ravel(), bins=80, color="tab:blue")
ax.set_title(f"{name} 初始化: 10 层后激活分布")
ax.set_xlim(-3, 3)
mean_abs = np.abs(h).mean()
ax.text(0.02, 0.95, f"均值|h| = {mean_abs:.3f}", transform=ax.transAxes,
va="top", fontsize=9)
fig.suptitle("参数初始化: 决定梯度能否流动 (He 适合 ReLU, 零初始化瘫痪)", fontsize=14)
fig.tight_layout(rect=(0, 0, 1, 0.93))
out = os.path.join(OUT, "fig_train_init.png")
fig.savefig(out, dpi=150)
print(f"-> {out}")
|
