|
|
|
|
@ -77,6 +77,56 @@ class SelectiveStandardScaler:
|
|
|
|
|
return self.fit(x).transform(x)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MaskedStandardScaler:
|
|
|
|
|
"""每个曲线列只使用实际物理时间覆盖范围内的有效值进行标准化。"""
|
|
|
|
|
|
|
|
|
|
def fit(self, x: np.ndarray, valid_mask: np.ndarray) -> "MaskedStandardScaler":
|
|
|
|
|
values = _ensure_2d("curve", x).astype(np.float64)
|
|
|
|
|
mask = np.asarray(valid_mask, dtype=bool)
|
|
|
|
|
if mask.shape != values.shape:
|
|
|
|
|
raise ValueError(f"curve mask shape mismatch: {mask.shape} != {values.shape}")
|
|
|
|
|
if not np.isfinite(values[mask]).all():
|
|
|
|
|
raise ValueError("valid curve values contain non-finite values")
|
|
|
|
|
|
|
|
|
|
counts = mask.sum(axis=0).astype(np.int64)
|
|
|
|
|
sums = np.where(mask, values, 0.0).sum(axis=0)
|
|
|
|
|
means = np.divide(sums, counts, out=np.zeros_like(sums), where=counts > 0)
|
|
|
|
|
centered = np.where(mask, values - means, 0.0)
|
|
|
|
|
variances = np.divide(
|
|
|
|
|
(centered * centered).sum(axis=0),
|
|
|
|
|
counts,
|
|
|
|
|
out=np.ones_like(sums),
|
|
|
|
|
where=counts > 0,
|
|
|
|
|
)
|
|
|
|
|
scales = np.sqrt(np.maximum(variances, 0.0))
|
|
|
|
|
scales[(counts == 0) | (scales < 1.0e-12)] = 1.0
|
|
|
|
|
|
|
|
|
|
self.n_features_in_ = int(values.shape[1])
|
|
|
|
|
self.n_samples_seen_ = counts
|
|
|
|
|
self.mean_ = means
|
|
|
|
|
self.var_ = variances
|
|
|
|
|
self.scale_ = scales
|
|
|
|
|
return self
|
|
|
|
|
|
|
|
|
|
def _validate(self, x: np.ndarray) -> np.ndarray:
|
|
|
|
|
if not hasattr(self, "n_features_in_"):
|
|
|
|
|
raise RuntimeError("MaskedStandardScaler is not fitted")
|
|
|
|
|
values = _ensure_2d("curve", x).astype(np.float64)
|
|
|
|
|
if values.shape[1] != self.n_features_in_:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
f"curve feature dimension mismatch: {values.shape[1]} != {self.n_features_in_}"
|
|
|
|
|
)
|
|
|
|
|
return values
|
|
|
|
|
|
|
|
|
|
def transform(self, x: np.ndarray) -> np.ndarray:
|
|
|
|
|
values = self._validate(x)
|
|
|
|
|
return (values - self.mean_) / self.scale_
|
|
|
|
|
|
|
|
|
|
def inverse_transform(self, x: np.ndarray) -> np.ndarray:
|
|
|
|
|
values = self._validate(x)
|
|
|
|
|
return values * self.scale_ + self.mean_
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _value_counts(values: np.ndarray | None, indices: np.ndarray) -> dict[str, int]:
|
|
|
|
|
"""按字符串键记录某个划分中的类别样本数。"""
|
|
|
|
|
if values is None:
|
|
|
|
|
@ -189,6 +239,47 @@ def _validate_optional_length(name: str, arr: np.ndarray | None, expected_n: int
|
|
|
|
|
raise ValueError(f"{name} row count does not match main arrays: {len(arr)} != {expected_n}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _validate_curve_time(
|
|
|
|
|
curve_time: np.ndarray,
|
|
|
|
|
expected_n: int,
|
|
|
|
|
expected_n_time_points: int,
|
|
|
|
|
) -> np.ndarray:
|
|
|
|
|
"""校验每条曲线对应的物理时间坐标。"""
|
|
|
|
|
values = np.asarray(curve_time, dtype=np.float64)
|
|
|
|
|
expected_shape = (expected_n, expected_n_time_points)
|
|
|
|
|
if values.ndim != 2 or values.shape != expected_shape:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
f"curve_time must have shape {expected_shape}, got shape={values.shape}"
|
|
|
|
|
)
|
|
|
|
|
if not np.all(np.isfinite(values)):
|
|
|
|
|
raise ValueError("curve_time contains non-finite values")
|
|
|
|
|
if np.any(values <= 0.0):
|
|
|
|
|
raise ValueError("curve_time values must be positive")
|
|
|
|
|
if np.any(np.diff(values, axis=1) <= 0.0):
|
|
|
|
|
raise ValueError("each curve_time row must be strictly increasing")
|
|
|
|
|
return values
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _validate_curve_valid_mask(
|
|
|
|
|
curve_valid_mask: np.ndarray,
|
|
|
|
|
expected_n: int,
|
|
|
|
|
expected_n_time_points: int,
|
|
|
|
|
) -> np.ndarray:
|
|
|
|
|
"""校验逐时间点记录物理覆盖范围的有效掩码。"""
|
|
|
|
|
values = np.asarray(curve_valid_mask)
|
|
|
|
|
expected_shape = (expected_n, expected_n_time_points)
|
|
|
|
|
if values.ndim != 2 or values.shape != expected_shape:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
f"curve_valid_mask must have shape {expected_shape}, got shape={values.shape}"
|
|
|
|
|
)
|
|
|
|
|
if not np.all(np.isfinite(values)) or not np.all((values == 0) | (values == 1)):
|
|
|
|
|
raise ValueError("curve_valid_mask must contain only 0/1 values")
|
|
|
|
|
values = values.astype(bool)
|
|
|
|
|
if np.any(values.sum(axis=1) < 2):
|
|
|
|
|
raise ValueError("each curve_valid_mask row must contain at least two valid points")
|
|
|
|
|
return values
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _curve_keep_indices(meta: dict, curve_dim: int) -> np.ndarray:
|
|
|
|
|
"""返回旧 processed 曲线中 pressure/derivative 两段的列索引。"""
|
|
|
|
|
layout = meta.get("curve_layout") or {}
|
|
|
|
|
@ -237,6 +328,46 @@ def migrate_processed_dataset_without_slope(input_path: Path, output_path: Path)
|
|
|
|
|
if keep_indices.size % 2 != 0:
|
|
|
|
|
raise ValueError(f"保留后的曲线维度必须能被 2 整除，当前为 {keep_indices.size}")
|
|
|
|
|
|
|
|
|
|
migration_n_time_points = int(keep_indices.size // 2)
|
|
|
|
|
migration_meta = dict(source.get("meta", {}))
|
|
|
|
|
if migration_meta.get("curve_time_mode") != "fixed":
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"processed migration requires meta.curve_time_mode='fixed'; "
|
|
|
|
|
"regenerate the H5 dataset on a fixed time grid and preprocess it again"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
prediction_curve_time = np.asarray(
|
|
|
|
|
migration_meta.get("prediction_curve_time"),
|
|
|
|
|
dtype=np.float64,
|
|
|
|
|
)
|
|
|
|
|
if prediction_curve_time.shape != (migration_n_time_points,):
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"processed migration requires meta.prediction_curve_time with shape "
|
|
|
|
|
f"({migration_n_time_points},); regenerate the H5 dataset and preprocess it again"
|
|
|
|
|
)
|
|
|
|
|
if (
|
|
|
|
|
not np.all(np.isfinite(prediction_curve_time))
|
|
|
|
|
or np.any(prediction_curve_time <= 0.0)
|
|
|
|
|
or np.any(np.diff(prediction_curve_time) <= 0.0)
|
|
|
|
|
):
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"processed migration requires a finite, positive, strictly increasing "
|
|
|
|
|
"meta.prediction_curve_time"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
for split in ("train", "val", "test"):
|
|
|
|
|
mask_key = f"curve_valid_mask_{split}"
|
|
|
|
|
if mask_key not in source:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
f"processed migration requires {mask_key}; regenerate the H5 dataset "
|
|
|
|
|
"on a fixed time grid and preprocess it again"
|
|
|
|
|
)
|
|
|
|
|
_validate_curve_valid_mask(
|
|
|
|
|
source[mask_key],
|
|
|
|
|
len(source[f"Y_curve_{split}"]),
|
|
|
|
|
migration_n_time_points,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
payload = dict(source)
|
|
|
|
|
for split in ("train", "val", "test"):
|
|
|
|
|
key = f"Y_curve_{split}"
|
|
|
|
|
@ -306,6 +437,8 @@ def preprocess_dataset(
|
|
|
|
|
x_params = _ensure_2d("params", f["params"][:])
|
|
|
|
|
x_schedule = _ensure_2d("schedule", f["schedule"][:])
|
|
|
|
|
y_curve = _ensure_2d("curve", f["curve"][:])
|
|
|
|
|
curve_time = _read_optional_numeric_dataset(f, "curve_time")
|
|
|
|
|
curve_valid_mask = _read_optional_numeric_dataset(f, "curve_valid_mask")
|
|
|
|
|
# 可选元数据会保留下来，供后续分析和排序验证使用；
|
|
|
|
|
# 基础正演训练循环不依赖这些字段。
|
|
|
|
|
param_names = _decode_attr_list(f.attrs.get("param_names"))
|
|
|
|
|
@ -347,19 +480,61 @@ def preprocess_dataset(
|
|
|
|
|
raise ValueError("params / schedule / curve row counts do not match")
|
|
|
|
|
|
|
|
|
|
raw_curve_dim = int(y_curve.shape[1])
|
|
|
|
|
if raw_curve_dim % 3 == 0:
|
|
|
|
|
# 兼容已有的 pressure/derivative/slope HDF5；预处理时丢弃未参与训练的 slope。
|
|
|
|
|
if curve_time is not None:
|
|
|
|
|
curve_time_shape = np.asarray(curve_time).shape
|
|
|
|
|
if len(curve_time_shape) != 2 or curve_time_shape[0] != n:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
f"curve_time must be a 2D array with {n} rows, got shape={curve_time_shape}"
|
|
|
|
|
)
|
|
|
|
|
n_time_points = int(curve_time_shape[1])
|
|
|
|
|
if raw_curve_dim == 3 * n_time_points:
|
|
|
|
|
# 兼容已有的压力/导数/斜率三通道 HDF5；训练时丢弃斜率通道。
|
|
|
|
|
y_curve = y_curve[:, : 2 * n_time_points]
|
|
|
|
|
elif raw_curve_dim != 2 * n_time_points:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"curve dimension does not match curve_time: "
|
|
|
|
|
f"curve_dim={raw_curve_dim}, n_time_points={n_time_points}; "
|
|
|
|
|
"expected 2*N or legacy 3*N"
|
|
|
|
|
)
|
|
|
|
|
elif raw_curve_dim % 3 == 0:
|
|
|
|
|
n_time_points = raw_curve_dim // 3
|
|
|
|
|
y_curve = y_curve[:, : 2 * n_time_points]
|
|
|
|
|
elif raw_curve_dim % 2 == 0:
|
|
|
|
|
n_time_points = raw_curve_dim // 2
|
|
|
|
|
else:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
f"curve 维度 {raw_curve_dim} 无法识别；期望 2*N（压力/导数）或 "
|
|
|
|
|
"3*N（旧版压力/导数/slope）"
|
|
|
|
|
f"curve dimension {raw_curve_dim} cannot be interpreted as 2*N or legacy 3*N"
|
|
|
|
|
)
|
|
|
|
|
curve_dim = int(y_curve.shape[1])
|
|
|
|
|
|
|
|
|
|
curve_time_mode = "missing"
|
|
|
|
|
prediction_curve_time = None
|
|
|
|
|
if curve_time is not None:
|
|
|
|
|
curve_time = _validate_curve_time(curve_time, n, n_time_points)
|
|
|
|
|
if np.allclose(curve_time, curve_time[0:1], rtol=1e-6, atol=1e-10):
|
|
|
|
|
curve_time_mode = "fixed"
|
|
|
|
|
prediction_curve_time = curve_time[0].copy()
|
|
|
|
|
else:
|
|
|
|
|
curve_time_mode = "per_sample"
|
|
|
|
|
|
|
|
|
|
if curve_time_mode != "fixed":
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"forward-surrogate preprocessing requires one fixed curve_time grid; "
|
|
|
|
|
f"got curve_time_mode={curve_time_mode!r}"
|
|
|
|
|
)
|
|
|
|
|
if curve_valid_mask is None:
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"fixed-time preprocessing requires curve_valid_mask; regenerate the H5 dataset"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
curve_valid_mask = _validate_curve_valid_mask(
|
|
|
|
|
curve_valid_mask,
|
|
|
|
|
n,
|
|
|
|
|
n_time_points,
|
|
|
|
|
)
|
|
|
|
|
curve_valid_mask_mode = "explicit"
|
|
|
|
|
curve_mask = np.concatenate([curve_valid_mask, curve_valid_mask], axis=1)
|
|
|
|
|
|
|
|
|
|
param_feature_transform = build_param_feature_transform(
|
|
|
|
|
param_names=param_names,
|
|
|
|
|
categorical_values=categorical_values,
|
|
|
|
|
@ -393,14 +568,20 @@ def preprocess_dataset(
|
|
|
|
|
x_params_train = x_params_features[idx_train]
|
|
|
|
|
x_schedule_train = x_schedule[idx_train]
|
|
|
|
|
y_curve_train = y_curve[idx_train]
|
|
|
|
|
curve_valid_mask_train = curve_valid_mask[idx_train]
|
|
|
|
|
curve_mask_train = curve_mask[idx_train]
|
|
|
|
|
|
|
|
|
|
x_params_val = x_params_features[idx_val]
|
|
|
|
|
x_schedule_val = x_schedule[idx_val]
|
|
|
|
|
y_curve_val = y_curve[idx_val]
|
|
|
|
|
curve_valid_mask_val = curve_valid_mask[idx_val]
|
|
|
|
|
curve_mask_val = curve_mask[idx_val]
|
|
|
|
|
|
|
|
|
|
x_params_test = x_params_features[idx_test]
|
|
|
|
|
x_schedule_test = x_schedule[idx_test]
|
|
|
|
|
y_curve_test = y_curve[idx_test]
|
|
|
|
|
curve_valid_mask_test = curve_valid_mask[idx_test]
|
|
|
|
|
curve_mask_test = curve_mask[idx_test]
|
|
|
|
|
|
|
|
|
|
schedule_meta_train = schedule_meta[idx_train] if schedule_meta is not None else None
|
|
|
|
|
schedule_meta_val = schedule_meta[idx_val] if schedule_meta is not None else None
|
|
|
|
|
@ -449,20 +630,32 @@ def preprocess_dataset(
|
|
|
|
|
)
|
|
|
|
|
scaler_params = SelectiveStandardScaler(scaled_param_indices)
|
|
|
|
|
scaler_schedule = StandardScaler()
|
|
|
|
|
scaler_curve = StandardScaler()
|
|
|
|
|
scaler_curve = MaskedStandardScaler()
|
|
|
|
|
|
|
|
|
|
valid_count_train = curve_valid_mask_train.sum(axis=0).astype(np.int64)
|
|
|
|
|
if prediction_curve_time is not None and np.any(valid_count_train == 0):
|
|
|
|
|
missing_columns = np.flatnonzero(valid_count_train == 0).tolist()
|
|
|
|
|
raise ValueError(
|
|
|
|
|
"fixed curve_time has training columns without physical coverage: "
|
|
|
|
|
f"{missing_columns}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# 只在训练集上拟合 scaler，避免验证/测试信息泄漏。
|
|
|
|
|
x_params_train_s = scaler_params.fit_transform(x_params_train)
|
|
|
|
|
x_schedule_train_s = scaler_schedule.fit_transform(x_schedule_train)
|
|
|
|
|
y_curve_train_s = scaler_curve.fit_transform(y_curve_train)
|
|
|
|
|
scaler_curve.fit(y_curve_train, curve_mask_train)
|
|
|
|
|
y_curve_train_s = scaler_curve.transform(y_curve_train)
|
|
|
|
|
y_curve_train_s[~curve_mask_train] = 0.0
|
|
|
|
|
|
|
|
|
|
x_params_val_s = scaler_params.transform(x_params_val)
|
|
|
|
|
x_schedule_val_s = scaler_schedule.transform(x_schedule_val)
|
|
|
|
|
y_curve_val_s = scaler_curve.transform(y_curve_val)
|
|
|
|
|
y_curve_val_s[~curve_mask_val] = 0.0
|
|
|
|
|
|
|
|
|
|
x_params_test_s = scaler_params.transform(x_params_test)
|
|
|
|
|
x_schedule_test_s = scaler_schedule.transform(x_schedule_test)
|
|
|
|
|
y_curve_test_s = scaler_curve.transform(y_curve_test)
|
|
|
|
|
y_curve_test_s[~curve_mask_test] = 0.0
|
|
|
|
|
|
|
|
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
|
|
|
|
@ -479,6 +672,14 @@ def preprocess_dataset(
|
|
|
|
|
"curve_dim": curve_dim,
|
|
|
|
|
"raw_curve_dim": raw_curve_dim,
|
|
|
|
|
"dropped_legacy_slope": bool(raw_curve_dim != curve_dim),
|
|
|
|
|
"curve_time_mode": curve_time_mode,
|
|
|
|
|
"curve_valid_mask_mode": curve_valid_mask_mode,
|
|
|
|
|
"curve_valid_fraction": {
|
|
|
|
|
"train": float(np.mean(curve_valid_mask_train)),
|
|
|
|
|
"val": float(np.mean(curve_valid_mask_val)),
|
|
|
|
|
"test": float(np.mean(curve_valid_mask_test)),
|
|
|
|
|
},
|
|
|
|
|
"curve_valid_count_train": valid_count_train.astype(int).tolist(),
|
|
|
|
|
"input_h5": str(input_path),
|
|
|
|
|
"param_names": param_names,
|
|
|
|
|
"param_feature_transform": param_feature_transform,
|
|
|
|
|
@ -508,6 +709,8 @@ def preprocess_dataset(
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
if prediction_curve_time is not None:
|
|
|
|
|
meta["prediction_curve_time"] = prediction_curve_time.tolist()
|
|
|
|
|
|
|
|
|
|
payload = {
|
|
|
|
|
"X_params_train": x_params_train_s.astype(np.float32),
|
|
|
|
|
@ -519,6 +722,9 @@ def preprocess_dataset(
|
|
|
|
|
"X_params_test": x_params_test_s.astype(np.float32),
|
|
|
|
|
"X_schedule_test": x_schedule_test_s.astype(np.float32),
|
|
|
|
|
"Y_curve_test": y_curve_test_s.astype(np.float32),
|
|
|
|
|
"curve_valid_mask_train": curve_valid_mask_train.astype(np.uint8),
|
|
|
|
|
"curve_valid_mask_val": curve_valid_mask_val.astype(np.uint8),
|
|
|
|
|
"curve_valid_mask_test": curve_valid_mask_test.astype(np.uint8),
|
|
|
|
|
"scaler_params": scaler_params,
|
|
|
|
|
"scaler_schedule": scaler_schedule,
|
|
|
|
|
"scaler_curve": scaler_curve,
|
|
|
|
|
@ -569,7 +775,8 @@ def preprocess_dataset(
|
|
|
|
|
print(
|
|
|
|
|
f"split_strategy={split_strategy}, "
|
|
|
|
|
f"solver_type_counts={meta['solver_type_counts']}, "
|
|
|
|
|
f"group_counts={meta['group_counts']}"
|
|
|
|
|
f"group_counts={meta['group_counts']}, "
|
|
|
|
|
f"curve_time_mode={curve_time_mode}"
|
|
|
|
|
)
|
|
|
|
|
if schedule_meta is not None:
|
|
|
|
|
print(f"schedule_meta_dim={schedule_meta.shape[1]}, family_name_saved={family_name is not None}")
|
|
|
|
|
|
