lixiaolong
b386f003a0
- 集成Qwen3.8-27B-FP8快速模型和Qwen3.8-27B精确模型作为SGLang服务 - 添加SGLang API配置选项(SGLANG_FAST_BASE_URL、SGLANG_ACCURATE_BASE_URL等) - 实现音频分类中的决策聚合算法(topK、nearestWeight、labelMaxDistance) - 添加视频采样帧限制(VIDEO_SAMPLING_FRAME_LIMIT)和上下文token限制 - 更新健康检查以监控SGLang服务状态 - 实现视频分析的双模式决策策略(快速+精确) - 添加音频参考文件导入工具(import_audio_references.py) - 扩展音频分类标签支持FIND_VEHICLE_HORN类别 - 优化视频分析的帧采样策略，始终包含视频尾部帧 - 添加决策策略参数(tuning、decisionPolicy)支持 - 更新配置类以支持新的SGLang和视频参数 - 修改compose配置以支持Qwen3.8模型部署 - 更新音频分类测试用例验证聚合逻辑 - 重构视频测试以支持SGLang API格式和决策策略
275 lines
11 KiB
Python
275 lines
11 KiB
Python
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
from scipy import signal
|
|
from scipy.fft import dct
|
|
from scipy.spatial.distance import cdist
|
|
|
|
from app.media import decode_audio
|
|
from app.profile_store import Profile, artifact_path
|
|
|
|
|
|
SUPPORTED_AUDIO_SUFFIXES = {".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg"}
|
|
|
|
|
|
def _hz_to_mel(hz: np.ndarray | float) -> np.ndarray | float:
|
|
return 2595.0 * np.log10(1.0 + np.asarray(hz) / 700.0)
|
|
|
|
|
|
def _mel_to_hz(mel: np.ndarray) -> np.ndarray:
|
|
return 700.0 * (np.power(10.0, mel / 2595.0) - 1.0)
|
|
|
|
|
|
def _mel_filterbank(sample_rate: int, n_fft: int, bands: int) -> np.ndarray:
|
|
mel_points = np.linspace(_hz_to_mel(40.0), _hz_to_mel(sample_rate / 2), bands + 2)
|
|
bins = np.floor((n_fft + 1) * _mel_to_hz(mel_points) / sample_rate).astype(int)
|
|
bins = np.clip(bins, 0, n_fft // 2)
|
|
filters = np.zeros((bands, n_fft // 2 + 1), dtype=np.float32)
|
|
for index in range(bands):
|
|
left, center, right = bins[index : index + 3]
|
|
if center > left:
|
|
filters[index, left:center] = np.linspace(0, 1, center - left, endpoint=False)
|
|
if right > center:
|
|
filters[index, center:right] = np.linspace(1, 0, right - center, endpoint=False)
|
|
return filters
|
|
|
|
|
|
def _trim_audio(samples: np.ndarray, sample_rate: int) -> np.ndarray:
|
|
if len(samples) < sample_rate // 4:
|
|
return samples
|
|
frame = max(1, int(sample_rate * 0.02))
|
|
energy = np.convolve(samples * samples, np.ones(frame) / frame, mode="same")
|
|
threshold = max(float(np.max(energy)) * 0.0025, 1e-7)
|
|
active = np.flatnonzero(energy >= threshold)
|
|
if not len(active):
|
|
return samples
|
|
padding = int(sample_rate * 0.15)
|
|
return samples[max(0, active[0] - padding) : min(len(samples), active[-1] + padding)]
|
|
|
|
|
|
def feature_sequence(
|
|
samples: np.ndarray, sample_rate: int, max_frames: int = 320
|
|
) -> np.ndarray:
|
|
samples = _trim_audio(samples.astype(np.float32), sample_rate)
|
|
samples -= float(np.mean(samples))
|
|
peak = float(np.max(np.abs(samples))) if len(samples) else 0.0
|
|
if peak < 1e-5:
|
|
raise ValueError("Audio contains no usable sound")
|
|
samples /= peak
|
|
|
|
n_fft = 512
|
|
if len(samples) < n_fft:
|
|
samples = np.pad(samples, (0, n_fft - len(samples)))
|
|
_, _, stft = signal.stft(
|
|
samples,
|
|
fs=sample_rate,
|
|
window="hann",
|
|
nperseg=n_fft,
|
|
noverlap=384,
|
|
boundary=None,
|
|
padded=False,
|
|
)
|
|
mel = _mel_filterbank(sample_rate, n_fft, 40) @ (np.abs(stft) ** 2)
|
|
coefficients = dct(np.log(mel + 1e-8), axis=0, norm="ortho")[1:21].T
|
|
|
|
mean = np.mean(coefficients, axis=0)
|
|
static = mean / max(float(np.linalg.norm(mean)), 1e-5)
|
|
normalized = (coefficients - mean) / np.maximum(np.std(coefficients, axis=0), 1e-5)
|
|
delta = np.gradient(normalized, axis=0)
|
|
sequence = np.concatenate(
|
|
[normalized, delta, np.tile(static * 0.75, (len(normalized), 1))], axis=1
|
|
).astype(np.float32)
|
|
sequence /= np.maximum(np.linalg.norm(sequence, axis=1, keepdims=True), 1e-5)
|
|
sequence = sequence[::2]
|
|
if len(sequence) > max_frames:
|
|
indices = np.linspace(0, len(sequence) - 1, max_frames).round().astype(int)
|
|
sequence = sequence[indices]
|
|
return sequence
|
|
|
|
|
|
def sequence_distance(first: np.ndarray, second: np.ndarray, band_ratio: float = 0.35) -> float:
|
|
distances = cdist(first, second, metric="cosine")
|
|
first_length, second_length = distances.shape
|
|
band = max(
|
|
abs(first_length - second_length) + 2,
|
|
int(max(first_length, second_length) * band_ratio),
|
|
)
|
|
previous = np.full(second_length + 1, np.inf, dtype=np.float32)
|
|
previous[0] = 0.0
|
|
for first_index in range(1, first_length + 1):
|
|
current = np.full(second_length + 1, np.inf, dtype=np.float32)
|
|
start = max(1, first_index - band)
|
|
end = min(second_length, first_index + band)
|
|
for second_index in range(start, end + 1):
|
|
current[second_index] = distances[first_index - 1, second_index - 1] + min(
|
|
current[second_index - 1],
|
|
previous[second_index],
|
|
previous[second_index - 1],
|
|
)
|
|
previous = current
|
|
return float(previous[second_length] / (first_length + second_length))
|
|
|
|
|
|
def _read_audio(path: Path) -> tuple[np.ndarray, int]:
|
|
raw, sample_rate = decode_audio(path)
|
|
return np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0, sample_rate
|
|
|
|
|
|
def build_audio_profile(profile: Profile) -> dict[str, Any]:
|
|
references = profile.directory / "references"
|
|
labels: list[str] = []
|
|
files: list[str] = []
|
|
durations: list[float] = []
|
|
sequences: list[np.ndarray] = []
|
|
for label_dir in sorted(path for path in references.glob("*") if path.is_dir()):
|
|
for path in sorted(label_dir.iterdir()):
|
|
if path.suffix.lower() not in SUPPORTED_AUDIO_SUFFIXES:
|
|
continue
|
|
samples, sample_rate = _read_audio(path)
|
|
labels.append(label_dir.name)
|
|
files.append(path.name)
|
|
durations.append(len(samples) / sample_rate)
|
|
sequences.append(feature_sequence(samples, sample_rate))
|
|
|
|
unique_labels = sorted(set(labels))
|
|
if len(unique_labels) < 2:
|
|
raise ValueError("At least two non-empty reference audio groups are required")
|
|
minimum_per_label = int(profile.config.get("minimumReferencesPerLabel", 1))
|
|
counts = {label: labels.count(label) for label in unique_labels}
|
|
insufficient = [label for label, count in counts.items() if count < minimum_per_label]
|
|
if insufficient:
|
|
raise ValueError(
|
|
f"Reference groups below minimum {minimum_per_label}: {', '.join(insufficient)}"
|
|
)
|
|
|
|
offsets = np.concatenate(([0], np.cumsum([len(sequence) for sequence in sequences])))
|
|
output = artifact_path(profile)
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
np.savez_compressed(
|
|
output,
|
|
sequence_features=np.concatenate(sequences),
|
|
sequence_offsets=offsets,
|
|
labels=np.asarray(labels),
|
|
files=np.asarray(files),
|
|
durations=np.asarray(durations, dtype=np.float32),
|
|
)
|
|
manifest = output.with_name("manifest.json")
|
|
manifest.write_text(
|
|
json.dumps(
|
|
{
|
|
"profileCode": profile.code,
|
|
"featureVersion": "mfcc-dtw-v2",
|
|
"labelCounts": counts,
|
|
"referenceCount": len(sequences),
|
|
"files": files,
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return {"referenceCount": len(sequences), "labelCounts": counts}
|
|
|
|
|
|
def load_reference_sequences(profile: Profile):
|
|
artifact = artifact_path(profile)
|
|
if not artifact.exists():
|
|
raise ValueError(
|
|
f"Profile {profile.code} has no feature library; add reference audio and rebuild it"
|
|
)
|
|
data = np.load(artifact, allow_pickle=False)
|
|
if "sequence_features" not in data or "sequence_offsets" not in data:
|
|
raise ValueError(f"Profile {profile.code} uses an obsolete feature library; rebuild it")
|
|
features = data["sequence_features"]
|
|
offsets = data["sequence_offsets"].astype(int)
|
|
sequences = [features[offsets[index] : offsets[index + 1]] for index in range(len(offsets) - 1)]
|
|
return sequences, data["labels"].astype(str), data["files"].astype(str)
|
|
|
|
|
|
def classify_sequence(
|
|
profile: Profile,
|
|
query: np.ndarray,
|
|
reference_sequences: list[np.ndarray],
|
|
labels: np.ndarray,
|
|
files: np.ndarray,
|
|
) -> dict[str, Any]:
|
|
decision = profile.config.get("decision", {})
|
|
top_k = max(1, int(decision.get("topK", 1)))
|
|
nearest_weight = float(decision.get("nearestWeight", 1.0))
|
|
nearest_weight = max(0.0, min(1.0, nearest_weight))
|
|
best_by_label: dict[str, dict[str, Any]] = {}
|
|
for reference, label, file_name in zip(reference_sequences, labels, files):
|
|
distance = sequence_distance(query, reference)
|
|
label = str(label)
|
|
current = best_by_label.setdefault(label, {"distances": []})
|
|
current["distances"].append((distance, str(file_name)))
|
|
|
|
for item in best_by_label.values():
|
|
ranked_distances = sorted(item.pop("distances"), key=lambda row: row[0])
|
|
nearest_distance, nearest_reference = ranked_distances[0]
|
|
neighbors = ranked_distances[:top_k]
|
|
aggregate_distance = (
|
|
nearest_weight * nearest_distance
|
|
+ (1.0 - nearest_weight) * float(np.mean([row[0] for row in neighbors]))
|
|
)
|
|
item.update(
|
|
{
|
|
"distance": aggregate_distance,
|
|
"nearestDistance": nearest_distance,
|
|
"reference": nearest_reference,
|
|
"neighborCount": len(neighbors),
|
|
}
|
|
)
|
|
|
|
ranking = sorted(best_by_label.items(), key=lambda item: item[1]["distance"])
|
|
winner_label, winner = ranking[0]
|
|
second_distance = ranking[1][1]["distance"] if len(ranking) > 1 else 1.0
|
|
distance_margin = second_distance - winner["distance"]
|
|
default_maximum_distance = float(decision.get("maxDistance", 0.45))
|
|
label_maximum_distances = decision.get("labelMaxDistance", {})
|
|
maximum_distance = float(
|
|
label_maximum_distances.get(winner_label, default_maximum_distance)
|
|
if isinstance(label_maximum_distances, dict)
|
|
else default_maximum_distance
|
|
)
|
|
minimum_margin = float(decision.get("minDistanceMargin", 0.015))
|
|
matched = winner["distance"] <= maximum_distance and distance_margin >= minimum_margin
|
|
output_label = winner_label if matched else str(decision.get("unknownLabel", "UNKNOWN"))
|
|
label_names = profile.config.get("labelNames", {})
|
|
return {
|
|
"label": output_label,
|
|
"labelName": label_names.get(output_label, output_label),
|
|
"matched": matched,
|
|
"confidence": round(max(0.0, min(1.0, 1.0 - winner["distance"])), 6),
|
|
"scores": {
|
|
label: round(max(0.0, min(1.0, 1.0 - item["distance"])), 6)
|
|
for label, item in ranking
|
|
},
|
|
"evidence": {
|
|
"reference": winner["reference"] if matched else None,
|
|
"nearestDistance": round(winner["nearestDistance"], 6),
|
|
"aggregateDistance": round(winner["distance"], 6),
|
|
"neighborCount": winner["neighborCount"],
|
|
},
|
|
"thresholds": {
|
|
"maxDistance": maximum_distance,
|
|
"minDistanceMargin": minimum_margin,
|
|
"topK": top_k,
|
|
"nearestWeight": nearest_weight,
|
|
},
|
|
}
|
|
|
|
|
|
def classify_audio(profile: Profile, media_path: Path) -> dict[str, Any]:
|
|
reference_sequences, labels, files = load_reference_sequences(profile)
|
|
samples, sample_rate = _read_audio(media_path)
|
|
maximum_seconds = float(profile.config.get("maximumAudioSeconds", 30))
|
|
if len(samples) / sample_rate > maximum_seconds:
|
|
raise ValueError(f"Audio duration exceeds the profile limit of {maximum_seconds:g} seconds")
|
|
query = feature_sequence(samples, sample_rate)
|
|
result = classify_sequence(profile, query, reference_sequences, labels, files)
|
|
result["evidence"].update({"startMs": 0, "endMs": round(len(samples) * 1000 / sample_rate)})
|
|
return result
|