diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..dabd5ad --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,30 @@ +SoundInputStream: + device: default + channels: 1 + sample_rate: 16000 + dtype: int16 + blocksize: 256 + latency: low + + +WakeWordRunner: + model_paths: + - models/hi_amy.onnx + - models/hey_amy.onnx + - models/hello_amy.onnx + sample_rate: 16000 + threshold: 0.3 + cooldown_sec: 1.5 + reset_sec: 3.0 + url: http://xxx.xxx.xxx.xxx:xxxx/xx/xx/xx + +ASRRunner: + asr_model: paraformer-zh-streaming + punc_model: ct-punc + disable_update: True + device: cpu + encoder_chunk_look_back: 4 + decoder_chunk_look_back: 1 + max_duration: 20 + max_empty_txt_count: 3 + url: http://xxx.xxx.xxx.xxx:xxxx/xx/xx/xx \ No newline at end of file diff --git a/models/hello_amy.onnx b/models/hello_amy.onnx new file mode 100644 index 0000000..bac8a19 Binary files /dev/null and b/models/hello_amy.onnx differ diff --git a/models/hello_amy.tflite b/models/hello_amy.tflite new file mode 100644 index 0000000..5f1d161 Binary files /dev/null and b/models/hello_amy.tflite differ diff --git a/models/hey_amy.onnx b/models/hey_amy.onnx new file mode 100644 index 0000000..03474d2 Binary files /dev/null and b/models/hey_amy.onnx differ diff --git a/models/hey_amy.tflite b/models/hey_amy.tflite new file mode 100644 index 0000000..9a41d15 Binary files /dev/null and b/models/hey_amy.tflite differ diff --git a/models/hi_amy.onnx b/models/hi_amy.onnx new file mode 100644 index 0000000..ae84a75 Binary files /dev/null and b/models/hi_amy.onnx differ diff --git a/models/hi_amy.tflite b/models/hi_amy.tflite new file mode 100644 index 0000000..456fb6d Binary files /dev/null and b/models/hi_amy.tflite differ diff --git a/run.py b/run.py new file mode 100644 index 0000000..bb2469b --- /dev/null +++ b/run.py @@ -0,0 +1,14 @@ +import yaml +from src.asr_runner import ASRRunner +from src.sound_input_stream import SoundInputStream +from src.wake_word_runner import WakeWordRunner + +with open(r"config/config.yaml", "r") as f: + cfg = yaml.load(f, Loader=yaml.FullLoader) + +s = SoundInputStream(cfg["SoundInputStream"]) +w = WakeWordRunner(cfg['WakeWordRunner']) +a = ASRRunner(cfg['ASRRunner']) + +import time +time.sleep(10) \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__pycache__/__init__.cpython-310.pyc b/src/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..ce0c22e Binary files /dev/null and b/src/__pycache__/__init__.cpython-310.pyc differ diff --git a/src/__pycache__/asr_runner.cpython-310.pyc b/src/__pycache__/asr_runner.cpython-310.pyc new file mode 100644 index 0000000..95f7036 Binary files /dev/null and b/src/__pycache__/asr_runner.cpython-310.pyc differ diff --git a/src/__pycache__/shared.cpython-310.pyc b/src/__pycache__/shared.cpython-310.pyc new file mode 100644 index 0000000..9daef22 Binary files /dev/null and b/src/__pycache__/shared.cpython-310.pyc differ diff --git a/src/__pycache__/sound_input_stream.cpython-310.pyc b/src/__pycache__/sound_input_stream.cpython-310.pyc new file mode 100644 index 0000000..6a73bbd Binary files /dev/null and b/src/__pycache__/sound_input_stream.cpython-310.pyc differ diff --git a/src/__pycache__/wake_word_runner.cpython-310.pyc b/src/__pycache__/wake_word_runner.cpython-310.pyc new file mode 100644 index 0000000..c2abf6a Binary files /dev/null and b/src/__pycache__/wake_word_runner.cpython-310.pyc differ diff --git a/src/asr_runner.py b/src/asr_runner.py new file mode 100644 index 0000000..adcd571 --- /dev/null +++ b/src/asr_runner.py @@ -0,0 +1,122 @@ +import time +import json +import queue +import requests +import threading +import datetime as dt +import numpy as np +from funasr import AutoModel +from .shared import audio_q, stop_flag, asr_flag + + +class ASRRunner: + def __init__(self, cfg): + self.chunk_size = [0, 10, 5] # [past_ms, cur_ms, subsample]，10*60ms=600ms窗口 + self.encoder_chunk_look_back = cfg['encoder_chunk_look_back'] + self.decoder_chunk_look_back = cfg['decoder_chunk_look_back'] + self.chunk_stride = self.chunk_size[1] * 960 # 600ms + self.chunk_stride = max(int(self.chunk_stride), 1600) # 不低于约100ms分辨率 + self.max_dur = cfg['max_duration'] + self.url = cfg['url'] + + self.model = AutoModel( + model=cfg['asr_model'], + disable_update=cfg['disable_update'], + device=cfg['device'], + ) + self.punc = AutoModel(model=cfg['punc_model'], disable_update=cfg['disable_update']) + self.cache = {} + self.start_time = None + self.buf = np.empty((0,), dtype=np.int16) + + self.empty_txt_count = 0 + self.max_empty_txt_count = cfg['max_empty_txt_count'] + self.text_buffer = "" + self.thread = threading.Thread(target=self.run) + self.thread.start() + + def run(self): + while not stop_flag.is_set(): + if not asr_flag.is_set(): + time.sleep(0.5) + continue + else: + if self.start_time is None: + self.start_time = dt.datetime.now() + + try: + chunk = audio_q.get(timeout=0.5) # int16 mono + except queue.Empty: + time.sleep(0.1) + continue + + # TODO: get audio frame from auido_q and concat them into fixed length (chunk_stride) + if chunk is None: + continue + if chunk.dtype != np.int16: + chunk = chunk.astype(np.int16, copy=False) + self.buf = np.concatenate((self.buf, chunk), dtype=np.int16) + if self.buf.shape[0] < self.chunk_stride: + continue + + res = None + while self.buf.shape[0] >= self.chunk_stride: + speech_chunk = self.buf[: self.chunk_stride] + self.buf = self.buf[self.chunk_stride :] + speech_chunk = np.ascontiguousarray( + speech_chunk.astype(np.float32, copy=False) + ) + res = self.model.generate( + input=speech_chunk, + cache=self.cache, # 关键：启用流式缓存 + is_final=False, + chunk_size=self.chunk_size, + encoder_chunk_look_back=self.encoder_chunk_look_back, + decoder_chunk_look_back=self.decoder_chunk_look_back, + fs=16000, # 明确采样率（SoundInputStream 已以 16k 推送） + ) + + if res is None: + continue + + print(res) + cur_time = dt.datetime.now() + if (cur_time - self.start_time).total_seconds() > self.max_dur: + self.switch() + + for r in res: + if r['text'] == '': + self.empty_txt_count += 1 + else: + self.empty_txt_count = 0 + self.text_buffer += r['text'] + + if self.empty_txt_count > self.max_empty_txt_count: + self.switch() + continue + + def do_task(self): + if len(self.text_buffer) == 0: + return + res = self.punc.generate(input=self.text_buffer) + data = "" + for r in res: + data += r['text'] + print(data) + # try: + # data_json= json.dumps({'text': data}) + # response = requests.post(self.url, data = data_json) + # # if response.result != "OK": + # # print("request Failed") + # except Exception as e: + # print("ASRRunner send request failed: ", e) + + def switch(self): + asr_flag.clear() + self.do_task() + self.cache = {} + self.text_buffer = "" + self.buf = np.empty((0,), dtype=np.int16) + self.start_time = None + self.empty_txt_count = 0 + diff --git a/src/shared.py b/src/shared.py new file mode 100644 index 0000000..5d10f25 --- /dev/null +++ b/src/shared.py @@ -0,0 +1,13 @@ +import queue +import threading +# from enum import Enum + +# class STATE(Enum): +# WAKEWORD = 1 +# ASR = 2 + +asr_flag = threading.Event() +stop_flag = threading.Event() + +asr_flag.clear() +audio_q = queue.Queue() \ No newline at end of file diff --git a/src/sound_input_stream.py b/src/sound_input_stream.py new file mode 100644 index 0000000..f437482 --- /dev/null +++ b/src/sound_input_stream.py @@ -0,0 +1,33 @@ +import numpy as np +import sounddevice as sd +from .shared import audio_q + + +class SoundInputStream: + def __init__(self, cfg: dict): + default_in = sd.default.device[0] + if cfg["device"] == "default": + cfg["device"] = None + else: + cfg["device"] = int(cfg["device"]) + # print(f"🎛️ 默认输入设备ID: {default_in} | 当前使用: {cfg["device"] if cfg["device"] is not None else cfg["device"]}") + + self.stream = sd.InputStream( + device=cfg["device"], + channels=cfg['channels'], + samplerate=cfg['sample_rate'], + dtype=cfg['dtype'], + blocksize=cfg['blocksize'], + latency=cfg['latency'], + callback=self.audio_callback + ) + self.stream.start() + print("SoundInputStream start") + + def audio_callback(self, indata, frames, time_info, status): + if status: + # 轻量报告即可，避免阻塞回调 + pass + # indata: shape (frames, channels), int16 + mono = indata[:, 0].copy().astype(np.int16) + audio_q.put(mono) diff --git a/src/wake_word_runner.py b/src/wake_word_runner.py new file mode 100644 index 0000000..e0116a6 --- /dev/null +++ b/src/wake_word_runner.py @@ -0,0 +1,64 @@ +import time +import json +import queue +import requests +import threading +import numpy as np +from openwakeword.model import Model +from .shared import audio_q, stop_flag, asr_flag + +class WakeWordRunner: + def __init__(self, cfg): + import openwakeword + openwakeword.utils.download_models() + self.model = Model(wakeword_models=cfg["model_paths"], inference_framework="onnx") + self.threshold = float(cfg['threshold']) + self.cooldown_sec = float(cfg['cooldown_sec']) + self.reset_sec = float(cfg['reset_sec']) + self.url = cfg['url'] + + self.last_trigger_time = 0.0 + self.last_activity_time = time.time() + self.thread = threading.Thread(target=self.run) + self.thread.start() + print("WakeWordRunner start") + + def run(self): + last_reset = time.time() + frame_buffer = [] # 累积音频块 + + while not stop_flag.is_set(): + if asr_flag.is_set(): + time.sleep(0.5) + continue + + try: + chunk = audio_q.get(timeout=0.5) # int16 mono + except queue.Empty: + time.sleep(0.1) + continue + + frame_buffer.append(chunk) + if len(frame_buffer) >= 1: + data = np.concatenate(frame_buffer) + scores = self.model.predict(data) + _, score = max(scores.items(), key=lambda kv: kv[1]) + now = time.time() + frame_buffer = [] # 清空缓冲区 + + if now - last_reset >= self.reset_sec: + self.model.reset() + last_reset = now + + if (now - self.last_trigger_time) < self.cooldown_sec: + continue + + if score >= self.threshold: + print(f"WordScores: {scores}") + asr_flag.set() + # try: + # data_json = json.dump({"msg": "wakeup"}) + # requests.post(self.url, data=data_json) + # self.last_trigger_time = now + # except Exception as e: + # print("WakeWordRunner send request failed: ", e) \ No newline at end of file