init

config/config.yaml

@@ -0,0 +1,30 @@
+SoundInputStream:
+  device: default
+  channels: 1
+  sample_rate: 16000
+  dtype: int16
+  blocksize: 256
+  latency: low
+
+
+WakeWordRunner:
+  model_paths: 
+    - models/hi_amy.onnx
+    - models/hey_amy.onnx
+    - models/hello_amy.onnx
+  sample_rate: 16000
+  threshold: 0.3
+  cooldown_sec: 1.5
+  reset_sec: 3.0
+  url: http://xxx.xxx.xxx.xxx:xxxx/xx/xx/xx
+
+ASRRunner:
+  asr_model: paraformer-zh-streaming
+  punc_model: ct-punc
+  disable_update: True
+  device: cpu
+  encoder_chunk_look_back: 4
+  decoder_chunk_look_back: 1
+  max_duration: 20
+  max_empty_txt_count: 3
+  url: http://xxx.xxx.xxx.xxx:xxxx/xx/xx/xx

run.py

@@ -0,0 +1,14 @@
+import yaml
+from src.asr_runner import ASRRunner
+from src.sound_input_stream import SoundInputStream
+from src.wake_word_runner import WakeWordRunner
+
+with open(r"config/config.yaml", "r") as f:
+    cfg = yaml.load(f, Loader=yaml.FullLoader)
+
+s = SoundInputStream(cfg["SoundInputStream"])
+w = WakeWordRunner(cfg['WakeWordRunner'])
+a = ASRRunner(cfg['ASRRunner'])
+
+import time
+time.sleep(10)

src/asr_runner.py

@@ -0,0 +1,122 @@
+import time
+import json
+import queue
+import requests
+import threading
+import datetime as dt
+import numpy as np
+from funasr import AutoModel
+from .shared import audio_q, stop_flag, asr_flag
+
+
+class ASRRunner:
+    def __init__(self, cfg):
+        self.chunk_size = [0, 10, 5]        # [past_ms, cur_ms, subsample]，10*60ms=600ms窗口
+        self.encoder_chunk_look_back = cfg['encoder_chunk_look_back']
+        self.decoder_chunk_look_back = cfg['decoder_chunk_look_back']
+        self.chunk_stride = self.chunk_size[1] * 960  # 600ms
+        self.chunk_stride = max(int(self.chunk_stride), 1600)  # 不低于约100ms分辨率
+        self.max_dur = cfg['max_duration']
+        self.url = cfg['url']
+
+        self.model = AutoModel(
+            model=cfg['asr_model'],
+            disable_update=cfg['disable_update'],  
+            device=cfg['device'],
+            )
+        self.punc = AutoModel(model=cfg['punc_model'], disable_update=cfg['disable_update'])
+        self.cache = {}
+        self.start_time = None
+        self.buf = np.empty((0,), dtype=np.int16)
+
+        self.empty_txt_count = 0
+        self.max_empty_txt_count = cfg['max_empty_txt_count']
+        self.text_buffer = ""
+        self.thread = threading.Thread(target=self.run)
+        self.thread.start()
+
+    def run(self):
+        while not stop_flag.is_set():
+            if not asr_flag.is_set():
+                time.sleep(0.5)
+                continue
+            else:
+                if self.start_time is None:
+                    self.start_time = dt.datetime.now()
+
+            try:
+                chunk = audio_q.get(timeout=0.5)            # int16 mono
+            except queue.Empty:
+                time.sleep(0.1)
+                continue
+            
+            # TODO: get audio frame from auido_q and concat them into fixed length (chunk_stride)
+            if chunk is None:
+                continue
+            if chunk.dtype != np.int16:
+                chunk = chunk.astype(np.int16, copy=False)
+            self.buf = np.concatenate((self.buf, chunk), dtype=np.int16)
+            if self.buf.shape[0] < self.chunk_stride:
+                continue
+
+            res = None
+            while self.buf.shape[0] >= self.chunk_stride:
+                speech_chunk = self.buf[: self.chunk_stride]
+                self.buf = self.buf[self.chunk_stride :]
+                speech_chunk = np.ascontiguousarray(
+                    speech_chunk.astype(np.float32, copy=False)
+                )
+                res = self.model.generate(
+                    input=speech_chunk,
+                    cache=self.cache,  # 关键：启用流式缓存
+                    is_final=False,
+                    chunk_size=self.chunk_size,
+                    encoder_chunk_look_back=self.encoder_chunk_look_back,
+                    decoder_chunk_look_back=self.decoder_chunk_look_back,
+                    fs=16000,  # 明确采样率（SoundInputStream 已以 16k 推送）
+                )
+
+            if res is None:
+                continue
+            
+            print(res)
+            cur_time = dt.datetime.now()
+            if (cur_time - self.start_time).total_seconds() > self.max_dur:
+                self.switch()
+            
+            for r in res:
+                if r['text'] == '':
+                    self.empty_txt_count += 1
+                else:
+                    self.empty_txt_count = 0
+                    self.text_buffer += r['text']
+
+                if self.empty_txt_count > self.max_empty_txt_count:
+                    self.switch()
+                    continue
+
+    def do_task(self):
+        if len(self.text_buffer) == 0:
+            return
+        res = self.punc.generate(input=self.text_buffer)
+        data = ""
+        for r in res:
+            data += r['text']
+        print(data)
+        # try:
+        #     data_json= json.dumps({'text': data})
+        #     response = requests.post(self.url, data = data_json)
+        #     # if response.result != "OK":
+        #     #     print("request Failed")
+        # except Exception as e:
+        #     print("ASRRunner send request failed: ", e)
+
+    def switch(self):
+        asr_flag.clear()
+        self.do_task()
+        self.cache = {}
+        self.text_buffer = ""
+        self.buf = np.empty((0,), dtype=np.int16)
+        self.start_time = None
+        self.empty_txt_count = 0
+        

src/shared.py

@@ -0,0 +1,13 @@
+import queue
+import threading
+# from enum import Enum
+
+# class STATE(Enum):
+#     WAKEWORD = 1
+#     ASR = 2
+
+asr_flag = threading.Event()
+stop_flag = threading.Event()
+
+asr_flag.clear()
+audio_q = queue.Queue()

src/sound_input_stream.py

@@ -0,0 +1,33 @@
+import numpy as np
+import sounddevice as sd
+from .shared import audio_q
+
+
+class SoundInputStream:
+    def __init__(self, cfg: dict):
+        default_in = sd.default.device[0]
+        if cfg["device"] == "default":
+            cfg["device"] = None
+        else:
+            cfg["device"] = int(cfg["device"])
+        # print(f"🎛️ 默认输入设备ID: {default_in} | 当前使用: {cfg["device"] if cfg["device"] is not None else cfg["device"]}")
+        
+        self.stream = sd.InputStream(
+            device=cfg["device"],
+            channels=cfg['channels'],
+            samplerate=cfg['sample_rate'],
+            dtype=cfg['dtype'],
+            blocksize=cfg['blocksize'],
+            latency=cfg['latency'],
+            callback=self.audio_callback
+            )
+        self.stream.start()
+        print("SoundInputStream start")
+    
+    def audio_callback(self, indata, frames, time_info, status):
+        if status:
+            # 轻量报告即可，避免阻塞回调
+            pass
+        # indata: shape (frames, channels), int16
+        mono = indata[:, 0].copy().astype(np.int16)
+        audio_q.put(mono)

src/wake_word_runner.py

@@ -0,0 +1,64 @@
+import time
+import json
+import queue
+import requests
+import threading
+import numpy as np
+from openwakeword.model import Model
+from .shared import audio_q, stop_flag, asr_flag
+
+class WakeWordRunner:
+    def __init__(self, cfg):
+        import openwakeword
+        openwakeword.utils.download_models()
+        self.model = Model(wakeword_models=cfg["model_paths"], inference_framework="onnx")
+        self.threshold = float(cfg['threshold'])
+        self.cooldown_sec = float(cfg['cooldown_sec'])
+        self.reset_sec = float(cfg['reset_sec'])
+        self.url = cfg['url']
+
+        self.last_trigger_time = 0.0
+        self.last_activity_time = time.time()
+        self.thread = threading.Thread(target=self.run)
+        self.thread.start()
+        print("WakeWordRunner start")
+
+    def run(self):
+        last_reset = time.time()
+        frame_buffer = []  # 累积音频块
+
+        while not stop_flag.is_set():
+            if asr_flag.is_set():
+                time.sleep(0.5)
+                continue
+
+            try:
+                chunk = audio_q.get(timeout=0.5)            # int16 mono
+            except queue.Empty:
+                time.sleep(0.1)
+                continue
+
+            frame_buffer.append(chunk)
+            if len(frame_buffer) >= 1:
+                data = np.concatenate(frame_buffer)
+                scores = self.model.predict(data)
+                _, score = max(scores.items(), key=lambda kv: kv[1])
+                now = time.time()
+                frame_buffer = []  # 清空缓冲区
+
+                if now - last_reset >= self.reset_sec:
+                    self.model.reset()
+                    last_reset = now
+
+                if (now - self.last_trigger_time) < self.cooldown_sec:
+                    continue
+
+                if score >= self.threshold:
+                    print(f"WordScores: {scores}")
+                    asr_flag.set()
+                    # try:
+                    #     data_json = json.dump({"msg": "wakeup"})
+                    #     requests.post(self.url, data=data_json)
+                    #     self.last_trigger_time = now
+                    # except Exception as e:
+                    #     print("WakeWordRunner send request failed: ", e)