feat(audio): 重构音频处理模块、音频流重采样测试成功

.gitignore

@@ -7,3 +7,4 @@ out
 __pycache__
 subenv
 caption-engine/build
+output.wav

caption-engine/audioprcs/__init__.py

@@ -1 +1 @@
-from .streamchnl import mergeStreamChannels
+from .process import mergeChunkChannels, resampleRawChunk

caption-engine/audioprcs/process.py

@@ -0,0 +1,49 @@
+import samplerate
+import numpy as np
+
+def mergeChunkChannels(chunk, channels):
+    """
+    将当前多通道音频数据块转换为单通道音频数据块
+
+    Args:
+        chunk: (bytes)多通道音频数据块
+        channels: 通道数
+
+    Returns:
+        (bytes)单通道音频数据块
+    """
+    # (length * channels,)
+    chunk_np = np.frombuffer(chunk, dtype=np.int16)
+    # (length, channels)
+    chunk_np = chunk_np.reshape(-1, channels)
+    # (length,)
+    chunk_mono_f = np.mean(chunk_np.astype(np.float32), axis=1)
+    chunk_mono = np.round(chunk_mono_f).astype(np.int16)
+    return chunk_mono.tobytes()
+
+
+def resampleRawChunk(chunk, channels, orig_sr, target_sr, mode="sinc_best"):
+    """
+    将当前多通道音频数据块转换成单通道音频数据块，然后进行重采样
+
+    Args:
+        chunk: (bytes)多通道音频数据块
+        channels: 通道数
+        orig_sr: 原始采样率
+        target_sr: 目标采样率
+        mode: 重采样模式，可选：'sinc_best' | 'sinc_medium' | 'sinc_fastest' | 'zero_order_hold' | 'linear'
+
+    Return:
+        (bytes)单通道音频数据块
+    """
+    # (length * channels,)
+    chunk_np = np.frombuffer(chunk, dtype=np.int16)
+    # (length, channels)
+    chunk_np = chunk_np.reshape(-1, channels)
+    # (length,)
+    chunk_mono_f = np.mean(chunk_np.astype(np.float32), axis=1)
+    chunk_mono = chunk_mono_f.astype(np.int16)
+    ratio = target_sr / orig_sr
+    chunk_mono_r =  samplerate.resample(chunk_mono, ratio, converter_type=mode)
+    chunk_mono_r = np.round(chunk_mono_r).astype(np.int16)
+    return chunk_mono_r.tobytes()

caption-engine/audioprcs/streamchnl.py

@@ -1,22 +0,0 @@
-import numpy as np
-
-def mergeStreamChannels(data, channels):
-    """
-    将当前多通道流数据合并为单通道流数据
-
-    Args:
-        data: 多通道数据
-        channels: 通道数
-
-    Returns:
-        mono_data_bytes: 单通道数据
-    """
-    # (length * channels,)
-    data_np = np.frombuffer(data, dtype=np.int16)
-    # (length, channels)
-    data_np_r = data_np.reshape(-1, channels)
-    # (length,)
-    mono_data = np.mean(data_np_r.astype(np.float32), axis=1)
-    mono_data = mono_data.astype(np.int16)
-    mono_data_bytes = mono_data.tobytes()
-    return mono_data_bytes

caption-engine/main-gummy.py

@@ -8,7 +8,7 @@ elif sys.platform == 'linux':
 else:
     raise NotImplementedError(f"Unsupported platform: {sys.platform}")
 
-from audioprcs import mergeStreamChannels
+from audioprcs import mergeChunkChannels
 from audio2text import InvalidParameter, GummyTranslator
 
 
@@ -26,13 +26,13 @@ def convert_audio_to_text(s_lang, t_lang, audio_type):
 
     while True:
         try:
-            data = stream.read_chunk()
+            chunk = stream.read_chunk()
-            data = mergeStreamChannels(data, stream.CHANNELS)
+            chunk_mono = mergeChunkChannels(chunk, stream.CHANNELS)
             try:
-                gummy.send_audio_frame(data)
+                gummy.send_audio_frame(chunk_mono)
             except InvalidParameter:
                 gummy.start()
-                gummy.send_audio_frame(data)
+                gummy.send_audio_frame(chunk_mono)
         except KeyboardInterrupt:
             stream.closeStream()
             gummy.stop()

caption-engine/requirements.txt

@@ -1,5 +1,6 @@
 dashscope==1.23.5
 numpy==2.2.6
+samplerate==0.2.1
 PyAudio==0.2.14
 PyAudioWPatch==0.2.12.7 # Windows only
 pyinstaller==6.14.1

caption-engine/sysaudio/linux.py

@@ -34,7 +34,7 @@ class AudioStream:
 
         音频样本块大小：{self.CHUNK}
         样本位宽：{self.SAMP_WIDTH}
-        音频数据格式：{self.FORMAT}
+        采样格式：{self.FORMAT}
         音频通道数：{self.CHANNELS}
         音频采样率：{self.RATE}
         """

caption-engine/sysaudio/win.py

@@ -65,7 +65,7 @@ class AudioStream:
     def printInfo(self):
         dev_info = f"""
         采样设备：
-            - 设备类型：{ "音频输入" if self.audio_type == 0 else "音频输出" }
+            - 设备类型：{ "音频输出" if self.audio_type == 0 else "音频输入" }
             - 序号：{self.device['index']}
             - 名称：{self.device['name']}
             - 最大输入通道数：{self.device['maxInputChannels']}
@@ -76,7 +76,7 @@ class AudioStream:
 
         音频样本块大小：{self.CHUNK}
         样本位宽：{self.SAMP_WIDTH}
-        音频数据格式：{self.FORMAT}
+        采样格式：{self.FORMAT}
         音频通道数：{self.CHANNELS}
         音频采样率：{self.RATE}
         """

engine-test/gummy.ipynb

@@ -2,11 +2,11 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from dashscope.audio.asr import *\n",
+    "from dashscope.audio.asr import * # type: ignore\n",
     "import pyaudiowpatch as pyaudio\n",
     "import numpy as np\n",
     "\n",

engine-test/process.ipynb

@@ -1,82 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "1e12f3ef",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "        采样设备：\n",
-      "            - 设备类型：音频输入\n",
-      "            - 序号：20\n",
-      "            - 名称：扬声器 (Realtek(R) Audio) [Loopback]\n",
-      "            - 最大输入通道数：2\n",
-      "            - 默认低输入延迟：0.003s\n",
-      "            - 默认高输入延迟：0.01s\n",
-      "            - 默认采样率：48000.0Hz\n",
-      "            - 是否回环设备：True\n",
-      "\n",
-      "        音频样本块大小：2400\n",
-      "        样本位宽：2\n",
-      "        音频数据格式：8\n",
-      "        音频通道数：2\n",
-      "        音频采样率：48000\n",
-      "        \n"
-     ]
-    }
-   ],
-   "source": [
-    "import sys\n",
-    "import os\n",
-    "\n",
-    "current_dir = os.getcwd() \n",
-    "sys.path.append(os.path.join(current_dir, '../caption-engine'))\n",
-    "\n",
-    "from sysaudio.win import AudioStream\n",
-    "\n",
-    "stream = AudioStream()\n",
-    "stream.printInfo()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d4c8ad80",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "stream.openStream()\n",
-    "SEC = 2\n",
-    "for i in range(SEC * 20):\n",
-    "    data = stream.stream.read(stream.CHUNK) # type: ignore\n",
-    "    "
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "mystd",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

engine-test/resample.ipynb

@@ -0,0 +1,191 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "1e12f3ef",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "        采样设备：\n",
+      "            - 设备类型：音频输出\n",
+      "            - 序号：26\n",
+      "            - 名称：耳机 (HUAWEI FreeLace 活力版) [Loopback]\n",
+      "            - 最大输入通道数：2\n",
+      "            - 默认低输入延迟：0.003s\n",
+      "            - 默认高输入延迟：0.01s\n",
+      "            - 默认采样率：48000.0Hz\n",
+      "            - 是否回环设备：True\n",
+      "\n",
+      "        音频样本块大小：2400\n",
+      "        样本位宽：2\n",
+      "        采样格式：8\n",
+      "        音频通道数：2\n",
+      "        音频采样率：48000\n",
+      "        \n"
+     ]
+    }
+   ],
+   "source": [
+    "import sys\n",
+    "import os\n",
+    "import pyaudio\n",
+    "import wave\n",
+    "\n",
+    "current_dir = os.getcwd() \n",
+    "sys.path.append(os.path.join(current_dir, '../caption-engine'))\n",
+    "\n",
+    "from sysaudio.win import AudioStream\n",
+    "from audioprcs import resampleRawChunk, mergeChunkChannels\n",
+    "\n",
+    "stream = AudioStream(0)\n",
+    "stream.printInfo()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "a72914f4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Recording...\n",
+      "Done\n"
+     ]
+    }
+   ],
+   "source": [
+    "\"\"\"获取系统音频输出5秒，然后保存为wav文件\"\"\"\n",
+    "\n",
+    "with wave.open('output.wav', 'wb') as wf:\n",
+    "    wf.setnchannels(stream.CHANNELS)\n",
+    "    wf.setsampwidth(stream.SAMP_WIDTH)\n",
+    "    wf.setframerate(stream.RATE)\n",
+    "    stream.openStream()\n",
+    "\n",
+    "    print('Recording...')\n",
+    "\n",
+    "    for _ in range(0, 100):\n",
+    "        chunk = stream.read_chunk()\n",
+    "        if isinstance(chunk, bytes):\n",
+    "            wf.writeframes(chunk)\n",
+    "        else:\n",
+    "            raise Exception('Error: chunk is not bytes')\n",
+    "        \n",
+    "    stream.closeStream()    \n",
+    "    print('Done')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6e8a098",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Recording...\n",
+      "Done\n"
+     ]
+    }
+   ],
+   "source": [
+    "\"\"\"获取系统音频输入，转换为单通道音频，持续5秒，然后保存为wav文件\"\"\"\n",
+    "\n",
+    "with wave.open('output.wav', 'wb') as wf:\n",
+    "    wf.setnchannels(1)\n",
+    "    wf.setsampwidth(stream.SAMP_WIDTH)\n",
+    "    wf.setframerate(stream.RATE)\n",
+    "    stream.openStream()\n",
+    "\n",
+    "    print('Recording...')\n",
+    "\n",
+    "    for _ in range(0, 100):\n",
+    "        chunk = mergeChunkChannels(\n",
+    "            stream.read_chunk(),\n",
+    "            stream.CHANNELS\n",
+    "        )\n",
+    "        if isinstance(chunk, bytes):\n",
+    "            wf.writeframes(chunk)\n",
+    "        else:\n",
+    "            raise Exception('Error: chunk is not bytes')\n",
+    "        \n",
+    "    stream.closeStream()    \n",
+    "    print('Done')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "aaca1465",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Recording...\n",
+      "Done\n"
+     ]
+    }
+   ],
+   "source": [
+    "\"\"\"获取系统音频输入，转换为单通道音频并重采样到16000Hz，持续5秒，然后保存为wav文件\"\"\"\n",
+    "\n",
+    "with wave.open('output.wav', 'wb') as wf:\n",
+    "    wf.setnchannels(1)\n",
+    "    wf.setsampwidth(stream.SAMP_WIDTH)\n",
+    "    wf.setframerate(16000)\n",
+    "    stream.openStream()\n",
+    "\n",
+    "    print('Recording...')\n",
+    "\n",
+    "    for _ in range(0, 100):\n",
+    "        chunk = resampleRawChunk(\n",
+    "            stream.read_chunk(),\n",
+    "            stream.CHANNELS,\n",
+    "            stream.RATE,\n",
+    "            16000,\n",
+    "            mode=\"sinc_best\"\n",
+    "        )\n",
+    "        if isinstance(chunk, bytes):\n",
+    "            wf.writeframes(chunk)\n",
+    "        else:\n",
+    "            raise Exception('Error: chunk is not bytes')\n",
+    "        \n",
+    "    stream.closeStream()    \n",
+    "    print('Done')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "mystd",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}