Ma/auto-caption
1
0
Fork 0
mirror of https://github.com/HiMeditator/auto-caption.git synced 2026-02-15 04:14:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(audio): 重构音频处理模块、音频流重采样测试成功

Browse Source
This commit is contained in:
himeditator
2025-07-07 00:47:36 +08:00
parent f2aa075e65
commit 0696651f04
11 changed files with 253 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -7,3 +7,4 @@ out
__pycache__
subenv
caption-engine/build
output.wav

2
caption-engine/audioprcs/__init__.py
View File

@@ -1 +1 @@
from .streamchnl import mergeStreamChannels
from .process import mergeChunkChannels, resampleRawChunk

49
caption-engine/audioprcs/process.py Normal file
View File

@@ -0,0 +1,49 @@
import samplerate
import numpy as np
def mergeChunkChannels(chunk, channels):
"""
将当前多通道音频数据块转换为单通道音频数据块
Args:
chunk: (bytes)多通道音频数据块
channels: 通道数
Returns:
(bytes)单通道音频数据块
"""
# (length * channels,)
chunk_np = np.frombuffer(chunk, dtype=np.int16)
# (length, channels)
chunk_np = chunk_np.reshape(-1, channels)
# (length,)
chunk_mono_f = np.mean(chunk_np.astype(np.float32), axis=1)
chunk_mono = np.round(chunk_mono_f).astype(np.int16)
return chunk_mono.tobytes()
def resampleRawChunk(chunk, channels, orig_sr, target_sr, mode="sinc_best"):
"""
将当前多通道音频数据块转换成单通道音频数据块，然后进行重采样
Args:
chunk: (bytes)多通道音频数据块
channels: 通道数
orig_sr: 原始采样率
target_sr: 目标采样率
mode: 重采样模式，可选：'sinc_best' | 'sinc_medium' | 'sinc_fastest' | 'zero_order_hold' | 'linear'
Return:
(bytes)单通道音频数据块
"""
# (length * channels,)
chunk_np = np.frombuffer(chunk, dtype=np.int16)
# (length, channels)
chunk_np = chunk_np.reshape(-1, channels)
# (length,)
chunk_mono_f = np.mean(chunk_np.astype(np.float32), axis=1)
chunk_mono = chunk_mono_f.astype(np.int16)
ratio = target_sr / orig_sr
chunk_mono_r = samplerate.resample(chunk_mono, ratio, converter_type=mode)
chunk_mono_r = np.round(chunk_mono_r).astype(np.int16)
return chunk_mono_r.tobytes()

22
caption-engine/audioprcs/streamchnl.py
View File

@@ -1,22 +0,0 @@
import numpy as np
def mergeStreamChannels(data, channels):
"""
将当前多通道流数据合并为单通道流数据
Args:
data: 多通道数据
channels: 通道数
Returns:
mono_data_bytes: 单通道数据
"""
# (length * channels,)
data_np = np.frombuffer(data, dtype=np.int16)
# (length, channels)
data_np_r = data_np.reshape(-1, channels)
# (length,)
mono_data = np.mean(data_np_r.astype(np.float32), axis=1)
mono_data = mono_data.astype(np.int16)
mono_data_bytes = mono_data.tobytes()
return mono_data_bytes

10
caption-engine/main-gummy.py
View File

@@ -8,7 +8,7 @@ elif sys.platform == 'linux':
else:
raise NotImplementedError(f"Unsupported platform: {sys.platform}")
from audioprcs import mergeStreamChannels
from audioprcs import mergeChunkChannels
from audio2text import InvalidParameter, GummyTranslator
@@ -26,13 +26,13 @@ def convert_audio_to_text(s_lang, t_lang, audio_type):
while True:
try:
data = stream.read_chunk()
data = mergeStreamChannels(data, stream.CHANNELS)
chunk = stream.read_chunk()
chunk_mono = mergeChunkChannels(chunk, stream.CHANNELS)
try:
gummy.send_audio_frame(data)
gummy.send_audio_frame(chunk_mono)
except InvalidParameter:
gummy.start()
gummy.send_audio_frame(data)
gummy.send_audio_frame(chunk_mono)
except KeyboardInterrupt:
stream.closeStream()
gummy.stop()

1
caption-engine/requirements.txt
View File

@@ -1,5 +1,6 @@
dashscope==1.23.5
numpy==2.2.6
samplerate==0.2.1
PyAudio==0.2.14
PyAudioWPatch==0.2.12.7 # Windows only
pyinstaller==6.14.1

2
caption-engine/sysaudio/linux.py
View File

@@ -34,7 +34,7 @@ class AudioStream:
音频样本块大小：{self.CHUNK}
样本位宽：{self.SAMP_WIDTH}
音频数据格式：{self.FORMAT}
采样格式：{self.FORMAT}
音频通道数：{self.CHANNELS}
音频采样率：{self.RATE}
"""

4
caption-engine/sysaudio/win.py
View File

@@ -65,7 +65,7 @@ class AudioStream:
def printInfo(self):
dev_info = f"""
采样设备：
- 设备类型：{ "音频输入" if self.audio_type == 0 else "音频输出" }
- 设备类型：{ "音频输出" if self.audio_type == 0 else "音频输入" }
- 序号：{self.device['index']}
- 名称：{self.device['name']}
- 最大输入通道数：{self.device['maxInputChannels']}
@@ -76,7 +76,7 @@ class AudioStream:
音频样本块大小：{self.CHUNK}
样本位宽：{self.SAMP_WIDTH}
音频数据格式：{self.FORMAT}
采样格式：{self.FORMAT}
音频通道数：{self.CHANNELS}
音频采样率：{self.RATE}
"""

4
engine-test/gummy.ipynb
View File

@@ -2,11 +2,11 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from dashscope.audio.asr import *\n",
"from dashscope.audio.asr import * # type: ignore\n",
"import pyaudiowpatch as pyaudio\n",
"import numpy as np\n",
"\n",

82
engine-test/process.ipynb
View File

@@ -1,82 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"id": "1e12f3ef",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
" 采样设备：\n",
" - 设备类型：音频输入\n",
" - 序号：20\n",
" - 名称：扬声器 (Realtek(R) Audio) [Loopback]\n",
" - 最大输入通道数：2\n",
" - 默认低输入延迟：0.003s\n",
" - 默认高输入延迟：0.01s\n",
" - 默认采样率：48000.0Hz\n",
" - 是否回环设备：True\n",
"\n",
" 音频样本块大小：2400\n",
" 样本位宽：2\n",
" 音频数据格式：8\n",
" 音频通道数：2\n",
" 音频采样率：48000\n",
" \n"
]
}
],
"source": [
"import sys\n",
"import os\n",
"\n",
"current_dir = os.getcwd() \n",
"sys.path.append(os.path.join(current_dir, '../caption-engine'))\n",
"\n",
"from sysaudio.win import AudioStream\n",
"\n",
"stream = AudioStream()\n",
"stream.printInfo()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d4c8ad80",
"metadata": {},
"outputs": [],
"source": [
"stream.openStream()\n",
"SEC = 2\n",
"for i in range(SEC * 20):\n",
" data = stream.stream.read(stream.CHUNK) # type: ignore\n",
" "
]
}
],
"metadata": {
"kernelspec": {
"display_name": "mystd",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

191
engine-test/resample.ipynb Normal file
View File

@@ -0,0 +1,191 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "1e12f3ef",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
" 采样设备：\n",
" - 设备类型：音频输出\n",
" - 序号：26\n",
" - 名称：耳机 (HUAWEI FreeLace 活力版) [Loopback]\n",
" - 最大输入通道数：2\n",
" - 默认低输入延迟：0.003s\n",
" - 默认高输入延迟：0.01s\n",
" - 默认采样率：48000.0Hz\n",
" - 是否回环设备：True\n",
"\n",
" 音频样本块大小：2400\n",
" 样本位宽：2\n",
" 采样格式：8\n",
" 音频通道数：2\n",
" 音频采样率：48000\n",
" \n"
]
}
],
"source": [
"import sys\n",
"import os\n",
"import pyaudio\n",
"import wave\n",
"\n",
"current_dir = os.getcwd() \n",
"sys.path.append(os.path.join(current_dir, '../caption-engine'))\n",
"\n",
"from sysaudio.win import AudioStream\n",
"from audioprcs import resampleRawChunk, mergeChunkChannels\n",
"\n",
"stream = AudioStream(0)\n",
"stream.printInfo()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "a72914f4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Recording...\n",
"Done\n"
]
}
],
"source": [
"\"\"\"获取系统音频输出5秒，然后保存为wav文件\"\"\"\n",
"\n",
"with wave.open('output.wav', 'wb') as wf:\n",
" wf.setnchannels(stream.CHANNELS)\n",
" wf.setsampwidth(stream.SAMP_WIDTH)\n",
" wf.setframerate(stream.RATE)\n",
" stream.openStream()\n",
"\n",
" print('Recording...')\n",
"\n",
" for _ in range(0, 100):\n",
" chunk = stream.read_chunk()\n",
" if isinstance(chunk, bytes):\n",
" wf.writeframes(chunk)\n",
" else:\n",
" raise Exception('Error: chunk is not bytes')\n",
" \n",
" stream.closeStream() \n",
" print('Done')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a6e8a098",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Recording...\n",
"Done\n"
]
}
],
"source": [
"\"\"\"获取系统音频输入，转换为单通道音频，持续5秒，然后保存为wav文件\"\"\"\n",
"\n",
"with wave.open('output.wav', 'wb') as wf:\n",
" wf.setnchannels(1)\n",
" wf.setsampwidth(stream.SAMP_WIDTH)\n",
" wf.setframerate(stream.RATE)\n",
" stream.openStream()\n",
"\n",
" print('Recording...')\n",
"\n",
" for _ in range(0, 100):\n",
" chunk = mergeChunkChannels(\n",
" stream.read_chunk(),\n",
" stream.CHANNELS\n",
" )\n",
" if isinstance(chunk, bytes):\n",
" wf.writeframes(chunk)\n",
" else:\n",
" raise Exception('Error: chunk is not bytes')\n",
" \n",
" stream.closeStream() \n",
" print('Done')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "aaca1465",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Recording...\n",
"Done\n"
]
}
],
"source": [
"\"\"\"获取系统音频输入，转换为单通道音频并重采样到16000Hz，持续5秒，然后保存为wav文件\"\"\"\n",
"\n",
"with wave.open('output.wav', 'wb') as wf:\n",
" wf.setnchannels(1)\n",
" wf.setsampwidth(stream.SAMP_WIDTH)\n",
" wf.setframerate(16000)\n",
" stream.openStream()\n",
"\n",
" print('Recording...')\n",
"\n",
" for _ in range(0, 100):\n",
" chunk = resampleRawChunk(\n",
" stream.read_chunk(),\n",
" stream.CHANNELS,\n",
" stream.RATE,\n",
" 16000,\n",
" mode=\"sinc_best\"\n",
" )\n",
" if isinstance(chunk, bytes):\n",
" wf.writeframes(chunk)\n",
" else:\n",
" raise Exception('Error: chunk is not bytes')\n",
" \n",
" stream.closeStream() \n",
" print('Done')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "mystd",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}