feat(control): 重构项目，增加字幕引擎配置

python-prototype/gummy.ipynb

@@ -156,112 +156,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "流式翻译开始...\n",
-      "\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n",
-      "(4410, 2)\n"
-     ]
-    },
-    {
-     "ename": "KeyboardInterrupt",
-     "evalue": "",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
-      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_29036\\3259296939.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     21\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mRATE\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mCHUNK\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mRECORD_SECONDS\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m     \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstream\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCHUNK\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m     \u001b[0mdata_np\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrombuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mint16\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[0mdata_np_r\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdata_np\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCHANNELS\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
-      "\u001b[1;32md:\\ML\\anaconda3\\envs\\mystd\\lib\\site-packages\\pyaudiowpatch\\__init__.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, num_frames, exception_on_overflow)\u001b[0m\n\u001b[0;32m    638\u001b[0m                           paCanNotReadFromAnOutputOnlyStream)\n\u001b[0;32m    639\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 640\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mpa\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_stream\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stream\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_frames\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexception_on_overflow\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    641\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    642\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget_read_available\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
-      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
-     ]
-    }
-   ],
    "source": [
     "RECORD_SECONDS = 20 # 监听时长(s)\n",
     "\n",

python-subprocess/main.py

@@ -1,23 +1,27 @@
 from sysaudio.win import LoopbackStream, mergeStreamChannels
 from audio2text.gummy import GummyTranslator
 import sys
+import argparse
 
-def main():
+def convert_audio_to_text(s_lang, t_lang, audio_source):
     sys.stdout.reconfigure(line_buffering=True)
     loopback = LoopbackStream()
     loopback.openStream()
 
-    gummy = GummyTranslator(loopback.RATE, "zh", "en")
+    gummy = GummyTranslator(loopback.RATE, s_lang, t_lang)
     gummy.translator.start()
 
-    for _ in range(0, 400):
+    while True:
         if not loopback.stream: continue
         data = loopback.stream.read(loopback.CHUNK)
         data = mergeStreamChannels(data, loopback.CHANNELS)
         gummy.translator.send_audio_frame(data)
 
-    gummy.translator.stop()
-    loopback.closeStream()
 
 if __name__ == "__main__":
-    main()
+    parser = argparse.ArgumentParser(description='Convert system audio stream to text')
+    parser.add_argument('-s', '--s_lang', default='en', help='Source language code')
+    parser.add_argument('-t', '--t_lang', default='zh', help='Target language code')
+    parser.add_argument('-a', '--audio', default=0, help='Audio stream source: 0 for output audio stream, 1 for input audio stream')
+    args = parser.parse_args()
+    convert_audio_to_text(args.s_lang, args.t_lang, args.audio)

src/main/caption.ts

@@ -3,8 +3,7 @@ import path from 'path'
 import { is } from '@electron-toolkit/utils'
 import icon from '../../resources/icon.png?asset'
 import { controlWindow } from './control'
-import { sendStyles, sendCaptionLog } from './data'
+import { sendStyles, sendCaptionLog } from './utils/config'
-import { send } from 'vite'
 
 class CaptionWindow { 
   window: BrowserWindow | undefined;
@@ -40,7 +39,6 @@ class CaptionWindow {
     })
 
     this.window.on('closed', () => {
-      console.log('INFO caption window closed')
       this.window = undefined
     })
 
@@ -63,7 +61,6 @@ class CaptionWindow {
     ipcMain.on('caption.controlWindow.activate', () => {
       if(!controlWindow.window){
         controlWindow.createWindow()
-        console.log('GET caption.controlWindow.activate')
       }
       else {
         controlWindow.window.show()
@@ -71,21 +68,18 @@ class CaptionWindow {
     })
     // 字幕窗口高度发生变化
     ipcMain.on('caption.windowHeight.change', (_, height) => {
-      console.log('GET caption.window.height.change', height)
       if(this.window){
         this.window.setSize(this.window.getSize()[0], height) 
       }
     })
     // 关闭字幕窗口
     ipcMain.on('caption.window.close', () => {
-      console.log('GET caption.window.close')
       if(this.window){
         this.window.close()
       }
     })
     // 是否固定在最前面
     ipcMain.on('caption.pin.set', (_, pinned) => {
-      console.log('GET caption.pin.set', pinned)
       if(this.window){
         this.window.setAlwaysOnTop(pinned)
       }

src/main/control.ts

@@ -2,8 +2,13 @@ import { shell, BrowserWindow, ipcMain } from 'electron'
 import path from 'path'
 import { is } from '@electron-toolkit/utils'
 import icon from '../../resources/icon.png?asset'
-import { setStyles, sendStyles, sendCaptionLog } from './data'
 import { captionWindow } from './caption'
+import {
+  setStyles,
+  sendStyles,
+  sendCaptionLog,
+  setControls
+} from './utils/config'
 
 class ControlWindow {
   window: BrowserWindow | undefined;
@@ -38,7 +43,6 @@ class ControlWindow {
     })
   
     this.window.on('closed', () => {
-      console.log('INFO control window closed')
       this.window = undefined
     })
   
@@ -57,7 +61,6 @@ class ControlWindow {
   public handleMessage() {
     // 控制窗口样式更新
     ipcMain.on('control.style.change', (_, args) => {
-      console.log('GET control.style.change', args)
       setStyles(args)
       if(captionWindow.window){
         sendStyles(captionWindow.window)
@@ -67,12 +70,15 @@ class ControlWindow {
     ipcMain.on('control.captionWindow.activate', () => {
       if(!captionWindow.window){
         captionWindow.createWindow()
-        console.log('GET control.captionWindow.activate')
       }
       else {
         captionWindow.window.show()
       }
     })
+    // 字幕引擎控制配置更新
+    ipcMain.on('control.control.change', (_, args) => {
+      setControls(args)
+    })
   }
 }
 

src/main/index.ts

@@ -3,7 +3,7 @@ import { electronApp, optimizer } from '@electron-toolkit/utils'
 import { controlWindow } from './control'
 import { captionWindow } from './caption'
 
-import { PythonProcess } from './pythonProcess'
+import { PythonProcess } from './utils/pythonProcess'
 const pySubProcess = new PythonProcess()
 
 app.whenReady().then(() => {
@@ -18,7 +18,7 @@ app.whenReady().then(() => {
 
   controlWindow.createWindow()
 
-  pySubProcess.start()
+  // pySubProcess.start()
 
   app.on('activate', function () {
     if (BrowserWindow.getAllWindows().length === 0){

src/main/types/index.ts

@@ -0,0 +1,29 @@
+export interface Styles {
+  fontFamily: string,
+  fontSize: number,
+  fontColor: string,
+  background: string,
+  opacity: number,
+  transDisplay: boolean,
+  transFontFamily: string,
+  transFontSize: number,
+  transFontColor: string
+}
+
+export interface CaptionItem {
+  index: number,
+  time_s: string,
+  time_t: string,
+  text: string,
+  translation: string
+}
+
+export interface Controls {
+  sourceLang: string,
+  targetLang: string,
+  engine: string,
+  translation: boolean,
+  customized: boolean,
+  customizedApp: string,
+  customizedCommand: string
+}

src/main/utils/config.ts

@@ -1,26 +1,7 @@
+import { Styles, CaptionItem, Controls } from '../types'
 import { BrowserWindow } from 'electron'
 
-export interface Styles {
+export const styles: Styles = {
-  fontFamily: string,
-  fontSize: number,
-  fontColor: string,
-  background: string,
-  opacity: number,
-  transDisplay: boolean,
-  transFontFamily: string,
-  transFontSize: number,
-  transFontColor: string
-}
-
-export interface CaptionItem {
-  index: number,
-  time_s: string,
-  time_t: string,
-  text: string,
-  translation: string
-}
-
-export let styles: Styles = {
   fontFamily: 'sans-serif',
   fontSize: 24,
   fontColor: '#000000',
@@ -32,6 +13,18 @@ export let styles: Styles = {
   transFontColor: '#000000'
 }
 
+export const captionLog: CaptionItem[] = []
+
+export const controls: Controls = {
+  sourceLang: 'en',
+  targetLang: 'zh',
+  engine: 'gummy',
+  translation: true,
+  customized: false,
+  customizedApp: '',
+  customizedCommand: ''
+}
+
 export function setStyles(args: any) {
   styles.fontFamily = args.fontFamily
   styles.fontSize = args.fontSize
@@ -42,18 +35,16 @@ export function setStyles(args: any) {
   styles.transFontFamily = args.transFontFamily
   styles.transFontSize = args.transFontSize
   styles.transFontColor = args.transFontColor
+  console.log('[INFO] Set Styles:', styles)
 }
 
 export function sendStyles(window: BrowserWindow) {
   window.webContents.send('caption.style.set', styles)
-  console.log('SNED caption.style.set')
+  console.log('[INFO] Send Styles:', styles)
 }
 
-export let captionLog: CaptionItem[] = []
-
 export function sendCaptionLog(window: BrowserWindow) {
   window.webContents.send('both.log.set', captionLog)
-  console.log('SEND both.log.set')
 }
 
 export function addCaptionLog(log: CaptionItem) {
@@ -64,8 +55,18 @@ export function addCaptionLog(log: CaptionItem) {
   else {
     captionLog.push(log)
   }
-  console.log('ADD caption')
   for(const window of BrowserWindow.getAllWindows()){
     sendCaptionLog(window)
   }
+}
+
+export function setControls(args: any) {
+  controls.sourceLang = args.sourceLang
+  controls.targetLang = args.targetLang
+  controls.engine = args.engine
+  controls.translation = args.translation
+  controls.customized = args.customized
+  controls.customizedApp = args.customizedApp
+  controls.customizedCommand = args.customizedCommand
+  console.log('[INFO] Set Controls:', controls)
 }

src/main/utils/pythonProcess.ts

@@ -1,7 +1,7 @@
 import { spawn } from 'child_process'
 import { app } from 'electron'
 import path from 'path'
-import { addCaptionLog } from './data'
+import { addCaptionLog } from './config'
 
 export class PythonProcess { 
     public start() { 

src/renderer/index.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <meta charset="UTF-8" />
-    <title>Auto Caption Player</title>
+    <title>Auto Caption</title>
     <!-- https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP -->
     <meta
       http-equiv="Content-Security-Policy"

src/renderer/src/components/CaptionControl.vue

@@ -9,7 +9,6 @@
       <span class="control-label">源语言</span>
       <a-select
         class="control-input"
-        ref="select"
         v-model:value="currentSourceLang"
         :options="langList"
       ></a-select>
@@ -18,7 +17,6 @@
       <span class="control-label">翻译语言</span>
       <a-select
         class="control-input"
-        ref="select"
         v-model:value="currentTargetLang"
         :options="langList.filter((item) => item.value !== 'auto')"
       ></a-select>
@@ -27,23 +25,34 @@
       <span class="control-label">字幕引擎</span>
       <a-select
         class="control-input"
-        ref="select"
         v-model:value="currentEngine"
         :options="captionEngine"
       ></a-select>
     </div>
-    <div class="control-item">
-      <span class="control-label">端口号</span>
-      <a-input
-        class="control-input"
-        ref="select"
-        type="number"
-        v-model:value="currentPort"
-      ></a-input>
-    </div>
     <div class="control-item">
       <span class="control-label">启用翻译</span>
       <a-switch v-model:checked="currentTranslation" />
+      <span class="control-label">自定义引擎</span>
+      <a-switch v-model:checked="currentCustomized" />
+    </div>
+    <div v-show="currentCustomized">
+      <a-card size="small" title="自定义字幕引擎">
+        <p class="customize-note">说明：允许用户使用自定义字幕引擎提供字幕。提供的引擎要能通过 <code>child_process.spawn()</code> 进行启动，且需要通过 IPC 与项目 node.js 后端进行通信。具体通信接口见后端实现。</p>
+        <div class="control-item">
+          <span class="control-label">引擎路径</span>
+          <a-input
+            class="control-input"
+            v-model:value="currentCustomizedApp"
+          ></a-input>
+        </div>
+        <div class="control-item">
+          <span class="control-label">引擎指令</span>
+          <a-input
+            class="control-input"
+            v-model:value="currentCustomizedCommand"
+          ></a-input>
+        </div>
+      </a-card>
     </div>
   </a-card>
   <div style="height: 20px;"></div>
@@ -60,9 +69,12 @@ const { captionEngine } = storeToRefs(captionControl)
 const currentSourceLang = ref('auto')
 const currentTargetLang = ref('zh')
 const currentEngine = ref('gummy')
-const currentPort = ref(8765)
 const currentTranslation = ref<boolean>(false)
 
+const currentCustomized = ref<boolean>(false)
+const currentCustomizedApp = ref('')
+const currentCustomizedCommand = ref('')
+
 const langList = computed(() => {
   for(let item of captionEngine.value){
     if(item.value === currentEngine.value) {
@@ -76,16 +88,24 @@ function applyChange(){
   captionControl.sourceLang = currentSourceLang.value
   captionControl.targetLang = currentTargetLang.value
   captionControl.engine = currentEngine.value
-  captionControl.port = currentPort.value
   captionControl.translation = currentTranslation.value
+
+  captionControl.customized = currentCustomized.value
+  captionControl.customizedApp = currentCustomizedApp.value
+  captionControl.customizedCommand = currentCustomizedCommand.value
+
+  captionControl.sendControlChange()
 }
 
 function cancelChange(){
   currentSourceLang.value = captionControl.sourceLang
   currentTargetLang.value = captionControl.targetLang
   currentEngine.value = captionControl.engine
-  currentPort.value = captionControl.port
   currentTranslation.value = captionControl.translation
+
+  currentCustomized.value = captionControl.customized
+  currentCustomizedApp.value = captionControl.customizedApp
+  currentCustomizedCommand.value = captionControl.customizedCommand
 }
 </script>
 
@@ -101,6 +121,12 @@ function cancelChange(){
   margin-right: 10px;
 }
 
+.customize-note {
+  padding: 0 20px;
+  color: red;
+  font-size: 12px;
+}
+
 .control-input {
   width: calc(100% - 100px);
   min-width: 100px;

src/renderer/src/main.ts

@@ -1,4 +1,4 @@
-import './assets/styles/reset.css'
+import './assets/reset.css'
 import { createPinia } from 'pinia'
 import { createApp } from 'vue'
 

src/renderer/src/stores/captionControl.ts

@@ -26,17 +26,36 @@ export const useCaptionControlStore = defineStore('captionControl', () => {
   ])
 
   const sourceLang = ref<string>('auto')
-  const targetLang = ref<string>('')
+  const targetLang = ref<string>('zh')
   const engine = ref<string>('gummy')
-  const port = ref<number>(8765)
   const translation = ref<boolean>(false)
+  const customized = ref<boolean>(false)
+  const customizedApp = ref<string>('')
+  const customizedCommand = ref<string>('')
+
+
+  function sendControlChange() {
+    const controls = {
+      sourceLang: sourceLang.value,
+      targetLang: targetLang.value,
+      engine: engine.value,
+      translation: translation.value,
+      customized: customized.value,
+      customizedApp: customizedApp.value,
+      customizedCommand: customizedCommand.value
+    }
+    window.electron.ipcRenderer.send('control.control.change', controls)
+  }
 
   return {
-    captionEngine,  // 字幕引擎
+    captionEngine,      // 字幕引擎
-    sourceLang,     // 源语言
+    sourceLang,         // 源语言
-    targetLang,     // 目标语言
+    targetLang,         // 目标语言
-    engine,         // 字幕引擎
+    engine,             // 字幕引擎
-    port,           // 端口
+    translation,        // 是否启用翻译
-    translation     // 是否启用翻译
+    customized,         // 是否使用自定义字幕引擎
+    customizedApp,      // 自定义字幕引擎的应用程序
+    customizedCommand,  // 自定义字幕引擎的命令
+    sendControlChange   // 发送最新控制消息到后端
   }
 })

src/renderer/src/stores/captionLog.ts

@@ -14,7 +14,6 @@ export const useCaptionLogStore = defineStore('captionLog', () => {
 
   window.electron.ipcRenderer.on('both.log.set', (_, logs) => {
     captionData.value = logs
-    console.log('GET both.log.set', logs)
   })
 
   return {

src/renderer/src/stores/captionStyle.ts

@@ -36,7 +36,6 @@ export const useCaptionStyleStore = defineStore('captionStyle', () => {
       transFontColor: transFontColor.value
     }
     window.electron.ipcRenderer.send('control.style.change', styles)
-    console.log('SEND control.style.change', styles)
   }
 
   window.electron.ipcRenderer.on('caption.style.set', (_, args) => {
@@ -50,7 +49,6 @@ export const useCaptionStyleStore = defineStore('captionStyle', () => {
     transFontSize.value = args.transFontSize
     transFontColor.value = args.transFontColor
     changeSignal.value = true
-    console.log('GET caption.style.set', args)
   })
 
   return {

src/renderer/src/views/CaptionPage.vue

@@ -58,7 +58,6 @@ onMounted(() => {
     for (const entry of entries) {
       if(windowHeight.value !== Math.floor(entry.contentRect.height) + 2) {
         windowHeight.value = Math.floor(entry.contentRect.height) + 2;
-        console.log('INFO window height change', windowHeight.value);
         window.electron.ipcRenderer.send('caption.windowHeight.change', windowHeight.value)
       }
     }