Upload folder using huggingface_hub

.gitattributes

@@ -40,3 +40,4 @@ wav/vad_example.wav filter=lfs diff=lfs merge=lfs -text
 *.json filter=lfs diff=lfs merge=lfs -text
 assert/gradio_demo.JPG filter=lfs diff=lfs merge=lfs -text
 wav/20200327_2P.wav filter=lfs diff=lfs merge=lfs -text
+dist/ax_meeting-0.1.0-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1 +1,2 @@
-__pycache__
+__pycache__
+ax_meeting.egg-info

MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include ax_meeting/static *
+recursive-include ax_meeting/ax_model *
+recursive-include ax_meeting/certs *
+recursive-include ax_meeting/vendor *
+include README.md

README.md

@@ -60,7 +60,7 @@ pip3 install -r requirements.txt
 启动：
 
 ```bash
-python -m app.server
+python -m ax_meeting.server
 ```
 
 浏览器访问：
@@ -79,6 +79,7 @@ HOST=0.0.0.0
 PORT=8000
 SSL_CERT=cert.pem
 SSL_KEY=key.pem
+AX_MODEL_DIR=/path/to/ax_model
 ```
 
 依赖提示（WebSocket）：
@@ -87,6 +88,9 @@ SSL_KEY=key.pem
 设备权限提示：
 - 如果遇到 `/dev/axcl_host` 权限错误，请用有权限的账号或 `sudo` 运行
 
+axengine 依赖提示：
+- 如果无法从 pip 获取 `pyaxengine`，请将本地 wheel 放到 `ax_meeting/vendor/`，或设置 `AXENGINE_WHEEL=/path/to/pyaxengine.whl`
+
 HTTPS（推荐，便于浏览器麦克风权限）：
 
 ```bash
@@ -96,9 +100,23 @@ openssl req -x509 -newkey rsa:2048 -nodes \\
 ```
 
 ```bash
-SSL_CERT=cert.pem SSL_KEY=key.pem python -m app.server
+SSL_CERT=cert.pem SSL_KEY=key.pem python -m ax_meeting.server
+```
+
+使用包内自签证书（默认打包在 `ax_meeting/certs/`）：
+
+```bash
+SSL_CERT=ax_meeting/certs/cert.pem SSL_KEY=ax_meeting/certs/key.pem python -m ax_meeting.server
 ```
 
+## 生成 wheel 包
+
+```bash
+./build_wheel.sh
+```
+
+生成结果在 `dist/` 目录。
+
 ![meeting_demo.png](assert/meeting_demo.png)
 
 
@@ -107,21 +125,51 @@ SSL_CERT=cert.pem SSL_KEY=key.pem python -m app.server
 对单个会议音频文件执行说话人聚类 + ASR，并导出文本，可选会议总结（LLM 通过参数配置）：
 
 ```bash
-python app/cli_batch.py --wav_file wav/vad_example.wav --output_dir output_dir
+python -m ax_meeting.vad_asr_cli --input wav/vad_example.wav --output_dir output_dir
 ```
 
-带会议总结（可选参数覆盖 LLM 配置）：
+说话人 + ASR（离线）：
 
 ```bash
-python app/cli_batch.py \\
+python -m ax_meeting.diar_asr_cli \\
   --wav_file wav/vad_example.wav \\
-  --output_dir output_dir \\
-  --summary \\
+  --output_dir output_dir
+```
+
+会议总结：
+
+```bash
+python -m ax_meeting.summarize_cli \\
+  --input output_dir/vad_example.txt \\
   --openai_base_url http://127.0.0.1:8001/v1 \\
   --openai_model AXERA-TECH/Qwen3-1.7B \\
   --openai_api_key xxx
 ```
 
+## Python API（轻量）
+
+```python
+from ax_meeting import VadAsrEngine, DiarAsrEngine, IncrementalSummarizer
+
+# VAD + ASR（流式）
+vad_asr = VadAsrEngine(stream=True)
+vad_asr.feed(audio_chunk)  # numpy / bytes / path / list
+segments = vad_asr.poll()   # 可能为空
+
+# 说话人 + ASR（离线）
+diar = DiarAsrEngine()
+text = diar.transcribe("wav/vad_example.wav")
+
+# 会议总结
+summarizer = IncrementalSummarizer()
+summary = summarizer.summarize_incrementally(text)
+```
+
+示例脚本：
+- `examples/vad_asr_stream.py`
+- `examples/diar_asr_offline.py`
+- `examples/summarize_text.py`
+
 ## Latency
 
 AX650N  

ax_meeting/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+__version__ = "0.1.0"
+
+from ax_meeting.engines import VadAsrEngine, DiarAsrEngine  # noqa: F401
+from ax_meeting.summarizer import IncrementalSummarizer  # noqa: F401

ax_meeting/ax_model/.gitattributes

@@ -0,0 +1,2 @@
+*.axmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text

ax_meeting/ax_model/auto.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d0997706b30274f7ff3b157ca90df50b7ed8ced35091a0231700355d5ee1374
+size 2368

ax_meeting/ax_model/campplus.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e85b4d59a94488caa727ca76c7fe2ba408de02587cd8c9a6680b83488338b16
+size 10744358

ax_meeting/ax_model/chn_jpn_yue_eng_ko_spectok.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa87f86064c3730d799ddf7af3c04659151102cba548bce325cf06ba4da4e6a8
+size 377341

ax_meeting/ax_model/en.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f21c6d5a0b876c696b15ee80317ee558b6a19a084c1147a13a2871900f6c72e
+size 2368

ax_meeting/ax_model/event_emo.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d22e3df5d192fdc3e73e368a2cb576975a5a43a114a8432a91c036adf8e2263
+size 4608

ax_meeting/ax_model/ja.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75a83ded838de7d02a56de32b774a50d2f02b5ec69880e7acdce600666f77517
+size 2368

ax_meeting/ax_model/ko.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f371fffcecd75c1268e2ef8043c276ec9b231177b5138b39e5732c65401749db
+size 2368

ax_meeting/ax_model/sensevoice.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b64a36fa15e75ab5e3b75f18ae87a058970cff76219407e503b54fb53dd8e38
+size 262170623

ax_meeting/ax_model/sensevoice/am.mvn

@@ -0,0 +1,8 @@
+<Nnet> 
+<Splice> 560 560
+[ 0 ]
+<AddShift> 560 560 
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 560 560
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet> 

ax_meeting/ax_model/sensevoice/config.yaml

@@ -0,0 +1,97 @@
+encoder: SenseVoiceEncoderSmall
+encoder_conf:
+    output_size: 512
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 50
+    tp_blocks: 20
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: pe
+    pos_enc_class: SinusoidalPositionEncoder
+    normalize_before: true
+    kernel_size: 11
+    sanm_shfit: 0
+    selfattention_layer_type: sanm
+
+
+model: SenseVoiceSmall
+model_conf:
+    length_normalized_loss: true
+    sos: 1
+    eos: 2
+    ignore_id: -1
+
+tokenizer: SentencepiecesTokenizer
+tokenizer_conf:
+  bpemodel: null
+  unk_symbol: <unk>
+  split_with_space: true
+
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 7
+    lfr_n: 6
+    cmvn_file: null
+
+
+dataset: SenseVoiceCTCDataset
+dataset_conf:
+  index_ds: IndexDSJsonl
+  batch_sampler: EspnetStyleBatchSampler
+  data_split_num: 32
+  batch_type: token
+  batch_size: 14000
+  max_token_length: 2000
+  min_token_length: 60
+  max_source_length: 2000
+  min_source_length: 60
+  max_target_length: 200
+  min_target_length: 0
+  shuffle: true
+  num_workers: 4
+  sos: ${model_conf.sos}
+  eos: ${model_conf.eos}
+  IndexDSJsonl: IndexDSJsonl
+  retry: 20
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 20
+  keep_nbest_models: 10
+  avg_nbest_model: 10
+  log_interval: 100
+  resume: true
+  validate_interval: 10000
+  save_checkpoint_interval: 10000
+
+optim: adamw
+optim_conf:
+  lr: 0.00002
+scheduler: warmuplr
+scheduler_conf:
+  warmup_steps: 25000
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 6
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1

ax_meeting/ax_model/vad.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d89d1087f14597bc93ae5cf89e1c60d8ee76d7bdffbe679ef6dc002635e6f6b
+size 1150874

ax_meeting/ax_model/vad/am.mvn

@@ -0,0 +1,8 @@
+<Nnet>
+<Splice> 400 400
+[ 0 ]
+<AddShift> 400 400
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 400 400
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet>

ax_meeting/ax_model/vad/config.yaml

@@ -0,0 +1,56 @@
+frontend: WavFrontendOnline
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    dither: 0.0
+    lfr_m: 5
+    lfr_n: 1
+
+model: FsmnVADStreaming
+model_conf:
+    sample_rate: 16000
+    detect_mode: 1 
+    snr_mode: 0
+    max_end_silence_time: 800
+    max_start_silence_time: 3000
+    do_start_point_detection: True
+    do_end_point_detection: True
+    window_size_ms: 200
+    sil_to_speech_time_thres: 150
+    speech_to_sil_time_thres: 150
+    speech_2_noise_ratio: 1.0
+    do_extend: 1
+    lookback_time_start_point: 200
+    lookahead_time_end_point: 100
+    max_single_segment_time: 60000
+    snr_thres: -100.0
+    noise_frame_num_used_for_snr: 100
+    decibel_thres: -100.0
+    speech_noise_thres: 0.6
+    fe_prior_thres: 0.0001
+    silence_pdf_num: 1
+    sil_pdf_ids: [0]
+    speech_noise_thresh_low: -0.1
+    speech_noise_thresh_high: 0.3
+    output_frame_probs: False
+    frame_in_ms: 10
+    frame_length_ms: 25
+    
+encoder: FSMN
+encoder_conf:
+    input_dim: 400
+    input_affine_dim: 140
+    fsmn_layers: 4
+    linear_dim: 250
+    proj_dim: 128
+    lorder: 20
+    rorder: 0
+    lstride: 1
+    rstride: 0
+    output_affine_dim: 140
+    output_dim: 248
+
+

ax_meeting/ax_model/withitn.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39bf02586f59237894fc2918ab2db4f12ec3c084c41465718832fbd7646ea729
+size 2368

ax_meeting/ax_model/yue.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:319c3d30e6b176338a6d8871f81ec57af5c3c112cecbdce4a0bf36bec51f3ceb
+size 2368

ax_meeting/ax_model/zh.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c952f976ae70642c8f682c88711abce3a2b64ba20a6d8ca0b706f227de816eb
+size 2368

ax_meeting/axengine_loader.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+import os
+import glob
+import subprocess
+import sys
+
+
+def ensure_axengine():
+    try:
+        import axengine  # noqa: F401
+        return
+    except Exception:
+        pass
+
+    wheel_path = os.getenv("AXENGINE_WHEEL")
+    if wheel_path and os.path.exists(wheel_path):
+        _install_wheel(wheel_path)
+        return
+
+    # Try bundled wheel
+    here = os.path.dirname(__file__)
+    candidates = glob.glob(os.path.join(here, "vendor", "pyaxengine*.whl"))
+    candidates += glob.glob(os.path.join(here, "vendor", "axengine*.whl"))
+    if candidates:
+        _install_wheel(candidates[0])
+        return
+
+    raise RuntimeError(
+        "axengine not found. Please install pyaxengine wheel or set AXENGINE_WHEEL."
+    )
+
+
+def _install_wheel(path: str):
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", path])

ax_meeting/certs/cert.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDETCCAfmgAwIBAgIUPI3rAC24IoJ7FUydYijEOp7GeygwDQYJKoZIhvcNAQEL
+BQAwGDEWMBQGA1UEAwwNMTAuMTI2LjMzLjE0MDAeFw0yNjAyMjcwMzE3MzRaFw0y
+NzAyMjcwMzE3MzRaMBgxFjAUBgNVBAMMDTEwLjEyNi4zMy4xNDAwggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQChzhKivPsGUSjfzFWwocMg1FT56iDFo8yy
+tba/LvbP2i2BpujTqK5/6iInLh6N9ZptJg4PsLSEQ2HWfLdYupEYvMrXDy4nYwsX
+gAbrjAz3uhsJV2+LSVF/0g8PNhwidDN4WNWLQoVf5g9FCxl0SneCoyKpQdpIB10r
+J0ZXtKe5SY9Ydq0EdjS+5898U83XgOIFQfKDRdakPuxKLX00DFd6S5Xz4Yw148As
+ypAOfNSXCqX0+2wtSyfAednwlPea+VxQPExQBpx7yOYe0eDrMDDPP7z6UUdzMdoD
+ZOlWTgtIarBCthD9HnhZhqK6iom9sxm9hvHOV+kM/pje1iMeaFbpAgMBAAGjUzBR
+MB0GA1UdDgQWBBRhENjhWth7MneI5GxGFzCu2RjUbTAfBgNVHSMEGDAWgBRhENjh
+Wth7MneI5GxGFzCu2RjUbTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA
+A4IBAQAXi8HfQF1LyACZSkilE6emnB0zh31NMR7oimhc9rlKgX5yPD4RQp+OKr8l
+APeIFeWLiHadvxKpjnT+MQwzODs5i+Qai0vDaRoy/OLPiaI3rH8yaAMxMI0hrh4c
+Rfyf6bWBvcTpuUTFO/hC1KxfVVCjCwMcV/Jmv0xukCMCXCHNqDdkG2m24RSmKUHl
+Af85bduX8PabtZxEDESkLc2Z8G8w5VvEWJizQ7OF0EcYJIA3JEDcr50o5kFpu7OQ
+ZkRfWopQ3I1DOUOfSrc4d5gMy3b1rvDBbeLTRsNnec9WciwP4U/+Hbxs2P+BALo+
+WSq6D7FC0WTTXtYdGFv2oY5s38Dr
+-----END CERTIFICATE-----

ax_meeting/certs/key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQChzhKivPsGUSjf
+zFWwocMg1FT56iDFo8yytba/LvbP2i2BpujTqK5/6iInLh6N9ZptJg4PsLSEQ2HW
+fLdYupEYvMrXDy4nYwsXgAbrjAz3uhsJV2+LSVF/0g8PNhwidDN4WNWLQoVf5g9F
+Cxl0SneCoyKpQdpIB10rJ0ZXtKe5SY9Ydq0EdjS+5898U83XgOIFQfKDRdakPuxK
+LX00DFd6S5Xz4Yw148AsypAOfNSXCqX0+2wtSyfAednwlPea+VxQPExQBpx7yOYe
+0eDrMDDPP7z6UUdzMdoDZOlWTgtIarBCthD9HnhZhqK6iom9sxm9hvHOV+kM/pje
+1iMeaFbpAgMBAAECggEAQjovPYX9dy3z/XpM3pGvZPoT2AEBLfQn/kPLS4CFDDlg
+o+814CBsYDXsib3iSreq4B8R5VEt6e8MljaQ8xPV/NqVaaYwfXWYHiPMcU/vJNx7
+YXz0zn2RirBncpHyvRVz1cACk9AD+GcZe+iZoBQ0y3dLYhzuo8nD1Dxsmcx7VCaF
+VTAVKGsCSW3ZHfXXxMDkB/3tiRknb4KNpvVzLo2GNgj5fiygkkgf9hM1hRfqt7/P
+nejF3A0laWQc0N0OluM6SF+2A4uNnWQwIeyLK/AdM14Me5rvYjiOG/xE1hNVBvup
+utkjVyVYP1pCoK0eCqaApd61DuDcagFPi6gSmKeY4wKBgQDXF0cGE6tbohNUa/GU
+r8jCb51F1gWuLJNl4y6SMDr9/iku1OKkJtbK0InMmfeyeKeg/xeuXSc5QJY5hcrd
+A/Y4mTRWlpfzpYCj5qos7f1HVl/mtwBqN2VUVi2Tid1OrfDgTup0ZZlN1IjszH3/
+37IQnJTs/sfVxwwh+ij+VKvP6wKBgQDAlFDTq8lRtbc/nVAqLPnnc2HnLtrAzeLg
+N1qD7QQvcfWP9gxSg0ZBYJf52uYRjLDB4b6LBPV5Ys9SIcXz8G6VYwUESkJcnQ+s
+yTNUuJolzAFq0NoMLAAN0vKV2yyIdGP/oupTYEfSRz1bRtIGchhY/XMdVjj5Ppxc
+w3CfAAcTewKBgCI4nuE1oe7bU437+py4dw2Qaopg6dhzWSQ9x/wUVl5w4KaF0mVh
+lI0CLtpxqLopfiocS+0+/u2Z/Ay837DYX4VTwsMABL8MFvJ80ZiCaOi/slRny1Ya
+6DFJ4Mh3h9Fr1UYq6ByKyaBbb0mVo3phYdhIwV0PkEXP/HsvbPRCDm/vAoGBALng
+bgNgk/giBLWKCY4ryyny3FRfjRT7pDf2NY+QfbGttO829b3Op0kDCq1G8zmNKi54
+zYkxSB3ZmXIU1xQUxSe7Y2Q4qMTrc+26ZakoZOCGf/exjkShU4wER9EMs3choENl
+4/aFv8zepgIr4RwHlCiQuUNfra4lGJcQrOtLA4lxAoGAWn8ovOGTEo5Seuq5CoCQ
+HbDdMmd6z9kLInKcqaGsuclFaEZjG6CHznD7B+SRtJFBN3+L0K3UbwNeAka6yEqg
+3TUXNBXMypOKYPYDAbGE8t7wrLL7WEtgHiqpkN/zZoxX61f3nLOIoku817dW3KtW
+PTUCWpm7u5IQWl6T+ThOJmU=
+-----END PRIVATE KEY-----

ax_meeting/config.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+SAMPLE_RATE = 16000
+
+# How often to run VAD checks on the rolling audio buffer
+VAD_CHECK_INTERVAL_SEC = 0.8
+
+# If end time is earlier than (now - PAUSE_MS), treat as a completed segment
+PAUSE_MS = 800
+
+# Ignore too-short segments
+MIN_SEGMENT_MS = 300
+
+# Merge VAD segments shorter than this in offline diarization
+MERGE_VAD_MAX_LEN_MS = 15 * 1000
+
+# LLM summarization chunking
+SUMMARY_CHUNK_CHARS = 1000
+SUMMARY_TARGET_CHARS = 100

ax_meeting/diar_asr_cli.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+import argparse
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+
+from ax_meeting.model_bundle import ModelBundle
+from ax_meeting.utils.vad_utils import merge_vad
+from ax_meeting.utils.ax_cam_bin import do_clustering, chunk
+from ax_meeting.diar_utils import pick_speaker
+from ax_meeting.config import MERGE_VAD_MAX_LEN_MS
+
+
+def load_audio(path: str, target_sr: int = 16000) -> np.ndarray:
+    audio, sr = sf.read(path, dtype="float32")
+    if audio.ndim > 1:
+        audio = audio.mean(axis=1)
+    if sr != target_sr:
+        ratio = target_sr / float(sr)
+        new_len = int(len(audio) * ratio)
+        if new_len <= 0:
+            return np.zeros(0, dtype=np.float32)
+        x_old = np.linspace(0, 1, num=len(audio), endpoint=False)
+        x_new = np.linspace(0, 1, num=new_len, endpoint=False)
+        audio = np.interp(x_new, x_old, audio).astype(np.float32)
+    return audio
+
+
+def diar_asr(bundle: ModelBundle, speech: np.ndarray, fs: int = 16000) -> str:
+    if speech.size == 0:
+        return ""
+
+    res_vad = bundle.vad_infer(speech)
+    if not res_vad:
+        return ""
+
+    vad_segments = merge_vad(res_vad, MERGE_VAD_MAX_LEN_MS)
+    vad_time = [[vad_t[0] / 1000, vad_t[1] / 1000] for vad_t in res_vad]
+    chunks = [c for (st, ed) in vad_time for c in chunk(st, ed)]
+
+    if not chunks:
+        return ""
+
+    embeddings = bundle.speaker_infer(speech, fs, chunks=chunks)
+    _, diar_results = do_clustering(chunks, embeddings, speaker_num=None)
+
+    lines = []
+    for i, segment in enumerate(vad_segments):
+        segment_start, segment_end = segment
+        start_sample = int(segment_start / 1000 * fs)
+        end_sample = min(int(segment_end / 1000 * fs), speech.shape[0])
+        segment_speech = speech[start_sample:end_sample]
+
+        text, _ = bundle.asr_infer(
+            segment_speech,
+            output_timestamp=False,
+            key=f"segment_{i}",
+        )
+        if not text or not text.strip():
+            continue
+
+        spk = pick_speaker(segment_start / 1000.0, segment_end / 1000.0, diar_results)
+        lines.append(
+            f"Speaker_{spk}: [{segment_start/1000.0:.3f} {segment_end/1000.0:.3f}] {text.strip()}"
+        )
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Offline diarization + ASR")
+    parser.add_argument("--wav_file", required=True, help="Input audio file")
+    parser.add_argument("--output_dir", default="output_dir", help="Output directory")
+    args = parser.parse_args()
+
+    out_dir = Path(args.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    speech = load_audio(args.wav_file, target_sr=16000)
+
+    bundle = ModelBundle()
+    bundle.ensure_loaded()
+
+    transcript = diar_asr(bundle, speech, fs=16000)
+    transcript_path = out_dir / (Path(args.wav_file).stem + ".txt")
+    transcript_path.write_text(transcript, encoding="utf-8")
+    print(f"Transcript saved: {transcript_path}")
+
+    # Summary is handled by ax_meeting.summarize_cli
+
+
+if __name__ == "__main__":
+    main()

ax_meeting/diar_utils.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+def pick_speaker(seg_st: float, seg_ed: float, diar_results) -> int:
+    if not diar_results:
+        return 0
+    best_spk = diar_results[0][2]
+    best_overlap = 0.0
+    for st_spk, ed_spk, spk in diar_results:
+        overlap = min(seg_ed, ed_spk) - max(seg_st, st_spk)
+        if overlap > best_overlap:
+            best_overlap = overlap
+            best_spk = spk
+    return int(best_spk)

ax_meeting/engines.py

@@ -0,0 +1,207 @@
+# -*- coding: utf-8 -*-
+import os
+import shutil
+import subprocess
+from dataclasses import dataclass
+from typing import List, Iterable, Optional, Union, Tuple
+
+import numpy as np
+import soundfile as sf
+
+from ax_meeting.model_bundle import ModelBundle
+from ax_meeting.config import SAMPLE_RATE, PAUSE_MS, MIN_SEGMENT_MS, MERGE_VAD_MAX_LEN_MS
+from ax_meeting.diar_utils import pick_speaker
+from ax_meeting.utils.vad_utils import merge_vad
+from ax_meeting.utils.ax_cam_bin import do_clustering, chunk
+
+
+@dataclass
+class VadAsrSegment:
+    start_ms: int
+    end_ms: int
+    text: str
+    audio: np.ndarray
+
+
+def _resample(audio: np.ndarray, src_sr: int, tgt_sr: int) -> np.ndarray:
+    if src_sr == tgt_sr:
+        return audio
+    ratio = tgt_sr / float(src_sr)
+    new_len = int(len(audio) * ratio)
+    if new_len <= 0:
+        return np.zeros(0, dtype=np.float32)
+    x_old = np.linspace(0, 1, num=len(audio), endpoint=False)
+    x_new = np.linspace(0, 1, num=new_len, endpoint=False)
+    return np.interp(x_new, x_old, audio).astype(np.float32)
+
+
+def _load_audio_file(path: str, target_sr: int = SAMPLE_RATE) -> np.ndarray:
+    try:
+        audio, sr = sf.read(path, dtype="float32")
+        if audio.ndim > 1:
+            audio = audio.mean(axis=1)
+        return _resample(audio, sr, target_sr)
+    except Exception:
+        # Try ffmpeg for formats like mp4/m4a
+        if shutil.which("ffmpeg") is None:
+            raise
+        cmd = [
+            "ffmpeg",
+            "-i",
+            path,
+            "-f",
+            "f32le",
+            "-ac",
+            "1",
+            "-ar",
+            str(target_sr),
+            "-",
+        ]
+        proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True)
+        return np.frombuffer(proc.stdout, dtype=np.float32)
+
+
+def _normalize_inputs(inputs: Union[np.ndarray, bytes, str, Iterable]) -> List[np.ndarray]:
+    if isinstance(inputs, np.ndarray):
+        return [inputs.astype(np.float32)]
+    if isinstance(inputs, (bytes, bytearray)):
+        audio = np.frombuffer(inputs, dtype=np.int16).astype(np.float32) / 32768.0
+        return [audio]
+    if isinstance(inputs, str):
+        return [_load_audio_file(inputs)]
+    if isinstance(inputs, Iterable):
+        out = []
+        for item in inputs:
+            out.extend(_normalize_inputs(item))
+        return out
+    raise TypeError(f"Unsupported input type: {type(inputs)}")
+
+
+class VadAsrEngine:
+    def __init__(
+        self,
+        model_bundle: Optional[ModelBundle] = None,
+        stream: bool = False,
+        sample_rate: int = SAMPLE_RATE,
+        pause_ms: int = PAUSE_MS,
+        min_segment_ms: int = MIN_SEGMENT_MS,
+    ):
+        self.bundle = model_bundle or ModelBundle()
+        self.bundle.ensure_loaded()
+        self.stream = stream
+        self.sample_rate = sample_rate
+        self.pause_ms = pause_ms
+        self.min_segment_ms = min_segment_ms
+        self.audio_chunks: List[np.ndarray] = []
+        self.total_samples: int = 0
+        self.last_processed_ms: int = 0
+
+    def feed(self, audio: Union[np.ndarray, bytes, str, Iterable]) -> None:
+        chunks = _normalize_inputs(audio)
+        for ch in chunks:
+            if ch.size == 0:
+                continue
+            self.audio_chunks.append(ch)
+            self.total_samples += ch.size
+
+    def _audio_all(self) -> np.ndarray:
+        if not self.audio_chunks:
+            return np.zeros(0, dtype=np.float32)
+        return np.concatenate(self.audio_chunks, axis=0)
+
+    def _current_ms(self) -> int:
+        return int(self.total_samples / self.sample_rate * 1000)
+
+    def poll(self) -> List[VadAsrSegment]:
+        if not self.stream:
+            raise RuntimeError("poll() only available in stream mode")
+        audio = self._audio_all()
+        if audio.size == 0:
+            return []
+        current_ms = self._current_ms()
+        vad_segments = self.bundle.vad_infer(audio)
+        out: List[VadAsrSegment] = []
+        for start_ms, end_ms in vad_segments:
+            if end_ms <= self.last_processed_ms:
+                continue
+            if (end_ms - start_ms) < self.min_segment_ms:
+                continue
+            if end_ms > current_ms - self.pause_ms:
+                continue
+            start_sample = int(start_ms / 1000 * self.sample_rate)
+            end_sample = int(end_ms / 1000 * self.sample_rate)
+            seg_audio = audio[start_sample:end_sample]
+            text, _ = self.bundle.asr_infer(seg_audio, output_timestamp=False, key="stream")
+            if text.strip():
+                out.append(VadAsrSegment(start_ms, end_ms, text.strip(), seg_audio))
+            self.last_processed_ms = max(self.last_processed_ms, int(end_ms))
+        return out
+
+    def transcribe(self, audio: Union[np.ndarray, bytes, str, Iterable]) -> List[VadAsrSegment]:
+        chunks = _normalize_inputs(audio)
+        if not chunks:
+            return []
+        speech = np.concatenate(chunks, axis=0)
+        vad_segments = self.bundle.vad_infer(speech)
+        vad_segments = merge_vad(vad_segments, MERGE_VAD_MAX_LEN_MS)
+        out: List[VadAsrSegment] = []
+        for start_ms, end_ms in vad_segments:
+            if (end_ms - start_ms) < self.min_segment_ms:
+                continue
+            start_sample = int(start_ms / 1000 * self.sample_rate)
+            end_sample = int(end_ms / 1000 * self.sample_rate)
+            seg_audio = speech[start_sample:end_sample]
+            text, _ = self.bundle.asr_infer(seg_audio, output_timestamp=False, key="offline")
+            if text.strip():
+                out.append(VadAsrSegment(start_ms, end_ms, text.strip(), seg_audio))
+        return out
+
+
+class DiarAsrEngine:
+    def __init__(self, model_bundle: Optional[ModelBundle] = None, sample_rate: int = SAMPLE_RATE):
+        self.bundle = model_bundle or ModelBundle()
+        self.bundle.ensure_loaded()
+        self.sample_rate = sample_rate
+        self.audio_chunks: List[np.ndarray] = []
+
+    def feed(self, audio: Union[np.ndarray, bytes, str, Iterable]) -> None:
+        chunks = _normalize_inputs(audio)
+        for ch in chunks:
+            if ch.size == 0:
+                continue
+            self.audio_chunks.append(ch)
+
+    def finalize(self) -> str:
+        if not self.audio_chunks:
+            return ""
+        speech = np.concatenate(self.audio_chunks, axis=0)
+        return self.transcribe(speech)
+
+    def transcribe(self, audio: Union[np.ndarray, bytes, str, Iterable]) -> str:
+        chunks = _normalize_inputs(audio)
+        if not chunks:
+            return ""
+        speech = np.concatenate(chunks, axis=0)
+        res_vad = self.bundle.vad_infer(speech)
+        vad_segments = merge_vad(res_vad, MERGE_VAD_MAX_LEN_MS)
+        vad_time = [[vad_t[0] / 1000, vad_t[1] / 1000] for vad_t in res_vad]
+        chunks = [c for (st, ed) in vad_time for c in chunk(st, ed)]
+        if not chunks:
+            return ""
+        embeddings = self.bundle.speaker_infer(speech, self.sample_rate, chunks=chunks)
+        _, diar_results = do_clustering(chunks, embeddings, speaker_num=None)
+
+        lines = []
+        for i, segment in enumerate(vad_segments):
+            segment_start, segment_end = segment
+            start_sample = int(segment_start / 1000 * self.sample_rate)
+            end_sample = min(int(segment_end / 1000 * self.sample_rate), speech.shape[0])
+            segment_speech = speech[start_sample:end_sample]
+            text, _ = self.bundle.asr_infer(segment_speech, output_timestamp=False, key=f"segment_{i}")
+            if not text or not text.strip():
+                continue
+            spk = pick_speaker(segment_start / 1000.0, segment_end / 1000.0, diar_results)
+            lines.append(
+                f"Speaker_{spk}: [{segment_start/1000.0:.3f} {segment_end/1000.0:.3f}] {text.strip()}"
+            )
+        return "\n".join(lines)

ax_meeting/model_bundle.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+import os
+from pathlib import Path
+import threading
+import numpy as np
+
+from ax_meeting.positional import SinusoidalPositionEncoder
+from ax_meeting.utils.ax_model_bin import AX_SenseVoiceSmall
+from ax_meeting.utils.ax_vad_bin import AX_Fsmn_vad
+from ax_meeting.utils.ax_cam_bin import AX_SpeakerEmbeddingInference
+from ax_meeting.utils.sentencepiece_tokenizer import SentencepiecesTokenizer
+from ax_meeting.text_cleaner import clean_asr_text
+
+
+class ModelBundle:
+    def __init__(self, ax_model_dir: str | None = None, seq_len: int = 132):
+        if ax_model_dir is None:
+            env_dir = os.getenv("AX_MODEL_DIR")
+            if env_dir:
+                ax_model_dir = env_dir
+            else:
+                ax_model_dir = str(Path(__file__).resolve().parent / "ax_model")
+        self.ax_model_dir = ax_model_dir
+        self.seq_len = seq_len
+
+        self._lock = threading.Lock()
+        self._loaded = False
+        self._load_error = None
+
+        self.vad = None
+        self.speaker = None
+        self.position_encoding = None
+        self.asr = None
+        self.tokenizer = None
+        self.language = "zh"
+        self.withitn = True
+
+    def load(self):
+        if self._loaded:
+            return
+        try:
+            self.vad = AX_Fsmn_vad(self.ax_model_dir)
+            self.speaker = AX_SpeakerEmbeddingInference(model_dir=self.ax_model_dir)
+
+            embed = SinusoidalPositionEncoder()
+            dummy = np.zeros((1, self.seq_len, 560), dtype=np.float32)
+            self.position_encoding = embed.get_position_encoding(dummy)
+            self.asr = AX_SenseVoiceSmall(self.ax_model_dir, seq_len=self.seq_len)
+
+            tokenizer_path = os.path.join(self.ax_model_dir, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+            self.tokenizer = SentencepiecesTokenizer(bpemodel=tokenizer_path)
+
+            self._loaded = True
+        except Exception as e:
+            self._load_error = e
+            raise
+
+    def ensure_loaded(self):
+        if not self._loaded:
+            self.load()
+
+    def vad_infer(self, audio: np.ndarray):
+        self.ensure_loaded()
+        with self._lock:
+            return self.vad(audio)[0]
+
+    def speaker_infer(self, audio: np.ndarray, fs: int, chunks):
+        self.ensure_loaded()
+        with self._lock:
+            return self.speaker(audio, fs, chunks=chunks)
+
+    def asr_infer(self, audio: np.ndarray, output_timestamp: bool = False, key: str = "segment"):
+        self.ensure_loaded()
+        with self._lock:
+            results, meta = self.asr(
+                audio,
+                self.language,
+                self.withitn,
+                self.position_encoding,
+                tokenizer=self.tokenizer,
+                output_timestamp=output_timestamp,
+                ban_emo_unk=False,
+                key=[key],
+            )
+        text = "".join([r.get("text", "") for r in results])
+        text = clean_asr_text(text)
+        return text, meta

ax_meeting/pipeline.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+import asyncio
+import time
+from dataclasses import dataclass, field
+from typing import List, Dict, Any, Optional
+
+import numpy as np
+
+from ax_meeting.config import (
+    SAMPLE_RATE,
+    VAD_CHECK_INTERVAL_SEC,
+    PAUSE_MS,
+    MIN_SEGMENT_MS,
+    MERGE_VAD_MAX_LEN_MS,
+)
+from ax_meeting.utils.vad_utils import merge_vad
+from ax_meeting.utils.ax_cam_bin import do_clustering, chunk
+from ax_meeting.diar_utils import pick_speaker
+
+
+@dataclass
+class TranscriptSegment:
+    start_ms: int
+    end_ms: int
+    text: str
+
+
+@dataclass
+class StreamingMeetingSession:
+    model_bundle: Any
+    session_id: str
+    sample_rate: int = SAMPLE_RATE
+    pause_ms: int = PAUSE_MS
+    min_segment_ms: int = MIN_SEGMENT_MS
+    vad_check_interval_sec: float = VAD_CHECK_INTERVAL_SEC
+
+    audio_chunks: List[np.ndarray] = field(default_factory=list)
+    total_samples: int = 0
+    last_processed_ms: int = 0
+    last_vad_check_ts: float = field(default_factory=lambda: 0.0)
+    processing: bool = False
+
+    async def add_audio(self, pcm_bytes: bytes, websocket):
+        if not pcm_bytes:
+            return
+        audio = np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+        if audio.size == 0:
+            return
+        self.audio_chunks.append(audio)
+        self.total_samples += audio.size
+
+        now = time.time()
+        if (now - self.last_vad_check_ts) >= self.vad_check_interval_sec and not self.processing:
+            self.last_vad_check_ts = now
+            self.processing = True
+            asyncio.create_task(self._run_vad_and_asr(websocket))
+
+    def _current_ms(self) -> int:
+        return int(self.total_samples / self.sample_rate * 1000)
+
+    def _audio_all(self) -> np.ndarray:
+        if not self.audio_chunks:
+            return np.zeros(0, dtype=np.float32)
+        return np.concatenate(self.audio_chunks, axis=0)
+
+    async def _run_vad_and_asr(self, websocket):
+        try:
+            segments = await asyncio.to_thread(self._detect_ready_segments)
+            for seg in segments:
+                await websocket.send_json(
+                    {
+                        "type": "transcript",
+                        "start_ms": seg.start_ms,
+                        "end_ms": seg.end_ms,
+                        "text": seg.text,
+                    }
+                )
+        finally:
+            self.processing = False
+
+    def _detect_ready_segments(self) -> List[TranscriptSegment]:
+        audio = self._audio_all()
+        if audio.size == 0:
+            return []
+
+        current_ms = self._current_ms()
+        vad_segments = self.model_bundle.vad_infer(audio)
+        ready_segments: List[TranscriptSegment] = []
+
+        for start_ms, end_ms in vad_segments:
+            if end_ms <= self.last_processed_ms:
+                continue
+            if (end_ms - start_ms) < self.min_segment_ms:
+                continue
+            if end_ms > current_ms - self.pause_ms:
+                continue
+
+            start_sample = int(start_ms / 1000 * self.sample_rate)
+            end_sample = int(end_ms / 1000 * self.sample_rate)
+            seg_audio = audio[start_sample:end_sample]
+            text, _ = self.model_bundle.asr_infer(seg_audio, output_timestamp=False, key="live")
+            if text.strip():
+                ready_segments.append(TranscriptSegment(start_ms, end_ms, text.strip()))
+            self.last_processed_ms = max(self.last_processed_ms, int(end_ms))
+
+        return ready_segments
+
+    async def finalize(self) -> Dict[str, Any]:
+        # Flush remaining segments
+        await asyncio.to_thread(self._flush_remaining_segments)
+
+        # Full diarization + ASR
+        transcript = await asyncio.to_thread(self._full_diarization_asr)
+        return {"transcript": transcript}
+
+    def _flush_remaining_segments(self):
+        audio = self._audio_all()
+        if audio.size == 0:
+            return
+        vad_segments = self.model_bundle.vad_infer(audio)
+        for start_ms, end_ms in vad_segments:
+            if end_ms <= self.last_processed_ms:
+                continue
+            start_sample = int(start_ms / 1000 * self.sample_rate)
+            end_sample = int(end_ms / 1000 * self.sample_rate)
+            seg_audio = audio[start_sample:end_sample]
+            text, _ = self.model_bundle.asr_infer(seg_audio, output_timestamp=False, key="final")
+            self.last_processed_ms = max(self.last_processed_ms, int(end_ms))
+
+    def _full_diarization_asr(self) -> str:
+        speech = self._audio_all()
+        if speech.size == 0:
+            return ""
+
+        res_vad = self.model_bundle.vad_infer(speech)
+        vad_segments = merge_vad(res_vad, MERGE_VAD_MAX_LEN_MS)
+
+        vad_time = [[vad_t[0] / 1000, vad_t[1] / 1000] for vad_t in res_vad]
+        chunks = [c for (st, ed) in vad_time for c in chunk(st, ed)]
+
+        embeddings = self.model_bundle.speaker_infer(speech, self.sample_rate, chunks=chunks)
+        _, diar_results = do_clustering(chunks, embeddings, speaker_num=None)
+
+        lines = []
+        for i, segment in enumerate(vad_segments):
+            segment_start, segment_end = segment
+            start_sample = int(segment_start / 1000 * self.sample_rate)
+            end_sample = min(int(segment_end / 1000 * self.sample_rate), speech.shape[0])
+            segment_speech = speech[start_sample:end_sample]
+
+            text, _ = self.model_bundle.asr_infer(
+                segment_speech,
+                output_timestamp=False,
+                key=f"segment_{i}",
+            )
+            if not text or not text.strip():
+                continue
+
+            spk = pick_speaker(segment_start / 1000.0, segment_end / 1000.0, diar_results)
+            lines.append(
+                f"Speaker_{spk}: [{segment_start/1000.0:.3f} {segment_end/1000.0:.3f}] {text.strip()}"
+            )
+        return "\n".join(lines)

ax_meeting/positional.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+import numpy as np
+
+
+class SinusoidalPositionEncoder:
+    def __init__(self, base: float = 10000.0):
+        self.base = base
+
+    def get_position_encoding(self, dummy: np.ndarray) -> np.ndarray:
+        # dummy shape: [1, seq_len, d_model]
+        seq_len = int(dummy.shape[1])
+        d_model = int(dummy.shape[2])
+        position = np.arange(seq_len)[:, None]
+        div_term = np.exp(np.arange(0, d_model, 2) * -(np.log(self.base) / d_model))
+        pe = np.zeros((seq_len, d_model), dtype=np.float32)
+        pe[:, 0::2] = np.sin(position * div_term)
+        pe[:, 1::2] = np.cos(position * div_term)
+        return pe[None, :, :]

ax_meeting/server.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+import asyncio
+import json
+import os
+import uuid
+from pathlib import Path
+
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect, UploadFile, File, Form
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import HTMLResponse, PlainTextResponse
+
+from ax_meeting.model_bundle import ModelBundle
+from ax_meeting.engines import DiarAsrEngine
+from ax_meeting.pipeline import StreamingMeetingSession
+from ax_meeting.summarizer import IncrementalSummarizer
+
+APP_DIR = Path(__file__).parent
+STATIC_DIR = APP_DIR / "static"
+
+app = FastAPI()
+models = ModelBundle()
+app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
+TRANSCRIPT_STORE = {}
+
+
+@app.get("/")
+def index():
+    html_path = STATIC_DIR / "index.html"
+    return HTMLResponse(html_path.read_text(encoding="utf-8"))
+
+
+@app.websocket("/ws")
+async def ws_endpoint(ws: WebSocket):
+    await ws.accept()
+    session_id = str(uuid.uuid4())
+    # Lazy model init to avoid failing import on machines without device access
+    try:
+        await asyncio.to_thread(models.ensure_loaded)
+    except Exception as e:
+        await ws.send_json({"type": "error", "message": f"模型初始化失败: {e}"})
+        await ws.close()
+        return
+
+    session = StreamingMeetingSession(models, session_id=session_id)
+    await ws.send_json({"type": "ready", "session_id": session_id})
+
+    try:
+        while True:
+            message = await ws.receive()
+
+            if "text" in message and message["text"]:
+                try:
+                    data = json.loads(message["text"])
+                except json.JSONDecodeError:
+                    continue
+
+                msg_type = data.get("type")
+                if msg_type == "end":
+                    result = await session.finalize()
+
+                    # Summarize
+                    try:
+                        summarizer = IncrementalSummarizer()
+                        summary = await asyncio.to_thread(
+                            summarizer.summarize_incrementally, result.get("transcript", "")
+                        )
+                    except Exception as e:
+                        summary = f"LLM 总结失败: {e}"
+
+                    final_text = result.get("transcript", "")
+                    TRANSCRIPT_STORE[session_id] = final_text
+                    await ws.send_json({"type": "final_transcript", "text": final_text})
+                    await ws.send_json({"type": "summary", "text": summary})
+                    await ws.send_json({"type": "end_ack"})
+
+                elif msg_type == "ping":
+                    await ws.send_json({"type": "pong"})
+
+            if "bytes" in message and message["bytes"]:
+                await session.add_audio(message["bytes"], ws)
+
+    except WebSocketDisconnect:
+        return
+
+
+@app.post("/diar_asr")
+async def diar_asr_api(file: UploadFile = File(...), session_id: str = Form(default="")):
+    if not session_id:
+        session_id = str(uuid.uuid4())
+    temp_path = APP_DIR / f"_upload_{session_id}_{file.filename}"
+    content = await file.read()
+    temp_path.write_bytes(content)
+    try:
+        engine = DiarAsrEngine(models)
+        text = engine.transcribe(str(temp_path))
+    finally:
+        try:
+            temp_path.unlink(missing_ok=True)
+        except Exception:
+            pass
+    TRANSCRIPT_STORE[session_id] = text
+    return {"session_id": session_id, "text": text}
+
+
+@app.get("/export/{session_id}")
+async def export_transcript(session_id: str):
+    text = TRANSCRIPT_STORE.get(session_id, "")
+    return PlainTextResponse(text, media_type="text/plain; charset=utf-8")
+
+
+if __name__ == "__main__":
+    import uvicorn
+    import socket
+
+    host = os.getenv("HOST", "0.0.0.0")
+    port = int(os.getenv("PORT", "8000"))
+    ssl_cert = os.getenv("SSL_CERT")
+    ssl_key = os.getenv("SSL_KEY")
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(("8.8.8.8", 80))
+        local_ip = s.getsockname()[0]
+        s.close()
+    except Exception:
+        local_ip = "127.0.0.1"
+    scheme = "https" if ssl_cert and ssl_key else "http"
+    print(f"Local URL:  {scheme}://{local_ip}:{port}")
+    uvicorn.run(
+        "ax_meeting.server:app",
+        host=host,
+        port=port,
+        reload=False,
+        ssl_certfile=ssl_cert,
+        ssl_keyfile=ssl_key,
+    )

ax_meeting/static/app.js

@@ -0,0 +1,187 @@
+const startBtn = document.getElementById('startBtn');
+const stopBtn = document.getElementById('stopBtn');
+const liveLog = document.getElementById('liveLog');
+const summary = document.getElementById('summary');
+const finalTranscript = document.getElementById('finalTranscript');
+const fileInput = document.getElementById('fileInput');
+const diarBtn = document.getElementById('diarBtn');
+const exportBtn = document.getElementById('exportBtn');
+
+let ws = null;
+let audioCtx = null;
+let processor = null;
+let sourceNode = null;
+let mediaStream = null;
+let started = false;
+let sessionId = null;
+
+function logLine(text) {
+  const line = document.createElement('div');
+  line.textContent = text;
+  liveLog.appendChild(line);
+  liveLog.scrollTop = liveLog.scrollHeight;
+}
+
+function formatTime(ms) {
+  const totalSec = ms / 1000;
+  const hours = Math.floor(totalSec / 3600);
+  const minutes = Math.floor((totalSec % 3600) / 60);
+  const seconds = totalSec % 60;
+  const h = String(hours).padStart(2, '0');
+  const m = String(minutes).padStart(2, '0');
+  const s = seconds.toFixed(2).padStart(5, '0');
+  return `${h}:${m}:${s}`;
+}
+
+function downsampleBuffer(buffer, inputRate, outputRate) {
+  if (outputRate === inputRate) {
+    return buffer;
+  }
+  const ratio = inputRate / outputRate;
+  const newLength = Math.round(buffer.length / ratio);
+  const result = new Float32Array(newLength);
+  let offset = 0;
+  for (let i = 0; i < newLength; i++) {
+    const nextOffset = Math.round((i + 1) * ratio);
+    let accum = 0;
+    let count = 0;
+    for (let j = offset; j < nextOffset && j < buffer.length; j++) {
+      accum += buffer[j];
+      count++;
+    }
+    result[i] = accum / Math.max(1, count);
+    offset = nextOffset;
+  }
+  return result;
+}
+
+function floatTo16BitPCM(float32Array) {
+  const out = new Int16Array(float32Array.length);
+  for (let i = 0; i < float32Array.length; i++) {
+    let s = Math.max(-1, Math.min(1, float32Array[i]));
+    out[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
+  }
+  return out;
+}
+
+async function startMeeting() {
+  if (started) return;
+  summary.textContent = '会议进行中...';
+  finalTranscript.textContent = '';
+  liveLog.innerHTML = '';
+
+  if (!window.isSecureContext && location.hostname !== 'localhost' && location.hostname !== '127.0.0.1') {
+    logLine('麦克风权限需要 HTTPS 或 localhost 访问。');
+    alert('麦克风权限需要 HTTPS 或 localhost 访问。请使用 https 或在本机用 127.0.0.1 访问。');
+  }
+
+  const wsProtocol = location.protocol === 'https:' ? 'wss' : 'ws';
+  ws = new WebSocket(`${wsProtocol}://${location.host}/ws`);
+  ws.binaryType = 'arraybuffer';
+
+  ws.onmessage = (event) => {
+    if (typeof event.data === 'string') {
+      const data = JSON.parse(event.data);
+      if (data.type === 'transcript') {
+        const start = formatTime(data.start_ms);
+        const end = formatTime(data.end_ms);
+        const t = `${start}-${end}  ${data.text}`;
+        logLine(t);
+      } else if (data.type === 'ready') {
+        sessionId = data.session_id;
+      } else if (data.type === 'final_transcript') {
+        finalTranscript.textContent = data.text || '';
+      } else if (data.type === 'summary') {
+        summary.textContent = data.text || '';
+      }
+    }
+  };
+
+  ws.onopen = async () => {
+    try {
+      mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    } catch (err) {
+      logLine(`麦克风权限请求失败: ${err && err.name ? err.name : err}`);
+      alert('麦克风权限请求失败。请检查浏览器权限或使用 HTTPS/localhost。');
+      return;
+    }
+    audioCtx = new (window.AudioContext || window.webkitAudioContext)();
+    sourceNode = audioCtx.createMediaStreamSource(mediaStream);
+
+    const bufferSize = 4096;
+    processor = audioCtx.createScriptProcessor(bufferSize, 1, 1);
+
+    processor.onaudioprocess = (event) => {
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      const input = event.inputBuffer.getChannelData(0);
+      const downsampled = downsampleBuffer(input, audioCtx.sampleRate, 16000);
+      const pcm16 = floatTo16BitPCM(downsampled);
+      ws.send(pcm16.buffer);
+    };
+
+    sourceNode.connect(processor);
+    processor.connect(audioCtx.destination);
+
+    started = true;
+    startBtn.disabled = true;
+    stopBtn.disabled = false;
+  };
+}
+
+async function stopMeeting() {
+  if (!started) return;
+  stopBtn.disabled = true;
+  if (ws && ws.readyState === WebSocket.OPEN) {
+    ws.send(JSON.stringify({ type: 'end' }));
+  }
+
+  if (processor) {
+    processor.disconnect();
+    processor.onaudioprocess = null;
+  }
+  if (sourceNode) {
+    sourceNode.disconnect();
+  }
+  if (audioCtx) {
+    await audioCtx.close();
+  }
+  if (mediaStream) {
+    mediaStream.getTracks().forEach((t) => t.stop());
+  }
+
+  started = false;
+  startBtn.disabled = false;
+}
+
+startBtn.addEventListener('click', startMeeting);
+stopBtn.addEventListener('click', stopMeeting);
+
+diarBtn.addEventListener('click', async () => {
+  if (!fileInput.files || fileInput.files.length === 0) {
+    alert('请选择音频文件');
+    return;
+  }
+  const file = fileInput.files[0];
+  const form = new FormData();
+  form.append('file', file);
+  if (sessionId) form.append('session_id', sessionId);
+
+  diarBtn.disabled = true;
+  try {
+    const resp = await fetch('/diar_asr', { method: 'POST', body: form });
+    const data = await resp.json();
+    if (!resp.ok) throw new Error(data.message || '识别失败');
+    if (data.session_id) sessionId = data.session_id;
+    finalTranscript.textContent = data.text || '';
+    exportBtn.disabled = !data.text;
+  } catch (err) {
+    alert(`识别失败: ${err.message || err}`);
+  } finally {
+    diarBtn.disabled = false;
+  }
+});
+
+exportBtn.addEventListener('click', () => {
+  if (!sessionId) return;
+  window.location.href = `/export/${sessionId}`;
+});

ax_meeting/static/index.html

@@ -0,0 +1,50 @@
+<!doctype html>
+<html lang="zh">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <title>会议纪要流式 Demo</title>
+  <link rel="stylesheet" href="/static/style.css" />
+</head>
+<body>
+  <div class="page">
+    <header class="header">
+      <div>
+        <h1>会议纪要流式 Demo</h1>
+        <p>实时分段转录 + 会议结束后一键总结</p>
+      </div>
+      <div class="controls">
+        <button id="startBtn" class="btn primary">开始会议</button>
+        <button id="stopBtn" class="btn" disabled>结束会议</button>
+      </div>
+    </header>
+
+    <section class="panel">
+      <h2>实时转录</h2>
+      <div id="liveLog" class="log"></div>
+    </section>
+
+    <section class="panel">
+      <h2>离线说话人识别 + ASR</h2>
+      <div class="upload-row">
+        <input id="fileInput" class="file-input" type="file" accept=".wav,.flac,.mp3,.mp4,.m4a" />
+        <button id="diarBtn" class="btn">导入音频识别</button>
+        <button id="exportBtn" class="btn" disabled>导出 TXT</button>
+      </div>
+      <div class="hint">支持 wav/flac/mp3/mp4/m4a。处理完成后可导出文本。</div>
+    </section>
+
+    <section class="panel">
+      <h2>最终会议纪要</h2>
+      <div id="summary" class="summary">等待会议结束...</div>
+    </section>
+
+    <section class="panel">
+      <h2>完整转写(含说话人)</h2>
+      <pre id="finalTranscript" class="final"></pre>
+    </section>
+  </div>
+
+  <script src="/static/app.js"></script>
+</body>
+</html>

ax_meeting/static/style.css

@@ -0,0 +1,147 @@
+:root {
+  --bg: #f5f3ee;
+  --ink: #1f1b16;
+  --accent: #e3693b;
+  --panel: #ffffff;
+  --muted: #666056;
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  font-family: "Noto Serif SC", "Source Han Serif SC", "Songti SC", serif;
+  background: var(--bg);
+  color: var(--ink);
+}
+
+.page {
+  max-width: 1100px;
+  margin: 32px auto;
+  padding: 0 20px 60px;
+}
+
+.header {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 20px;
+  margin-bottom: 24px;
+}
+
+.header h1 {
+  margin: 0 0 6px 0;
+  font-size: 28px;
+}
+
+.header p {
+  margin: 0;
+  color: var(--muted);
+}
+
+.controls {
+  display: flex;
+  gap: 10px;
+}
+
+.btn {
+  border: 1px solid var(--ink);
+  background: transparent;
+  color: var(--ink);
+  padding: 10px 16px;
+  border-radius: 8px;
+  cursor: pointer;
+}
+
+.btn.primary {
+  background: var(--accent);
+  border-color: var(--accent);
+  color: #fff;
+}
+
+.btn:disabled {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+.panel {
+  background: var(--panel);
+  border-radius: 12px;
+  padding: 16px;
+  box-shadow: 0 1px 6px rgba(0, 0, 0, 0.05);
+  margin-bottom: 16px;
+}
+
+.panel h2 {
+  margin: 0 0 12px 0;
+  font-size: 18px;
+}
+
+.upload-row {
+  display: flex;
+  gap: 10px;
+  align-items: center;
+  flex-wrap: wrap;
+}
+
+.file-input {
+  flex: 1 1 360px;
+  padding: 6px 8px;
+  border: 1px solid #d9d2c7;
+  border-radius: 8px;
+  background: #fffaf2;
+  font-family: "Noto Sans SC", "Source Han Sans SC", "PingFang SC", sans-serif;
+  color: var(--ink);
+}
+
+.file-input::file-selector-button {
+  margin-right: 10px;
+  padding: 8px 12px;
+  border: 1px solid #c9c1b5;
+  border-radius: 8px;
+  background: #efe7dc;
+  color: var(--ink);
+  cursor: pointer;
+}
+
+.file-input::file-selector-button:hover {
+  background: #e6ddcf;
+}
+
+.hint {
+  margin-top: 8px;
+  color: var(--muted);
+  font-size: 13px;
+}
+
+.log {
+  min-height: 160px;
+  max-height: 280px;
+  overflow: auto;
+  font-family: "JetBrains Mono", "SFMono-Regular", "Menlo", "Consolas", monospace;
+  white-space: pre-wrap;
+}
+
+.summary {
+  font-family: "Noto Sans SC", "Source Han Sans SC", "PingFang SC", sans-serif;
+  min-height: 120px;
+  white-space: pre-wrap;
+}
+
+.final {
+  min-height: 140px;
+  max-height: 320px;
+  overflow: auto;
+  background: #f6f6f6;
+  padding: 12px;
+  border-radius: 8px;
+}
+
+@media (max-width: 720px) {
+  .header {
+    flex-direction: column;
+    align-items: flex-start;
+  }
+}

ax_meeting/summarize_cli.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+import argparse
+from pathlib import Path
+
+from ax_meeting.summarizer import IncrementalSummarizer
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Meeting summarization (OpenAI-style API)")
+    parser.add_argument("--input", required=True, help="Transcript text file")
+    parser.add_argument("--output", default=None, help="Output summary file")
+    parser.add_argument("--openai_api_key", default=None)
+    parser.add_argument("--openai_base_url", default=None)
+    parser.add_argument("--openai_model", default=None)
+    args = parser.parse_args()
+
+    text = Path(args.input).read_text(encoding="utf-8")
+    summarizer = IncrementalSummarizer(
+        api_key=args.openai_api_key,
+        base_url=args.openai_base_url,
+        model=args.openai_model,
+    )
+    summary = summarizer.summarize_incrementally(text)
+
+    if args.output:
+        Path(args.output).write_text(summary, encoding="utf-8")
+        print(f"Summary saved: {args.output}")
+    else:
+        print(summary)
+
+
+if __name__ == "__main__":
+    main()

ax_meeting/summarizer.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+import os
+import re
+from typing import List
+
+from ax_meeting.config import SUMMARY_CHUNK_CHARS, SUMMARY_TARGET_CHARS
+
+try:
+    from openai import OpenAI
+except Exception as e:  # pragma: no cover
+    OpenAI = None
+
+
+def _split_text(text: str, max_chars: int) -> List[str]:
+    text = text.strip()
+    if not text:
+        return []
+    chunks = []
+    buf = []
+    cur = 0
+    for line in text.splitlines():
+        if cur + len(line) + 1 > max_chars and buf:
+            chunks.append("\n".join(buf))
+            buf = []
+            cur = 0
+        buf.append(line)
+        cur += len(line) + 1
+    if buf:
+        chunks.append("\n".join(buf))
+    # Fallback if a single line is too long
+    if len(chunks) == 1 and len(chunks[0]) > max_chars:
+        chunks = [chunks[0][i:i + max_chars] for i in range(0, len(chunks[0]), max_chars)]
+    return chunks
+
+
+class IncrementalSummarizer:
+    def __init__(self, api_key: str | None = None, base_url: str | None = None, model: str | None = None):
+        if OpenAI is None:
+            raise RuntimeError("openai package not installed")
+        api_key = api_key if api_key is not None else os.getenv("OPENAI_API_KEY", "")
+        base_url = base_url if base_url is not None else os.getenv("OPENAI_BASE_URL")
+        if base_url:
+            self.client = OpenAI(api_key=api_key, base_url=base_url)
+        else:
+            self.client = OpenAI(api_key=api_key)
+        self.model = model if model is not None else os.getenv("OPENAI_MODEL", "AXERA-TECH/Qwen3-1.7B")
+
+    def summarize_incrementally(self, transcript: str) -> str:
+        chunks = _split_text(transcript, SUMMARY_CHUNK_CHARS)
+        if not chunks:
+            return ""
+
+        summary = ""
+        for idx, chunk in enumerate(chunks):
+            prompt = (
+                "你是会议纪要助手。"\
+                f"\n前情提要(可为空): {summary}"\
+                f"\n本段会议文本(第{idx + 1}段):\n{chunk}"\
+                f"\n请将本段内容总结为约{SUMMARY_TARGET_CHARS}字中文摘要。"\
+                "输出要求: 只输出摘要正文, 不要标题。/no_think"
+            )
+
+            resp = self.client.chat.completions.create(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": "你擅长从会议记录中抽取关键信息并总结。"},
+                    {"role": "user", "content": prompt},
+                ],
+                temperature=0.2,
+            )
+
+            summary = (resp.choices[0].message.content or "").strip()
+            summary = re.sub(r"<think>.*?</think>", "", summary, flags=re.DOTALL).strip()
+            print(f"Summary chunk {idx + 1}: {summary}")
+        return summary

ax_meeting/text_cleaner.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+import re
+
+
+def clean_asr_text(text: str) -> str:
+    if not text:
+        return ""
+    # Remove token tags like <|zh|>, <|SPEECH|> or broken sequences
+    text = re.sub(r"<\|[^>]*?\|>", "", text)
+    text = re.sub(r"<\|[^|>]*\|", "", text)
+    text = text.replace("<|", "").replace("|>", "")
+
+    # Remove plain token chains like zh|NEUTRAL|Speech|withitn|
+    text = re.sub(
+        r"(?:\b(zh|en|yue|ja|ko|speech|happy|sad|angry|neutral|emo_unknown|withitn|woitn)\b\|)+",
+        "",
+        text,
+        flags=re.IGNORECASE,
+    )
+
+    # Remove leftover meta tokens joined by pipes anywhere
+    text = re.sub(
+        r"\b(zh|en|yue|ja|ko|speech|happy|sad|angry|neutral|emo_unknown|withitn|woitn)\b",
+        "",
+        text,
+        flags=re.IGNORECASE,
+    )
+    text = text.replace("|", "")
+
+    # Remove standalone metadata tokens (case-insensitive)
+    meta_re = r"\b(zh|en|yue|ja|ko|speech|happy|sad|angry|neutral|emo_unknown|withitn|woitn)\b"
+    text = re.sub(meta_re, "", text, flags=re.IGNORECASE)
+
+    # Remove concatenated metadata tokens like zhEMO_UNKNOWNSpeechwithitn
+    meta_tokens = ["zh","en","yue","ja","ko","speech","happy","sad","angry","neutral","emo_unknown","withitn","woitn"]
+    meta_concat_re = r"(?:%s)+" % "|".join(meta_tokens)
+    text = re.sub(meta_concat_re, "", text, flags=re.IGNORECASE)
+
+    # Normalize spaces
+    text = re.sub(r"\s+", " ", text).strip()
+    return text

ax_meeting/utils/ax_cam_bin.py

@@ -0,0 +1,231 @@
+import os
+import sys
+import numpy as np
+
+sys.path.append('%s'%os.path.dirname(__file__))
+
+from ax_meeting.utils.speaker_fbank import compute_fbank
+from ax_meeting.utils.cluster_utils import CommonClustering
+from ax_meeting.axengine_loader import ensure_axengine
+ensure_axengine()
+import axengine as axe
+
+def get_trans_sentence_sensevoice(output_asr):
+    """Get transcription with timestamps from ASR"""
+    sentence_info = [[]]
+    punc_pattern = r'[,.!?;:"\-—…、，。！？；：""'']'
+    
+    words = output_asr['merged_words']
+    #text = asr_result[0]['text']
+    timestamp = output_asr['merged_timestamps']
+    assert len(timestamp) == len(words)
+    text_pt = 0
+    
+    # 遍历每个单词及其时间戳
+    for i, wd in enumerate(words):
+        # 如果当前单词是标点符号，将其与前一个单词合并
+        if wd in punc_pattern and sentence_info and sentence_info[-1]:
+            # 合并标点符号到前一个单词
+            prev_word, prev_ts = sentence_info[-1][-1]
+            sentence_info[-1][-1] = [prev_word + wd, [prev_ts[0], timestamp[i][1]]]
+            
+            # # 如果标点是句子结束标点，开始新句子
+            if i < len(words) - 1:
+                sentence_info.append([])
+        else:
+            # 对于非标点单词，直接添加到当前句子
+            sentence_info[-1].append([wd, timestamp[i]])
+    return sentence_info
+
+
+def match_spk(sentence, output_field_labels):
+    """Match speaker ID with transcription segments"""
+    if len(sentence) == 0:
+        return []
+        
+    st_sent = sentence[0][1][0]
+    ed_sent = sentence[-1][1][1]
+    overlap_per_spk = {}
+    
+    for st_spk, ed_spk, spk in output_field_labels:
+        overlap_dur = min(ed_sent, ed_spk) - max(st_sent, st_spk)
+        if spk not in overlap_per_spk:
+            overlap_per_spk[spk] = 0
+        if overlap_dur > 0:
+            overlap_per_spk[spk] += overlap_dur
+            
+    overlap_per_spk_list = [[spk, overlap_per_spk[spk]] for spk in overlap_per_spk if overlap_per_spk[spk] > 0]
+    overlap_per_spk_list = sorted(overlap_per_spk_list, key=lambda x:x[1], reverse=True)
+    overlap_per_spk_list = [i[0] for i in overlap_per_spk_list]
+    
+    return overlap_per_spk_list
+def distribute_spk(sentence_info, output_field_labels):
+    """Distribute speaker IDs to transcription"""
+    last_spk = 0
+    for sentence in sentence_info:
+        main_spks = match_spk(sentence, output_field_labels)
+        main_spk = main_spks[0] if len(main_spks) > 0 else last_spk
+        
+        for i, wd in enumerate(sentence):
+            wd_spks = match_spk([wd], output_field_labels)
+            if main_spk in wd_spks:
+                sentence[i].append(main_spk)
+            elif len(wd_spks) > 0:
+                sentence[i].append(wd_spks[0])
+            else:
+                sentence[i].append(last_spk)
+        last_spk = sentence[-1][2]
+        
+    if len(sentence_info) == 0:
+        return []
+        
+    # Merge consecutive segments from same speaker
+    sentence_info = [j for i in sentence_info for j in i]
+    sentence_info_with_spk_merge = [sentence_info[0]]
+    
+    for i in sentence_info[1:]:
+        if (i[2] == sentence_info_with_spk_merge[-1][2] and 
+            i[1][0] < sentence_info_with_spk_merge[-1][1][1] + 2):
+            sentence_info_with_spk_merge[-1][0] += i[0]
+            sentence_info_with_spk_merge[-1][1][1] = i[1][1]
+        else:
+            sentence_info_with_spk_merge.append(i)
+            
+    return sentence_info_with_spk_merge
+
+def chunk(st, ed, dur=1.5, step=0.75):
+        chunks = []
+        subseg_st = st
+        while subseg_st + dur < ed + step:
+            subseg_ed = min(subseg_st + dur, ed)
+            chunks.append([subseg_st, subseg_ed])
+            subseg_st += step
+        return chunks
+
+def compressed_seg(seg_list):
+    new_seg_list = []
+    for i, seg in enumerate(seg_list):
+        seg_st, seg_ed, cluster_id = seg
+        if i == 0:
+            new_seg_list.append([seg_st, seg_ed, cluster_id])
+        elif cluster_id == new_seg_list[-1][2]:
+            if seg_st > new_seg_list[-1][1]:
+                new_seg_list.append([seg_st, seg_ed, cluster_id])
+            else:
+                new_seg_list[-1][1] = seg_ed
+        else:
+            if seg_st < new_seg_list[-1][1]:
+                p = (new_seg_list[-1][1]+seg_st) / 2
+                new_seg_list[-1][1] = p
+                seg_st = p
+            new_seg_list.append([seg_st, seg_ed, cluster_id])
+    return new_seg_list
+
+def do_clustering(chunks, embeddings, speaker_num=None):
+
+        # kmeans 和 DBSCAN 聚类效果都不太好，pca降维无法提升聚类速度
+
+        # 对嵌入向量进行降维处理
+        # from sklearn.decomposition import PCA
+        # if embeddings.shape[1] > 50:
+        #     pca = PCA(n_components=50)
+        #     embeddings = pca.fit_transform(embeddings)
+
+        cluster = CommonClustering(
+                cluster_type='spectral',
+                mer_cos=0.8,
+                min_num_spks=1,
+                max_num_spks=15,
+                min_cluster_size=4,
+                oracle_num=None,
+                pval=0.012
+            )
+        cluster_labels = cluster(
+            embeddings, 
+            speaker_num = speaker_num if speaker_num is not None else speaker_num
+        )
+
+        # from sklearn.cluster import DBSCAN
+        # from sklearn.neighbors import NearestNeighbors
+        # neigh = NearestNeighbors(n_neighbors=2)
+        # nbrs = neigh.fit(embeddings)
+        # distances, _ = nbrs.kneighbors(embeddings)
+        # distances = np.sort(distances, axis=0)
+        # distances = distances[:,1]
+        # eps = np.percentile(distances, 90)
+        # cluster_labels = DBSCAN(eps=eps, min_samples=5).fit_predict(embeddings)
+
+        # from sklearn.cluster import KMeans
+        # if speaker_num is None:
+        #     # 如果没有提供说话人数量，使用默认值2
+        #     speaker_num = 4
+        # cluster_labels = KMeans(n_clusters=speaker_num, random_state=0, n_init=10).fit_predict(embeddings)
+
+        speaker_num = cluster_labels.max()+1
+        output_field_labels = [[i[0], i[1], int(j)] for i, j in zip(chunks, cluster_labels)]
+        output_field_labels = compressed_seg(output_field_labels)
+        return speaker_num, output_field_labels
+
+
+
+class AX_SpeakerEmbeddingInference:
+    def __init__(self, model_dir):
+        #"Initialize speaker embedding model for inference"          
+        model_file = os.path.join(model_dir, "campplus.axmodel")
+        # model_file = os.path.join(model_dir, "res2netv2.axmodel")
+        
+        self.session = axe.InferenceSession(model_file)
+
+    def infer(self, feats: np.ndarray) -> np.ndarray:
+        # Run inference with ONNX Runtime
+        # Run inference
+        # feats = np.expand_dims(feats, axis=-1).astype(np.float32)
+        outputs = self.session.run(None, {'feature': feats})
+        return outputs[0]
+
+    def __call__(self, wav_file, fs, chunks=None, **kwargs):
+        """Process audio file with chunks
+        Args:
+            wav_file: path to wav file
+            chunks: list of [start_time, end_time] in seconds
+        """
+        if chunks is None or len(chunks) == 0:
+            return np.zeros((0, 192), dtype=np.float32)
+
+        wav = wav_file.astype(np.float32)
+        if wav.ndim > 1:
+            wav = wav.reshape(-1)
+
+        wavs = [wav[int(st * fs):int(ed * fs)] for st, ed in chunks]
+        max_len = max([x.shape[0] for x in wavs])
+        max_len = max(max_len, 57900)
+
+        def circle_pad_np(x: np.ndarray, target_len: int) -> np.ndarray:
+            if x.shape[0] >= target_len:
+                return x[:target_len]
+            n = int(np.ceil(target_len / x.shape[0]))
+            xcat = np.tile(x, n)
+            return xcat[:target_len]
+
+        wavs = [circle_pad_np(x, max_len) for x in wavs]
+
+        batch_size = 1
+        embeddings = []
+        for i in range(0, len(wavs), batch_size):
+            batch_wavs = wavs[i:i+batch_size]
+            feats_list = []
+            for w in batch_wavs:
+                feat = compute_fbank(w, fs, n_mels=80, mean_nor=True)
+                if feat.shape[0] >= 360:
+                    feat = feat[:360]
+                else:
+                    pad = np.zeros((360 - feat.shape[0], 80), dtype=np.float32)
+                    feat = np.concatenate([feat, pad], axis=0)
+                feats_list.append(feat)
+            feats_batch = np.stack(feats_list, axis=0).astype(np.float32)
+            embeddings_batch = self.infer(feats_batch)
+            embeddings.append(embeddings_batch)
+        
+        # Concatenate all embeddings
+        embeddings = np.concatenate(embeddings, axis=0)
+        return embeddings

ax_meeting/utils/ax_model_bin.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/FunAudioLLM/SenseVoice). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+import os.path
+from pathlib import Path
+from typing import List, Union, Tuple
+TORCH_AVAILABLE = False
+import numpy as np
+from ax_meeting.axengine_loader import ensure_axengine
+ensure_axengine()
+import axengine as axe
+
+try:
+    import librosa
+except ImportError:
+    print("Warning: librosa not found. Please install it using 'pip install librosa'.")
+    # Provide a fallback implementation if needed
+    def load_wav_fallback(path, sr=None):
+        import wave
+        import numpy as np
+        with wave.open(path, 'rb') as wf:
+            num_frames = wf.getnframes()
+            frames = wf.readframes(num_frames)
+            return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0, wf.getframerate()
+
+from ax_meeting.utils.infer_utils import (
+    CharTokenizer,
+    get_logger,
+    read_yaml,
+)
+from ax_meeting.utils.frontend import WavFrontend
+
+logging = get_logger()
+
+
+def sequence_mask_np(lengths, maxlen=None, dtype=np.float32):
+    lengths = np.asarray(lengths).astype(np.int64)
+    if maxlen is None:
+        maxlen = int(lengths.max())
+    row_vector = np.arange(0, maxlen, 1)
+    matrix = lengths.reshape(-1, 1)
+    mask = row_vector < matrix
+    return mask.astype(dtype)
+
+
+def unique_consecutive_np(arr: np.ndarray) -> np.ndarray:
+    if arr.size == 0:
+        return arr
+    out = [arr[0]]
+    for v in arr[1:]:
+        if v != out[-1]:
+            out.append(v)
+    return np.asarray(out, dtype=arr.dtype)
+
+
+class AX_SenseVoiceSmall:
+    """
+    Author: Speech Lab of DAMO Academy, Alibaba Group
+    Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
+    https://arxiv.org/abs/2206.08317
+    """
+
+    def __init__(
+        self,
+        model_dir: Union[str, Path] = None,
+        batch_size: int = 1,
+        seq_len: int = 68
+    ):
+
+        model_file = os.path.join(model_dir, "sensevoice.axmodel")
+        config_file = os.path.join(model_dir, "sensevoice/config.yaml")
+        cmvn_file = os.path.join(model_dir, "sensevoice/am.mvn")
+        config = read_yaml(config_file)
+        self.model_dir = model_dir
+        # token_list = os.path.join(model_dir, "tokens.json")
+        # with open(token_list, "r", encoding="utf-8") as f:
+        #     token_list = json.load(f)
+
+        # self.converter = TokenIDConverter(token_list)
+        self.tokenizer = CharTokenizer()
+        config["frontend_conf"]['cmvn_file'] = cmvn_file
+        self.frontend = WavFrontend(**config["frontend_conf"])
+        # self.ort_infer = OrtInferSession(
+        #     model_file, device_id, intra_op_num_threads=intra_op_num_threads
+        # )
+        self.session = axe.InferenceSession(model_file)
+        self.batch_size = batch_size
+        self.blank_id = 0
+        self.seq_len = seq_len
+
+        self.lid_dict = {"auto": 0, "zh": 3, "en": 4, "yue": 7, "ja": 11, "ko": 12, "nospeech": 13}
+        self.lid_int_dict = {24884: 3, 24885: 4, 24888: 7, 24892: 11, 24896: 12, 24992: 13}
+        self.textnorm_dict = {"withitn": 14, "woitn": 15}
+        self.textnorm_int_dict = {25016: 14, 25017: 15}
+        self.emo_dict = {"unk": 25009, "happy": 25001, "sad": 25002, "angry": 25003, "neutral": 25004}
+
+    def __call__(self, 
+                 wav_content: Union[str, np.ndarray, List[str]], 
+                 language: str,
+                 withitn: bool,
+                 position_encoding: np.ndarray,
+                 tokenizer=None,
+                 **kwargs) -> List:
+        """Enhanced model inference with additional features from model.py
+        
+        Args:
+            wav_content: Audio data or path
+            language: Language code for processing
+            withitn: Whether to use ITN (inverse text normalization)
+            position_encoding: Position encoding tensor
+            tokenizer: Tokenizer for text conversion
+            **kwargs: Additional arguments
+        """
+        # Start time tracking for metadata
+        import time
+        meta_data = {}
+        time_start = time.perf_counter()
+        
+        # Load waveform data
+        waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq)
+        waveform_nums = len(waveform_list)
+        time_load = time.perf_counter()
+        meta_data["load_data"] = f"{time_load - time_start:0.3f}"
+        # Get key for result identification
+        key = kwargs.get("key", ["wav_file"])
+        if isinstance(wav_content, str):
+            wav_name = os.path.splitext(os.path.basename(wav_content))[0]
+            if key == ["wav_file"]:
+                key = [wav_name]
+        
+        # Load queries from saved numpy files
+        language_query = np.load(os.path.join(self.model_dir, f"{language}.npy"))
+        textnorm_query = np.load(os.path.join(self.model_dir, "withitn.npy") if withitn 
+                                 else os.path.join(self.model_dir, "woitn.npy"))
+        event_emo_query = np.load(os.path.join(self.model_dir, "event_emo.npy"))
+
+        # Concatenate queries to form input_query
+        input_query = np.concatenate((language_query, event_emo_query, textnorm_query), axis=1)
+
+        # Setup dataset directories for saving intermediate files
+        dataset = "dataset"
+        os.makedirs(dataset, exist_ok=True)
+        speech_dir = os.path.join(dataset, "speech", language, "withitn" if withitn else "woitn")
+        mask_dir = os.path.join(dataset, "masks", language, "withitn" if withitn else "woitn")
+        pe_dir = os.path.join(dataset, "position_encoding", language, "withitn" if withitn else "woitn")
+        os.makedirs(speech_dir, exist_ok=True)
+        os.makedirs(mask_dir, exist_ok=True)
+        os.makedirs(pe_dir, exist_ok=True)
+        
+        # Process features
+        results = []
+        output_timestamp = kwargs.get("output_timestamp", False)
+        ban_emo_unk = kwargs.get("ban_emo_unk", False)
+        ibest_writer = None
+        # 添加时间偏移变量，用于跟踪连续的时间戳
+        time_offset = 0.0
+        # 添加合并时间戳变量，用于存储同一音频文件的所有时间戳，仅保留必要功能
+        merged_timestamps = []
+        merged_words = []
+        
+        
+        # Handle output_dir without using DatadirWriter (which is not available)
+        output_dir = kwargs.get("output_dir")
+
+        slice_len = self.seq_len - 4
+        time_pre = time.perf_counter()
+        meta_data["preprocess"] = f"{time_pre - time_load:0.3f}"
+        for beg_idx in range(0, waveform_nums, self.batch_size):
+            end_idx = min(waveform_nums, beg_idx + self.batch_size)
+            feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx])
+            
+            time_feat = time.perf_counter()
+            meta_data["extract_feat"] = f"{time_feat - time_pre:0.3f}"
+
+            for i in range(int(np.ceil(feats.shape[1] / slice_len))):
+                sub_feats = np.concatenate([input_query, feats[:, i*slice_len : (i+1)*slice_len, :]], axis=1)
+                feats_len[0] = sub_feats.shape[1]
+                
+                # 计算当前片段的实际长度（帧数）
+                actual_slice_length = min(slice_len, feats.shape[1] - i*slice_len)
+
+                if feats_len[0] < self.seq_len:
+                    sub_feats = np.concatenate([sub_feats, np.zeros((1, self.seq_len - feats_len[0], 560), dtype=np.float32)], axis=1)
+
+                masks = sequence_mask_np([self.seq_len], maxlen=self.seq_len, dtype=np.float32)[:, None, :]
+                
+                # Run inference
+
+                ctc_logits, encoder_out_lens = self.infer(sub_feats, masks, position_encoding)
+                
+                
+                # Ban emotion unknown token if requested
+                if ban_emo_unk:
+                    ctc_logits[:, :, self.emo_dict["unk"]] = -float("inf")
+
+                # Process results for each batch
+                b, n, d = ctc_logits.shape
+                if isinstance(key, (list, tuple)) and len(key) < b:
+                    key = key * b
+                    
+                for j in range(b):
+                    enc_len = int(encoder_out_lens[j]) if not hasattr(encoder_out_lens[j], "item") else int(encoder_out_lens[j].item())
+                    x = ctc_logits[j, : enc_len, :]
+                    yseq = np.argmax(x, axis=-1)
+                    yseq = unique_consecutive_np(yseq)
+
+                    mask = yseq != self.blank_id
+                    token_int = yseq[mask].tolist()
+
+                    # Convert tokens to text
+                    text = tokenizer.decode(token_int) if tokenizer is not None else str(token_int)
+                    # 文本处理
+                    # 简化文本处理，不执行合并操作
+                    
+                    # Write to output directory if provided
+                    if ibest_writer is not None:
+                        ibest_writer["text"][key[j]] = text
+                    
+                    if output_timestamp:
+                        # Torch-free build: skip timestamp generation
+                        output_timestamp = False
+                    result_i = {"key": key[j] if j < len(key) else f"result_{j}", "text": text}
+                    
+                    # 直接添加结果，重复处理将在export.py中进行
+                    results.append(result_i)
+
+                time_offset = round(time_offset + (actual_slice_length * 60 ) / 1000, 2)
+        # 结果处理完成
+        time_end = time.perf_counter()
+        meta_data["total_time"] = f"{time_end - time_start:0.3f}"
+        meta_data["inference_time"] = f"{time_end - time_feat:0.3f}"
+        
+        # 简化元数据处理，只保留时间戳
+        if len(results) > 0 and output_timestamp and merged_timestamps:
+            # 只保留时间戳数据，以便在export.py中处理
+            meta_data["merged_timestamps"] = merged_timestamps
+            meta_data["merged_words"] = merged_words
+   
+        return results, meta_data
+
+    def load_data(self, wav_content: Union[str, np.ndarray, List[str]], fs: int = None) -> List:
+        def load_wav(path: str) -> np.ndarray:
+            try:
+                # Use librosa if available
+                if 'librosa' in globals():
+                    waveform, _ = librosa.load(path, sr=fs)
+                else:
+                    # Use fallback implementation
+                    waveform, native_sr = load_wav_fallback(path)
+                    if fs is not None and native_sr != fs:
+                        # Implement resampling if needed
+                        print(f"Warning: Resampling from {native_sr} to {fs} is not implemented in fallback mode")
+                return waveform
+            except Exception as e:
+                print(f"Error loading audio file {path}: {e}")
+                # Return empty audio in case of error
+                return np.zeros(1600, dtype=np.float32)
+
+        if isinstance(wav_content, np.ndarray):
+            return [wav_content]
+
+        if isinstance(wav_content, str):
+            return [load_wav(wav_content)]
+
+        if isinstance(wav_content, list):
+            return [load_wav(path) for path in wav_content]
+
+        raise TypeError(f"The type of {wav_content} is not in [str, np.ndarray, list]")
+
+    def extract_feat(self, waveform_list: List[np.ndarray]) -> Tuple[np.ndarray, np.ndarray]:
+        feats, feats_len = [], []
+        for waveform in waveform_list:
+            speech, _ = self.frontend.fbank(waveform)
+
+            feat, feat_len = self.frontend.lfr_cmvn(speech)
+
+            feats.append(feat)
+            feats_len.append(feat_len)
+
+        feats = self.pad_feats(feats, np.max(feats_len))
+        feats_len = np.array(feats_len).astype(np.int32)
+        return feats, feats_len
+
+    @staticmethod
+    def pad_feats(feats: List[np.ndarray], max_feat_len: int) -> np.ndarray:
+        def pad_feat(feat: np.ndarray, cur_len: int) -> np.ndarray:
+            pad_width = ((0, max_feat_len - cur_len), (0, 0))
+            return np.pad(feat, pad_width, "constant", constant_values=0)
+
+        feat_res = [pad_feat(feat, feat.shape[0]) for feat in feats]
+        feats = np.array(feat_res).astype(np.float32)
+        return feats
+
+    def infer(self, 
+              feats: np.ndarray, 
+              masks: np.ndarray,
+              position_encoding: np.ndarray,
+              ) -> Tuple[np.ndarray, np.ndarray]:
+        #outputs = self.ort_infer([feats, masks, position_encoding])
+        outputs =self.session.run(None, {
+            'speech': feats,
+            'masks': masks,
+            'position_encoding': position_encoding
+        })
+        return outputs

ax_meeting/utils/ax_vad_bin.py

@@ -0,0 +1,158 @@
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+import os.path
+from typing import List,  Tuple
+
+import numpy as np
+
+from ax_meeting.utils.utils.utils import read_yaml
+from ax_meeting.utils.utils.frontend import WavFrontend
+from ax_meeting.utils.utils.e2e_vad import E2EVadModel
+from ax_meeting.axengine_loader import ensure_axengine
+ensure_axengine()
+import axengine as axe
+
+class AX_Fsmn_vad:
+    def __init__(self, model_dir, batch_size=1, max_end_sil=None):
+            """Initialize VAD model for inference"""
+            
+            # Export model if needed
+            model_file = os.path.join(model_dir, "vad.axmodel")
+            
+            # Load config and frontend
+            config_file = os.path.join(model_dir, "vad/config.yaml")
+            cmvn_file = os.path.join(model_dir, "vad/am.mvn")
+            self.config = read_yaml(config_file)
+            self.frontend = WavFrontend(cmvn_file=cmvn_file, **self.config["frontend_conf"])
+            self.session = axe.InferenceSession(model_file)
+            self.batch_size = batch_size
+            self.vad_scorer = E2EVadModel(self.config["model_conf"])
+            self.max_end_sil = max_end_sil if max_end_sil is not None else self.config["model_conf"]["max_end_silence_time"]
+
+    def extract_feat(self, waveform_list):
+        """Extract features from waveform"""
+        feats, feats_len = [], []
+        for waveform in waveform_list:
+            speech, _ = self.frontend.fbank(waveform)
+            feat, feat_len = self.frontend.lfr_cmvn(speech)
+            feats.append(feat)
+            feats_len.append(feat_len)
+
+        max_len = max(feats_len)
+        padded_feats = [np.pad(f, ((0, max_len - f.shape[0]), (0, 0)), 'constant') for f in feats]
+        feats = np.array(padded_feats).astype(np.float32)
+        feats_len = np.array(feats_len).astype(np.int32)
+        return feats, feats_len
+
+    def infer(self, feats: List) -> Tuple[np.ndarray, np.ndarray]:
+        """Run inference with ONNX Runtime"""
+        # Get all input names from the model
+        input_names = [input.name for input in self.session.get_inputs()]
+        output_names = [x.name for x in self.session.get_outputs()]
+        
+        # Create input dictionary for all inputs
+        input_dict = {}
+        for i, (name, tensor) in enumerate(zip(input_names, feats)):
+            input_dict[name] = tensor
+            
+        # Run inference with all inputs
+        outputs = self.session.run(output_names, input_dict)
+        scores, out_caches = outputs[0], outputs[1:]
+        return scores, out_caches
+
+    def __call__(self, wav_file, **kwargs):
+        """Process audio file with sliding window approach"""
+        # Load audio and prepare data
+        # waveform = self.load_wav(wav_file)
+        # waveform, _ = librosa.load(wav_file, sr=16000)
+        waveform_list = [wav_file]
+        waveform_nums = len(waveform_list)
+        is_final = kwargs.get("kwargs", False)
+        segments = [[]] * self.batch_size
+
+        for beg_idx in range(0, waveform_nums, self.batch_size):
+            vad_scorer = E2EVadModel(self.config["model_conf"])
+            end_idx = min(waveform_nums, beg_idx + self.batch_size)
+            waveform = waveform_list[beg_idx:end_idx]
+            feats, feats_len = self.extract_feat(waveform)
+            waveform = np.array(waveform)
+            param_dict = kwargs.get("param_dict", dict())
+            in_cache = param_dict.get("in_cache", list())
+            in_cache = self.prepare_cache(in_cache)
+
+            t_offset = 0
+            feats_len_max = int(feats_len.max()) if hasattr(feats_len, "max") else int(feats_len)
+            step = int(min(feats_len_max, 6000))
+            for t_offset in range(0, feats_len_max, min(step, feats_len_max - t_offset)):
+                if t_offset + step >= feats_len_max - 1:
+                    step = feats_len_max - t_offset
+                    is_final = True
+                else:
+                    is_final = False
+
+                # Extract feature segment
+                feats_package = feats[:, t_offset:int(t_offset + step), :]
+
+                # Pad if it's the final segment
+                if is_final:
+                    pad_length = 6000 - int(step)
+                    feats_package = np.pad(
+                        feats_package,
+                        ((0, 0), (0, pad_length), (0, 0)),
+                        mode='constant',
+                        constant_values=0
+                    )
+
+                # Extract corresponding waveform segment
+                waveform_package = waveform[
+                    :,
+                    t_offset * 160:min(waveform.shape[-1], (int(t_offset + step) - 1) * 160 + 400),
+                ]
+
+                # Pad waveform if it's the final segment
+                if is_final:
+                    expected_wave_length = 6000 * 160 + 240
+                    current_wave_length = waveform_package.shape[-1]
+                    pad_wave_length = expected_wave_length - current_wave_length
+                    if pad_wave_length > 0:
+                        waveform_package = np.pad(
+                            waveform_package,
+                            ((0, 0), (0, pad_wave_length)),
+                            mode='constant',
+                            constant_values=0
+                        )
+
+                # Run inference
+                inputs = [feats_package]
+                inputs.extend(in_cache)
+                scores, out_caches = self.infer(inputs)
+                in_cache = out_caches
+                
+                # Get VAD segments for this chunk
+                segments_part = vad_scorer(
+                    scores,
+                    waveform_package,
+                    is_final=is_final,
+                    max_end_sil=self.max_end_sil,
+                    online=False,
+                )
+
+                # Accumulate segments
+                if segments_part:
+                    for batch_num in range(0, self.batch_size):
+                        segments[batch_num] += segments_part[batch_num]
+
+        return segments
+
+    def prepare_cache(self, in_cache: list = []):
+        if len(in_cache) > 0:
+            return in_cache
+        fsmn_layers = 4
+        proj_dim = 128
+        lorder = 20
+        for i in range(fsmn_layers):
+            cache = np.zeros((1, proj_dim, lorder - 1, 1)).astype(np.float32)
+            in_cache.append(cache)
+        return in_cache

ax_meeting/utils/cluster_utils.py

@@ -0,0 +1,241 @@
+import numpy as np
+import scipy
+from sklearn.cluster._kmeans import k_means
+from sklearn.metrics.pairwise import cosine_similarity
+
+import fastcluster
+from scipy.cluster.hierarchy import fcluster
+from scipy.spatial.distance import squareform
+
+
+
+class SpectralCluster:
+    """A spectral clustering method using unnormalized Laplacian of affinity matrix.
+    This implementation is adapted from https://github.com/speechbrain/speechbrain.
+    """
+
+    def __init__(self, min_num_spks=1, max_num_spks=10, pval=0.02, min_pnum=6, oracle_num=None):
+        self.min_num_spks = min_num_spks
+        self.max_num_spks = max_num_spks
+        self.min_pnum = min_pnum
+        self.pval = pval
+        self.k = oracle_num
+
+    def __call__(self, X, **kwargs):
+        pval = kwargs.get('pval', None)
+        oracle_num = kwargs.get('speaker_num', None)
+
+        # Similarity matrix computation
+        sim_mat = self.get_sim_mat(X)
+
+        # Refining similarity matrix with pval
+        prunned_sim_mat = self.p_pruning(sim_mat, pval)
+
+        # Symmetrization
+        sym_prund_sim_mat = 0.5 * (prunned_sim_mat + prunned_sim_mat.T)
+
+        # Laplacian calculation
+        laplacian = self.get_laplacian(sym_prund_sim_mat)
+
+        # Get Spectral Embeddings
+        emb, num_of_spk = self.get_spec_embs(laplacian, oracle_num)
+
+        # Perform clustering
+        labels = self.cluster_embs(emb, num_of_spk)
+
+        return labels
+
+    def get_sim_mat(self, X):
+        # Cosine similarities
+        M = cosine_similarity(X, X)
+        return M
+
+    def p_pruning(self, A, pval=None):
+        if pval is None:
+            pval = self.pval
+        n_elems = int((1 - pval) * A.shape[0])
+        n_elems = min(n_elems, A.shape[0]-self.min_pnum)
+
+        # For each row in a affinity matrix
+        for i in range(A.shape[0]):
+            low_indexes = np.argsort(A[i, :])
+            low_indexes = low_indexes[0:n_elems]
+
+            # Replace smaller similarity values by 0s
+            A[i, low_indexes] = 0
+        return A
+
+    def get_laplacian(self, M):
+        M[np.diag_indices(M.shape[0])] = 0
+        D = np.sum(np.abs(M), axis=1)
+        D = np.diag(D)
+        L = D - M
+        return L
+
+    def get_spec_embs(self, L, k_oracle=None):
+        if k_oracle is None:
+            k_oracle = self.k
+
+        lambdas, eig_vecs = scipy.sparse.linalg.eigsh(L, k=min(self.max_num_spks+1, L.shape[0]), which='SM')
+
+        if k_oracle is not None:
+            num_of_spk = k_oracle
+        else:
+            lambda_gap_list = self.getEigenGaps(
+                lambdas[self.min_num_spks - 1:self.max_num_spks + 1])
+            num_of_spk = np.argmax(lambda_gap_list) + self.min_num_spks
+
+        emb = eig_vecs[:, :num_of_spk]
+        return emb, num_of_spk
+
+    def cluster_embs(self, emb, k):
+        # k-means
+        _, labels, _ = k_means(emb, k)
+        return labels
+
+    def getEigenGaps(self, eig_vals):
+        eig_vals_gap_list = []
+        for i in range(len(eig_vals) - 1):
+            gap = float(eig_vals[i + 1]) - float(eig_vals[i])
+            eig_vals_gap_list.append(gap)
+        return eig_vals_gap_list
+
+
+class UmapHdbscan:
+    """
+    Reference:
+    - Siqi Zheng, Hongbin Suo. Reformulating Speaker Diarization as Community Detection With 
+      Emphasis On Topological Structure. ICASSP2022
+    """
+
+    def __init__(self, n_neighbors=20, n_components=60, min_samples=20, min_cluster_size=10, metric='euclidean'):
+        self.n_neighbors = n_neighbors
+        self.n_components = n_components
+        self.min_samples = min_samples
+        self.min_cluster_size = min_cluster_size
+        self.metric = metric
+        import umap  # type: ignore
+        import hdbscan  # type: ignore
+        self._umap = umap
+        self._hdbscan = hdbscan
+
+    def __call__(self, X, **kwargs):
+        umap_X = self._umap.UMAP(
+            n_neighbors=self.n_neighbors,
+            min_dist=0.0,
+            n_components=min(self.n_components, X.shape[0]-2),
+            metric=self.metric,
+        ).fit_transform(X)
+        labels = self._hdbscan.HDBSCAN(
+            min_samples=self.min_samples,
+            min_cluster_size=self.min_cluster_size
+        ).fit_predict(umap_X)
+        return labels
+
+class AHCluster:
+    """
+    Agglomerative Hierarchical Clustering, a bottom-up approach which iteratively merges 
+    the closest clusters until a termination condition is reached.
+    This implementation is adapted from https://github.com/BUTSpeechFIT/VBx.
+    """
+
+    def __init__(self, fix_cos_thr=0.4):
+        self.fix_cos_thr = fix_cos_thr
+
+    def __call__(self, X, **kwargs):
+        scr_mx = cosine_similarity(X)
+        scr_mx = squareform(-scr_mx, checks=False)
+        lin_mat = fastcluster.linkage(scr_mx, method='average', preserve_input='False')
+        adjust = abs(lin_mat[:, 2].min())
+        lin_mat[:, 2] += adjust
+        labels = fcluster(lin_mat, -self.fix_cos_thr + adjust, criterion='distance') - 1
+        return labels
+
+
+class CommonClustering:
+    """Perfom clustering for input embeddings and output the labels.
+    """
+
+    def __init__(self, cluster_type, cluster_line=40, mer_cos=None, min_cluster_size=4, **kwargs):
+        self.cluster_type = cluster_type
+        self.cluster_line = cluster_line
+        self.min_cluster_size = min_cluster_size
+        self.mer_cos = mer_cos
+        
+        # Initialize main cluster
+        if self.cluster_type == 'spectral':
+            self.cluster = SpectralCluster(**kwargs)
+        elif self.cluster_type == 'umap_hdbscan':
+            kwargs['min_cluster_size'] = min_cluster_size
+            self.cluster = UmapHdbscan(**kwargs)
+        elif self.cluster_type == 'AHC':
+            self.cluster = AHCluster(**kwargs)
+        else:
+            raise ValueError(
+                '%s is not currently supported.' % self.cluster_type
+            )
+            
+        # Initialize short cluster
+        if self.cluster_type != 'AHC':
+            self.cluster_for_short = AHCluster()
+        else:
+            self.cluster_for_short = self.cluster
+        
+
+    def __call__(self, X, **kwargs):
+        # clustering and return the labels
+        assert len(X.shape) == 2, 'Shape of input should be [N, C]'
+        if X.shape[0] <= 1:
+            return np.zeros(X.shape[0], dtype=int)
+
+        if X.shape[0] < self.cluster_line:
+            labels = self.cluster_for_short(X)
+        else:
+            labels = self.cluster(X, **kwargs)
+
+        # remove extremely minor cluster
+        labels = self.filter_minor_cluster(labels, X, self.min_cluster_size)
+
+        # merge similar speaker
+        if self.mer_cos is not None:
+            labels = self.merge_by_cos(labels, X, self.mer_cos)
+        return labels
+
+    def filter_minor_cluster(self, labels, x, min_cluster_size):
+        cset = np.unique(labels)
+        csize = np.array([(labels == i).sum() for i in cset])
+        minor_idx = np.where(csize <= self.min_cluster_size)[0]
+        if len(minor_idx) == 0:
+            return labels
+
+        minor_cset = cset[minor_idx]
+        major_idx = np.where(csize > self.min_cluster_size)[0]
+        if len(major_idx) == 0:
+            return np.zeros_like(labels)
+        major_cset = cset[major_idx]
+        major_center = np.stack([x[labels == i].mean(0) \
+            for i in major_cset])
+        for i in range(len(labels)):
+            if labels[i] in minor_cset:
+                cos_sim = cosine_similarity(x[i][np.newaxis], major_center)
+                labels[i] = major_cset[cos_sim.argmax()]
+
+        return labels
+
+    def merge_by_cos(self, labels, x, cos_thr):
+        # merge the similar speakers by cosine similarity
+        assert cos_thr > 0 and cos_thr <= 1
+        while True:
+            cset = np.unique(labels)
+            if len(cset) == 1:
+                break
+            centers = np.stack([x[labels == i].mean(0) \
+                for i in cset])
+            affinity = cosine_similarity(centers, centers)
+            affinity = np.triu(affinity, 1)
+            idx = np.unravel_index(np.argmax(affinity), affinity.shape)
+            if affinity[idx] < cos_thr:
+                break
+            c1, c2 = cset[np.array(idx)]
+            labels[labels==c2]=c1
+        return labels

ax_meeting/utils/ctc_alignment.py

@@ -0,0 +1,76 @@
+import torch
+
+def ctc_forced_align(
+    log_probs: torch.Tensor,
+    targets: torch.Tensor,
+    input_lengths: torch.Tensor,
+    target_lengths: torch.Tensor,
+    blank: int = 0,
+    ignore_id: int = -1,
+) -> torch.Tensor:
+    """Align a CTC label sequence to an emission.
+
+    Args:
+        log_probs (Tensor): log probability of CTC emission output.
+            Tensor of shape `(B, T, C)`. where `B` is the batch size, `T` is the input length,
+            `C` is the number of characters in alphabet including blank.
+        targets (Tensor): Target sequence. Tensor of shape `(B, L)`,
+            where `L` is the target length.
+        input_lengths (Tensor):
+            Lengths of the inputs (max value must each be <= `T`). 1-D Tensor of shape `(B,)`.
+        target_lengths (Tensor):
+            Lengths of the targets. 1-D Tensor of shape `(B,)`.
+        blank_id (int, optional): The index of blank symbol in CTC emission. (Default: 0)
+        ignore_id (int, optional): The index of ignore symbol in CTC emission. (Default: -1)
+    """
+    targets[targets == ignore_id] = blank
+
+    batch_size, input_time_size, _ = log_probs.size()
+    bsz_indices = torch.arange(batch_size, device=input_lengths.device)
+
+    _t_a_r_g_e_t_s_ = torch.cat(
+        (
+            torch.stack((torch.full_like(targets, blank), targets), dim=-1).flatten(start_dim=1),
+            torch.full_like(targets[:, :1], blank),
+        ),
+        dim=-1,
+    )
+    diff_labels = torch.cat(
+        (
+            torch.as_tensor([[False, False]], device=targets.device).expand(batch_size, -1),
+            _t_a_r_g_e_t_s_[:, 2:] != _t_a_r_g_e_t_s_[:, :-2],
+        ),
+        dim=1,
+    )
+
+    neg_inf = torch.tensor(float("-inf"), device=log_probs.device, dtype=log_probs.dtype)
+    padding_num = 2
+    padded_t = padding_num + _t_a_r_g_e_t_s_.size(-1)
+    best_score = torch.full((batch_size, padded_t), neg_inf, device=log_probs.device, dtype=log_probs.dtype)
+    best_score[:, padding_num + 0] = log_probs[:, 0, blank]
+    best_score[:, padding_num + 1] = log_probs[bsz_indices, 0, _t_a_r_g_e_t_s_[:, 1]]
+
+    backpointers = torch.zeros((batch_size, input_time_size, padded_t), device=log_probs.device, dtype=targets.dtype)
+
+    for t in range(1, input_time_size):
+        prev = torch.stack(
+            (best_score[:, 2:], best_score[:, 1:-1], torch.where(diff_labels, best_score[:, :-2], neg_inf))
+        )
+        prev_max_value, prev_max_idx = prev.max(dim=0)
+        best_score[:, padding_num:] = log_probs[:, t].gather(-1, _t_a_r_g_e_t_s_) + prev_max_value
+        backpointers[:, t, padding_num:] = prev_max_idx
+
+    l1l2 = best_score.gather(
+        -1, torch.stack((padding_num + target_lengths * 2 - 1, padding_num + target_lengths * 2), dim=-1)
+    )
+
+    path = torch.zeros((batch_size, input_time_size), device=best_score.device, dtype=torch.long)
+    path[bsz_indices, input_lengths - 1] = padding_num + target_lengths * 2 - 1 + l1l2.argmax(dim=-1)
+
+    for t in range(input_time_size - 1, 0, -1):
+        target_indices = path[:, t]
+        prev_max_idx = backpointers[bsz_indices, t, target_indices]
+        path[:, t - 1] += target_indices - prev_max_idx
+
+    alignments = _t_a_r_g_e_t_s_.gather(dim=-1, index=(path - padding_num).clamp(min=0))
+    return alignments

ax_meeting/utils/frontend.py

@@ -0,0 +1,433 @@
+# -*- encoding: utf-8 -*-
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union
+import copy
+
+import numpy as np
+import kaldi_native_fbank as knf
+
+root_dir = Path(__file__).resolve().parent
+
+logger_initialized = {}
+
+
+class WavFrontend:
+    """Conventional frontend structure for ASR."""
+
+    def __init__(
+        self,
+        cmvn_file: str = None,
+        fs: int = 16000,
+        window: str = "hamming",
+        n_mels: int = 80,
+        frame_length: int = 25,
+        frame_shift: int = 10,
+        lfr_m: int = 1,
+        lfr_n: int = 1,
+        dither: float = 1.0,
+        **kwargs,
+    ) -> None:
+
+        opts = knf.FbankOptions()
+        opts.frame_opts.samp_freq = fs
+        opts.frame_opts.dither = dither
+        opts.frame_opts.window_type = window
+        opts.frame_opts.frame_shift_ms = float(frame_shift)
+        opts.frame_opts.frame_length_ms = float(frame_length)
+        opts.mel_opts.num_bins = n_mels
+        opts.energy_floor = 0
+        opts.frame_opts.snip_edges = True
+        opts.mel_opts.debug_mel = False
+        self.opts = opts
+
+        self.lfr_m = lfr_m
+        self.lfr_n = lfr_n
+        self.cmvn_file = cmvn_file
+
+        if self.cmvn_file:
+            self.cmvn = self.load_cmvn()
+        self.fbank_fn = None
+        self.fbank_beg_idx = 0
+        self.reset_status()
+
+    def fbank(self, waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        waveform = waveform * (1 << 15)
+        self.fbank_fn = knf.OnlineFbank(self.opts)
+        self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
+        frames = self.fbank_fn.num_frames_ready
+        mat = np.empty([frames, self.opts.mel_opts.num_bins])
+        for i in range(frames):
+            mat[i, :] = self.fbank_fn.get_frame(i)
+        feat = mat.astype(np.float32)
+        feat_len = np.array(mat.shape[0]).astype(np.int32)
+        return feat, feat_len
+
+    def fbank_online(self, waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        waveform = waveform * (1 << 15)
+        # self.fbank_fn = knf.OnlineFbank(self.opts)
+        self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
+        frames = self.fbank_fn.num_frames_ready
+        mat = np.empty([frames, self.opts.mel_opts.num_bins])
+        for i in range(self.fbank_beg_idx, frames):
+            mat[i, :] = self.fbank_fn.get_frame(i)
+        # self.fbank_beg_idx += (frames-self.fbank_beg_idx)
+        feat = mat.astype(np.float32)
+        feat_len = np.array(mat.shape[0]).astype(np.int32)
+        return feat, feat_len
+
+    def reset_status(self):
+        self.fbank_fn = knf.OnlineFbank(self.opts)
+        self.fbank_beg_idx = 0
+
+    def lfr_cmvn(self, feat: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        if self.lfr_m != 1 or self.lfr_n != 1:
+            feat = self.apply_lfr(feat, self.lfr_m, self.lfr_n)
+
+        if self.cmvn_file:
+            feat = self.apply_cmvn(feat)
+
+        feat_len = np.array(feat.shape[0]).astype(np.int32)
+        return feat, feat_len
+
+    @staticmethod
+    def apply_lfr(inputs: np.ndarray, lfr_m: int, lfr_n: int) -> np.ndarray:
+        LFR_inputs = []
+
+        T = inputs.shape[0]
+        T_lfr = int(np.ceil(T / lfr_n))
+        left_padding = np.tile(inputs[0], ((lfr_m - 1) // 2, 1))
+        inputs = np.vstack((left_padding, inputs))
+        T = T + (lfr_m - 1) // 2
+        for i in range(T_lfr):
+            if lfr_m <= T - i * lfr_n:
+                LFR_inputs.append((inputs[i * lfr_n : i * lfr_n + lfr_m]).reshape(1, -1))
+            else:
+                # process last LFR frame
+                num_padding = lfr_m - (T - i * lfr_n)
+                frame = inputs[i * lfr_n :].reshape(-1)
+                for _ in range(num_padding):
+                    frame = np.hstack((frame, inputs[-1]))
+
+                LFR_inputs.append(frame)
+        LFR_outputs = np.vstack(LFR_inputs).astype(np.float32)
+        return LFR_outputs
+
+    def apply_cmvn(self, inputs: np.ndarray) -> np.ndarray:
+        """
+        Apply CMVN with mvn data
+        """
+        frame, dim = inputs.shape
+        means = np.tile(self.cmvn[0:1, :dim], (frame, 1))
+        vars = np.tile(self.cmvn[1:2, :dim], (frame, 1))
+        inputs = (inputs + means) * vars
+        return inputs
+
+    def load_cmvn(
+        self,
+    ) -> np.ndarray:
+        with open(self.cmvn_file, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+
+        means_list = []
+        vars_list = []
+        for i in range(len(lines)):
+            line_item = lines[i].split()
+            if line_item[0] == "<AddShift>":
+                line_item = lines[i + 1].split()
+                if line_item[0] == "<LearnRateCoef>":
+                    add_shift_line = line_item[3 : (len(line_item) - 1)]
+                    means_list = list(add_shift_line)
+                    continue
+            elif line_item[0] == "<Rescale>":
+                line_item = lines[i + 1].split()
+                if line_item[0] == "<LearnRateCoef>":
+                    rescale_line = line_item[3 : (len(line_item) - 1)]
+                    vars_list = list(rescale_line)
+                    continue
+
+        means = np.array(means_list).astype(np.float64)
+        vars = np.array(vars_list).astype(np.float64)
+        cmvn = np.array([means, vars])
+        return cmvn
+
+
+class WavFrontendOnline(WavFrontend):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # self.fbank_fn = knf.OnlineFbank(self.opts)
+        # add variables
+        self.frame_sample_length = int(
+            self.opts.frame_opts.frame_length_ms * self.opts.frame_opts.samp_freq / 1000
+        )
+        self.frame_shift_sample_length = int(
+            self.opts.frame_opts.frame_shift_ms * self.opts.frame_opts.samp_freq / 1000
+        )
+        self.waveform = None
+        self.reserve_waveforms = None
+        self.input_cache = None
+        self.lfr_splice_cache = []
+
+    @staticmethod
+    # inputs has catted the cache
+    def apply_lfr(
+        inputs: np.ndarray, lfr_m: int, lfr_n: int, is_final: bool = False
+    ) -> Tuple[np.ndarray, np.ndarray, int]:
+        """
+        Apply lfr with data
+        """
+
+        LFR_inputs = []
+        T = inputs.shape[0]  # include the right context
+        T_lfr = int(
+            np.ceil((T - (lfr_m - 1) // 2) / lfr_n)
+        )  # minus the right context: (lfr_m - 1) // 2
+        splice_idx = T_lfr
+        for i in range(T_lfr):
+            if lfr_m <= T - i * lfr_n:
+                LFR_inputs.append((inputs[i * lfr_n : i * lfr_n + lfr_m]).reshape(1, -1))
+            else:  # process last LFR frame
+                if is_final:
+                    num_padding = lfr_m - (T - i * lfr_n)
+                    frame = (inputs[i * lfr_n :]).reshape(-1)
+                    for _ in range(num_padding):
+                        frame = np.hstack((frame, inputs[-1]))
+                    LFR_inputs.append(frame)
+                else:
+                    # update splice_idx and break the circle
+                    splice_idx = i
+                    break
+        splice_idx = min(T - 1, splice_idx * lfr_n)
+        lfr_splice_cache = inputs[splice_idx:, :]
+        LFR_outputs = np.vstack(LFR_inputs)
+        return LFR_outputs.astype(np.float32), lfr_splice_cache, splice_idx
+
+    @staticmethod
+    def compute_frame_num(
+        sample_length: int, frame_sample_length: int, frame_shift_sample_length: int
+    ) -> int:
+        frame_num = int((sample_length - frame_sample_length) / frame_shift_sample_length + 1)
+        return frame_num if frame_num >= 1 and sample_length >= frame_sample_length else 0
+
+    def fbank(
+        self, input: np.ndarray, input_lengths: np.ndarray
+    ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+        self.fbank_fn = knf.OnlineFbank(self.opts)
+        batch_size = input.shape[0]
+        if self.input_cache is None:
+            self.input_cache = np.empty((batch_size, 0), dtype=np.float32)
+        input = np.concatenate((self.input_cache, input), axis=1)
+        frame_num = self.compute_frame_num(
+            input.shape[-1], self.frame_sample_length, self.frame_shift_sample_length
+        )
+        # update self.in_cache
+        self.input_cache = input[
+            :, -(input.shape[-1] - frame_num * self.frame_shift_sample_length) :
+        ]
+        waveforms = np.empty(0, dtype=np.float32)
+        feats_pad = np.empty(0, dtype=np.float32)
+        feats_lens = np.empty(0, dtype=np.int32)
+        if frame_num:
+            waveforms = []
+            feats = []
+            feats_lens = []
+            for i in range(batch_size):
+                waveform = input[i]
+                waveforms.append(
+                    waveform[
+                        : (
+                            (frame_num - 1) * self.frame_shift_sample_length
+                            + self.frame_sample_length
+                        )
+                    ]
+                )
+                waveform = waveform * (1 << 15)
+
+                self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
+                frames = self.fbank_fn.num_frames_ready
+                mat = np.empty([frames, self.opts.mel_opts.num_bins])
+                for i in range(frames):
+                    mat[i, :] = self.fbank_fn.get_frame(i)
+                feat = mat.astype(np.float32)
+                feat_len = np.array(mat.shape[0]).astype(np.int32)
+                feats.append(feat)
+                feats_lens.append(feat_len)
+
+            waveforms = np.stack(waveforms)
+            feats_lens = np.array(feats_lens)
+            feats_pad = np.array(feats)
+        self.fbanks = feats_pad
+        self.fbanks_lens = copy.deepcopy(feats_lens)
+        return waveforms, feats_pad, feats_lens
+
+    def get_fbank(self) -> Tuple[np.ndarray, np.ndarray]:
+        return self.fbanks, self.fbanks_lens
+
+    def lfr_cmvn(
+        self, input: np.ndarray, input_lengths: np.ndarray, is_final: bool = False
+    ) -> Tuple[np.ndarray, np.ndarray, List[int]]:
+        batch_size = input.shape[0]
+        feats = []
+        feats_lens = []
+        lfr_splice_frame_idxs = []
+        for i in range(batch_size):
+            mat = input[i, : input_lengths[i], :]
+            lfr_splice_frame_idx = -1
+            if self.lfr_m != 1 or self.lfr_n != 1:
+                # update self.lfr_splice_cache in self.apply_lfr
+                mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(
+                    mat, self.lfr_m, self.lfr_n, is_final
+                )
+            if self.cmvn_file is not None:
+                mat = self.apply_cmvn(mat)
+            feat_length = mat.shape[0]
+            feats.append(mat)
+            feats_lens.append(feat_length)
+            lfr_splice_frame_idxs.append(lfr_splice_frame_idx)
+
+        feats_lens = np.array(feats_lens)
+        feats_pad = np.array(feats)
+        return feats_pad, feats_lens, lfr_splice_frame_idxs
+
+    def extract_fbank(
+        self, input: np.ndarray, input_lengths: np.ndarray, is_final: bool = False
+    ) -> Tuple[np.ndarray, np.ndarray]:
+        batch_size = input.shape[0]
+        assert (
+            batch_size == 1
+        ), "we support to extract feature online only when the batch size is equal to 1 now"
+        waveforms, feats, feats_lengths = self.fbank(input, input_lengths)  # input shape: B T D
+        if feats.shape[0]:
+            self.waveforms = (
+                waveforms
+                if self.reserve_waveforms is None
+                else np.concatenate((self.reserve_waveforms, waveforms), axis=1)
+            )
+            if not self.lfr_splice_cache:
+                for i in range(batch_size):
+                    self.lfr_splice_cache.append(
+                        np.expand_dims(feats[i][0, :], axis=0).repeat((self.lfr_m - 1) // 2, axis=0)
+                    )
+
+            if feats_lengths[0] + self.lfr_splice_cache[0].shape[0] >= self.lfr_m:
+                lfr_splice_cache_np = np.stack(self.lfr_splice_cache)  # B T D
+                feats = np.concatenate((lfr_splice_cache_np, feats), axis=1)
+                feats_lengths += lfr_splice_cache_np[0].shape[0]
+                frame_from_waveforms = int(
+                    (self.waveforms.shape[1] - self.frame_sample_length)
+                    / self.frame_shift_sample_length
+                    + 1
+                )
+                minus_frame = (self.lfr_m - 1) // 2 if self.reserve_waveforms is None else 0
+                feats, feats_lengths, lfr_splice_frame_idxs = self.lfr_cmvn(
+                    feats, feats_lengths, is_final
+                )
+                if self.lfr_m == 1:
+                    self.reserve_waveforms = None
+                else:
+                    reserve_frame_idx = lfr_splice_frame_idxs[0] - minus_frame
+                    # print('reserve_frame_idx:  ' + str(reserve_frame_idx))
+                    # print('frame_frame:  ' + str(frame_from_waveforms))
+                    self.reserve_waveforms = self.waveforms[
+                        :,
+                        reserve_frame_idx
+                        * self.frame_shift_sample_length : frame_from_waveforms
+                        * self.frame_shift_sample_length,
+                    ]
+                    sample_length = (
+                        frame_from_waveforms - 1
+                    ) * self.frame_shift_sample_length + self.frame_sample_length
+                    self.waveforms = self.waveforms[:, :sample_length]
+            else:
+                # update self.reserve_waveforms and self.lfr_splice_cache
+                self.reserve_waveforms = self.waveforms[
+                    :, : -(self.frame_sample_length - self.frame_shift_sample_length)
+                ]
+                for i in range(batch_size):
+                    self.lfr_splice_cache[i] = np.concatenate(
+                        (self.lfr_splice_cache[i], feats[i]), axis=0
+                    )
+                return np.empty(0, dtype=np.float32), feats_lengths
+        else:
+            if is_final:
+                self.waveforms = (
+                    waveforms if self.reserve_waveforms is None else self.reserve_waveforms
+                )
+                feats = np.stack(self.lfr_splice_cache)
+                feats_lengths = np.zeros(batch_size, dtype=np.int32) + feats.shape[1]
+                feats, feats_lengths, _ = self.lfr_cmvn(feats, feats_lengths, is_final)
+        if is_final:
+            self.cache_reset()
+        return feats, feats_lengths
+
+    def get_waveforms(self):
+        return self.waveforms
+
+    def cache_reset(self):
+        self.fbank_fn = knf.OnlineFbank(self.opts)
+        self.reserve_waveforms = None
+        self.input_cache = None
+        self.lfr_splice_cache = []
+
+
+def load_bytes(input):
+    middle_data = np.frombuffer(input, dtype=np.int16)
+    middle_data = np.asarray(middle_data)
+    if middle_data.dtype.kind not in "iu":
+        raise TypeError("'middle_data' must be an array of integers")
+    dtype = np.dtype("float32")
+    if dtype.kind != "f":
+        raise TypeError("'dtype' must be a floating point type")
+
+    i = np.iinfo(middle_data.dtype)
+    abs_max = 2 ** (i.bits - 1)
+    offset = i.min + abs_max
+    array = np.frombuffer((middle_data.astype(dtype) - offset) / abs_max, dtype=np.float32)
+    return array
+
+
+class SinusoidalPositionEncoderOnline:
+    """Streaming Positional encoding."""
+
+    def encode(self, positions: np.ndarray = None, depth: int = None, dtype: np.dtype = np.float32):
+        batch_size = positions.shape[0]
+        positions = positions.astype(dtype)
+        log_timescale_increment = np.log(np.array([10000], dtype=dtype)) / (depth / 2 - 1)
+        inv_timescales = np.exp(np.arange(depth / 2).astype(dtype) * (-log_timescale_increment))
+        inv_timescales = np.reshape(inv_timescales, [batch_size, -1])
+        scaled_time = np.reshape(positions, [1, -1, 1]) * np.reshape(inv_timescales, [1, 1, -1])
+        encoding = np.concatenate((np.sin(scaled_time), np.cos(scaled_time)), axis=2)
+        return encoding.astype(dtype)
+
+    def forward(self, x, start_idx=0):
+        batch_size, timesteps, input_dim = x.shape
+        positions = np.arange(1, timesteps + 1 + start_idx)[None, :]
+        position_encoding = self.encode(positions, input_dim, x.dtype)
+
+        return x + position_encoding[:, start_idx : start_idx + timesteps]
+
+
+def test():
+    path = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav"
+    import librosa
+
+    cmvn_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/am.mvn"
+    config_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/config.yaml"
+    from funasr.runtime.python.onnxruntime.rapid_paraformer.utils.utils import read_yaml
+
+    config = read_yaml(config_file)
+    waveform, _ = librosa.load(path, sr=None)
+    frontend = WavFrontend(
+        cmvn_file=cmvn_file,
+        **config["frontend_conf"],
+    )
+    speech, _ = frontend.fbank_online(waveform)  # 1d, (sample,), numpy
+    feat, feat_len = frontend.lfr_cmvn(
+        speech
+    )  # 2d, (frame, 450), np.float32 -> torch, torch.from_numpy(), dtype, (1, frame, 450)
+
+    frontend.reset_status()  # clear cache
+    return feat, feat_len
+
+
+if __name__ == "__main__":
+    test()

ax_meeting/utils/infer_func.py

@@ -0,0 +1,273 @@
+import torch
+import numpy as np
+from typing import List, Tuple
+from tqdm import tqdm
+from axengine import InferenceSession
+import os
+import re
+from ml_dtypes import bfloat16
+
+
+# Discover model files automatically from model_dir.
+# We expect files like: <prefix>_p128_l<idx>_together.axmodel and <prefix>_post.axmodel
+# we try to detect model prefix and layer files automatically
+def _find_axmodel_files(base_dir: str, expected_layers: int = None, expected_prefill: int = 128):
+    files = os.listdir(base_dir)
+    # match prefix, prefill size (dynamic), and layer index
+    layer_pattern = re.compile(r"^(?P<prefix>.*)_p(?P<prefill>\d+)_l(?P<idx>\d+)_together\.axmodel$")
+    post_pattern = re.compile(r"^(?P<prefix>.*)_post\.axmodel$")
+
+    # collect prefix -> [(idx, fname)]
+    prefix_map = {}
+    for fname in files:
+        m = layer_pattern.match(fname)
+        if m:
+            prefix = m.group("prefix")
+            idx = int(m.group("idx"))
+            prefix_map.setdefault(prefix, []).append((idx, fname))
+
+    if not prefix_map:
+        # fallback to hardcoded pattern if nothing detected
+        prefix = "gemma3_text"
+        layer_files = [(
+            i, f"{prefix}_p{expected_prefill}_l{i}_together.axmodel"
+        ) for i in range(expected_layers or 0)]
+    else:
+        # choose the prefix with the most layers (most likely the correct one)
+        prefix = max(prefix_map.items(), key=lambda kv: len(kv[1]))[0]
+        # debug info
+        print(f"Detected prefixes: {list(prefix_map.keys())}, chosen: {prefix}, layers: {len(prefix_map[prefix])}")
+        layer_files = sorted(prefix_map[prefix], key=lambda it: it[0])
+
+    # find post process file
+    post_file = None
+    for fname in files:
+        m = post_pattern.match(fname)
+        if m and m.group("prefix") == prefix:
+            post_file = fname
+            break
+    if post_file is None:
+        candidate = os.path.join(base_dir, f"{prefix}_post.axmodel")
+        if os.path.exists(candidate):
+            post_file = f"{prefix}_post.axmodel"
+        else:
+            for fname in files:
+                if fname.endswith("_post.axmodel"):
+                    post_file = fname
+                    break
+
+    return layer_files, post_file, prefix
+
+class InferManager:
+    def __init__(self, config, model_dir, max_seq_len=4095, max_prefill_len=1023):
+
+        self.config = config
+        self.max_seq_len = max_seq_len
+        self.max_prefill_len = max_prefill_len
+
+        self.sub_dim = config.hidden_size // config.num_attention_heads if not config.head_dim else config.head_dim
+        self.kv_dim = self.sub_dim * config.num_key_value_heads
+
+        self.k_caches = [
+            np.zeros((1, self.max_seq_len, self.kv_dim), dtype=bfloat16)
+            for _ in range(config.num_hidden_layers)
+        ]
+        self.v_caches = [
+            np.zeros((1, self.max_seq_len, self.kv_dim), dtype=bfloat16)
+            for _ in range(config.num_hidden_layers)
+        ]
+
+        layer_files, post_file, prefix = _find_axmodel_files(model_dir, config.num_hidden_layers)
+
+        self.decoder_sessions = []
+        for _, fname in tqdm(layer_files, desc="Init InferenceSession"):
+            session = InferenceSession(os.path.join(model_dir, fname))
+            self.decoder_sessions.append(session)
+
+        # post_file was returned by _find_axmodel_files; ensure it was found
+        if post_file is None:
+            raise FileNotFoundError("Cannot find post process .axmodel file in model_dir")
+        self.post_process_session = InferenceSession(os.path.join(model_dir, post_file))
+        print("Model loaded successfully!")
+
+    @staticmethod
+    def _top_p(probs: np.ndarray, p: float) -> np.ndarray:
+        sorted_indices = np.argsort(probs)
+        filtered = probs.copy()
+        cumulative = 0
+        for idx in sorted_indices[::-1]:
+            if cumulative >= p:
+                filtered[idx] = 0
+            cumulative += filtered[idx]
+        return filtered / cumulative
+
+    @staticmethod
+    def _softmax(logits: np.ndarray) -> np.ndarray:
+        logits = logits - logits.max()
+        exp_logits = np.exp(logits)
+        return (exp_logits / np.sum(exp_logits)).astype(np.float64)
+
+    def post_process(self, logits, top_k=1, top_p=0.9, temperature=0.6):
+        logits = logits.astype(np.float32).flatten()
+        candidate_indices = np.argpartition(logits, -top_k)[-top_k:]
+        candidate_logits = logits[candidate_indices] / temperature
+        candidate_probs = self._softmax(candidate_logits)
+        candidate_probs = self._top_p(candidate_probs, top_p)
+        candidate_probs = candidate_probs.astype(np.float64) / candidate_probs.sum()
+        chosen_idx = np.random.multinomial(1, candidate_probs).argmax()
+        next_token = candidate_indices[chosen_idx]
+        return next_token, candidate_indices, candidate_probs
+
+    def gen_slice_indices(self, token_len, prefill=128, expand=128):
+        remaining = max(0, token_len - prefill)
+        extra_blocks = (remaining + expand - 1) // expand
+        return list(range(extra_blocks + 1))
+
+    def prefill(
+        self,
+        tokenizer,
+        token_ids,
+        embed_data,
+        slice_len=128,
+    ):
+        """
+        Prefill step for chunked inference.
+        """
+        seq_len = len(token_ids)
+        slice_indices = [i for i in range(min(seq_len, self.max_prefill_len) // slice_len + 1)]
+        print(f"slice_indices: {slice_indices}")
+        # total_prefill_len = (
+        #     slice_len * slice_indices[-1]
+        #     if slice_indices[-1] != 0
+        #     else slice_len
+        # )
+        total_prefill_len = slice_len * (slice_indices[-1] + 1)
+        # slice_indices = self.gen_slice_indices(seq_len)
+
+        if total_prefill_len > 0:
+            for slice_idx in slice_indices:
+                indices = np.arange(
+                    slice_idx * slice_len,
+                    (slice_idx + 1) * slice_len,
+                    dtype=np.uint32
+                ).reshape((1, slice_len))
+
+                mask = (
+                    np.zeros((1, slice_len, slice_len * (slice_idx + 1)))
+                    - 65536
+                )
+                data = np.zeros((1, slice_len, self.config.hidden_size)).astype(bfloat16)
+                for i, t in enumerate(
+                    range(
+                        slice_idx * slice_len,
+                        (slice_idx + 1) * slice_len,
+                    )
+                ):
+                    if t < len(token_ids):
+                        mask[:, i, : slice_idx * slice_len + i + 1] = 0
+                        data[:, i : i + 1, :] = (
+                            embed_data[t]
+                            .reshape((1, 1, self.config.hidden_size))
+                            .astype(bfloat16)
+                        )
+
+                remain_len = (
+                    seq_len - slice_idx * slice_len
+                    if slice_idx == slice_indices[-1]
+                    else slice_len
+                )
+                mask = mask.astype(bfloat16)
+                for layer_idx in range(self.config.num_hidden_layers):
+                    input_feed = {
+                        "K_cache": (
+                            self.k_caches[layer_idx][:, 0 : slice_len * slice_idx, :]
+                            if slice_idx
+                            else np.zeros((1, 1, self.config.hidden_size), dtype=bfloat16)
+                        ),
+                        "V_cache": (
+                            self.v_caches[layer_idx][:, 0 : slice_len * slice_idx, :]
+                            if slice_idx
+                            else np.zeros((1, 1, self.config.hidden_size), dtype=bfloat16)
+                        ),
+                        "indices": indices,
+                        "input": data,
+                        "mask": mask,
+                    }
+                    outputs = self.decoder_sessions[layer_idx].run(None, input_feed, shape_group=slice_idx + 1)
+                    self.k_caches[layer_idx][
+                        :,
+                        slice_idx * slice_len : slice_idx * slice_len + remain_len,
+                        :,
+                    ] = outputs[0][:, :remain_len, :]
+                    self.v_caches[layer_idx][
+                        :,
+                        slice_idx * slice_len : slice_idx * slice_len + remain_len,
+                        :,
+                    ] = outputs[1][:, :remain_len, :]
+                    data = outputs[2]
+
+                print("Slice prefill done:", slice_idx)
+            
+            # return data[:, :remain_len, :]
+            post_out = self.post_process_session.run(
+                None,
+                {
+                    "input": data[
+                        :, seq_len - (len(slice_indices) - 1) * slice_len - 1, None, :
+                    ]
+                }
+            )[0]
+            next_token, possible_tokens, possible_probs = self.post_process(post_out)
+            possible_decoded = [tokenizer.decode([t]) for t in possible_tokens]
+            possible_probs_str = [str((t, p)) for t, p in zip(possible_decoded, possible_probs)]
+            token_ids.append(next_token)
+            return token_ids
+
+    def decode(
+        self,
+        tokenizer,
+        token_ids,
+        embed_matrix,
+        prefill_len=128,
+        slice_len=128,
+        eos_token_id=None, # 某些模型有多个 eos_token_id
+    ):
+        print("answer >>", tokenizer.decode(token_ids[-1], skip_special_tokens=True), end='', flush=True)
+        mask = np.zeros((1, 1, self.max_seq_len + 1), dtype=np.float32).astype(bfloat16)
+        mask[:, :, :self.max_seq_len] -= 65536
+        seq_len = len(token_ids) - 1
+        if prefill_len > 0:
+            mask[:, :, :seq_len] = 0
+        for step_idx in range(self.max_seq_len):
+            if prefill_len > 0 and step_idx < seq_len:
+                continue
+            cur_token = token_ids[step_idx]
+            indices = np.array([step_idx], np.uint32).reshape((1, 1))
+            data = embed_matrix[cur_token, :].reshape((1, 1, self.config.hidden_size)).astype(bfloat16)
+            for layer_idx in range(self.config.num_hidden_layers):
+                input_feed = {
+                    "K_cache": self.k_caches[layer_idx],
+                    "V_cache": self.v_caches[layer_idx],
+                    "indices": indices,
+                    "input": data,
+                    "mask": mask,
+                }
+                outputs = self.decoder_sessions[layer_idx].run(None, input_feed, shape_group=0)
+                self.k_caches[layer_idx][:, step_idx, :] = outputs[0][:, :, :]
+                self.v_caches[layer_idx][:, step_idx, :] = outputs[1][:, :, :]
+                data = outputs[2]
+            mask[..., step_idx] = 0
+            if step_idx < seq_len - 1:
+                continue
+            else:
+                post_out = self.post_process_session.run(None, {"input": data})[0]
+                next_token, possible_tokens, possible_probs = self.post_process(post_out)
+                if eos_token_id is not None and next_token in eos_token_id:
+                    break
+                elif next_token == tokenizer.eos_token_id:
+                    break
+                else:
+                    pass
+                token_ids.append(next_token)
+            print(tokenizer.decode(next_token, skip_special_tokens=True), end='', flush=True)
+

ax_meeting/utils/infer_utils.py

@@ -0,0 +1,312 @@
+# -*- encoding: utf-8 -*-
+
+import functools
+import logging
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union
+
+import re
+import numpy as np
+import yaml
+
+import jieba
+import warnings
+
+root_dir = Path(__file__).resolve().parent
+
+logger_initialized = {}
+
+
+def pad_list(xs, pad_value, max_len=None):
+    n_batch = len(xs)
+    if max_len is None:
+        max_len = max(x.size(0) for x in xs)
+    # pad = xs[0].new(n_batch, max_len, *xs[0].size()[1:]).fill_(pad_value)
+    # numpy format
+    pad = (np.zeros((n_batch, max_len)) + pad_value).astype(np.int32)
+    for i in range(n_batch):
+        pad[i, : xs[i].shape[0]] = xs[i]
+
+    return pad
+
+
+"""
+def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None):
+    if length_dim == 0:
+        raise ValueError("length_dim cannot be 0: {}".format(length_dim))
+
+    if not isinstance(lengths, list):
+        lengths = lengths.tolist()
+    bs = int(len(lengths))
+    if maxlen is None:
+        if xs is None:
+            maxlen = int(max(lengths))
+        else:
+            maxlen = xs.size(length_dim)
+    else:
+        assert xs is None
+        assert maxlen >= int(max(lengths))
+
+    seq_range = torch.arange(0, maxlen, dtype=torch.int64)
+    seq_range_expand = seq_range.unsqueeze(0).expand(bs, maxlen)
+    seq_length_expand = seq_range_expand.new(lengths).unsqueeze(-1)
+    mask = seq_range_expand >= seq_length_expand
+
+    if xs is not None:
+        assert xs.size(0) == bs, (xs.size(0), bs)
+
+        if length_dim < 0:
+            length_dim = xs.dim() + length_dim
+        # ind = (:, None, ..., None, :, , None, ..., None)
+        ind = tuple(
+            slice(None) if i in (0, length_dim) else None for i in range(xs.dim())
+        )
+        mask = mask[ind].expand_as(xs).to(xs.device)
+    return mask
+"""
+
+
+class TokenIDConverter:
+    def __init__(
+        self,
+        token_list: Union[List, str],
+    ):
+
+        self.token_list = token_list
+        self.unk_symbol = token_list[-1]
+        self.token2id = {v: i for i, v in enumerate(self.token_list)}
+        self.unk_id = self.token2id[self.unk_symbol]
+
+    def get_num_vocabulary_size(self) -> int:
+        return len(self.token_list)
+
+    def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]:
+        if isinstance(integers, np.ndarray) and integers.ndim != 1:
+            raise TokenIDConverterError(f"Must be 1 dim ndarray, but got {integers.ndim}")
+        return [self.token_list[i] for i in integers]
+
+    def tokens2ids(self, tokens: Iterable[str]) -> List[int]:
+
+        return [self.token2id.get(i, self.unk_id) for i in tokens]
+
+
+class CharTokenizer:
+    def __init__(
+        self,
+        symbol_value: Union[Path, str, Iterable[str]] = None,
+        space_symbol: str = "<space>",
+        remove_non_linguistic_symbols: bool = False,
+    ):
+
+        self.space_symbol = space_symbol
+        self.non_linguistic_symbols = self.load_symbols(symbol_value)
+        self.remove_non_linguistic_symbols = remove_non_linguistic_symbols
+
+    @staticmethod
+    def load_symbols(value: Union[Path, str, Iterable[str]] = None) -> Set:
+        if value is None:
+            return set()
+
+        if isinstance(value, Iterable[str]):
+            return set(value)
+
+        file_path = Path(value)
+        if not file_path.exists():
+            logging.warning("%s doesn't exist.", file_path)
+            return set()
+
+        with file_path.open("r", encoding="utf-8") as f:
+            return set(line.rstrip() for line in f)
+
+    def text2tokens(self, line: Union[str, list]) -> List[str]:
+        tokens = []
+        while len(line) != 0:
+            for w in self.non_linguistic_symbols:
+                if line.startswith(w):
+                    if not self.remove_non_linguistic_symbols:
+                        tokens.append(line[: len(w)])
+                    line = line[len(w) :]
+                    break
+            else:
+                t = line[0]
+                if t == " ":
+                    t = "<space>"
+                tokens.append(t)
+                line = line[1:]
+        return tokens
+
+    def tokens2text(self, tokens: Iterable[str]) -> str:
+        tokens = [t if t != self.space_symbol else " " for t in tokens]
+        return "".join(tokens)
+
+    def __repr__(self):
+        return (
+            f"{self.__class__.__name__}("
+            f'space_symbol="{self.space_symbol}"'
+            f'non_linguistic_symbols="{self.non_linguistic_symbols}"'
+            f")"
+        )
+
+
+class Hypothesis(NamedTuple):
+    """Hypothesis data type."""
+
+    yseq: np.ndarray
+    score: Union[float, np.ndarray] = 0
+    scores: Dict[str, Union[float, np.ndarray]] = dict()
+    states: Dict[str, Any] = dict()
+
+    def asdict(self) -> dict:
+        """Convert data to JSON-friendly dict."""
+        return self._replace(
+            yseq=self.yseq.tolist(),
+            score=float(self.score),
+            scores={k: float(v) for k, v in self.scores.items()},
+        )._asdict()
+
+
+class TokenIDConverterError(Exception):
+    pass
+
+
+class ONNXRuntimeError(Exception):
+    pass
+
+
+def split_to_mini_sentence(words: list, word_limit: int = 20):
+    assert word_limit > 1
+    if len(words) <= word_limit:
+        return [words]
+    sentences = []
+    length = len(words)
+    sentence_len = length // word_limit
+    for i in range(sentence_len):
+        sentences.append(words[i * word_limit : (i + 1) * word_limit])
+    if length % word_limit > 0:
+        sentences.append(words[sentence_len * word_limit :])
+    return sentences
+
+
+def code_mix_split_words(text: str):
+    words = []
+    segs = text.split()
+    for seg in segs:
+        # There is no space in seg.
+        current_word = ""
+        for c in seg:
+            if len(c.encode()) == 1:
+                # This is an ASCII char.
+                current_word += c
+            else:
+                # This is a Chinese char.
+                if len(current_word) > 0:
+                    words.append(current_word)
+                    current_word = ""
+                words.append(c)
+        if len(current_word) > 0:
+            words.append(current_word)
+    return words
+
+
+def isEnglish(text: str):
+    if re.search("^[a-zA-Z']+$", text):
+        return True
+    else:
+        return False
+
+
+def join_chinese_and_english(input_list):
+    line = ""
+    for token in input_list:
+        if isEnglish(token):
+            line = line + " " + token
+        else:
+            line = line + token
+
+    line = line.strip()
+    return line
+
+
+def code_mix_split_words_jieba(seg_dict_file: str):
+    jieba.load_userdict(seg_dict_file)
+
+    def _fn(text: str):
+        input_list = text.split()
+        token_list_all = []
+        langauge_list = []
+        token_list_tmp = []
+        language_flag = None
+        for token in input_list:
+            if isEnglish(token) and language_flag == "Chinese":
+                token_list_all.append(token_list_tmp)
+                langauge_list.append("Chinese")
+                token_list_tmp = []
+            elif not isEnglish(token) and language_flag == "English":
+                token_list_all.append(token_list_tmp)
+                langauge_list.append("English")
+                token_list_tmp = []
+
+            token_list_tmp.append(token)
+
+            if isEnglish(token):
+                language_flag = "English"
+            else:
+                language_flag = "Chinese"
+
+        if token_list_tmp:
+            token_list_all.append(token_list_tmp)
+            langauge_list.append(language_flag)
+
+        result_list = []
+        for token_list_tmp, language_flag in zip(token_list_all, langauge_list):
+            if language_flag == "English":
+                result_list.extend(token_list_tmp)
+            else:
+                seg_list = jieba.cut(join_chinese_and_english(token_list_tmp), HMM=False)
+                result_list.extend(seg_list)
+
+        return result_list
+
+    return _fn
+
+
+def read_yaml(yaml_path: Union[str, Path]) -> Dict:
+    if not Path(yaml_path).exists():
+        raise FileExistsError(f"The {yaml_path} does not exist.")
+
+    with open(str(yaml_path), "rb") as f:
+        data = yaml.load(f, Loader=yaml.Loader)
+    return data
+
+
+@functools.lru_cache()
+def get_logger(name="funasr_onnx"):
+    """Initialize and get a logger by name.
+    If the logger has not been initialized, this method will initialize the
+    logger by adding one or two handlers, otherwise the initialized logger will
+    be directly returned. During initialization, a StreamHandler will always be
+    added.
+    Args:
+        name (str): Logger name.
+    Returns:
+        logging.Logger: The expected logger.
+    """
+    logger = logging.getLogger(name)
+    if name in logger_initialized:
+        return logger
+
+    for logger_name in logger_initialized:
+        if name.startswith(logger_name):
+            return logger
+
+    formatter = logging.Formatter(
+        "[%(asctime)s] %(name)s %(levelname)s: %(message)s", datefmt="%Y/%m/%d %H:%M:%S"
+    )
+
+    sh = logging.StreamHandler()
+    sh.setFormatter(formatter)
+    logger.addHandler(sh)
+    logger_initialized[name] = True
+    logger.propagate = False
+    logging.basicConfig(level=logging.ERROR)
+    return logger

ax_meeting/utils/sentencepiece_tokenizer.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+from pathlib import Path
+from typing import Iterable, List, Union
+
+import sentencepiece as spm
+
+
+class SentencepiecesTokenizer:
+    def __init__(self, bpemodel: Union[Path, str], **kwargs):
+        self.bpemodel = str(bpemodel)
+        self.sp = None
+        self._build()
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}(model="{self.bpemodel}")'
+
+    def _build(self):
+        if self.sp is None:
+            self.sp = spm.SentencePieceProcessor()
+            self.sp.load(self.bpemodel)
+
+    def text2tokens(self, line: str) -> List[str]:
+        self._build()
+        return self.sp.EncodeAsPieces(line)
+
+    def tokens2text(self, tokens: Iterable[str]) -> str:
+        self._build()
+        return self.sp.DecodePieces(list(tokens))
+
+    def encode(self, line: str) -> List[int]:
+        self._build()
+        return self.sp.EncodeAsIds(line)
+
+    def decode(self, line: List[int]):
+        self._build()
+        return self.sp.DecodeIds(line)
+
+    def get_vocab_size(self):
+        self._build()
+        return self.sp.GetPieceSize()
+
+    def ids2tokens(self, *args, **kwargs):
+        return self.decode(*args, **kwargs)
+
+    def tokens2ids(self, *args, **kwargs):
+        return self.encode(*args, **kwargs)

ax_meeting/utils/speaker_fbank.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+import numpy as np
+import kaldi_native_fbank as knf
+
+
+def compute_fbank(wav: np.ndarray, sample_rate: int, n_mels: int = 80, mean_nor: bool = True) -> np.ndarray:
+    if wav.ndim != 1:
+        wav = wav.reshape(-1)
+    opts = knf.FbankOptions()
+    opts.frame_opts.samp_freq = sample_rate
+    opts.mel_opts.num_bins = n_mels
+    opts.frame_opts.dither = 0.0
+    fbank = knf.OnlineFbank(opts)
+    fbank.accept_waveform(sample_rate, wav.astype(np.float32))
+    fbank.input_finished()
+    num_frames = fbank.num_frames_ready
+    if num_frames == 0:
+        return np.zeros((0, n_mels), dtype=np.float32)
+    feats = np.stack([fbank.get_frame(i) for i in range(num_frames)]).astype(np.float32)
+    if mean_nor and feats.size > 0:
+        feats = feats - feats.mean(axis=0, keepdims=True)
+    return feats