Upload Ming-Lite-Omni-1.5 safetensors

talker/{audio_detokenizer.yaml → audio_detokenizer_stream.yaml}

@@ -5,21 +5,30 @@ __set_seed3: !apply:torch.manual_seed [1986]
 __set_seed4: !apply:torch.cuda.manual_seed_all [1986]
 
 # fixed params
-sample_rate: 22050
-text_encoder_input_size: 512
-llm_input_size: 1024
-llm_output_size: 1024
+sample_rate: 24000
+llm_input_size: 896
+llm_output_size: 896
 spk_embed_dim: 192
+qwen_pretrain_path: ''
+token_frame_rate: 50
+token_mel_ratio: 1
 
-flow: !new:.audio_detokenizer.flow.flow.MaskedDiffWithXvec
+# stream related params
+chunk_size: 50 # streaming inference chunk size, in token
+num_decoding_left_chunks: -1 # streaming inference flow decoder left chunk size, <0 means use all left chunks
+
+
+flow: !new:.audio_detokenizer.flow.flow.CausalMaskedDiffWithXvec
     input_size: 512
     output_size: 80
     spk_embed_dim: !ref <spk_embed_dim>
     output_type: 'mel'
     vocab_size: 4096
-    input_frame_rate: 50
+    input_frame_rate: !ref <token_frame_rate>
     only_mask_loss: True
-    encoder: !new:.audio_detokenizer.transformer.encoder.ConformerEncoder
+    token_mel_ratio: !ref <token_mel_ratio>
+    pre_lookahead_len: 3
+    encoder: !new:.audio_detokenizer.transformer.upsample_encoder_new_mel.UpsampleConformerEncoder
         output_size: 512
         attention_heads: 8
         linear_units: 2048
@@ -34,14 +43,11 @@ flow: !new:.audio_detokenizer.flow.flow.MaskedDiffWithXvec
         input_size: 512
         use_cnn_module: False
         macaron_style: False
-    length_regulator: !new:.audio_detokenizer.flow.length_regulator.InterpolateRegulator
-        channels: 80
-        sampling_ratios: [1, 1, 1, 1]
-    decoder: !new:.audio_detokenizer.flow.flow_matching.ConditionalCFM
+        static_chunk_size: !ref <chunk_size>
+    decoder: !new:.audio_detokenizer.flow.flow_matching.CausalConditionalCFM
         in_channels: 240
         n_spks: 1
         spk_emb_dim: 80
-        tensorrt_model_path: 'bin/ckpt_300M/estimator_fp16.plan'
         cfm_params: !new:omegaconf.DictConfig
             content:
                 sigma_min: 1e-06
@@ -50,16 +56,18 @@ flow: !new:.audio_detokenizer.flow.flow.MaskedDiffWithXvec
                 training_cfg_rate: 0.2
                 inference_cfg_rate: 0.7
                 reg_loss_type: 'l1'
-        estimator: !new:.audio_detokenizer.flow.decoder.ConditionalDecoder
+        estimator: !new:.audio_detokenizer.flow.decoder.CausalConditionalDecoder
             in_channels: 320
             out_channels: 80
-            channels: [256, 256]
-            dropout: 0
+            channels: [256]
+            dropout: 0.0
             attention_head_dim: 64
             n_blocks: 4
             num_mid_blocks: 12
             num_heads: 8
             act_fn: 'gelu'
+            static_chunk_size: !ref <chunk_size> * <token_mel_ratio>
+            num_decoding_left_chunks: !ref <num_decoding_left_chunks>
 
 hift: !new:.audio_detokenizer.hifigan.generator.HiFTGenerator
     in_channels: 80
@@ -69,15 +77,15 @@ hift: !new:.audio_detokenizer.hifigan.generator.HiFTGenerator
     nsf_alpha: 0.1
     nsf_sigma: 0.003
     nsf_voiced_threshold: 10
-    upsample_rates: [8, 8]
-    upsample_kernel_sizes: [16, 16]
+    upsample_rates: [8, 5, 3]
+    upsample_kernel_sizes: [16, 11, 7]
     istft_params:
         n_fft: 16
         hop_len: 4
     resblock_kernel_sizes: [3, 7, 11]
     resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
-    source_resblock_kernel_sizes: [7, 11]
-    source_resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5]]
+    source_resblock_kernel_sizes: [7, 7, 11]
+    source_resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
     lrelu_slope: 0.1
     audio_limit: 0.99
     f0_predictor: !new:.audio_detokenizer.hifigan.f0_predictor.ConvRNNF0Predictor
@@ -85,12 +93,13 @@ hift: !new:.audio_detokenizer.hifigan.generator.HiFTGenerator
         in_channels: 80
         cond_channels: 512
 
+
 feat_extractor: !name:matcha.utils.audio.mel_spectrogram
-    n_fft: 1024
+    n_fft: 1920
     num_mels: 80
     sampling_rate: !ref <sample_rate>
-    hop_size: 256
-    win_size: 1024
+    hop_size: 480
+    win_size: 1920
     fmin: 0
     fmax: 8000
     center: False

transformer/diffusion_pytorch_model-00002-of-00002.safetensors → talker/flow_stream.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e386f056ca05cf1b83af9360a79e7316552f41c984aa34b3b8c58fff5c7b52a2
-size 1473408512
+oid sha256:b907e9d1567f633d2015081ac170b2a9c367ea750a0456e5863344ee1cbe1aaf
+size 1329720739

talker/{hift.pt → hift_v2.pt}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91e679b6ca1eff71187ffb4f3ab0444935594cdcc20a9bd12afad111ef8d6012
-size 81896716
+oid sha256:3386cc880324d4e98e05987b99107f49e40ed925b8ecc87c1f4939432d429879
+size 83390254

talker/ossutil_output/ossutil_report_20250716_170346.report

@@ -0,0 +1 @@
+# ossutil64 cp -r /video_hy2/workspace/weilong.cwl/metax_models/bailingv4_moe_lite_addmetax_0716/ oss://moe-opensource-hy/multimodal/bailingv4_moe_lite_addmetax_0716/

talker/flow.pt → transformer/diffusion_pytorch_model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21eae78c105b5e1c6c337b04f667843377651b4bcfb2d43247ed3ad7fd0a3470
-size 419900943
+oid sha256:02893d422480a4252bf2b7e46a2816b3a86596062aef6fb52e2316192b53f99b
+size 2956851104

transformer/{diffusion_pytorch_model-00001-of-00002.safetensors → diffusion_pytorch_model-00002-of-00004.safetensors}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fecc17c89bb0d9ca82c3d0e9ea9ac079d066a073e8858a4545520c987322a19d
-size 9991573456
+oid sha256:8d0c69da15b07b70b85ffc511df3991ffa30fb45e594e54c1d6f5576a5c7e321
+size 2993322680

transformer/diffusion_pytorch_model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac5730f943ac8b9ff96f8a78fc6a22d1f4453027e96a3d937b8f8881c4200fd
+size 2955511536

transformer/diffusion_pytorch_model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04babdf93bbe81237a78591d4c0396c0872946eee6a725da37adaac7574fd0be
+size 2559296136