{
"cells": [
{
"cell_type": "markdown",
"id": "f2c16396",
"metadata": {},
"source": [
"### Universal Audio Understanding for Qwen2.5-Omni\n",
"\n",
"This notebook demonstrates how to use Qwen2.5-Omni to finish audio tasks such as speech recongnition, speech-to-text translation and audio analysis."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "638e9082-c1ef-4efd-9a10-e35507e25363",
"metadata": {
"execution": {
"iopub.execute_input": "2025-01-29T12:40:04.049566Z",
"iopub.status.busy": "2025-01-29T12:40:04.049365Z"
},
"tags": []
},
"outputs": [],
"source": [
"!pip install git+https://github.com/huggingface/transformers@f742a644ca32e65758c3adb36225aef1731bd2a8\n",
"!pip install qwen-omni-utils\n",
"!pip install openai\n",
"!pip install flash-attn --no-build-isolation"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "9596c50d-80a8-433f-b846-1fbf61145ccc",
"metadata": {
"ExecutionIndicator": {
"show": true
},
"execution": {
"iopub.execute_input": "2025-01-29T12:40:16.511701Z",
"iopub.status.busy": "2025-01-29T12:40:16.510916Z",
"iopub.status.idle": "2025-01-29T12:40:16.878038Z",
"shell.execute_reply": "2025-01-29T12:40:16.877543Z",
"shell.execute_reply.started": "2025-01-29T12:40:16.511678Z"
},
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/envs/omni/lib/python3.10/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml\n",
" warnings.warn(\n"
]
}
],
"source": [
"from qwen_omni_utils import process_mm_info\n",
"\n",
"# @title inference function\n",
"def inference(audio_path, prompt, sys_prompt):\n",
" messages = [\n",
" {\"role\": \"system\", \"content\": sys_prompt},\n",
" {\"role\": \"user\", \"content\": [\n",
" {\"type\": \"text\", \"text\": prompt},\n",
" {\"type\": \"audio\", \"audio\": audio_path},\n",
" ]\n",
" },\n",
" ]\n",
" text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
" print(\"text:\", text)\n",
" # image_inputs, video_inputs = process_vision_info([messages])\n",
" audios, images, videos = process_mm_info(messages, use_audio_in_video=True)\n",
" inputs = processor(text=text, audios=audios, images=images, videos=videos, return_tensors=\"pt\", padding=True, use_audio_in_video=True)\n",
" inputs = inputs.to(model.device).to(model.dtype)\n",
"\n",
" output = model.generate(**inputs, use_audio_in_video=True, return_audio=False)\n",
"\n",
" text = processor.batch_decode(output, skip_special_tokens=True, clean_up_tokenization_spaces=False)\n",
" return text"
]
},
{
"cell_type": "markdown",
"id": "386e4cd8",
"metadata": {},
"source": [
"Load model and processors."
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e829b782-0be7-4bc6-a576-6b815323376e",
"metadata": {
"ExecutionIndicator": {
"show": false
},
"execution": {
"iopub.execute_input": "2025-01-29T12:40:18.337731Z",
"iopub.status.busy": "2025-01-29T12:40:18.337470Z",
"iopub.status.idle": "2025-01-29T12:40:47.760976Z",
"shell.execute_reply": "2025-01-29T12:40:47.760220Z",
"shell.execute_reply.started": "2025-01-29T12:40:18.337713Z"
},
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/envs/omni/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n",
"2025-03-29 12:32:42.948355: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
"2025-03-29 12:32:42.983611: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
"2025-03-29 12:32:42.983643: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
"2025-03-29 12:32:42.984557: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
"2025-03-29 12:32:42.990843: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
"To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
"2025-03-29 12:32:43.751920: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
"You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour\n",
"Qwen2_5OmniToken2WavModel must inference with fp32, but flash_attention_2 only supports fp16 and bf16, attention implementation of Qwen2_5OmniToken2WavModel will fallback to sdpa.\n",
"Loading checkpoint shards: 100%|██████████| 5/5 [00:06<00:00, 1.30s/it]\n",
"/opt/conda/envs/omni/lib/python3.10/site-packages/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py:4641: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
" for key, value in torch.load(path).items():\n"
]
}
],
"source": [
"import torch\n",
"from transformers import Qwen2_5OmniModel, Qwen2_5OmniProcessor\n",
"\n",
"model_path = \"Qwen/Qwen2.5-Omni-7B\"\n",
"model = Qwen2_5OmniModel.from_pretrained(\n",
" model_path,\n",
" torch_dtype=torch.bfloat16,\n",
" device_map=\"auto\",\n",
" attn_implementation=\"flash_attention_2\",\n",
")\n",
"processor = Qwen2_5OmniProcessor.from_pretrained(model_path)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "ed93fb82",
"metadata": {},
"outputs": [],
"source": [
"import librosa\n",
"\n",
"from io import BytesIO\n",
"from urllib.request import urlopen\n",
"\n",
"from IPython.display import Audio"
]
},
{
"cell_type": "markdown",
"id": "6a47ad45",
"metadata": {},
"source": [
"#### 1. Speeh Recognition"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "1935af5e",
"metadata": {
"ExecutionIndicator": {
"show": true
},
"execution": {
"iopub.execute_input": "2025-01-29T12:41:18.150397Z",
"iopub.status.busy": "2025-01-29T12:41:18.149631Z",
"iopub.status.idle": "2025-01-29T12:41:19.978329Z",
"shell.execute_reply": "2025-01-29T12:41:19.977054Z",
"shell.execute_reply.started": "2025-01-29T12:41:18.150371Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING:root:System prompt modified, audio output may not work as expected. Audio output mode only works when using default system prompt 'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"text: ['<|im_start|>system\\nYou are a speech recognition model.<|im_end|>\\n<|im_start|>user\\nTranscribe the English audio into text without any punctuation marks.<|audio_bos|><|AUDIO|><|audio_eos|><|im_end|>\\n<|im_start|>assistant\\n']\n",
"system\n",
"You are a speech recognition model.\n",
"user\n",
"Transcribe the English audio into text without any punctuation marks.\n",
"assistant\n",
"mr quilter is the apostle of the middle classes and we are glad to welcome his gospel\n"
]
}
],
"source": [
"audio_path = \"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/1272-128104-0000.flac\"\n",
"prompt = \"Transcribe the English audio into text without any punctuation marks.\"\n",
"\n",
"audio = librosa.load(BytesIO(urlopen(audio_path).read()), sr=16000)[0]\n",
"display(Audio(audio, rate=16000))\n",
"\n",
"## Use a local HuggingFace model to inference.\n",
"response = inference(audio_path, prompt=prompt, sys_prompt=\"You are a speech recognition model.\")\n",
"print(response[0])"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "0496ff75",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING:root:System prompt modified, audio output may not work as expected. Audio output mode only works when using default system prompt 'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"text: ['<|im_start|>system\\nYou are a speech recognition model.<|im_end|>\\n<|im_start|>user\\n请将这段中文语音转换为纯文本，去掉标点符号。<|audio_bos|><|AUDIO|><|audio_eos|><|im_end|>\\n<|im_start|>assistant\\n']\n",
"system\n",
"You are a speech recognition model.\n",
"user\n",
"请将这段中文语音转换为纯文本，去掉标点符号。\n",
"assistant\n",
"甚至出现交易几乎停滞的情况\n"
]
}
],
"source": [
"audio_path = \"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/BAC009S0764W0121.wav\"\n",
"prompt = \"请将这段中文语音转换为纯文本，去掉标点符号。\"\n",
"\n",
"audio = librosa.load(BytesIO(urlopen(audio_path).read()), sr=16000)[0]\n",
"display(Audio(audio, rate=16000))\n",
"\n",
"## Use a local HuggingFace model to inference.\n",
"response = inference(audio_path, prompt=prompt, sys_prompt=\"You are a speech recognition model.\")\n",
"print(response[0])"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "7e50a541",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING:root:System prompt modified, audio output may not work as expected. Audio output mode only works when using default system prompt 'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"text: ['<|im_start|>system\\nYou are a speech recognition model.<|im_end|>\\n<|im_start|>user\\nTranscribe the Russian audio into text without including any punctuation marks.<|audio_bos|><|AUDIO|><|audio_eos|><|im_end|>\\n<|im_start|>assistant\\n']\n",
"system\n",
"You are a speech recognition model.\n",
"user\n",
"Transcribe the Russian audio into text without including any punctuation marks.\n",
"assistant\n",
"в древнем китае использовали уникальный способ обозначения периодов времени каждый этап китая или каждая семья находившаяся у власти были особой династии\n"
]
}
],
"source": [
"audio_path = \"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/10000611681338527501.wav\"\n",
"prompt = \"Transcribe the Russian audio into text without including any punctuation marks.\"\n",
"\n",
"audio = librosa.load(BytesIO(urlopen(audio_path).read()), sr=16000)[0]\n",
"display(Audio(audio, rate=16000))\n",
"\n",
"## Use a local HuggingFace model to inference.\n",
"response = inference(audio_path, prompt=prompt, sys_prompt=\"You are a speech recognition model.\")\n",
"print(response[0])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "89ee4aed",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING:root:System prompt modified, audio output may not work as expected. Audio output mode only works when using default system prompt 'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"text: ['<|im_start|>system\\nYou are a speech recognition model.<|im_end|>\\n<|im_start|>user\\nTranscribe the French audio into text without including any punctuation marks.<|audio_bos|><|AUDIO|><|audio_eos|><|im_end|>\\n<|im_start|>assistant\\n']\n",
"system\n",
"You are a speech recognition model.\n",
"user\n",
"Transcribe the French audio into text without including any punctuation marks.\n",
"assistant\n",
"les voyageurs à destination de pays où les taxes sont élevées peuvent parfois faire des économies considérables en particulier sur des produits comme les boissons alcoolisées ou le tabac\n"
]
}
],
"source": [
"audio_path = \"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/7105431834829365765.wav\"\n",
"prompt = \"Transcribe the French audio into text without including any punctuation marks.\"\n",
"\n",
"audio = librosa.load(BytesIO(urlopen(audio_path).read()), sr=16000)[0]\n",
"display(Audio(audio, rate=16000))\n",
"\n",
"## Use a local HuggingFace model to inference.\n",
"response = inference(audio_path, prompt=prompt, sys_prompt=\"You are a speech recognition model.\")\n",
"print(response[0])"
]
},
{
"cell_type": "markdown",
"id": "f9961aae",
"metadata": {},
"source": [
"#### 2. Speech Translation"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "0894f5f1",
"metadata": {
"ExecutionIndicator": {
"show": true
},
"execution": {
"iopub.execute_input": "2025-01-29T12:44:01.387553Z",
"iopub.status.busy": "2025-01-29T12:44:01.386725Z",
"iopub.status.idle": "2025-01-29T12:44:09.671782Z",
"shell.execute_reply": "2025-01-29T12:44:09.671200Z",
"shell.execute_reply.started": "2025-01-29T12:44:01.387530Z"
},
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING:root:System prompt modified, audio output may not work as expected. Audio output mode only works when using default system prompt 'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"text: ['<|im_start|>system\\nYou are a speech translation model.<|im_end|>\\n<|im_start|>user\\nListen to the provided English speech and produce a translation in Chinese text.<|audio_bos|><|AUDIO|><|audio_eos|><|im_end|>\\n<|im_start|>assistant\\n']\n",
"system\n",
"You are a speech translation model.\n",
"user\n",
"Listen to the provided English speech and produce a translation in Chinese text.\n",
"assistant\n",
"奎尔特先生是中产阶级的使徒,我们很高兴欢迎他的福音。\n"
]
}
],
"source": [
"audio_path = \"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/1272-128104-0000.flac\"\n",
"prompt = \"Listen to the provided English speech and produce a translation in Chinese text.\"\n",
"\n",
"audio = librosa.load(BytesIO(urlopen(audio_path).read()), sr=16000)[0]\n",
"display(Audio(audio, rate=16000))\n",
"\n",
"## Use a local HuggingFace model to inference.\n",
"response = inference(audio_path, prompt=prompt, sys_prompt=\"You are a speech translation model.\")\n",
"print(response[0])"
]
},
{
"cell_type": "markdown",
"id": "4d434aa8",
"metadata": {},
"source": [
"#### 3. Vocal Sound Classification"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "609b8b22",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING:root:System prompt modified, audio output may not work as expected. Audio output mode only works when using default system prompt 'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"text: ['<|im_start|>system\\nYou are a vocal sound classification model.<|im_end|>\\n<|im_start|>user\\nClassify the given human vocal sound in English.<|audio_bos|><|AUDIO|><|audio_eos|><|im_end|>\\n<|im_start|>assistant\\n']\n",
"system\n",
"You are a vocal sound classification model.\n",
"user\n",
"Classify the given human vocal sound in English.\n",
"assistant\n",
"Cough\n"
]
}
],
"source": [
"audio_path = \"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/cough.wav\"\n",
"prompt = \"Classify the given human vocal sound in English.\"\n",
"\n",
"audio = librosa.load(BytesIO(urlopen(audio_path).read()), sr=16000)[0]\n",
"display(Audio(audio, rate=16000))\n",
"\n",
"## Use a local HuggingFace model to inference.\n",
"response = inference(audio_path, prompt=prompt, sys_prompt=\"You are a vocal sound classification model.\")\n",
"print(response[0])"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "omni",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.16"
}
},
"nbformat": 4,
"nbformat_minor": 5
}