Skip to content

vllm.assets.video

Classes:

Functions:

VideoAsset dataclass

Methods:

  • get_audio –

    Read audio data from the video asset, used in Qwen2.5-Omni examples.

Source code in vllm/assets/video.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
@dataclass(frozen=True)
class VideoAsset:
    name: VideoAssetName
    num_frames: int = -1

    _NAME_TO_FILE: ClassVar[dict[VideoAssetName, str]] = {
        "baby_reading": "sample_demo_1.mp4",
    }

    @property
    def filename(self) -> str:
        return self._NAME_TO_FILE[self.name]

    @property
    def video_path(self) -> str:
        return download_video_asset(self.filename)

    @property
    def pil_images(self) -> list[Image.Image]:
        ret = video_to_pil_images_list(self.video_path, self.num_frames)
        return ret

    @property
    def np_ndarrays(self) -> npt.NDArray:
        ret = video_to_ndarrays(self.video_path, self.num_frames)
        return ret

    @property
    def metadata(self) -> dict[str, Any]:
        ret = video_get_metadata(self.video_path, self.num_frames)
        return ret

    def get_audio(self, sampling_rate: float | None = None) -> npt.NDArray:
        """
        Read audio data from the video asset, used in Qwen2.5-Omni examples.

        See also: examples/generate/multimodal/qwen2_5_omni/only_thinker.py
        """
        return load_audio_pyav(self.video_path, sr=sampling_rate)[0]

get_audio(sampling_rate=None)

Read audio data from the video asset, used in Qwen2.5-Omni examples.

See also: examples/generate/multimodal/qwen2_5_omni/only_thinker.py

Source code in vllm/assets/video.py 
138
139
140
141
142
143
144
def get_audio(self, sampling_rate: float | None = None) -> npt.NDArray:
    """
    Read audio data from the video asset, used in Qwen2.5-Omni examples.

    See also: examples/generate/multimodal/qwen2_5_omni/only_thinker.py
    """
    return load_audio_pyav(self.video_path, sr=sampling_rate)[0]

download_video_asset(filename) cached

Download and open an image from huggingface repo: raushan-testing-hf/videos-test

Source code in vllm/assets/video.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
@lru_cache
def download_video_asset(filename: str) -> str:
    """
    Download and open an image from huggingface
    repo: raushan-testing-hf/videos-test
    """
    video_directory = get_cache_dir() / "video-example-data"
    video_directory.mkdir(parents=True, exist_ok=True)

    video_path = video_directory / filename
    video_path_str = str(video_path)
    if not video_path.exists():
        video_path_str = hf_api().hf_hub_download(
            repo_id="raushan-testing-hf/videos-test",
            filename=filename,
            repo_type="dataset",
            cache_dir=video_directory,
        )
    return video_path_str