Skip to content

vllm.transformers_utils.runai_utils

Classes:

Functions:

  • list_safetensors –

    List full file names from object path and filter by allow pattern.

ObjectStorageModel

A class representing an ObjectStorage model mirrored into a temporary directory.

Attributes:

  • dir –

    The temporary created directory.

Methods:

  • pull_files –

    Pull model from object storage to the temporary directory.

Source code in vllm/transformers_utils/runai_utils.py 
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
class ObjectStorageModel:
    """
    A class representing an ObjectStorage model mirrored into a
    temporary directory.

    Attributes:
        dir: The temporary created directory.

    Methods:
        pull_files(): Pull model from object storage to the temporary directory.
    """

    def __init__(self, url: str) -> None:
        if envs.VLLM_ASSETS_CACHE_MODEL_CLEAN:
            for sig in (signal.SIGINT, signal.SIGTERM):
                existing_handler = signal.getsignal(sig)
                signal.signal(sig, self._close_by_signal(existing_handler))

        dir_name = os.path.join(
            get_cache_dir(),
            "model_streamer",
            hashlib.sha256(str(url).encode()).hexdigest()[:8],
        )
        os.makedirs(dir_name, exist_ok=True)
        self.dir = dir_name
        logger.debug("Init object storage, model cache path is: %s", dir_name)

    def _close(self) -> None:
        if os.path.exists(self.dir):
            shutil.rmtree(self.dir)

    def _close_by_signal(self, existing_handler=None):
        def new_handler(signum, frame):
            self._close()
            if existing_handler:
                existing_handler(signum, frame)

        return new_handler

    def pull_files(
        self,
        model_path: str = "",
        allow_pattern: list[str] | None = None,
        ignore_pattern: list[str] | None = None,
    ) -> None:
        """
        Pull files from object storage into the temporary directory.

        Args:
            model_path: The object storage path of the model.
            allow_pattern: A list of patterns of which files to pull.
            ignore_pattern: A list of patterns of which files not to pull.

        """
        if not model_path.endswith("/"):
            model_path = model_path + "/"
        runai_pull_files(model_path, self.dir, allow_pattern, ignore_pattern)

pull_files(model_path='', allow_pattern=None, ignore_pattern=None)

Pull files from object storage into the temporary directory.

Parameters:

  • model_path

    (str, default: '' ) –

    The object storage path of the model.

  • allow_pattern

    (list[str] | None, default: None ) –

    A list of patterns of which files to pull.

  • ignore_pattern

    (list[str] | None, default: None ) –

    A list of patterns of which files not to pull.

Source code in vllm/transformers_utils/runai_utils.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def pull_files(
    self,
    model_path: str = "",
    allow_pattern: list[str] | None = None,
    ignore_pattern: list[str] | None = None,
) -> None:
    """
    Pull files from object storage into the temporary directory.

    Args:
        model_path: The object storage path of the model.
        allow_pattern: A list of patterns of which files to pull.
        ignore_pattern: A list of patterns of which files not to pull.

    """
    if not model_path.endswith("/"):
        model_path = model_path + "/"
    runai_pull_files(model_path, self.dir, allow_pattern, ignore_pattern)

list_safetensors(path='')

List full file names from object path and filter by allow pattern.

Parameters:

  • path

    (str, default: '' ) –

    The object storage path to list from.

Returns:

  • list[str] –

    list[str]: List of full object storage paths allowed by the pattern

Source code in vllm/transformers_utils/runai_utils.py 
27
28
29
30
31
32
33
34
35
36
37
def list_safetensors(path: str = "") -> list[str]:
    """
    List full file names from object path and filter by allow pattern.

    Args:
        path: The object storage path to list from.

    Returns:
        list[str]: List of full object storage paths allowed by the pattern
    """
    return runai_list_safetensors(path)