Skip to content

vllm.entrypoints.serve.dev.cache.api_router

Functions:

reset_encoder_cache(raw_request) async

Reset the encoder cache. Note that we currently do not check if the encoder cache is successfully reset in the API server.

Source code in vllm/entrypoints/serve/dev/cache/api_router.py 
57
58
59
60
61
62
63
64
65
@router.post("/reset_encoder_cache")
async def reset_encoder_cache(raw_request: Request):
    """
    Reset the encoder cache. Note that we currently do not check if the
    encoder cache is successfully reset in the API server.
    """
    logger.info("Resetting encoder cache...")
    await engine_client(raw_request).reset_encoder_cache()
    return Response(status_code=200)

reset_mm_cache(raw_request) async

Reset the multi-modal cache. Note that we currently do not check if the multi-modal cache is successfully reset in the API server.

Source code in vllm/entrypoints/serve/dev/cache/api_router.py 
46
47
48
49
50
51
52
53
54
@router.post("/reset_mm_cache")
async def reset_mm_cache(raw_request: Request):
    """
    Reset the multi-modal cache. Note that we currently do not check if the
    multi-modal cache is successfully reset in the API server.
    """
    logger.info("Resetting multi-modal cache...")
    await engine_client(raw_request).reset_mm_cache()
    return Response(status_code=200)

reset_prefix_cache(raw_request, reset_running_requests=Query(default=False), reset_external=Query(default=False)) async

Reset the local prefix cache.

Optionally, if the query parameter reset_external=true also resets the external (connector-managed) prefix cache.

Note that we currently do not check if the prefix cache is successfully reset in the API server.

Example

POST /reset_prefix_cache?reset_external=true

Source code in vllm/entrypoints/serve/dev/cache/api_router.py 
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
@router.post("/reset_prefix_cache")
async def reset_prefix_cache(
    raw_request: Request,
    reset_running_requests: bool = Query(default=False),
    reset_external: bool = Query(default=False),
):
    """
    Reset the local prefix cache.

    Optionally, if the query parameter `reset_external=true`
    also resets the external (connector-managed) prefix cache.

    Note that we currently do not check if the prefix cache
    is successfully reset in the API server.

    Example:
       POST /reset_prefix_cache?reset_external=true
    """
    logger.info("Resetting prefix cache...")

    await engine_client(raw_request).reset_prefix_cache(
        reset_running_requests, reset_external
    )
    return Response(status_code=200)