Only turn on FastIncrementalDetokenizer when tokenizers >= 0.21.1

Summary: Turn off FastIncrementalDetokenizer if the tokenizers version is < 0.21.1

Reviewed By: zhewenl

Differential Revision: D73638322

Author: houseroad

vllm/v1/engine/detokenizer.py

@@ -1,7 +1,9 @@
 # SPDX-License-Identifier: Apache-2.0
 from abc import ABC, abstractmethod
+from packaging import version
 from typing import Optional
 
+import tokenizers
 from tokenizers import Tokenizer
 from tokenizers.decoders import DecodeStream
 from transformers import PreTrainedTokenizerFast
@@ -43,7 +45,8 @@ def from_new_request(
             # No tokenizer => skipping detokenization.
             return IncrementalDetokenizer()
 
-        if isinstance(tokenizer, PreTrainedTokenizerFast):
+        if isinstance(tokenizer, PreTrainedTokenizerFast) and \
+                version.parse(tokenizers.__version__) >= version.parse("0.21.1"):
             # Fast tokenizer => use tokenizers library DecodeStream.
             return FastIncrementalDetokenizer(tokenizer, request)