bump to transformer v5 (#689)

Author: awni

mlx_lm/tokenizer_utils.py

@@ -89,11 +89,7 @@ def finalize(self):
     def text(self):
         if self._current_tokens:
             self._current_text = self._tokenizer.decode(self._current_tokens)
-            if self._current_text.endswith("\ufffd") or (
-                self._tokenizer.clean_up_tokenization_spaces
-                and len(self._current_text) > 0
-                and self._current_text[-1] == " "
-            ):
+            if self._current_text.endswith("\ufffd"):
                 self._current_text = self._current_text[:-1]
         if self._current_text and self._current_text[-1] == "\n":
             self._text += self._current_text
@@ -161,8 +157,6 @@ class BPEStreamingDetokenizer(StreamingDetokenizer):
     _space_matches = (".", "?", "!", ",", "n't", "'m", "'s", "'ve", "'re")
 
     def __init__(self, tokenizer):
-        self.clean_spaces = tokenizer.clean_up_tokenization_spaces
-
         # Extract the tokens in a list from id to text
         self.tokenmap = [None] * len(tokenizer.vocab)
         for value, tokenid in tokenizer.vocab.items():
@@ -197,8 +191,6 @@ def _maybe_trim_space(self, current_text):
             return current_text
         elif not self.text:
             return current_text[1:]
-        elif self.clean_spaces and current_text[1:].startswith(self._space_matches):
-            return current_text[1:]
         return current_text
 
     def add_token(self, token):
@@ -208,10 +200,7 @@ def add_token(self, token):
         text = self._decode_bytes(self._unflushed)
 
         # For multi-byte utf-8 wait until they are complete
-        # For single spaces wait until the next token to clean it if needed
-        if not text.endswith("\ufffd") and not (
-            len(v) == 1 and self._byte_decoder.get(v[0]) == 32
-        ):
+        if not text.endswith("\ufffd"):
             self.text += self._maybe_trim_space(text)
             self._unflushed = ""
 

setup.py

@@ -26,7 +26,7 @@
     install_requires=[
         f"mlx>={MIN_MLX_VERSION}; platform_system == 'Darwin'",
         "numpy",
-        "transformers>=4.39.3",
+        "transformers==5.0.0rc1",
         "sentencepiece",
         "protobuf",
         "pyyaml",