Phi-4 bug fix

unslothai · Jan 8, 2025 · 8a5cff1 · 8a5cff1
1 parent 562cfd3
commit 8a5cff1
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "2025.1.2"
+__version__ = "2025.1.3"
 
 __all__ = [
     "SUPPORTS_BFLOAT16",

diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
@@ -265,6 +265,10 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
     )))
     all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens))
 
+    # Remove replacement char for false positive
+    replacement_char = b"\xc3\xaf\xc2\xbf\xc2\xbd".decode("utf-8")
+    all_special_tokens = [x for x in all_special_tokens if x != replacement_char]
+
     # Check if chat template is enabled!
     check_chat_template1 = True
     check_chat_template2 = True