mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-11-22 13:37:32 +08:00
LiteLLM's win_amd64 wheels ship the bundled cl100k/o200k/p50k encoding files with CRLF line endings. They fail tiktoken's SHA-256 check, so tiktoken deletes them and downloads the encoding again. Where openaipublic.blob.core.windows.net is slow or blocked, the first token count hangs (the 30s count_tokens timeouts in LANG-392), and LiteLLM's cost calculation on the event loop waits on the same import lock, which froze Core during the 4.11.6 Windows release gate. Before LiteLLM is imported, write LF copies into data/tiktoken_cache and point CUSTOM_TIKTOKEN_CACHE_DIR at them. No-op when the variable is set or the bundled files are already LF (Linux/macOS wheels, Docker). Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: multica-agent <github@multica.ai>
80 lines
2.8 KiB
Python
80 lines
2.8 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from langbot.pkg.utils import tiktoken_cache
|
|
|
|
NAME = '9b5ad71b2ce5302211f9c61530b329a4922fc6a4'
|
|
OTHER = 'fb374d419588a4632f3f557e76b4b70aebbca790'
|
|
|
|
|
|
@pytest.fixture
|
|
def bundled(tmp_path, monkeypatch):
|
|
directory = tmp_path / 'litellm' / 'litellm_core_utils' / 'tokenizers'
|
|
directory.mkdir(parents=True)
|
|
(directory / '__init__.py').write_text('')
|
|
(directory / 'anthropic_tokenizer.json').write_bytes(b'{}\r\n')
|
|
monkeypatch.setattr(tiktoken_cache, '_bundled_tokenizer_dir', lambda: directory)
|
|
monkeypatch.delenv(tiktoken_cache.CUSTOM_CACHE_ENV, raising=False)
|
|
return directory
|
|
|
|
|
|
def test_crlf_files_are_copied_as_lf_and_litellm_is_pointed_at_them(bundled, tmp_path, monkeypatch):
|
|
(bundled / NAME).write_bytes(b'IQ== 0\r\nIg== 1\r\n')
|
|
(bundled / OTHER).write_bytes(b'Iw== 0\n')
|
|
target = tmp_path / 'data' / 'tiktoken_cache'
|
|
|
|
assert tiktoken_cache.ensure_offline_tiktoken_cache(target) == target
|
|
|
|
assert (target / NAME).read_bytes() == b'IQ== 0\nIg== 1\n'
|
|
assert (target / OTHER).read_bytes() == b'Iw== 0\n'
|
|
assert not (target / 'anthropic_tokenizer.json').exists()
|
|
assert not list(target.glob('*.tmp'))
|
|
import os
|
|
|
|
assert os.environ[tiktoken_cache.CUSTOM_CACHE_ENV] == str(target)
|
|
|
|
|
|
def test_repeated_start_rewrites_only_changed_files(bundled, tmp_path, monkeypatch):
|
|
(bundled / NAME).write_bytes(b'IQ== 0\r\n')
|
|
target = tmp_path / 'cache'
|
|
tiktoken_cache.ensure_offline_tiktoken_cache(target)
|
|
mtime = (target / NAME).stat().st_mtime_ns
|
|
monkeypatch.delenv(tiktoken_cache.CUSTOM_CACHE_ENV)
|
|
|
|
tiktoken_cache.ensure_offline_tiktoken_cache(target)
|
|
|
|
assert (target / NAME).stat().st_mtime_ns == mtime
|
|
|
|
|
|
def test_lf_files_need_nothing(bundled, tmp_path):
|
|
import os
|
|
|
|
(bundled / NAME).write_bytes(b'IQ== 0\n')
|
|
|
|
assert tiktoken_cache.ensure_offline_tiktoken_cache(tmp_path / 'cache') is None
|
|
assert not (tmp_path / 'cache').exists()
|
|
assert tiktoken_cache.CUSTOM_CACHE_ENV not in os.environ
|
|
|
|
|
|
def test_user_override_is_respected(bundled, tmp_path, monkeypatch):
|
|
(bundled / NAME).write_bytes(b'IQ== 0\r\n')
|
|
monkeypatch.setenv(tiktoken_cache.CUSTOM_CACHE_ENV, str(tmp_path / 'mine'))
|
|
|
|
assert tiktoken_cache.ensure_offline_tiktoken_cache(tmp_path / 'cache') is None
|
|
assert not (tmp_path / 'cache').exists()
|
|
|
|
|
|
def test_missing_litellm_is_ignored(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(tiktoken_cache, '_bundled_tokenizer_dir', lambda: None)
|
|
monkeypatch.delenv(tiktoken_cache.CUSTOM_CACHE_ENV, raising=False)
|
|
|
|
assert tiktoken_cache.ensure_offline_tiktoken_cache(tmp_path / 'cache') is None
|
|
|
|
|
|
def test_finds_the_installed_litellm_bundle():
|
|
directory = tiktoken_cache._bundled_tokenizer_dir()
|
|
|
|
assert directory is not None
|
|
assert (directory / NAME).is_file()
|