Files
huanghuoguoguo 98ab9ce355 feat(evaluation): add public Pipeline comparisons and memory fixes (#3040)
* fix(powercontext): declare empty host permissions

* fix(longtermmemory): declare required host capabilities

* feat(evaluation): add public memory retrieval metrics

* feat(evaluation): add answer metrics to public scorer

* docs(evaluation): pin public locomo dataset manifest

* docs(evaluation): pin locomo source revision

* fix(memory): deliver turn context and preserve recall authorization

* feat(evaluation): add reproducible public Pipeline comparisons

* fix(langrag): preserve Unicode text during fallback parsing

* fix(evaluation): verify raw model and judge provenance

* docs(evaluation): publish real Pipeline calibration evidence

* style(evaluation): align package formatting with Ruff

* docs(evaluation): record full tools suite validation
2026-11-20 22:19:03 +08:00

72 lines
2.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import codecs
import unittest
from unittest.mock import patch
from components.knowledge_engine.parser import FileParser
class FileParserEncodingTests(unittest.IsolatedAsyncioTestCase):
async def test_utf8_punctuation_is_preserved_without_encoding_guesses(self):
# Public LoCoMo conv-26 includes curly punctuation in session 14.
text = "He's so cute! What’s the funniest thing Oliver's done?"
parser = FileParser()
for filename in ('conversation.txt', 'conversation.unknown'):
for prefix in (b'', codecs.BOM_UTF8):
with self.subTest(filename=filename, bom=bool(prefix)):
with patch(
'components.knowledge_engine.parser.chardet.detect',
return_value={'encoding': 'Windows-1252', 'confidence': 0.73},
) as detect:
parsed = await parser.parse(prefix + text.encode('utf-8'), filename)
self.assertEqual(parsed, text)
detect.assert_not_called()
async def test_unicode_boms_are_decoded_before_utf8(self):
text = 'Meeting at 10 today.'
parser = FileParser()
for prefix, encoding in (
(codecs.BOM_UTF16_LE, 'utf-16-le'),
(codecs.BOM_UTF16_BE, 'utf-16-be'),
(codecs.BOM_UTF32_LE, 'utf-32-le'),
(codecs.BOM_UTF32_BE, 'utf-32-be'),
):
with self.subTest(encoding=encoding):
with patch('components.knowledge_engine.parser.chardet.detect') as detect:
parsed = await parser.parse(prefix + text.encode(encoding), 'conversation.txt')
self.assertEqual(parsed, text)
detect.assert_not_called()
async def test_unmarked_unicode_with_nuls_uses_encoding_detection(self):
text = 'Meeting at 10 today.'
parser = FileParser()
for encoding in ('utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be'):
with self.subTest(encoding=encoding):
with patch(
'components.knowledge_engine.parser.chardet.detect',
return_value={'encoding': encoding, 'confidence': 0.9},
) as detect:
raw = text.encode(encoding)
parsed = await parser.parse(raw, 'conversation.txt')
self.assertEqual(parsed, text)
detect.assert_called_once_with(raw)
async def test_legacy_encoding_uses_detection_after_utf8_fails(self):
text = 'My friend’s café is nearby.'
raw = text.encode('cp1252')
parser = FileParser()
with patch(
'components.knowledge_engine.parser.chardet.detect',
return_value={'encoding': 'Windows-1252', 'confidence': 0.73},
) as detect:
parsed = await parser.parse(raw, 'conversation.txt')
self.assertEqual(parsed, text)
detect.assert_called_once_with(raw)