blank/tests/cleaning/test_cleaner.py

"""
Unit tests for TextCleaner.
"""

import pytest
from src.cleaning.cleaner import TextCleaner


class TestTextCleaner:
    """Test suite for TextCleaner."""

    @pytest.fixture
    def cleaner(self):
        """Create a TextCleaner instance."""
        return TextCleaner()

    def test_clean_empty_text(self, cleaner):
        """Test cleaning empty text."""
        assert cleaner.clean("") == ""
        assert cleaner.clean(None) if None else "" == cleaner.clean("")

    def test_remove_extra_whitespace(self, cleaner):
        """Test removing extra whitespace."""
        text = "这是  一段    有很多空格的  文本。"
        result = cleaner.remove_extra_whitespace_func(text)
        assert "  " not in result
        assert "很多" in result

    def test_remove_multiple_newlines(self, cleaner):
        """Test removing multiple consecutive newlines."""
        text = "第一行\n\n\n\n\n第二行"
        result = cleaner.remove_extra_whitespace_func(text)
        # Should keep up to 2 consecutive empty lines (3 newlines = 2 empty lines)
        assert "第一行" in result
        assert "第二行" in result
        # Should reduce 5 newlines (4 empty lines) to 3 newlines (2 empty lines)
        assert result.count("\n") < text.count("\n")

    def test_preserve_paragraph_structure(self, cleaner):
        """Test that paragraph structure is preserved."""
        text = "第一段\n\n第二段\n\n第三段"
        result = cleaner.remove_extra_whitespace_func(text)
        assert "\n\n" in result  # Paragraph breaks should be kept

    def test_fix_multiple_periods(self, cleaner):
        """Test fixing multiple Chinese periods."""
        text = "这是第一句。。这是第二句。。。"
        result = cleaner.fix_punctuation_func(text)
        assert "。。" not in result

    def test_fix_mixed_punctuation(self, cleaner):
        """Test fixing mixed Chinese/English punctuation."""
        text = "这是句子。，也是句子。!"
        result = cleaner.fix_punctuation_func(text)
        # Note: mixed punctuation is complex, just check that some fix was attempted
        assert "。" in result  # Chinese period should be preserved

    def test_fix_multiple_exclamations(self, cleaner):
        """Test fixing multiple exclamation marks."""
        text = "太棒了！！！！！"
        result = cleaner.fix_punctuation_func(text)
        assert "！！" not in result

    def test_fix_multiple_question_marks(self, cleaner):
        """Test fixing multiple question marks."""
        text = "真的吗？？？"
        result = cleaner.fix_punctuation_func(text)
        assert "？？" not in result

    def test_remove_invalid_chars(self, cleaner):
        """Test removing invalid characters."""
        text = "正常文本\x00\x01\x02更多文本"
        result = cleaner.remove_invalid_chars_func(text)
        assert "\x00" not in result
        assert "\x01" not in result
        assert "正常文本" in result

    def test_remove_bom_character(self, cleaner):
        """Test removing BOM character."""
        text = "\ufeff这是文本"
        result = cleaner.remove_invalid_chars_func(text)
        assert "\ufeff" not in result

    def test_normalize_quotes(self, cleaner):
        """Test quote normalization."""
        text = '这是\'引号\'和"双引号"内容'
        result = cleaner.normalize_quotes_func(text)
        # Quotes should be normalized to ASCII
        assert "'" in result or '"' in result

    def test_full_cleaning_pipeline(self, cleaner):
        """Test the full cleaning pipeline."""
        text = "  这是  一段  有问题\x00的文本。。\n\n\n还有多余的空格！  "
        result = cleaner.clean(text)
        assert "\x00" not in result
        assert "。。" not in result
        assert not result.startswith(" ")
        assert not result.endswith(" ")

    def test_remove_urls(self, cleaner):
        """Test URL removal."""
        text = "访问 https://example.com 查看更多信息"
        result = cleaner.clean(text)
        assert "https://" not in result

    def test_remove_email_addresses(self, cleaner):
        """Test email address removal."""
        text = "联系 test@example.com 获取更多信息"
        result = cleaner.clean(text)
        assert "@" not in result

    def test_custom_removal_patterns(self):
        """Test custom removal patterns."""
        text = "这是 [ISBN:123] 一些文字 [ISBN:456] 更多文字"
        cleaner = TextCleaner(custom_removals=[r'\[ISBN:\d+\]'])
        result = cleaner.clean(text)
        assert "[ISBN:" not in result

    def test_remove_ads(self, cleaner):
        """Test advertisement removal."""
        text = "这是小说内容。本章完。请收藏本站。更多精彩内容。"
        result = cleaner.remove_ads(text)
        assert "本章完" not in result
        assert "请收藏" not in result

    def test_extract_numbers(self, cleaner):
        """Test number extraction."""
        text = "林风今年18岁，身高175.5厘米，有3个朋友。"
        numbers = cleaner.extract_numbers(text)
        assert "18" in numbers
        assert "175.5" in numbers  # Decimal is returned as whole number
        assert "3" in numbers

    def test_count_words_chinese(self, cleaner):
        """Test word counting for Chinese text."""
        text = "这是一段中文文本用于测试字数统计。"
        count = cleaner.count_words(text)
        assert count > 0

    def test_count_words_mixed(self, cleaner):
        """Test word counting for mixed text."""
        text = "这里有 Chinese 和 English 123 混合"
        count = cleaner.count_words(text)
        assert count > 0

    def test_truncate_short_text(self, cleaner):
        """Test truncating short text (no change)."""
        text = "短文本"
        result = cleaner.truncate(text, 100)
        assert result == text

    def test_truncate_long_text(self, cleaner):
        """Test truncating long text."""
        text = "这是一段很长的文本需要被截断"
        result = cleaner.truncate(text, 10)
        # Each Chinese character is 1 byte in Python string length
        assert len(result) <= 13  # 10 chars + "..." (but might be less due to multibyte)
        assert result.endswith("...")

    def test_split_into_sentences_chinese(self, cleaner):
        """Test splitting Chinese text into sentences."""
        text = "这是第一句。这是第二句！这是第三句？"
        sentences = cleaner.split_into_sentences(text)
        assert len(sentences) == 3
        assert "第一句" in sentences[0]

    def test_split_into_sentences_english(self, cleaner):
        """Test splitting English text into sentences."""
        text = "This is first. This is second! This is third?"
        sentences = cleaner.split_into_sentences(text)
        assert len(sentences) >= 2

    def test_cleaning_preserves_content(self, cleaner):
        """Test that cleaning doesn't remove important content."""
        text = "第一章 开始\n\n林风站在山顶，看着远方的城市。\n\n" \
               "\"你好，\"他说道。\n\n这是重要的对话内容。"
        result = cleaner.clean(text)
        assert "第一章" in result
        assert "林风" in result
        assert "山顶" in result

    def test_fix_punctuation_spacing(self, cleaner):
        """Test fixing spacing around punctuation."""
        text = "这是句子 ，还有句子 。 还有感叹号 ！"
        result = cleaner.fix_punctuation_func(text)
        assert " ，" not in result  # No space before Chinese comma
        assert " 。" not in result  # No space before Chinese period

    def test_mismatched_parentheses(self, cleaner):
        """Test fixing mismatched parentheses."""
        text = "这是（左括号和)右括号"
        result = cleaner.fix_punctuation_func(text)
        # Should normalize to matching pairs

    def test_disabled_options(self):
        """Test cleaner with options disabled."""
        text = "  文本  。。\x00"
        cleaner = TextCleaner(
            remove_extra_whitespace=False,
            fix_punctuation=False,
            remove_invalid_chars=False
        )
        result = cleaner.clean(text)
        # Should preserve most of the original
        assert "  " in result  # Extra spaces preserved

    def test_trailing_whitespace_removal(self, cleaner):
        """Test removal of trailing whitespace."""
        text = "第一行  \n第二行\t\n第三行   "
        result = cleaner.remove_extra_whitespace_func(text)
        assert not result.endswith(" ")
        assert not result.endswith("\t")

    def test_empty_lines_preservation(self, cleaner):
        """Test that single empty lines are preserved."""
        text = "第一段\n\n第二段"
        result = cleaner.remove_extra_whitespace_func(text)
        assert "\n\n" in result

    def test_multiple_consecutive_punctuation(self, cleaner):
        """Test handling of multiple consecutive punctuation marks."""
        text = "什么！？？真的。。。好吧。。。"
        result = cleaner.fix_punctuation_func(text)
        assert "！？" in result or "？" in result
        assert "。。" not in result

    def test_colon_and_semicolon_fix(self, cleaner):
        """Test fixing colon and semicolon issues."""
        text = "这是：：测试；；内容"
        result = cleaner.fix_punctuation_func(text)
        assert "：：" not in result
        assert "；；" not in result