NLP基础与词嵌入：让AI理解文字（superior哥深度学习系列第13期）

13_NLP基础与词嵌入：让AI理解文字

superior哥深度学习系列第十三篇
从像素到文字，从视觉到语言——让AI跨越认知的桥梁

🎯 前言：当AI学会"读懂"文字

各位小伙伴们，欢迎来到superior哥深度学习系列的第十三篇！前面我们深入学习了计算机视觉领域的各种技术，从图像分类到目标检测，再到图像分割。今天我们要跨越一个全新的领域——自然语言处理（NLP）。

如果说计算机视觉是让AI拥有"眼睛"，那么自然语言处理就是让AI拥有"语言能力"。想象一下，当AI不仅能看懂图片，还能理解文字、对话、甚至创作文章，那将是多么激动人心的事情！

在这个信息爆炸的时代，文本数据无处不在——新闻、社交媒体、客服对话、法律文件…如何让AI理解这些文字的含义，成为了人工智能发展的关键一环。

🌟 本文亮点预览：

• 📚 从零开始理解NLP基础理论和文本预处理技术
• 🔤 深度解析Word2Vec、GloVe、FastText三大词嵌入技术
• 💻 完整实现新闻分类系统，包含特征工程和模型优化
• 📊 掌握NLP任务评估指标和性能调优策略
• 🎯 提供实战项目模板和最佳实践指南

📊 知识架构图

NLP基础与词嵌入技术
├── NLP基础概念
│   ├── 文本预处理
│   ├── 语言模型基础
│   ├── 词汇表示方法
│   └── 评估指标体系
├── 词嵌入技术
│   ├── Word2Vec
│   │   ├── Skip-gram模型
│   │   ├── CBOW模型
│   │   └── 负采样技术
│   ├── GloVe
│   │   ├── 全局统计信息
│   │   ├── 矩阵分解
│   │   └── 局部上下文
│   └── FastText
│       ├── 子词信息
│       ├── N-gram特征
│       └── 未登录词处理
├── 文本分类应用
│   ├── 特征工程
│   ├── 模型架构
│   ├── 训练技巧
│   └── 性能优化
└── 实战项目
    ├── 新闻分类系统
    ├── 情感分析应用
    ├── 文本相似度计算
    └── 词汇语义分析

🧠 第一章：NLP基础理论与文本预处理

1.1 什么是自然语言处理？

自然语言处理（Natural Language Processing, NLP）是人工智能和语言学的交叉领域，致力于让计算机能够理解、处理和生成人类语言。

🎯 NLP的核心挑战

import re
import jieba
import pandas as pd
import numpy as np
from collections import Counter
import matplotlib.pyplot as plt
import seaborn as sns
from wordcloud import WordCloud
import warnings
warnings.filterwarnings('ignore')

# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False   # 用来正常显示负号

class NLPChallengesDemo:
    """NLP核心挑战演示"""
    
    def __init__(self):
        self.challenges = {
            '歧义性': {
                '词汇歧义': ['银行', '苹果', '春天'],
                '句法歧义': ['我看见了望远镜里的人', '飞机上的人很多'],
                '语义歧义': ['时间过得真快', '这个苹果很甜']
            },
            '多样性': {
                '同义词': ['快乐-高兴-愉悦', '汽车-轿车-车辆'],
                '方言差异': ['你好-您好-你好呀'],
                '语言风格': ['正式-非正式-网络语言']
            },
            '上下文依赖': {
                '代词指代': ['他很聪明', '这个很好'],
                '省略现象': ['去吗？', '好的'],
                '语境理解': ['反话', '暗示', '隐喻']
            }
        }
    
    def demonstrate_ambiguity(self):
        """演示语言歧义性"""
        print("=== NLP的核心挑战：语言歧义性 ===\n")
        
        # 词汇歧义示例
        ambiguous_examples = {
            '银行': ['金融机构', '河岸边缘'],
            '苹果': ['水果', '科技公司'],
            '春天': ['季节', '青春年华', '人名'],
            '打球': ['运动', '制作球体'],
            '花': ['植物', '花费', '花纹']
        }
        
        print("🔍 词汇歧义示例：")
        for word, meanings in ambiguous_examples.items():
            print(f"'{word}' 可能的含义：{' | '.join(meanings)}")
        
        # 句法歧义示例
        syntactic_examples = [
            {
                '句子': '我看见了望远镜里的人',
                '理解1': '我通过望远镜看见了人',
                '理解2': '我看见了在望远镜里面的人'
            },
            {
                '句子': '漂亮的女孩的衣服',
                '理解1': '(漂亮的女孩)的衣服',
                '理解2': '漂亮的(女孩的衣服)'
            }
        ]
        
        print(f"\n🔍 句法歧义示例：")
        for example in syntactic_examples:
            print(f"句子：{example['句子']}")
            print(f"  理解1：{example['理解1']}")
            print(f"  理解2：{example['理解2']}")
            print()
    
    def demonstrate_diversity(self):
        """演示语言多样性"""
        print("=== NLP的核心挑战：语言多样性 ===\n")
        
        # 同义词表达
        synonyms_groups = [
            ['快乐', '高兴', '愉悦', '欢喜', '开心', '兴奋'],
            ['汽车', '轿车', '车辆', '座驾', '车子', '代步工具'],
            ['房子', '住宅', '房屋', '居所', '家', '住所'],
            ['食物', '食品', '美食', '吃的', '食材', '餐食']
        ]
        
        print("🔍 同义词表达多样性：")
        for i, group in enumerate(synonyms_groups, 1):
            print(f"组{i}：{' | '.join(group)}")
        
        # 语言风格差异
        style_examples = {
            '正式': '尊敬的客户，您好！感谢您对我们产品的关注。',
            '非正式': '亲，你好！感谢你关注我们的产品哦～',
            '网络语言': '宝贝，你好！谢谢关注我们产品，么么哒！',
            '古文风格': '客官，有礼了！感谢垂青敝司产品。'
        }
        
        print(f"\n🔍 语言风格多样性：")
        for style, example in style_examples.items():
            print(f"{style}：{example}")
    
    def demonstrate_context_dependency(self):
        """演示上下文依赖"""
        print("=== NLP的核心挑战：上下文依赖 ===\n")
        
        # 代词指代示例
        pronoun_examples = [
            {
                '上下文': '小明和小红一起去图书馆。',
                '句子': '他很喜欢看书。',
                '分析': '"他"指代"小明"，需要上下文才能确定'
            },
            {
                '上下文': '这款手机很好用，那款手机也不错。',
                '句子': '这个更便宜一些。',
                '分析': '"这个"指代前面提到的"这款手机"'
            }
        ]
        
        print("🔍 代词指代示例：")
        for example in pronoun_examples:
            print(f"上下文：{example['上下文']}")
            print(f"句子：{example['句子']}")
            print(f"分析：{example['分析']}")
            print()
        
        # 语境理解示例
        context_examples = [
            {
                '场景': '朋友迟到了',
                '话语': '你可真准时啊！',
                '真实含义': '反话，表示不满'
            },
            {
                '场景': '夏天很热',
                '话语': '今天天气真好啊！',
                '真实含义': '反话，表示抱怨'
            }
        ]
        
        print("🔍 语境理解示例：")
        for example in context_examples:
            print(f"场景：{example['场景']}")
            print(f"话语：{example['话语']}")
            print(f"真实含义：{example['真实含义']}")
            print()

# 运行NLP挑战演示
challenges_demo = NLPChallengesDemo()
challenges_demo.demonstrate_ambiguity()
challenges_demo.demonstrate_diversity()
challenges_demo.demonstrate_context_dependency()

1.2 文本预处理：数据清洗的艺术

文本预处理是NLP的第一步，就像做菜前要洗菜一样重要。原始文本数据往往包含各种噪声和不规范的格式，需要通过预处理来提高数据质量。

🧹 文本预处理工具包

import string
import re
from collections import Counter
import jieba
import jieba.posseg as pseg
from zhon.hanzi import punctuation as zh_punctuation

class TextPreprocessor:
    """文本预处理工具包"""
    
    def __init__(self):
        self.stop_words = self._load_stop_words()
        self.punctuation = string.punctuation + zh_punctuation
        
    def _load_stop_words(self):
        """加载停用词表"""
        # 常用中文停用词
        stop_words = {
            '的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好', '自己', '这', '那', '它', '他', '她', '我们', '你们', '他们', '这个', '那个', '这些', '那些', '这样', '那样', '什么', '怎么', '为什么', '因为', '所有', '可以', '如果', '虽然', '但是', '然而', '而且', '还有', '或者', '以及', '等等', '比如', '例如', '包括', '关于', '对于', '由于', '根据', '按照', '通过', '经过', '进行', '实现', '完成', '开始', '结束', '继续', '保持', '成为', '变成', '属于', '位于', '来自', '用于', '作为', '当做', '称为', '叫做'
        }
        return stop_words
    
    def clean_text(self, text):
        """基础文本清洗"""
        if not isinstance(text, str):
            return ""
        
        # 移除多余空白字符
        text = re.sub(r'\s+', ' ', text)
        
        # 移除特殊字符和数字（保留中英文）
        text = re.sub(r'[^\u4e00-\u9fa5a-zA-Z\s]', '', text)
        
        # 转为小写（英文）
        text = text.lower()
        
        # 去除首尾空格
        text = text.strip()
        
        return text
    
    def advanced_clean(self, text):
        """高级文本清洗"""
        if not isinstance(text, str):
            return ""
        
        # 移除URL
        text = re.sub(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '', text)
        
        # 移除邮箱
        text = re.sub(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}', '', text)
        
        # 移除HTML标签
        text = re.sub(r'<[^>]+>', '', text)
        
        # 移除表情符号
        text = re.sub(r'[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF\U00002702-\U000027B0\U000024C2-\U0001F251]+', '', text)
        
        # 标准化空白字符
        text = re.sub(r'\s+', ' ', text)
        
        return text.strip()
    
    def tokenize_chinese(self, text):
        """中文分词"""
        if not text:
            return []
        
        # 使用jieba进行分词
        tokens = jieba.lcut(text)
        
        # 过滤掉空字符串和单字符
        tokens = [token for token in tokens if len(token) > 1]
        
        return tokens
    
    def remove_stopwords(self, tokens):
        """移除停用词"""
        if not tokens:
            return []
        
        filtered_tokens = [token for token in tokens if token not in self.stop_words]
        return filtered_tokens
    
    def pos_tagging(self, text):
        """词性标注"""
        if not text:
            return []
        
        # 使用jieba进行词性标注
        words = pseg.lcut(text)
        
        # 返回词汇和词性的元组列表
        result = [(word, flag) for word, flag in words if len(word) > 1]
        
        return result
    
    def extract_keywords(self, tokens, top_k=10):
        """提取关键词"""
        if not tokens:
            return []
        
        # 计算词频
        word_freq = Counter(tokens)
        
        # 返回频率最高的top_k个词汇
        keywords = word_freq.most_common(top_k)
        
        return keywords
    
    def preprocess_pipeline(self, text, include_pos=False):
        """完整的预处理管道"""
        if not text:
            return []
        
        # 步骤1：高级清洗
        cleaned_text = self.advanced_clean(text)
        
        # 步骤2：分词
        tokens = self.tokenize_chinese(cleaned_text)
        
        # 步骤3：移除停用词
        filtered_tokens = self.remove_stopwords(tokens)
        
        # 步骤4：词性标注（可选）
        if include_pos:
            pos_result = self.pos_tagging(cleaned_text)
            return {
                'tokens': filtered_tokens,
                'pos_tags': pos_result
            }
        
        return filtered_tokens
    
    def analyze_text_statistics(self, texts):
        """分析文本统计信息"""
        if not texts:
            return {}
        
        # 确保texts是列表格式
        if isinstance(texts, str):
            texts = [texts]
        
        all_tokens = []
        char_counts = []
        word_counts = []
        
        for text in texts:
            tokens = self.preprocess_pipeline(text)
            all_tokens.extend(tokens)
            char_counts.append(len(text))
            word_counts.append(len(tokens))
        
        # 计算统计信息
        stats = {
            'total_documents': len(texts),
            'total_characters': sum(char_counts),
            'total_words': sum(word_counts),
            'avg_char_per_doc': np.mean(char_counts),
            'avg_word_per_doc': np.mean(word_counts),
            'vocabulary_size': len(set(all_tokens)),
            'most_common_words': Counter(all_tokens).most_common(10)
        }
        
        return stats

# 演示文本预处理
preprocessor = TextPreprocessor()

# 示例文本
sample_texts = [
    "这是一个很好的例子！我们来看看NLP的文本预处理效果。😊",
    "今天天气不错，适合出去走走。但是工作还有很多。",
    "人工智能技术正在快速发展，特别是在自然语言处理领域。",
    "机器学习和深度学习是AI的重要分支。",
    "文本分类、情感分析、机器翻译等都是NLP的经典应用。"
]

print("=== 文本预处理演示 ===\n")

# 对每个文本进行预处理
for i, text in enumerate(sample_texts, 1):
    print(f"原始文本 {i}: {text}")
    
    # 基础清洗
    cleaned = preprocessor.clean_text(text)
    print(f"清洗后: {cleaned}")
    
    # 分词
    tokens = preprocessor.tokenize_chinese(cleaned)
    print(f"分词结果: {tokens}")
    
    # 移除停用词
    filtered = preprocessor.remove_stopwords(tokens)
    print(f"移除停用词: {filtered}")
    
    # 词性标注
    pos_tags = preprocessor.pos_tagging(cleaned)
    print(f"词性标注: {pos_tags[:5]}...")  # 只显示前5个
    
    print("-" * 50)

# 统计分析
stats = preprocessor.analyze_text_statistics(sample_texts)
print("\n=== 文本统计分析 ===")
for key, value in stats.items():
    print(f"{key}: {value}")

1.3 语言模型基础

语言模型是NLP的核心概念，它试图估计一个句子出现的概率，或者给定前面的词汇，预测下一个词汇的概率。

📊 N-gram语言模型

import math
from collections import defaultdict, Counter

class NgramLanguageModel:
    """N-gram语言模型实现"""
    
    def __init__(self, n=2):
        self.n = n
        self.ngram_counts = defaultdict(int)
        self.context_counts = defaultdict(int)
        self.vocabulary = set()
        
    def train(self, texts):
        """训练N-gram模型"""
        print(f"开始训练 {self.n}-gram 语言模型...")
        
        for text in texts:
            # 文本预处理
            tokens = preprocessor.preprocess_pipeline(text)
            
            if len(tokens) < self.n:
                continue
                
            # 添加开始和结束标记
            tokens = ['<START>'] * (self.n - 1) + tokens + ['<END>']
            
            # 更新词汇表
            self.vocabulary.update(tokens)
            
            # 统计n-gram
            for i in range(len(tokens) - self.n + 1):
                ngram = tuple(tokens[i:i + self.n])
                context = ngram[:-1] if self.n > 1 else ()
                
                self.ngram_counts[ngram] += 1
                if context:
                    self.context_counts[context] += 1
        
        print(f"训练完成！词汇表大小: {len(self.vocabulary)}")
        print(f"N-gram数量: {len(self.ngram_counts)}")
    
    def get_probability(self, ngram):
        """计算n-gram概率"""
        if isinstance(ngram, str):
            ngram = tuple(ngram.split())
        
        if len(ngram) != self.n:
            return 0.0
        
        ngram_count = self.ngram_counts[ngram]
        
        if self.n == 1:
            # Unigram概率
            total_count = sum(self.ngram_counts.values())
            return ngram_count / total_count if total_count > 0 else 0.0
        else:
            # 条件概率 P(w_n | w_1, ..., w_{n-1})
            context = ngram[:-1]
            context_count = self.context_counts[context]
            return ngram_count / context_count if context_count > 0 else 0.0
    
    def predict_next_word(self, context, top_k=5):
        """预测下一个词"""
        if isinstance(context, str):
            context = tuple(context.split())
        
        if len(context) != self.n - 1:
            return []
        
        # 找出所有以该上下文开始的n-gram
        candidates = []
        for ngram, count in self.ngram_counts.items():
            if ngram[:-1] == context:
                word = ngram[-1]
                prob = self.get_probability(ngram)
                candidates.append((word, prob))
        
        # 按概率排序
        candidates.sort(key=lambda x: x[1], reverse=True)
        
        return candidates[:top_k]
    
    def calculate_perplexity(self, test_texts):
        """计算困惑度"""
        total_log_prob = 0.0
        total_words = 0
        
        for text in test_texts:
            tokens = preprocessor.preprocess_pipeline(text)
            if len(tokens) < self.n:
                continue
            
            tokens = ['<START>'] * (self.n - 1) + tokens + ['<END>']
            
            for i in range(len(tokens) - self.n + 1):
                ngram = tuple(tokens[i:i + self.n])
                prob = self.get_probability(ngram)
                
                if prob > 0:
                    total_log_prob += math.log2(prob)
                    total_words += 1
                else:
                    # 处理未见过的n-gram（简单平滑）
                    total_log_prob += math.log2(1e-10)
                    total_words += 1
        
        if total_words == 0:
            return float('inf')
        
        avg_log_prob = total_log_prob / total_words
        perplexity = 2 ** (-avg_log_prob)
        
        return perplexity
    
    def generate_text(self, start_context, max_length=20):
        """生成文本"""
        if isinstance(start_context, str):
            start_context = start_context.split()
        
        if len(start_context) != self.n - 1:
            return "上下文长度不匹配"
        
        generated = list(start_context)
        current_context = tuple(start_context)
        
        for _ in range(max_length):
            predictions = self.predict_next_word(current_context, top_k=3)
            if not predictions or predictions[0][0] == '<END>':
                break
            
            # 选择概率最高的词（可以改为随机选择）
            next_word = predictions[0][0]
            generated.append(next_word)
            
            # 更新上下文
            current_context = tuple(generated[-(self.n-1):])
        
        return ' '.join(generated)

# 训练和测试N-gram语言模型
training_texts = [
    "机器学习是人工智能的重要分支",
    "深度学习正在改变世界",
    "自然语言处理让计算机理解文本",
    "词嵌入技术将词汇转换为向量",
    "神经网络模型在语言任务中表现出色",
    "文本分类是自然语言处理的基础任务",
    "情感分析可以判断文本的情感倾向",
    "机器翻译帮助人们跨越语言障碍",
    "语言模型能够预测下一个词汇",
    "预训练模型在各种NLP任务中都很有效"
]

# 训练不同的N-gram模型
models = {}
for n in [1, 2, 3]:
    print(f"\n=== 训练 {n}-gram 模型 ===")
    model = NgramLanguageModel(n=n)
    model.train(training_texts)
    models[n] = model
    
    # 测试概率计算
    if n == 2:
        test_bigrams = [
            ('机器', '学习'),
            ('自然', '语言'),
            ('深度', '学习'),
            ('文本', '分类')
        ]
        
        print(f"\n{n}-gram 概率示例:")
        for bigram in test_bigrams:
            prob = model.get_probability(bigram)
            print(f"P({bigram[1]} | {bigram[0]}) = {prob:.4f}")
    
    # 测试文本生成
    if n == 2:
        print(f"\n{n}-gram 文本生成示例:")
        start_contexts = ['机器', '自然', '深度']
        for context in start_contexts:
            generated = model.generate_text([context], max_length=10)
            print(f"起始: {context} -> {generated}")

# 计算困惑度比较
test_texts = [
    "人工智能技术快速发展",
    "机器学习算法不断进步",
    "自然语言处理应用广泛"
]

print(f"\n=== 模型困惑度比较 ===")
for n, model in models.items():
    perplexity = model.calculate_perplexity(test_texts)
    print(f"{n}-gram 模型困惑度: {perplexity:.2f}")

通过这个第一章，我们建立了NLP的基础理论框架，理解了文本预处理的重要性，并实现了基础的N-gram语言模型。接下来我们将深入学习词嵌入技术，这是现代NLP的核心技术之一。

🔤 第二章：词嵌入技术深度解析

词嵌入（Word Embedding）是将词汇转换为稠密向量表示的技术，是现代NLP的基石。它解决了传统one-hot编码的稀疏性问题，让AI能够理解词汇之间的语义关系。

2.1 从One-hot到词嵌入的演进

🎯 传统词汇表示方法的局限性

import numpy as np
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots

class WordRepresentationEvolution:
    """词汇表示方法演进演示"""
    
    def __init__(self):
        self.vocab = ['国王', '王后', '男人', '女人', '苹果', '橙子', '汽车', '飞机']
        self.vocab_size = len(self.vocab)
        
    def demonstrate_onehot(self):
        """演示One-hot编码"""
        print("=== One-hot编码演示 ===\n")
        
        # 创建One-hot编码
        onehot_matrix = np.eye(self.vocab_size)
        
        print("词汇表:", self.vocab)
        print("\nOne-hot编码矩阵:")
        print("词汇    ", end="")
        for i in range(self.vocab_size):
            print(f"{i:3d}", end="")
        print()
        
        for i, word in enumerate(self.vocab):
            print(f"{word:4s} [{' '.join(f'{int(x):2d}' for x in onehot_matrix[i])}]")
        
        # 计算词汇间的相似度
        print(f"\n词汇间余弦相似度（One-hot）:")
        print("所有词汇对的相似度都是 0.0（正交）")
        
        # One-hot的问题
        problems = [
            "1. 稀疏性：每个向量只有一个1，其余都是0",
            "2. 高维度：词汇表大小等于向量维度",
            "3. 无语义：无法表示词汇间的语义关系",
            "4. 存储效率低：大量零值占用空间"
        ]
        
        print(f"\nOne-hot编码的问题:")
        for problem in problems:
            print(f"  {problem}")
            
        return onehot_matrix
    
    def create_sample_embeddings(self):
        """创建示例词嵌入"""
        # 手工设计的词嵌入（体现语义关系）
        embeddings = {
            '国王': [0.8, 0.2, 0.1, 0.9],    # 男性、权力
            '王后': [0.2, 0.8, 0.1, 0.9],    # 女性、权力  
            '男人': [0.9, 0.1, 0.5, 0.3],    # 男性、普通
            '女人': [0.1, 0.9, 0.5, 0.3],    # 女性、普通
            '苹果': [0.5, 0.5, 0.9, 0.1],    # 中性、食物
            '橙子': [0.5, 0.5, 0.8, 0.1],    # 中性、食物
            '汽车': [0.6, 0.4, 0.2, 0.8],    # 略男性、交通工具
            '飞机': [0.5, 0.5, 0.1, 0.9]     # 中性、交通工具
        }
        
        return embeddings
    
    def demonstrate_embeddings(self):
        """演示词嵌入的优势"""
        print("\n=== 词嵌入演示（4维示例）===\n")
        
        embeddings = self.create_sample_embeddings()
        
        print("维度含义: [男性度, 女性度, 生活性, 权力度]")
        print("\n词汇嵌入向量:")
        for word, vector in embeddings.items():
            print(f"{word:4s} [{', '.join(f'{x:.1f}' for x in vector)}]")
        
        # 计算相似度
        def cosine_similarity(v1, v2):
            dot_product = np.dot(v1, v2)
            norm1 = np.linalg.norm(v1)
            norm2 = np.linalg.norm(v2)
            return dot_product / (norm1 * norm2)
        
        print(f"\n词汇间余弦相似度（词嵌入）:")
        
        # 展示一些有趣的相似度对比
        interesting_pairs = [
            ('国王', '王后'),
            ('男人', '女人'), 
            ('苹果', '橙子'),
            ('汽车', '飞机'),
            ('国王', '男人'),
            ('王后', '女人'),
            ('国王', '苹果')
        ]
        
        for word1, word2 in interesting_pairs:
            sim = cosine_similarity(embeddings[word1], embeddings[word2])
            print(f"{word1} ↔ {word2}: {sim:.3f}")
        
        # 词汇关系分析
        print(f"\n语义关系分析:")
        
        # 国王 - 男人 + 女人 ≈ 王后
        king_vec = np.array(embeddings['国王'])
        man_vec = np.array(embeddings['男人'])
        woman_vec = np.array(embeddings['女人'])
        queen_vec = np.array(embeddings['王后'])
        
        analogy_result = king_vec - man_vec + woman_vec
        similarity_to_queen = cosine_similarity(analogy_result, queen_vec)
        
        print(f"国王 - 男人 + 女人 ≈ 王后")
        print(f"计算结果与王后的相似度: {similarity_to_queen:.3f}")
        
        return embeddings
    
    def visualize_embeddings(self, embeddings):
        """可视化词嵌入"""
        # 提取词汇和向量
        words = list(embeddings.keys())
        vectors = np.array(list(embeddings.values()))
        
        # PCA降维到2D
        pca = PCA(n_components=2)
        vectors_2d = pca.fit_transform(vectors)
        
        # 创建可视化
        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
        
        # 原始4维向量的前两个维度
        ax1.scatter(vectors[:, 0], vectors[:, 1], s=100, alpha=0.7)
        for i, word in enumerate(words):
            ax1.annotate(word, (vectors[i, 0], vectors[i, 1]), 
                        xytext=(5, 5), textcoords='offset points',
                        fontsize=12, ha='left')
        ax1.set_xlabel('维度1 (男性度)')
        ax1.set_ylabel('维度2 (女性度)')
        ax1.set_title('词嵌入可视化 - 原始维度')
        ax1.grid(True, alpha=0.3)
        
        # PCA降维后的2D可视化
        ax2.scatter(vectors_2d[:, 0], vectors_2d[:, 1], s=100, alpha=0.7)
        for i, word in enumerate(words):
            ax2.annotate(word, (vectors_2d[i, 0], vectors_2d[i, 1]), 
                        xytext=(5, 5), textcoords='offset points',
                        fontsize=12, ha='left')
        ax2.set_xlabel('PCA第1主成分')
        ax2.set_ylabel('PCA第2主成分')
        ax2.set_title('词嵌入可视化 - PCA降维')
        ax2.grid(True, alpha=0.3)
        
        plt.tight_layout()
        plt.savefig('word_embeddings_visualization.png', dpi=300, bbox_inches='tight')
        plt.show()
        
        # 创建交互式3D可视化
        if vectors.shape[1] >= 3:
            fig_3d = go.Figure(data=[go.Scatter3d(
                x=vectors[:, 0],
                y=vectors[:, 1], 
                z=vectors[:, 2],
                mode='markers+text',
                text=words,
                textposition="middle right",
                marker=dict(
                    size=10,
                    color=np.arange(len(words)),
                    colorscale='viridis',
                    showscale=True
                )
            )])
            
            fig_3d.update_layout(
                title='词嵌入3D可视化',
                scene=dict(
                    xaxis_title='维度1 (男性度)',
                    yaxis_title='维度2 (女性度)',
                    zaxis_title='维度3 (生活性)'
                )
            )
            
            fig_3d.show()

# 演示词汇表示的演进
demo = WordRepresentationEvolution()

print("词汇表示方法演进演示")
print("=" * 50)

# One-hot编码演示
onehot_matrix = demo.demonstrate_onehot()

# 词嵌入演示
embeddings = demo.demonstrate_embeddings()

# 可视化对比
demo.visualize_embeddings(embeddings)

2.2 Word2Vec：开启词嵌入新时代

Word2Vec是Google在2013年提出的词嵌入技术，通过神经网络学习词汇的分布式表示，包含两种主要架构：Skip-gram和CBOW。

⚡ Skip-gram模型实现

import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict, Counter
import random
from sklearn.manifold import TSNE
import warnings
warnings.filterwarnings('ignore')

class Word2VecSkipGram:
    """Word2Vec Skip-gram模型实现"""
    
    def __init__(self, embedding_dim=100, window_size=5, negative_samples=5, learning_rate=0.025):
        self.embedding_dim = embedding_dim
        self.window_size = window_size
        self.negative_samples = negative_samples
        self.learning_rate = learning_rate
        
        # 词汇表和映射
        self.word2idx = {}
        self.idx2word = {}
        self.vocab_size = 0
        
        # 嵌入矩阵
        self.W_input = None   # 输入词嵌入矩阵
        self.W_output = None  # 输出词嵌入矩阵
        
        # 训练统计
        self.loss_history = []
        
    def build_vocabulary(self, texts):
        """构建词汇表"""
        print("构建词汇表...")
        
        # 统计词频
        word_counts = Counter()
        for text in texts:
            tokens = preprocessor.preprocess_pipeline(text)
            word_counts.update(tokens)
        
        # 过滤低频词
        min_count = 2
        filtered_words = [word for word, count in word_counts.items() if count >= min_count]
        
        # 创建词汇映射
        self.word2idx = {word: idx for idx, word in enumerate(filtered_words)}
        self.idx2word = {idx: word for word, idx in self.word2idx.items()}
        self.vocab_size = len(filtered_words)
        
        print(f"词汇表大小: {self.vocab_size}")
        print(f"最高频词汇: {word_counts.most_common(10)}")
        
        return word_counts
    
    def initialize_embeddings(self):
        """初始化嵌入矩阵"""
        print("初始化嵌入矩阵...")
        
        # Xavier初始化
        bound = np.sqrt(6.0 / (self.vocab_size + self.embedding_dim))
        self.W_input = np.random.uniform(-bound, bound, (self.vocab_size, self.embedding_dim))
        self.W_output = np.random.uniform(-bound, bound, (self.vocab_size, self.embedding_dim))
        
        print(f"输入嵌入矩阵形状: {self.W_input.shape}")
        print(f"输出嵌入矩阵形状: {self.W_output.shape}")
    
    def generate_training_data(self, texts):
        """生成训练数据（中心词，上下文词）对"""
        print("生成训练数据...")
        
        training_data = []
        
        for text in texts:
            tokens = preprocessor.preprocess_pipeline(text)
            # 过滤掉不在词汇表中的词
            tokens = [token for token in tokens if token in self.word2idx]
            
            if len(tokens) < 2:
                continue
            
            # 为每个词生成上下文
            for i, center_word in enumerate(tokens):
                # 定义上下文窗口
                start = max(0, i - self.window_size)
                end = min(len(tokens), i + self.window_size + 1)
                
                # 收集上下文词
                for j in range(start, end):
                    if i != j:  # 排除中心词本身
                        context_word = tokens[j]
                        training_data.append((center_word, context_word))
        
        print(f"生成训练样本数: {len(training_data)}")
        return training_data
    
    def negative_sampling(self, positive_word_idx):
        """负采样"""
        negative_samples = []
        
        # 简单随机采样（实际应用中应该按词频采样）
        while len(negative_samples) < self.negative_samples:
            neg_idx = random.randint(0, self.vocab_size - 1)
            if neg_idx != positive_word_idx:
                negative_samples.append(neg_idx)
        
        return negative_samples
    
    def sigmoid(self, x):
        """Sigmoid激活函数"""
        # 防止溢出
        x = np.clip(x, -500, 500)
        return 1.0 / (1.0 + np.exp(-x))
    
    def train_step(self, center_word, context_word):
        """单步训练"""
        center_idx = self.word2idx[center_word]
        context_idx = self.word2idx[context_word]
        
        # 获取中心词的嵌入向量
        center_embedding = self.W_input[center_idx]
        
        # 正样本训练
        context_output = self.W_output[context_idx]
        score = np.dot(center_embedding, context_output)
        prob = self.sigmoid(score)
        
        # 计算梯度
        grad = (1 - prob) * self.learning_rate
        
        # 更新中心词嵌入
        self.W_input[center_idx] += grad * context_output
        # 更新上下文词输出嵌入
        self.W_output[context_idx] += grad * center_embedding
        
        # 负样本训练
        negative_samples = self.negative_sampling(context_idx)
        for neg_idx in negative_samples:
            neg_output = self.W_output[neg_idx]
            neg_score = np.dot(center_embedding, neg_output)
            neg_prob = self.sigmoid(neg_score)
            
            # 负样本梯度
            neg_grad = -neg_prob * self.learning_rate
            
            # 更新嵌入
            self.W_input[center_idx] += neg_grad * neg_output
            self.W_output[neg_idx] += neg_grad * center_embedding
        
        # 计算损失（简化版）
        loss = -np.log(prob) - sum(np.log(1 - self.sigmoid(np.dot(center_embedding, self.W_output[neg_idx]))) 
                                 for neg_idx in negative_samples)
        
        return loss
    
    def train(self, texts, epochs=10):
        """训练Word2Vec模型"""
        print("开始训练Word2Vec模型...")
        
        # 构建词汇表
        word_counts = self.build_vocabulary(texts)
        
        # 初始化嵌入矩阵
        self.initialize_embeddings()
        
        # 生成训练数据
        training_data = self.generate_training_data(texts)
        
        # 训练循环
        for epoch in range(epochs):
            total_loss = 0.0
            random.shuffle(training_data)
            
            for i, (center_word, context_word) in enumerate(training_data):
                loss = self.train_step(center_word, context_word)
                total_loss += loss
                
                # 打印进度
                if i % 1000 == 0:
                    print(f"Epoch {epoch+1}/{epochs}, Step {i}/{len(training_data)}, "
                          f"Loss: {loss:.4f}")
            
            avg_loss = total_loss / len(training_data)
            self.loss_history.append(avg_loss)
            print(f"Epoch {epoch+1} 完成，平均损失: {avg_loss:.4f}")
            
            # 动态调整学习率
            self.learning_rate *= 0.9
    
    def get_word_vector(self, word):
        """获取词向量"""
        if word in self.word2idx:
            return self.W_input[self.word2idx[word]]
        else:
            return None
    
    def find_similar_words(self, word, top_k=10):
        """找到相似词汇"""
        if word not in self.word2idx:
            return []
        
        word_vector = self.get_word_vector(word)
        similarities = []
        
        for other_word in self.word2idx:
            if other_word != word:
                other_vector = self.get_word_vector(other_word)
                # 计算余弦相似度
                similarity = np.dot(word_vector, other_vector) / (
                    np.linalg.norm(word_vector) * np.linalg.norm(other_vector)
                )
                similarities.append((other_word, similarity))
        
        # 按相似度排序
        similarities.sort(key=lambda x: x[1], reverse=True)
        
        return similarities[:top_k]
    
    def word_analogy(self, word_a, word_b, word_c, top_k=5):
        """词汇类比：A之于B，如C之于？"""
        if not all(word in self.word2idx for word in [word_a, word_b, word_c]):
            return []
        
        # 计算类比向量：B - A + C
        vec_a = self.get_word_vector(word_a)
        vec_b = self.get_word_vector(word_b)
        vec_c = self.get_word_vector(word_c)
        
        analogy_vector = vec_b - vec_a + vec_c
        
        # 找到最相似的词
        similarities = []
        for word in self.word2idx:
            if word not in [word_a, word_b, word_c]:
                word_vector = self.get_word_vector(word)
                similarity = np.dot(analogy_vector, word_vector) / (
                    np.linalg.norm(analogy_vector) * np.linalg.norm(word_vector)
                )
                similarities.append((word, similarity))
        
        similarities.sort(key=lambda x: x[1], reverse=True)
        return similarities[:top_k]
    
    def visualize_embeddings(self, words_to_plot=None):
        """可视化词嵌入"""
        if words_to_plot is None:
            # 选择一些高频词进行可视化
            words_to_plot = list(self.word2idx.keys())[:20]
        
        # 过滤存在的词
        words_to_plot = [word for word in words_to_plot if word in self.word2idx]
        
        if len(words_to_plot) < 2:
            print("没有足够的词汇进行可视化")
            return
        
        # 获取词向量
        vectors = np.array([self.get_word_vector(word) for word in words_to_plot])
        
        # 使用t-SNE降维
        tsne = TSNE(n_components=2, random_state=42, perplexity=min(30, len(words_to_plot)-1))
        vectors_2d = tsne.fit_transform(vectors)
        
        # 绘制可视化图
        plt.figure(figsize=(12, 8))
        plt.scatter(vectors_2d[:, 0], vectors_2d[:, 1], alpha=0.7, s=100)
        
        for i, word in enumerate(words_to_plot):
            plt.annotate(word, (vectors_2d[i, 0], vectors_2d[i, 1]), 
                        xytext=(5, 5), textcoords='offset points',
                        fontsize=10, ha='left')
        
        plt.title('Word2Vec词嵌入可视化 (t-SNE)')
        plt.xlabel('t-SNE维度1')
        plt.ylabel('t-SNE维度2')
        plt.grid(True, alpha=0.3)
        plt.tight_layout()
        plt.savefig('word2vec_visualization.png', dpi=300, bbox_inches='tight')
        plt.show()
    
    def plot_training_loss(self):
        """绘制训练损失曲线"""
        if not self.loss_history:
            print("没有训练历史数据")
            return
        
        plt.figure(figsize=(10, 6))
        plt.plot(self.loss_history)
        plt.title('Word2Vec训练损失曲线')
        plt.xlabel('Epoch')
        plt.ylabel('Average Loss')
        plt.grid(True, alpha=0.3)
        plt.tight_layout()
        plt.savefig('word2vec_training_loss.png', dpi=300, bbox_inches='tight')
        plt.show()

# 使用示例
print("=== Word2Vec Skip-gram 模型训练 ===\n")

# 准备训练数据
training_texts = [
    "机器学习是人工智能的重要分支，它让计算机能够自动学习和改进",
    "深度学习是机器学习的一个子领域，使用神经网络模型",
    "自然语言处理让计算机能够理解和生成人类语言",
    "词嵌入技术将词汇转换为数值向量表示",
    "卷积神经网络在图像识别任务中表现出色",
    "循环神经网络适合处理序列数据",
    "注意力机制提高了模型的性能",
    "预训练模型在各种任务中都很有效",
    "文本分类是自然语言处理的基础任务",
    "情感分析可以判断文本的情感倾向",
    "机器翻译帮助人们跨越语言障碍",
    "语音识别将声音转换为文本",
    "计算机视觉让机器能够理解图像",
    "数据挖掘从大量数据中发现有用信息",
    "人工智能正在改变我们的世界",
    "算法优化提高了计算效率",
    "数据科学结合了统计学和计算机科学",
    "机器人技术集成了多个学科的知识",
    "云计算提供了强大的计算资源",
    "大数据分析帮助企业做出更好的决策"
]

# 创建和训练模型
model = Word2VecSkipGram(embedding_dim=50, window_size=3, negative_samples=3, learning_rate=0.1)
model.train(training_texts, epochs=20)

# 绘制训练损失
model.plot_training_loss()

# 测试词汇相似度
test_words = ['机器', '学习', '神经', '网络', '数据']
print("\n=== 词汇相似度测试 ===")
for word in test_words:
    if word in model.word2idx:
        similar_words = model.find_similar_words(word, top_k=5)
        print(f"\n与 '{word}' 最相似的词汇:")
        for similar_word, similarity in similar_words:
            print(f"  {similar_word}: {similarity:.4f}")

# 测试词汇类比
print("\n=== 词汇类比测试 ===")
analogy_tests = [
    ('机器', '学习', '深度'),
    ('计算机', '视觉', '自然'),
    ('数据', '挖掘', '文本')
]

for word_a, word_b, word_c in analogy_tests:
    results = model.word_analogy(word_a, word_b, word_c, top_k=3)
    if results:
        print(f"\n{word_a} 之于 {word_b}，如 {word_c} 之于：")
        for result_word, similarity in results:
            print(f"  {result_word}: {similarity:.4f}")

# 可视化词嵌入
print("\n开始可视化词嵌入...")
model.visualize_embeddings()

print("\n✅ Word2Vec Skip-gram 模型训练完成！")

🎯 CBOW模型实现

CBOW（Continuous Bag of Words）模型与Skip-gram相反，它使用上下文词汇来预测中心词。

class Word2VecCBOW:
    """Word2Vec CBOW模型实现"""
    
    def __init__(self, embedding_dim=100, window_size=5, negative_samples=5, learning_rate=0.025):
        self.embedding_dim = embedding_dim
        self.window_size = window_size
        self.negative_samples = negative_samples
        self.learning_rate = learning_rate
        
        # 继承Skip-gram的基础结构
        self.word2idx = {}
        self.idx2word = {}
        self.vocab_size = 0
        self.W_input = None
        self.W_output = None
        self.loss_history = []
        
    def build_vocabulary(self, texts):
        """构建词汇表（与Skip-gram相同）"""
        print("构建词汇表...")
        
        word_counts = Counter()
        for text in texts:
            tokens = preprocessor.preprocess_pipeline(text)
            word_counts.update(tokens)
        
        min_count = 2
        filtered_words = [word for word, count in word_counts.items() if count >= min_count]
        
        self.word2idx = {word: idx for idx, word in enumerate(filtered_words)}
        self.idx2word = {idx: word for word, idx in self.word2idx.items()}
        self.vocab_size = len(filtered_words)
        
        print(f"词汇表大小: {self.vocab_size}")
        return word_counts
    
    def initialize_embeddings(self):
        """初始化嵌入矩阵"""
        bound = np.sqrt(6.0 / (self.vocab_size + self.embedding_dim))
        self.W_input = np.random.uniform(-bound, bound, (self.vocab_size, self.embedding_dim))
        self.W_output = np.random.uniform(-bound, bound, (self.vocab_size, self.embedding_dim))
    
    def generate_training_data(self, texts):
        """生成CBOW训练数据（上下文词列表，中心词）"""
        print("生成CBOW训练数据...")
        
        training_data = []
        
        for text in texts:
            tokens = preprocessor.preprocess_pipeline(text)
            tokens = [token for token in tokens if token in self.word2idx]
            
            if len(tokens) < 3:
                continue
            
            for i, center_word in enumerate(tokens):
                # 收集上下文词
                start = max(0, i - self.window_size)
                end = min(len(tokens), i + self.window_size + 1)
                
                context_words = []
                for j in range(start, end):
                    if i != j:
                        context_words.append(tokens[j])
                
                if len(context_words) >= 2:  # 确保有足够的上下文
                    training_data.append((context_words, center_word))
        
        print(f"生成CBOW训练样本数: {len(training_data)}")
        return training_data
    
    def sigmoid(self, x):
        """Sigmoid激活函数"""
        x = np.clip(x, -500, 500)
        return 1.0 / (1.0 + np.exp(-x))
    
    def train_step(self, context_words, center_word):
        """CBOW单步训练"""
        # 获取上下文词的索引
        context_indices = [self.word2idx[word] for word in context_words if word in self.word2idx]
        center_idx = self.word2idx[center_word]
        
        if len(context_indices) == 0:
            return 0.0
        
        # 计算上下文向量的平均值
        context_embeddings = self.W_input[context_indices]
        avg_context = np.mean(context_embeddings, axis=0)
        
        # 正样本训练
        center_output = self.W_output[center_idx]
        score = np.dot(avg_context, center_output)
        prob = self.sigmoid(score)
        
        # 计算梯度
        grad = (1 - prob) * self.learning_rate
        
        # 更新上下文词嵌入
        for ctx_idx in context_indices:
            self.W_input[ctx_idx] += grad * center_output / len(context_indices)
        
        # 更新中心词输出嵌入
        self.W_output[center_idx] += grad * avg_context
        
        # 负采样
        negative_samples = self.negative_sampling(center_idx)
        for neg_idx in negative_samples:
            neg_output = self.W_output[neg_idx]
            neg_score = np.dot(avg_context, neg_output)
            neg_prob = self.sigmoid(neg_score)
            
            neg_grad = -neg_prob * self.learning_rate
            
            # 更新嵌入
            for ctx_idx in context_indices:
                self.W_input[ctx_idx] += neg_grad * neg_output / len(context_indices)
            self.W_output[neg_idx] += neg_grad * avg_context
        
        # 计算损失
        loss = -np.log(prob) - sum(np.log(1 - self.sigmoid(np.dot(avg_context, self.W_output[neg_idx]))) 
                                 for neg_idx in negative_samples)
        
        return loss
    
    def negative_sampling(self, positive_word_idx):
        """负采样"""
        negative_samples = []
        while len(negative_samples) < self.negative_samples:
            neg_idx = random.randint(0, self.vocab_size - 1)
            if neg_idx != positive_word_idx:
                negative_samples.append(neg_idx)
        return negative_samples
    
    def train(self, texts, epochs=10):
        """训练CBOW模型"""
        print("开始训练CBOW模型...")
        
        self.build_vocabulary(texts)
        self.initialize_embeddings()
        training_data = self.generate_training_data(texts)
        
        for epoch in range(epochs):
            total_loss = 0.0
            random.shuffle(training_data)
            
            for i, (context_words, center_word) in enumerate(training_data):
                loss = self.train_step(context_words, center_word)
                total_loss += loss
                
                if i % 1000 == 0:
                    print(f"CBOW Epoch {epoch+1}/{epochs}, Step {i}/{len(training_data)}, "
                          f"Loss: {loss:.4f}")
            
            avg_loss = total_loss / len(training_data)
            self.loss_history.append(avg_loss)
            print(f"CBOW Epoch {epoch+1} 完成，平均损失: {avg_loss:.4f}")
            
            self.learning_rate *= 0.9
    
    def get_word_vector(self, word):
        """获取词向量"""
        if word in self.word2idx:
            return self.W_input[self.word2idx[word]]
        return None
    
    def find_similar_words(self, word, top_k=10):
        """找到相似词汇"""
        if word not in self.word2idx:
            return []
        
        word_vector = self.get_word_vector(word)
        similarities = []
        
        for other_word in self.word2idx:
            if other_word != word:
                other_vector = self.get_word_vector(other_word)
                similarity = np.dot(word_vector, other_vector) / (
                    np.linalg.norm(word_vector) * np.linalg.norm(other_vector)
                )
                similarities.append((other_word, similarity))
        
        similarities.sort(key=lambda x: x[1], reverse=True)
        return similarities[:top_k]

# 比较Skip-gram和CBOW
print("=== Skip-gram vs CBOW 对比训练 ===\n")

# 训练CBOW模型
cbow_model = Word2VecCBOW(embedding_dim=50, window_size=3, negative_samples=3, learning_rate=0.1)
cbow_model.train(training_texts, epochs=15)

# 比较两种模型的效果
test_word = '学习'
if test_word in model.word2idx and test_word in cbow_model.word2idx:
    print(f"\n=== 模型对比：'{test_word}' 的相似词 ===")
    
    print("\nSkip-gram模型结果:")
    skipgram_similar = model.find_similar_words(test_word, top_k=5)
    for word, sim in skipgram_similar:
        print(f"  {word}: {sim:.4f}")
    
    print("\nCBOW模型结果:")
    cbow_similar = cbow_model.find_similar_words(test_word, top_k=5)
    for word, sim in cbow_similar:
        print(f"  {word}: {sim:.4f}")

print("\n✅ CBOW模型训练完成！")

2.3 GloVe：全局词汇向量

GloVe（Global Vectors for Word Representation）结合了全局统计信息和局部上下文信息，通过共现矩阵分解来学习词向量。

🌐 GloVe模型实现

import numpy as np
from scipy.sparse import coo_matrix
from collections import defaultdict
import matplotlib.pyplot as plt

class GloVeModel:
    """GloVe模型实现"""
    
    def __init__(self, embedding_dim=100, x_max=100, alpha=0.75, learning_rate=0.05):
        self.embedding_dim = embedding_dim
        self.x_max = x_max  # 权重函数的截止值
        self.alpha = alpha  # 权重函数的指数
        self.learning_rate = learning_rate
        
        # 词汇表
        self.word2idx = {}
        self.idx2word = {}
        self.vocab_size = 0
        
        # 共现矩阵
        self.cooccurrence_matrix = None
        
        # 嵌入矩阵
        self.W_main = None      # 主词向量
        self.W_context = None   # 上下文词向量
        self.b_main = None      # 主词偏置
        self.b_context = None   # 上下文词偏置
        
        # 训练历史
        self.loss_history = []
    
    def build_vocabulary(self, texts):
        """构建词汇表"""
        print("构建GloVe词汇表...")
        
        word_counts = Counter()
        for text in texts:
            tokens = preprocessor.preprocess_pipeline(text)
            word_counts.update(tokens)
        
        # 过滤低频词
        min_count = 2
        filtered_words = [word for word, count in word_counts.items() if count >= min_count]
        
        self.word2idx = {word: idx for idx, word in enumerate(filtered_words)}
        self.idx2word = {idx: word for word, idx in self.word2idx.items()}
        self.vocab_size = len(filtered_words)
        
        print(f"GloVe词汇表大小: {self.vocab_size}")
        return word_counts
    
    def build_cooccurrence_matrix(self, texts, window_size=5):
        """构建共现矩阵"""
        print("构建共现矩阵...")
        
        # 使用字典存储共现计数
        cooccur_counts = defaultdict(float)
        
        for text in texts:
            tokens = preprocessor.preprocess_pipeline(text)
            tokens = [token for token in tokens if token in self.word2idx]
            
            if len(tokens) < 2:
                continue
            
            # 计算共现
            for i, center_word in enumerate(tokens):
                center_idx = self.word2idx[center_word]
                
                # 定义窗口
                start = max(0, i - window_size)
                end = min(len(tokens), i + window_size + 1)
                
                for j in range(start, end):
                    if i != j:
                        context_word = tokens[j]
                        context_idx = self.word2idx[context_word]
                        
                        # 距离权重（离中心词越远权重越小）
                        distance = abs(i - j)
                        weight = 1.0 / distance
                        
                        cooccur_counts[(center_idx, context_idx)] += weight
        
        # 转换为稀疏矩阵格式
        rows, cols, data = [], [], []
        for (i, j), count in cooccur_counts.items():
            rows.append(i)
            cols.append(j)
            data.append(count)
        
        self.cooccurrence_matrix = coo_matrix((data, (rows, cols)), 
                                            shape=(self.vocab_size, self.vocab_size))
        
        print(f"共现矩阵非零元素数: {len(data)}")
        print(f"共现矩阵密度: {len(data) / (self.vocab_size ** 2):.6f}")
    
    def weight_function(self, x):
        """GloVe权重函数"""
        if x < self.x_max:
            return (x / self.x_max) ** self.alpha
        else:
            return 1.0
    
    def initialize_parameters(self):
        """初始化模型参数"""
        print("初始化GloVe参数...")
        
        # Xavier初始化
        bound = 0.5 / self.embedding_dim
        
        self.W_main = np.random.uniform(-bound, bound, (self.vocab_size, self.embedding_dim))
        self.W_context = np.random.uniform(-bound, bound, (self.vocab_size, self.embedding_dim))
        
        # 偏置项初始化
        self.b_main = np.random.uniform(-bound, bound, self.vocab_size)
        self.b_context = np.random.uniform(-bound, bound, self.vocab_size)
        
        print(f"参数矩阵形状: {self.W_main.shape}")
    
    def train(self, texts, epochs=50, window_size=5):
        """训练GloVe模型"""
        print("开始训练GloVe模型...")
        
        # 构建词汇表和共现矩阵
        self.build_vocabulary(texts)
        self.build_cooccurrence_matrix(texts, window_size)
        self.initialize_parameters()
        
        # 将稀疏矩阵转换为COO格式以便迭代
        cooccur_coo = self.cooccurrence_matrix.tocoo()
        
        for epoch in range(epochs):
            total_loss = 0.0
            
            # 遍历所有非零共现对
            for idx in range(len(cooccur_coo.data)):
                i = cooccur_coo.row[idx]
                j = cooccur_coo.col[idx]
                x_ij = cooccur_coo.data[idx]
                
                # 计算权重
                weight = self.weight_function(x_ij)
                
                # 前向传播
                dot_product = np.dot(self.W_main[i], self.W_context[j])
                log_x_ij = np.log(x_ij)
                
                # 计算误差
                error = dot_product + self.b_main[i] + self.b_context[j] - log_x_ij
                loss = weight * (error ** 2)
                total_loss += loss
                
                # 计算梯度
                grad_factor = weight * error * self.learning_rate
                
                # 更新主词向量
                w_main_grad = grad_factor * self.W_context[j]
                self.W_main[i] -= w_main_grad
                
                # 更新上下文词向量
                w_context_grad = grad_factor * self.W_main[i]
                self.W_context[j] -= w_context_grad
                
                # 更新偏置
                self.b_main[i] -= grad_factor
                self.b_context[j] -= grad_factor
            
            avg_loss = total_loss / len(cooccur_coo.data)
            self.loss_history.append(avg_loss)
            
            if epoch % 10 == 0:
                print(f"GloVe Epoch {epoch+1}/{epochs}, 平均损失: {avg_loss:.4f}")
            
            # 学习率衰减
            self.learning_rate *= 0.99
        
        print(f"GloVe训练完成！最终损失: {self.loss_history[-1]:.4f}")
    
    def get_word_vector(self, word):
        """获取词向量（主向量和上下文向量的和）"""
        if word in self.word2idx:
            idx = self.word2idx[word]
            return self.W_main[idx] + self.W_context[idx]
        return None
    
    def find_similar_words(self, word, top_k=10):
        """找到相似词汇"""
        if word not in self.word2idx:
            return []
        
        word_vector = self.get_word_vector(word)
        similarities = []
        
        for other_word in self.word2idx:
            if other_word != word:
                other_vector = self.get_word_vector(other_word)
                similarity = np.dot(word_vector, other_vector) / (
                    np.linalg.norm(word_vector) * np.linalg.norm(other_vector)
                )
                similarities.append((other_word, similarity))
        
        similarities.sort(key=lambda x: x[1], reverse=True)
        return similarities[:top_k]
    
    def word_analogy(self, word_a, word_b, word_c, top_k=5):
        """词汇类比"""
        if not all(word in self.word2idx for word in [word_a, word_b, word_c]):
            return []
        
        vec_a = self.get_word_vector(word_a)
        vec_b = self.get_word_vector(word_b)
        vec_c = self.get_word_vector(word_c)
        
        analogy_vector = vec_b - vec_a + vec_c
        
        similarities = []
        for word in self.word2idx:
            if word not in [word_a, word_b, word_c]:
                word_vector = self.get_word_vector(word)
                similarity = np.dot(analogy_vector, word_vector) / (
                    np.linalg.norm(analogy_vector) * np.linalg.norm(word_vector)
                )
                similarities.append((word, similarity))
        
        similarities.sort(key=lambda x: x[1], reverse=True)
        return similarities[:top_k]
    
    def plot_training_loss(self):
        """绘制训练损失曲线"""
        plt.figure(figsize=(10, 6))
        plt.plot(self.loss_history)
        plt.title('GloVe训练损失曲线')
        plt.xlabel('Epoch')
        plt.ylabel('Average Loss')
        plt.grid(True, alpha=0.3)
        plt.yscale('log')  # 使用对数刻度
        plt.tight_layout()
        plt.savefig('glove_training_loss.png', dpi=300, bbox_inches='tight')
        plt.show()
    
    def analyze_cooccurrence(self, top_k=10):
        """分析共现统计"""
        print(f"\n=== 共现矩阵分析 ===")
        
        # 转换为COO格式
        coo = self.cooccurrence_matrix.tocoo()
        
        # 找出共现频率最高的词对
        cooccur_pairs = []
        for idx in range(len(coo.data)):
            i, j, count = coo.row[idx], coo.col[idx], coo.data[idx]
            word_i = self.idx2word[i]
            word_j = self.idx2word[j]
            cooccur_pairs.append(((word_i, word_j), count))
        
        # 按共现频率排序
        cooccur_pairs.sort(key=lambda x: x[1], reverse=True)
        
        print(f"共现频率最高的{top_k}个词对:")
        for (word_i, word_j), count in cooccur_pairs[:top_k]:
            print(f"  ({word_i}, {word_j}): {count:.2f}")

# 训练GloVe模型
print("=== GloVe模型训练 ===\n")

glove_model = GloVeModel(embedding_dim=50, x_max=100, alpha=0.75, learning_rate=0.05)
glove_model.train(training_texts, epochs=100, window_size=5)

# 绘制训练损失
glove_model.plot_training_loss()

# 分析共现矩阵
glove_model.analyze_cooccurrence()

# 测试GloVe模型效果
test_words = ['机器', '学习', '深度', '神经', '网络']
print(f"\n=== GloVe模型词汇相似度测试 ===")
for word in test_words:
    if word in glove_model.word2idx:
        similar_words = glove_model.find_similar_words(word, top_k=5)
        print(f"\n与 '{word}' 最相似的词汇 (GloVe):")
        for similar_word, similarity in similar_words:
            print(f"  {similar_word}: {similarity:.4f}")

# GloVe词汇类比测试
print(f"\n=== GloVe词汇类比测试 ===")
analogy_tests = [
    ('机器', '学习', '深度'),
    ('计算机', '视觉', '自然'),
    ('数据', '挖掘', '文本')
]

for word_a, word_b, word_c in analogy_tests:
    results = glove_model.word_analogy(word_a, word_b, word_c, top_k=3)
    if results:
        print(f"\nGloVe: {word_a} 之于 {word_b}，如 {word_c} 之于：")
        for result_word, similarity in results:
            print(f"  {result_word}: {similarity:.4f}")

print("\n✅ GloVe模型训练完成！")

### 2.4 FastText：考虑子词信息的词嵌入

#### 🚀 FastText的核心创新

FastText是Facebook开发的词嵌入模型，它的核心创新在于**考虑了单词的内部结构**。与Word2Vec和GloVe不同，FastText不仅考虑词汇级别的信息，还考虑**字符级n-gram**信息，这使得它能够：

1. **处理未登录词（OOV）**：通过子词信息推测新词的语义
2. **处理形态变化**：理解单词的词根、前缀、后缀
3. **提升稀有词性能**：通过子词信息增强稀有词的表示

#### 🧠 FastText模型实现

```python
import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict, Counter
import re
from typing import List, Tuple, Dict, Set
import jieba
import random

class FastTextModel:
    """FastText词嵌入模型实现"""
    
    def __init__(self, embedding_dim=100, window_size=5, min_count=5, 
                 min_n=3, max_n=6, negative_samples=5, learning_rate=0.025):
        """
        初始化FastText模型
        
        Args:
            embedding_dim: 词向量维度
            window_size: 上下文窗口大小
            min_count: 最小词频
            min_n: 最小n-gram长度
            max_n: 最大n-gram长度
            negative_samples: 负采样数量
            learning_rate: 学习率
        """
        self.embedding_dim = embedding_dim
        self.window_size = window_size
        self.min_count = min_count
        self.min_n = min_n
        self.max_n = max_n
        self.negative_samples = negative_samples
        self.learning_rate = learning_rate
        
        # 初始化词汇表和映射
        self.word2idx = {}
        self.idx2word = {}
        self.word_freq = Counter()
        
        # 初始化子词映射
        self.subword2idx = {}
        self.idx2subword = {}
        
        # 词向量和子词向量
        self.word_vectors = None
        self.subword_vectors = None
        
        # 训练历史
        self.training_history = {'loss': []}
        
    def get_subwords(self, word: str) -> List[str]:
        """
        获取单词的子词列表
        
        Args:
            word: 输入单词
            
        Returns:
            子词列表
        """
        # 添加词边界标记
        word = f"<{word}>"
        subwords = []
        
        # 生成n-gram子词
        for n in range(self.min_n, min(len(word) + 1, self.max_n + 1)):
            for i in range(len(word) - n + 1):
                subword = word[i:i + n]
                subwords.append(subword)
        
        return subwords
        
    def build_vocab(self, texts: List[str]):
        """构建词汇表和子词表"""
        print("🔧 构建词汇表...")
        
        # 统计词频
        for text in texts:
            words = list(jieba.cut(text))
            for word in words:
                if len(word.strip()) > 0:
                    self.word_freq[word] += 1
        
        # 过滤低频词
        filtered_words = {word for word, freq in self.word_freq.items() 
                         if freq >= self.min_count}
        
        # 构建词汇映射
        self.word2idx = {word: idx for idx, word in enumerate(filtered_words)}
        self.idx2word = {idx: word for word, idx in self.word2idx.items()}
        
        # 构建子词表
        all_subwords = set()
        for word in filtered_words:
            subwords = self.get_subwords(word)
            all_subwords.update(subwords)
        
        self.subword2idx = {subword: idx for idx, subword in enumerate(all_subwords)}
        self.idx2subword = {idx: subword for subword, idx in self.subword2idx.items()}
        
        # 初始化词向量和子词向量
        vocab_size = len(self.word2idx)
        subword_size = len(self.subword2idx)
        
        # Xavier初始化
        self.word_vectors = np.random.normal(0, 0.1, (vocab_size, self.embedding_dim))
        self.subword_vectors = np.random.normal(0, 0.1, (subword_size, self.embedding_dim))
        
        print(f"📊 词汇表大小: {vocab_size}")
        print(f"📊 子词表大小: {subword_size}")
        
    def get_word_vector(self, word: str) -> np.ndarray:
        """
        获取单词的向量表示（结合词向量和子词向量）
        
        Args:
            word: 输入单词
            
        Returns:
            单词向量
        """
        vector = np.zeros(self.embedding_dim)
        count = 0
        
        # 如果词在词汇表中，添加词向量
        if word in self.word2idx:
            word_idx = self.word2idx[word]
            vector += self.word_vectors[word_idx]
            count += 1
        
        # 添加子词向量
        subwords = self.get_subwords(word)
        for subword in subwords:
            if subword in self.subword2idx:
                subword_idx = self.subword2idx[subword]
                vector += self.subword_vectors[subword_idx]
                count += 1
        
        # 平均化
        if count > 0:
            vector /= count
            
        return vector
        
    def negative_sampling(self, target_word: str, context_words: List[str]) -> List[str]:
        """负采样"""
        # 基于词频的负采样概率
        word_probs = np.array([self.word_freq[word] ** 0.75 
                              for word in self.word2idx.keys()])
        word_probs = word_probs / word_probs.sum()
        
        negative_words = []
        words_list = list(self.word2idx.keys())
        
        while len(negative_words) < self.negative_samples:
            neg_word = np.random.choice(words_list, p=word_probs)
            if neg_word not in context_words and neg_word != target_word:
                negative_words.append(neg_word)
                
        return negative_words
        
    def sigmoid(self, x):
        """Sigmoid激活函数"""
        x = np.clip(x, -500, 500)  # 防止溢出
        return 1 / (1 + np.exp(-x))
        
    def train_pair(self, target_word: str, context_word: str, label: int):
        """训练单个词对"""
        # 获取向量
        target_vector = self.get_word_vector(target_word)
        context_vector = self.get_word_vector(context_word)
        
        # 计算预测值
        dot_product = np.dot(target_vector, context_vector)
        prediction = self.sigmoid(dot_product)
        
        # 计算梯度
        error = label - prediction
        gradient = error * self.learning_rate
        
        # 更新向量
        target_grad = gradient * context_vector
        context_grad = gradient * target_vector
        
        # 更新目标词的词向量和子词向量
        if target_word in self.word2idx:
            target_idx = self.word2idx[target_word]
            self.word_vectors[target_idx] += target_grad
            
        target_subwords = self.get_subwords(target_word)
        for subword in target_subwords:
            if subword in self.subword2idx:
                subword_idx = self.subword2idx[subword]
                self.subword_vectors[subword_idx] += target_grad / len(target_subwords)
        
        # 更新上下文词的向量
        if context_word in self.word2idx:
            context_idx = self.word2idx[context_word]
            self.word_vectors[context_idx] += context_grad
            
        context_subwords = self.get_subwords(context_word)
        for subword in context_subwords:
            if subword in self.subword2idx:
                subword_idx = self.subword2idx[subword]
                self.subword_vectors[subword_idx] += context_grad / len(context_subwords)
        
        return -np.log(prediction + 1e-8) if label == 1 else -np.log(1 - prediction + 1e-8)
        
    def train(self, texts: List[str], epochs: int = 5):
        """训练FastText模型"""
        print("🚀 开始训练FastText模型...")
        
        # 构建词汇表
        self.build_vocab(texts)
        
        # 准备训练数据
        training_pairs = []
        for text in texts:
            words = [word for word in jieba.cut(text) 
                    if word.strip() and word in self.word2idx]
            
            # 生成正样本
            for i, target_word in enumerate(words):
                for j in range(max(0, i - self.window_size), 
                              min(len(words), i + self.window_size + 1)):
                    if i != j:
                        context_word = words[j]
                        training_pairs.append((target_word, context_word, 1))
        
        print(f"📊 训练样本数: {len(training_pairs)}")
        
        # 训练
        for epoch in range(epochs):
            epoch_loss = 0
            random.shuffle(training_pairs)
            
            for target_word, context_word, label in training_pairs:
                # 正样本训练
                loss = self.train_pair(target_word, context_word, 1)
                epoch_loss += loss
                
                # 负采样训练
                negative_words = self.negative_sampling(target_word, [context_word])
                for neg_word in negative_words:
                    loss = self.train_pair(target_word, neg_word, 0)
                    epoch_loss += loss
            
            avg_loss = epoch_loss / (len(training_pairs) * (1 + self.negative_samples))
            self.training_history['loss'].append(avg_loss)
            
            # 学习率衰减
            self.learning_rate *= 0.98
            
            if epoch % 1 == 0:
                print(f"Epoch {epoch + 1}/{epochs}, Loss: {avg_loss:.4f}, LR: {self.learning_rate:.6f}")
        
        print("✅ FastText模型训练完成！")
        
    def find_similar_words(self, word: str, top_k: int = 10) -> List[Tuple[str, float]]:
        """寻找相似词汇"""
        if not self.word_vectors:
            return []
            
        word_vector = self.get_word_vector(word)
        similarities = []
        
        for other_word in self.word2idx.keys():
            if other_word != word:
                other_vector = self.get_word_vector(other_word)
                # 计算余弦相似度
                norm_product = np.linalg.norm(word_vector) * np.linalg.norm(other_vector)
                if norm_product > 0:
                    similarity = np.dot(word_vector, other_vector) / norm_product
                    similarities.append((other_word, similarity))
        
        similarities.sort(key=lambda x: x[1], reverse=True)
        return similarities[:top_k]
        
    def word_analogy(self, word_a: str, word_b: str, word_c: str, 
                    top_k: int = 5) -> List[Tuple[str, float]]:
        """词汇类比：A之于B，如C之于？"""
        try:
            vec_a = self.get_word_vector(word_a)
            vec_b = self.get_word_vector(word_b)
            vec_c = self.get_word_vector(word_c)
            
            # 计算目标向量：vec_b - vec_a + vec_c
            target_vector = vec_b - vec_a + vec_c
            
            similarities = []
            for word in self.word2idx.keys():
                if word not in [word_a, word_b, word_c]:
                    word_vector = self.get_word_vector(word)
                    norm_product = np.linalg.norm(target_vector) * np.linalg.norm(word_vector)
                    if norm_product > 0:
                        similarity = np.dot(target_vector, word_vector) / norm_product
                        similarities.append((word, similarity))
            
            similarities.sort(key=lambda x: x[1], reverse=True)
            return similarities[:top_k]
            
        except Exception as e:
            print(f"词汇类比计算出错: {e}")
            return []
            
    def plot_training_history(self):
        """绘制训练历史"""
        if not self.training_history['loss']:
            print("没有训练历史数据")
            return
            
        plt.figure(figsize=(12, 4))
        
        # 训练损失
        plt.subplot(1, 2, 1)
        plt.plot(self.training_history['loss'], 'b-', linewidth=2)
        plt.title('FastText训练损失', fontsize=14, fontweight='bold')
        plt.xlabel('Epoch')
        plt.ylabel('Loss')
        plt.grid(True, alpha=0.3)
        
        # 学习率变化
        plt.subplot(1, 2, 2)
        initial_lr = 0.025
        decay_rate = 0.98
        epochs = len(self.training_history['loss'])
        lr_values = [initial_lr * (decay_rate ** epoch) for epoch in range(epochs)]
        plt.plot(lr_values, 'r-', linewidth=2)
        plt.title('学习率衰减', fontsize=14, fontweight='bold')
        plt.xlabel('Epoch')
        plt.ylabel('Learning Rate')
        plt.grid(True, alpha=0.3)
        
        plt.tight_layout()
        plt.show()
        
    def analyze_subwords(self, word: str):
        """分析单词的子词结构"""
        if word not in self.word2idx:
            print(f"词汇 '{word}' 不在词汇表中")
            return
            
        subwords = self.get_subwords(word)
        print(f"\n🔍 单词 '{word}' 的子词分析:")
        print(f"原始单词: {word}")
        print(f"添加边界标记: <{word}>")
        print(f"子词列表 ({len(subwords)}个):")
        
        for i, subword in enumerate(subwords, 1):
            in_vocab = "✓" if subword in self.subword2idx else "✗"
            print(f"  {i:2d}. {subword:10s} {in_vocab}")
            
        # 计算子词贡献的向量
        word_vector = self.get_word_vector(word)
        print(f"\n📊 最终词向量维度: {word_vector.shape}")
        print(f"📊 词向量范数: {np.linalg.norm(word_vector):.4f}")

# 演示FastText模型
print("=" * 60)
print("🚀 FastText词嵌入模型演示")
print("=" * 60)

# 准备训练数据
training_texts = [
    "机器学习是人工智能的核心技术",
    "深度学习使用神经网络进行训练",
    "自然语言处理研究人机交互",
    "计算机视觉分析图像和视频",
    "数据挖掘从大数据中发现模式",
    "算法优化提高计算效率",
    "模式识别用于分类和预测",
    "人工智能改变世界的发展",
    "机器人技术结合多种学科",
    "智能系统具有自主决策能力",
    "深度神经网络学习复杂特征",
    "卷积神经网络处理图像数据",
    "递归神经网络处理序列数据",
    "强化学习通过奖励机制训练",
    "无监督学习发现数据结构",
    "监督学习使用标记数据训练",
    "特征工程提取有用信息",
    "模型评估验证算法性能",
    "交叉验证避免过拟合问题",
    "正则化技术控制模型复杂度"
]

# 创建和训练FastText模型
fasttext_model = FastTextModel(
    embedding_dim=50, 
    window_size=3, 
    min_count=2,
    min_n=2,
    max_n=4,
    negative_samples=3,
    learning_rate=0.01
)

fasttext_model.train(training_texts, epochs=10)

# 绘制训练历史
fasttext_model.plot_training_history()

# 分析子词结构
print(f"\n=== FastText子词分析 ===")
test_subword_words = ['机器学习', '深度', '神经网络']
for word in test_subword_words:
    fasttext_model.analyze_subwords(word)

# 测试FastText模型效果
test_words = ['机器', '学习', '深度', '神经', '网络']
print(f"\n=== FastText模型词汇相似度测试 ===")
for word in test_words:
    similar_words = fasttext_model.find_similar_words(word, top_k=5)
    if similar_words:
        print(f"\n与 '{word}' 最相似的词汇 (FastText):")
        for similar_word, similarity in similar_words:
            print(f"  {similar_word}: {similarity:.4f}")

# FastText词汇类比测试
print(f"\n=== FastText词汇类比测试 ===")
analogy_tests = [
    ('机器', '学习', '深度'),
    ('计算机', '视觉', '自然'),
    ('数据', '挖掘', '文本')
]

for word_a, word_b, word_c in analogy_tests:
    results = fasttext_model.word_analogy(word_a, word_b, word_c, top_k=3)
    if results:
        print(f"\nFastText: {word_a} 之于 {word_b}，如 {word_c} 之于：")
        for result_word, similarity in results:
            print(f"  {result_word}: {similarity:.4f}")

# 测试未登录词处理能力
print(f"\n=== FastText未登录词处理测试 ===")
oov_words = ['深度学', '机器习', '智能化']
for oov_word in oov_words:
    vector = fasttext_model.get_word_vector(oov_word)
    print(f"未登录词 '{oov_word}' 向量范数: {np.linalg.norm(vector):.4f}")
    
    # 寻找相似词
    similar_words = fasttext_model.find_similar_words(oov_word, top_k=3)
    if similar_words:
        print(f"  与 '{oov_word}' 最相似的词汇:")
        for similar_word, similarity in similar_words:
            print(f"    {similar_word}: {similarity:.4f}")

print("\n✅ FastText模型演示完成！")

### 2.5 三种词嵌入方法对比分析

#### 📊 模型特点对比

```python
import pandas as pd

def compare_embedding_methods():
    """对比三种词嵌入方法"""
    
    # 创建对比表格
    comparison_data = {
        '特征': ['训练方式', '向量维度', '语义理解', '计算复杂度', '内存占用', 
                '未登录词', '训练速度', '适用场景'],
        'Word2Vec': [
            '局部上下文', '可调节', '较好', '中等', '中等', 
            '无法处理', '较快', '通用文本'
        ],
        'GloVe': [
            '全局+局部', '可调节', '很好', '较高', '较高', 
            '无法处理', '较慢', '大规模语料'
        ],
        'FastText': [
            '子词+上下文', '可调节', '较好', '较高', '最高', 
            '可以处理', '中等', '形态丰富语言'
        ]
    }
    
    df = pd.DataFrame(comparison_data)
    print("📊 三种词嵌入方法对比:")
    print(df.to_string(index=False))
    
    # 优缺点分析
    methods_analysis = {
        'Word2Vec': {
            '优点': [
                '训练速度快',
                '模型简单易理解',
                '效果稳定可靠',
                '支持Skip-gram和CBOW两种架构'
            ],
            '缺点': [
                '无法处理未登录词',
                '忽略了全局统计信息',
                '对稀有词效果不佳'
            ]
        },
        'GloVe': {
            '优点': [
                '结合全局和局部信息',
                '语义表示更丰富',
                '理论基础扎实',
                '在词汇类比任务上表现优秀'
            ],
            '缺点': [
                '训练时间长',
                '内存消耗大',
                '无法处理未登录词'
            ]
        },
        'FastText': {
            '优点': [
                '可处理未登录词',
                '适合形态变化丰富的语言',
                '对稀有词效果好',
                '考虑子词信息'
            ],
            '缺点': [
                '模型复杂度高',
                '内存占用最大',
                '训练时间较长'
            ]
        }
    }
    
    print(f"\n🎯 详细优缺点分析:")
    for method, analysis in methods_analysis.items():
        print(f"\n{method}:")
        print(f"  ✅ 优点:")
        for advantage in analysis['优点']:
            print(f"    • {advantage}")
        print(f"  ❌ 缺点:")
        for disadvantage in analysis['缺点']:
            print(f"    • {disadvantage}")

# 运行对比分析
compare_embedding_methods()

# 性能测试对比
def performance_comparison():
    """性能对比测试"""
    print(f"\n🏃‍♂️ 性能对比测试:")
    
    # 模拟性能数据
    performance_data = {
        '指标': ['训练时间(秒)', '内存占用(MB)', '相似度准确率(%)', '类比准确率(%)', '未登录词处理'],
        'Word2Vec': [120, 256, 78.5, 65.2, '❌'],
        'GloVe': [280, 512, 82.3, 71.8, '❌'],
        'FastText': [180, 768, 79.1, 67.5, '✅']
    }
    
    df_performance = pd.DataFrame(performance_data)
    print(df_performance.to_string(index=False))
    
    # 推荐使用场景
    print(f"\n💡 使用场景推荐:")
    scenarios = {
        'Word2Vec': [
            '快速原型开发',
            '计算资源有限',
            '英文文本处理',
            '通用语义理解任务'
        ],
        'GloVe': [
            '大规模语料库',
            '需要高质量词向量',
            '词汇类比任务',
            '学术研究项目'
        ],
        'FastText': [
            '多语言文本处理',
            '形态变化丰富的语言',
            '需要处理未登录词',
            '专业领域文本'
        ]
    }
    
    for method, uses in scenarios.items():
        print(f"\n{method} 适用场景:")
        for use_case in uses:
            print(f"  🎯 {use_case}")

performance_comparison()