import jieba

# 精确模式（最常用）
words = jieba.cut("我在北京大学学习自然语言处理")
print("/".join(words))
# 我/在/北京/大学/学习/自然语言处理

# 全模式
words = jieba.cut("我在北京大学学习自然语言处理", cut_all=True)
print("/".join(words))
# 我/在/北京/大学/北京大/大学/学习/自然/语言/处理/自然语言

# 搜索引擎模式
words = jieba.cut_for_search("南京市长江大桥")
print("/".join(words))
# 南京市/南京/市长/长江/长江大桥/大桥

# 添加自定义词典
jieba.add_word("机器学习", freq=10000)
jieba.add_word("深度学习", freq=10000)

# 加载外部词典文件
# jieba.load_userdict("mydict.txt")
# 文件格式：词语 频率 词性
# 机器学习 10000 n
# 深度学习 10000 n

# 获取词语和词性
import jieba.posseg as pseg
words = pseg.cut("我爱自然语言处理")
for word, flag in words:
    print(f"{word}/{flag}", end=" ")
# 我/r 爱/v 自然语言处理/l

from sklearn.feature_extraction.text import TfidfVectorizer
import jieba

# 中文需要先分词
corpus = [
    "机器学习是人工智能的一个分支",
    "深度学习是机器学习的重要方法",
    "自然语言处理使用深度学习",
    "Python 是广泛使用的编程语言"
]

# 分词 + 去除停用词
stopwords = {"是", "的", "一个", "重要", "广泛", "使用"}
segmented = [
    " ".join([w for w in jieba.cut(text) if w not in stopwords and len(w.strip()) > 0])
    for text in corpus
]

# TF-IDF 向量化
vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(segmented)

# 查看结果
feature_names = vectorizer.get_feature_names_out()
print("特征词：", list(feature_names))
print("TF-IDF 矩阵：")
print(tfidf_matrix.toarray())

# 查看某个词的 IDF 值
for word, idf in zip(feature_names, vectorizer.idf_):
    print(f"  IDF({word}) = {idf:.4f}")

# 经典例子
vec("king") - vec("man") + vec("woman") ≈ vec("queen")

# 语义类比
vec("Paris") - vec("France") + vec("Japan") ≈ vec("Tokyo")
vec("bigger") - vec("big") + vec("small") ≈ vec("smaller")

from gensim.models import KeyedVectors

# 加载预训练的 Word2Vec 模型（中文需要下载对应模型）
# model = KeyedVectors.load_word2vec_format("sgns.baidubaike.bigram-char", binary=False)

# 使用示例（以模拟数据为例）
import numpy as np

# 模拟词向量（实际使用时加载预训练模型）
word_vectors = {
    "机器": np.array([0.2, 0.8, 0.1, 0.5]),
    "学习": np.array([0.3, 0.7, 0.2, 0.4]),
    "深度": np.array([0.1, 0.9, 0.3, 0.6]),
    "自然": np.array([0.4, 0.3, 0.7, 0.2]),
    "语言": np.array([0.5, 0.4, 0.6, 0.3]),
}

def cosine_similarity(v1, v2):
    """计算余弦相似度"""
    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))

# 语义相似度
print("机器 与 学习 的相似度:", cosine_similarity(word_vectors["机器"], word_vectors["学习"]))
print("机器 与 自然 的相似度:", cosine_similarity(word_vectors["机器"], word_vectors["自然"]))

"""
完整文本分类示例：新闻分类
使用 TF-IDF + 多种分类器对比
"""
import jieba
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, accuracy_score
from sklearn.pipeline import Pipeline

# ── 1. 准备训练数据 ──
train_data = [
    # 科技类
    ("苹果发布新款iPhone，搭载A18芯片", "科技"),
    ("华为推出鸿蒙操作系统新版本", "科技"),
    ("人工智能在医疗诊断领域取得突破", "科技"),
    ("特斯拉发布自动驾驶新功能", "科技"),
    ("谷歌推出新一代大语言模型", "科技"),
    ("量子计算机实现新里程碑", "科技"),
    ("5G网络覆盖范围进一步扩大", "科技"),
    ("SpaceX 成功发射星舰", "科技"),
    # 体育类
    ("中国女排夺得世界冠军", "体育"),
    ("世界杯预选赛中国队获胜", "体育"),
    ("姚明当选篮球协会主席", "体育"),
    ("奥运会筹备工作进展顺利", "体育"),
    ("刘翔退役后投身青少年体育教育", "体育"),
    ("足球联赛新赛季开幕", "体育"),
    ("游泳世锦赛中国队摘金夺银", "体育"),
    ("NBA总决赛湖人队夺冠", "体育"),
    # 财经类
    ("央行宣布降低利率", "财经"),
    ("股市今日大幅上涨", "财经"),
    ("房地产市场调控政策出台", "财经"),
    ("比特币价格突破历史新高", "财经"),
    ("新能源汽车企业获得融资", "财经"),
    ("外资持续流入A股市场", "财经"),
    ("央行发布货币政策报告", "财经"),
    ("科技股估值引发市场讨论", "财经"),
]

texts = [t[0] for t in train_data]
labels = [t[1] for t in train_data]

# ── 2. 文本预处理 ──
stopwords = {"的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
             "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
             "没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那", "被",
             "从", "把", "让", "给", "对", "与", "以", "及", "等", "但", "而",
             "如", "或", "之", "其", "已", "将", "更", "新", "为", "于", "中",
             "后", "前", "可", "能", "所", "此", "这个", "那个", "什么"}

def preprocess(text):
    """分词 + 去停用词"""
    words = jieba.cut(text)
    return " ".join([w for w in words if w not in stopwords and len(w.strip()) > 1])

processed_texts = [preprocess(t) for t in texts]

# 打印预处理结果
print("=" * 60)
print("📝 预处理示例")
print("=" * 60)
for i in range(3):
    print(f"原文: {texts[i]}")
    print(f"分词: {processed_texts[i]}")
    print()

# ── 3. 划分训练集和测试集 ──
X_train, X_test, y_train, y_test = train_test_split(
    processed_texts, labels, test_size=0.3, random_state=42, stratify=labels
)

# ── 4. 使用 Pipeline 构建分类器 ──
classifiers = {
    "朴素贝叶斯": Pipeline([
        ("tfidf", TfidfVectorizer(max_features=5000)),
        ("clf", MultinomialNB())
    ]),
    "逻辑回归": Pipeline([
        ("tfidf", TfidfVectorizer(max_features=5000)),
        ("clf", LogisticRegression(max_iter=1000))
    ]),
    "SVM": Pipeline([
        ("tfidf", TfidfVectorizer(max_features=5000)),
        ("clf", LinearSVC())
    ]),
}

print("=" * 60)
print("📊 分类器对比")
print("=" * 60)

for name, clf in classifiers.items():
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    acc = accuracy_score(y_test, y_pred)
    print(f"\n{name}: 准确率 = {acc:.2%}")
    if len(set(y_test)) > 1:
        print(classification_report(y_test, y_pred, zero_division=0))

# ── 5. 使用最佳模型进行预测 ──
best_clf = classifiers["逻辑回归"]  # 通常逻辑回归效果最好
print("=" * 60)
print("🔮 新文本预测")
print("=" * 60)

new_texts = [
    "研究人员开发出新型电池技术",
    "足球世界杯决赛将在今晚举行",
    "央行宣布下调存款准备金率",
]

for text in new_texts:
    processed = preprocess(text)
    pred = best_clf.predict([processed])[0]
    # 获取概率（如果支持）
    if hasattr(best_clf.named_steps['clf'], 'predict_proba'):
        proba = best_clf.predict_proba([processed])[0]
        classes = best_clf.classes_
        proba_str = ", ".join([f"{c}: {p:.1%}" for c, p in zip(classes, proba)])
        print(f"文本: {text}")
        print(f"  预测: {pred} | 概率: {proba_str}")
    else:
        print(f"文本: {text} → 预测: {pred}")
    print()