# 安装与数据下载
# pip install nltk
import nltk
nltk.download('punkt')          # 分词模型
nltk.download('averaged_perceptron_tagger')  # 词性标注
nltk.download('maxent_ne_chunker')  # 命名实体识别
nltk.download('words')          # 英语词典
nltk.download('stopwords')      # 停用词
nltk.download('wordnet')        # 词汇语义网络

from nltk.tokenize import word_tokenize, sent_tokenize

text = "Dr. Smith said: 'NLP is amazing!' Isn't it? Yes, it is."

# 句子分词
sentences = sent_tokenize(text)
# ["Dr. Smith said: 'NLP is amazing!'", "Isn't it?", "Yes, it is."]

# 词级分词
words = word_tokenize(text)
# ['Dr.', 'Smith', 'said', ':', "'NLP", 'is', 'amazing', '!', "'", "Isn't", 'it', '?', ...]

import jieba

text = "我爱自然语言处理"
words = list(jieba.cut(text))
# ['我', '爱', '自然语言处理'] 或 ['我', '爱', '自然', '语言', '处理']

from nltk import pos_tag

words = word_tokenize("The quick brown fox jumps over the lazy dog")
tags = pos_tag(words)
# [('The', 'DT'), ('quick', 'JJ'), ('brown', 'NN'), ('fox', 'NN'), 
#  ('jumps', 'VBZ'), ('over', 'IN'), ('the', 'DT'), ('lazy', 'JJ'), ('dog', 'NN')]

from nltk.corpus import stopwords

stop_words = set(stopwords.words('english'))
words = word_tokenize("The quick brown fox jumps over the lazy dog")
filtered = [w for w in words if w.lower() not in stop_words]
# ['quick', 'brown', 'fox', 'jumps', 'lazy', 'dog']

from nltk.stem import PorterStemmer, WordNetLemmatizer

stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

words = ["running", "flies", "better", "studies"]

# 词干提取
stems = [stemmer.stem(w) for w in words]
# ['run', 'fli', 'better', 'studi']

# 词形还原
lemmas = [lemmatizer.lemmatize(w) for w in words]
# ['running', 'fly', 'better', 'study']  （需指定 POS 效果更好）

from nltk import ne_chunk

text = "Barack Obama was born in Hawaii and worked at the White House"
words = word_tokenize(text)
tags = pos_tag(words)
tree = ne_chunk(tags)

# 遍历提取实体
for subtree in tree:
    if hasattr(subtree, 'label'):
        entity = ' '.join([word for word, tag in subtree.leaves()])
        print(f"{subtree.label()}: {entity}")
# PERSON: Barack Obama
# GPE: Hawaii
# ORGANIZATION: White House

# 安装与模型下载
# pip install spacy
# python -m spacy download en_core_web_sm   # 英文小模型
# python -m spacy download zh_core_web_sm   # 中文小模型

import spacy
nlp = spacy.load("en_core_web_sm")  # 加载英文模型

import spacy

nlp = spacy.load("en_core_web_sm")
doc = nlp("Apple is looking at buying U.K. startup for $1 billion")

# 分词
tokens = [token.text for token in doc]
# ['Apple', 'is', 'looking', 'at', 'buying', 'U.K.', 'startup', 'for', '$', '1', 'billion']

# 词性标注
pos = [(token.text, token.pos_) for token in doc]
# [('Apple', 'PROPN'), ('is', 'AUX'), ('looking', 'VERB'), ...]

# 依存句法分析
deps = [(token.text, token.dep_, token.head.text) for token in doc]
# [('Apple', 'nsubj', 'looking'), ('is', 'aux', 'looking'), ...]

# 命名实体识别
ents = [(ent.text, ent.label_) for ent in doc.ents]
# [('Apple', 'ORG'), ('U.K.', 'GPE'), ('$1 billion', 'MONEY')]

doc = nlp("Apple is looking at buying a startup")
token = doc[0]  # "Apple"

token.text      # "Apple"      - 原始文本
token.lemma_    # "apple"      - 词形还原
token.pos_      # "PROPN"      - 粗粒度词性
token.tag_      # "NNP"        - 细粒度词性
token.dep_      # "nsubj"      - 依存关系
token.morph     # Number=Sing   - 形态特征
token.is_stop   # False         - 是否停用词
token.is_alpha  # True          - 是否纯字母
token.is_punct  # False         - 是否标点

nlp = spacy.load("en_core_web_md")  # 需要中/大模型

doc1 = nlp("I love cats")
doc2 = nlp("I adore felines")
doc3 = nlp("I hate dogs")

print(doc1.similarity(doc2))  # 0.75 (高相似度)
print(doc1.similarity(doc3))  # 0.32 (低相似度)

from spacy.training import Example
import spacy

# 准备训练数据
TRAIN_DATA = [
    ("华为发布新手机", {"entities": [(0, 2, "ORG"), (4, 8, "PRODUCT")]}),
    ("我在北京工作", {"entities": [(2, 4, "LOC")]}),
]

nlp = spacy.blank("zh")  # 创建空白中文模型
ner = nlp.add_pipe("ner")
ner.add_label("ORG")
ner.add_label("PRODUCT")
ner.add_label("LOC")

# 训练循环（简化版）
optimizer = nlp.begin_training()
for i in range(20):
    for text, annotations in TRAIN_DATA:
        doc = nlp.make_doc(text)
        example = Example.from_dict(doc, annotations)
        nlp.update([example], sgd=optimizer)

"""
聊天机器人意图识别模块
使用 spaCy 进行基本的意图分类和实体提取
"""
import spacy

class IntentRecognizer:
    """基于规则的意图识别器"""
    
    def __init__(self):
        self.nlp = spacy.load("en_core_web_sm")
        # 意图关键词映射
        self.intent_keywords = {
            "greeting": ["hello", "hi", "hey", "good morning"],
            "weather": ["weather", "temperature", "rain", "sunny"],
            "time": ["time", "clock", "hour", "minute"],
            "goodbye": ["bye", "goodbye", "see you"],
        }
    
    def recognize(self, text: str) -> dict:
        """识别用户输入的意图和实体"""
        doc = self.nlp(text.lower())
        
        # 提取实体
        entities = {}
        for ent in doc.ents:
            entities[ent.label_] = ent.text
        
        # 匹配意图
        intent = "unknown"
        tokens = set([token.text for token in doc])
        
        for intent_name, keywords in self.intent_keywords.items():
            if tokens & set(keywords):
                intent = intent_name
                break
        
        return {
            "intent": intent,
            "entities": entities,
            "tokens": [token.text for token in doc]
        }

# 使用示例
recognizer = IntentRecognizer()
tests = [
    "Hello, how are you?",
    "What's the weather in Beijing?",
    "What time is it?",
    "Goodbye!",
]
for text in tests:
    result = recognizer.recognize(text)
    print(f"Input: {text}")
    print(f"  → Intent: {result['intent']}, Entities: {result['entities']}")
    print()

# ❌ 错误：用 NLTK 分中文
from nltk.tokenize import word_tokenize
word_tokenize("我爱Python")  # 可能报错或分词不准

# ✅ 正确：用 jieba 分中文
import jieba
list(jieba.cut("我爱Python"))  # ['我', '爱', 'Python']

# ❌ 会报 OSError: [E050]
nlp = spacy.load("en_core_web_sm")

# ✅ 先下载模型
# python -m spacy download en_core_web_sm

# ❌ 大小写敏感导致漏匹配
if "hello" in text:  # "Hello" 匹配不到

# ✅ 统一转小写
if "hello" in text.lower():

# ❌ 词干提取可能产生非词
stemmer.stem("better")  # "better" (没变，有时会变成 "bett")

# ✅ 词形还原更准确
lemmatizer.lemmatize("better", pos='a')  # "good"