┌─────────────────────────────────────────────────────────┐
│                    用户查询 (Query)                       │
└────────────────────────┬────────────────────────────────┘
                         │
                         ▼
┌─────────────────────────────────────────────────────────┐
│  第 1 层：文档处理 (Document Processing)                  │
│  格式解析 → 元数据提取 → 内容清洗                           │
└────────────────────────┬────────────────────────────────┘
                         │
                         ▼
┌─────────────────────────────────────────────────────────┐
│  第 2 层：切分 (Chunking)                                │
│  分块策略 → 块大小控制 → 重叠窗口                           │
└────────────────────────┬────────────────────────────────┘
                         │
                         ▼
┌─────────────────────────────────────────────────────────┐
│  第 3 层：向量化 (Embedding)                              │
│  文本向量化 → 批量处理 → 维度管理                           │
└────────────────────────┬────────────────────────────────┘
                         │
                         ▼
┌─────────────────────────────────────────────────────────┐
│  第 4 层：索引与检索 (Indexing & Retrieval)               │
│  向量存储 → 索引策略 → 相似度搜索 → 元数据过滤              │
└────────────────────────┬────────────────────────────────┘
                         │
                         ▼
┌─────────────────────────────────────────────────────────┐
│  第 5 层：生成 (Generation)                              │
│  上下文注入 → Prompt 构建 → 模型生成 → 引用追溯             │
└─────────────────────────────────────────────────────────┘

from unstructured.partition.auto import partition

elements = partition(
    filename="company_report.pdf",
    strategy="hi_res",
    chunking_strategy="by_title",
    languages=["chi_sim", "eng"]
)

for element in elements:
    print(f"[{element.category}] {element.text[:100]}")

from langchain.text_splitter import CharacterTextSplitter

splitter = CharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separator="\n"
)
chunks = splitter.split_text(document_text)

from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=500,
    chunk_overlap=100,
    separators=["\n\n", "\n", "。", "！", "？", ".", "!", "?", " "]
)
chunks = splitter.split_text(document_text)

from langchain_experimental.text_splitter import SemanticChunker
from langchain_openai import OpenAIEmbeddings

splitter = SemanticChunker(
    OpenAIEmbeddings(),
    breakpoint_threshold_type="percentile",
    breakpoint_threshold_amount=85
)
chunks = splitter.split_text(document_text)

from langchain.text_splitter import MarkdownHeaderTextSplitter

headers_to_split_on = [
    ("#", "h1"),
    ("##", "h2"),
    ("###", "h3"),
]

splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
chunks = splitter.split_text(markdown_text)

from openai import OpenAI

client = OpenAI()
response = client.embeddings.create(
    model="text-embedding-3-large",
    input="RAG 是一种检索增强生成技术",
    dimensions=1024  # 从 3072 缩减到 1024
)
vector = response.data[0].embedding

from sentence_transformers import SentenceTransformer
import numpy as np

model = SentenceTransformer("BAAI/bge-m3")

texts = ["文档片段1", "文档片段2", "文档片段3"]
embeddings = model.encode(
    texts,
    batch_size=32,
    show_progress_bar=True,
    normalize_embeddings=True  # L2 归一化，兼容余弦相似度
)

总向量数 = 100 万
nlist = 1024（聚类数）
nprobe = 32（搜索聚类数）
→ 搜索范围缩小到约 3%，精度损失约 1-2%

import faiss

dimension = 1024
index = faiss.IndexHNSWFlat(dimension, 32)  # M=32
index.hnsw.efConstruction = 200
index.hnsw.efSearch = 128

# Qdrant 示例：向量检索 + 元数据过滤
from qdrant_client import QdrantClient
from qdrant_client.models import Filter, FieldCondition, MatchValue

results = client.search(
    collection_name="knowledge_base",
    query_vector=query_embedding,
    query_filter=Filter(
        must=[
            FieldCondition(key="source", match=MatchValue(value="产品手册")),
            FieldCondition(key="year", match=MatchValue(value=2024)),
        ]
    ),
    limit=10
)

results = vector_store.similarity_search(
    query="如何配置单点登录",
    k=5
)

results = vector_store.max_marginal_relevance_search(
    query="如何配置单点登录",
    k=5,
    fetch_k=20,       # 先检索 20 个候选
    lambda_mult=0.7   # 0=纯多样性，1=纯相关性
)

from langchain.retrievers import MultiQueryRetriever

retriever = MultiQueryRetriever.from_llm(
    retriever=vector_store.as_retriever(),
    llm=ChatOpenAI(model="gpt-4o-mini", temperature=0.2)
)

results = retriever.invoke("单点登录配置")
# LLM 会自动生成多个查询如：
# - "SSO 配置步骤"
# - "SAML 单点登录集成方法"
# - "OAuth2 单点登录流程"

from langchain.retrievers.self_query.base import SelfQueryRetriever
from langchain.chains.query_constructor.base import AttributeInfo

metadata_fields = [
    FieldInfo(name="source", description="文档来源，如产品手册/内部Wiki/FAQ"),
    FieldInfo(name="department", description="所属部门，如技术/产品/运维"),
    FieldInfo(name="last_updated", description="最后更新日期"),
]

retriever = SelfQueryRetriever.from_llm(
    llm=ChatOpenAI(model="gpt-4o"),
    vectorstore=vector_store,
    document_contents="技术文档内容",
    metadata_field_info=metadata_fields,
)

# 混合检索的典型实现
from langchain.retrievers import EnsembleRetriever
from langchain_community.retrievers import BM25Retriever

bm25_retriever = BM25Retriever.from_texts(texts, k=5)
vector_retriever = vector_store.as_retriever(search_kwargs={"k": 5})

ensemble_retriever = EnsembleRetriever(
    retrievers=[bm25_retriever, vector_retriever],
    weights=[0.4, 0.6]  # 关键词权重 0.4，语义权重 0.6
)

results = ensemble_retriever.invoke("BGE-M3 模型的稀疏向量")

RAG_PROMPT_TEMPLATE = """你是一个专业的技术文档助手。请基于以下参考资料回答用户问题。

规则：
1. 仅基于提供的参考资料回答，不要编造信息
2. 如果参考资料中没有相关信息，明确告知用户
3. 在回答中标注信息来源（文档名称或编号）
4. 使用中文回答

参考资料：
{context}

用户问题：{question}
"""

# 检索时保留来源信息
search_results = vector_store.similarity_search_with_score(query, k=5)

# 构建带来源标记的上下文
context_parts = []
for doc, score in search_results:
    source = doc.metadata.get("source", "未知来源")
    context_parts.append(f"[来源: {source}] {doc.page_content}")

context = "\n\n---\n\n".join(context_parts)

VERIFICATION_PROMPT = """请验证以下回答是否完全基于提供的参考资料，找出任何编造或不准确的内容。

参考资料：
{context}

待验证回答：
{answer}

请逐条检查回答中的事实性陈述，标注每条陈述是否在参考资料中有依据。"""

Based on the following documents, answer the question.
For each claim, cite the source document using [Doc-X] format.
If documents don't contain enough information, say "根据现有资料无法回答" rather than guessing.

Step 1: Identify the relevant documents for this question.
Step 2: Extract key facts from each relevant document.
Step 3: Synthesize the facts into a coherent answer.
Step 4: Verify your answer against the extracted facts.

# 检索时的权限过滤示例
def authorized_search(query, user_role, top_k=5):
    results = vector_store.similarity_search_with_score(query, k=top_k * 2)
    
    authorized = []
    for doc, score in results:
        doc_access_level = doc.metadata.get("access_level", "public")
        if check_permission(user_role, doc_access_level):
            authorized.append((doc, score))
            if len(authorized) >= top_k:
                break
    
    return authorized