import json
import time
import random
from zhipuai import ZhipuAI
# 初始化模型
client = ZhipuAI(api_key="48b393b3474e43f3b7eafceb662db81a.xxxxxx")
style_config = {
    "小白": {
        "system_prompt": "你是一位區塊鏈科普專家，需要用最簡單易懂的方式向完全不了解技術的普通人解釋區塊鏈概念。要求：\n1. 使用生活化比喻（如'像大家一起記賬本'）\n2. 避免專業術語，必須使用時立即解釋\n3. 回答控制在3句話以內\n4. 可適當使用??????等表情符號",
        "examples": [
            {"role": "user", "content": "區塊鏈到底是什么？"},
            {"role": "assistant",
             "content": "想象一群人在共同維護一個公開賬本??，每筆交易都記錄在所有人的本子上，想作弊就得改掉大部分人的記錄，這幾乎不可能??！"},
            {"role": "user", "content": "比特幣為什么值錢？"},
            {"role": "assistant", "content": "就像限量版球星卡??，總量固定(2100萬枚)，越來越多人想要時價格就上漲啦！"}
        ],
        "temperature": 0.2  # 低溫度值保證回答穩定性
    },
    "資深": {
        "system_prompt": "你是區塊鏈核心技術研究員，需滿足：\n1. 使用專業術語（如'橢圓曲線數字簽名''梅克爾帕特里夏樹'）\n2. 包含技術實現細節\n3. 必要時給出公式/代碼片段\n4. 引用最新論文成果（如2023 IEEE S&P會議）",
        "examples": [
            {"role": "user", "content": "請解釋zk-SNARKs的工作原理"},
            {"role": "assistant",
             "content": "zk-SNARKs依賴QAP(二次算術程序)將計算轉化為多項式關系：\n1) Setup生成CRS(公共參考串)\n2) Prover構造π=(A,B,C)滿足A*B=C+H*Z\n3) Verifier通過雙線性配對驗證e(A,B)=e(C,g)?e(H,g^z)\n參考：Groth16方案優化后驗證僅需3個配對運算。"},
            {"role": "user", "content": "分析EIP-4844對L2的影響"},
            {"role": "assistant",
             "content": "Proto-danksharding通過引入blob攜帶交易(每個blob≈125KB)，使Rollups數據可用性成本降低10-100倍。實測顯示：\n- Optimism交易費從$0.3→$0.007\n- zkSync證明生成時間減少40%"}
        ],
        "temperature": 0.5  # 中等溫度值允許一定創造性
    }
}
def generate_style_data(style_name, num_samples=50):
    config = style_config[style_name]
    data = []
    # 構建對話上下文 = 系統提示 + 示例對話
    messages = [
        {"role": "system", "content": config["system_prompt"]},
        *config["examples"]
    ]
    # 預定義的用戶問題集（可擴展）
    user_inputs = [
        "比特幣和區塊鏈是什么關系？",
        "智能合約真的安全嗎？",
        "普通人怎么參與DeFi？",
        "解釋一下零知識證明",
        "為什么NFT能賣那么貴？",
        "區塊鏈能防止數據造假嗎？"
    ]
    for _ in range(num_samples):
        try:
            # 隨機選擇問題
            user_msg = random.choice(user_inputs)
            # 當前對話 = 歷史上下文 + 新問題
            current_messages = messages + [
                {"role": "user", "content": user_msg}
            ]
            # 調用API（修正模型名稱）
            response = client.chat.completions.create(
                model="glm-4-plus",
                messages=current_messages,
                temperature=config["temperature"],
                max_tokens=150
            )
            # 獲取回復內容
            reply = response.choices[0].message.content
            data.append({
                "user": user_msg,
                "assistant": reply,
                "style": style_name
            })
            print("獲取最新回復內容: ", data[-1])
            time.sleep(0.5)  # 限流
        except Exception as e:
            print(f"生成失敗：{str(e)}")
    return data
if __name__ == '__main__':
    all_data = []
    try:
        print("開始生成小白風格數據...")
        gentle_data = generate_style_data("小白", 10)
        all_data.extend(gentle_data)
        print("開始生成資深風格數據...")
        gentle_data = generate_style_data("資深", 10)
        all_data.extend(gentle_data)
    except KeyboardInterrupt:
        print("\n用戶中斷，保存已生成數據...")
    finally:
        with open("style_chat_data.json", "w", encoding="utf-8") as f:
            json.dump(all_data, f, ensure_ascii=False, indent=2)
        print(f"數據已保存，有效樣本數：{len(all_data)}")

import json
from typing import List, Dict


def is_meaningless(text: str) -> bool:
    """檢查是否是無意義內容"""
    # 空白或超短內容
    if len(text.strip()) < 5:
        return True

    # 常見無意義模式
    meaningless_phrases = [
        "我不知道", "無法回答",
        "這個問題", "請重新提問"
    ]
    return any(phrase in text for phrase in meaningless_phrases)


def simple_clean(data: List[Dict]) -> List[Dict]:
    """基礎清洗流程"""
    seen = set()  # 用于去重
    cleaned_data = []

    for item in data:
        text = item["assistant"].strip()

        # 檢查空白/重復/無意義
        if (not text or
                text in seen or
                is_meaningless(text)):
            continue

        seen.add(text)
        cleaned_data.append({
            "user": item["user"],
            "assistant": text,
            "style": item["style"]
        })

    return cleaned_data


if __name__ == '__main__':
    # 加載原始數據
    with open('style_chat_data.json', 'r', encoding='utf-8') as f:
        raw_data = json.load(f)

    # 執行清洗
    cleaned_data = simple_clean(raw_data)

    # 保存結果
    with open('cleaned_data.json', 'w', encoding='utf-8') as f:
        json.dump(cleaned_data, f, ensure_ascii=False, indent=2)

    # 打印簡單報告
    print(f"原始數據量: {len(raw_data)}")
    print(f"清洗后保留: {len(cleaned_data)}")
    print(f"過濾數量: {len(raw_data) - len(cleaned_data)}")

import json
from sentence_transformers import SentenceTransformer
import numpy as np
def deduplicate_data(data_list, threshold=0.9):
    
"""
    語義去重：對 assistant 回復去重
    :param data_list: List[dict], 每個 dict 包含 user / assistant / style
    :param threshold: float, 相似度閾值
    :return: 去重后的 List[dict]
    """
    
model = SentenceTransformer("/root/autodl-tmp/models/thomas/text2vec-base-chinese")
    replies = [item["assistant"] for item in data_list]
    embeddings = model.encode(replies)
    kept = []
    seen = set()
    for i, emb_i in enumerate(embeddings):
        if i in seen:
            continue
        keep = True
        for j, emb_j in enumerate(embeddings):
            if j == i or j in seen:
                continue
            sim = np.dot(emb_i, emb_j) / (np.linalg.norm(emb_i) * np.linalg.norm(emb_j))
            print(f"sim({i}, {j}) = {sim:.4f}")
            if sim > threshold:
                seen.add(j)
        kept.append(data_list[i])
    return kept
if __name__ == '__main__':
    with open("style_chat_data.json", "r", encoding="utf-8") as f:
        all_data = json.loads(f.read())
        cleaned_data = deduplicate_data(all_data, 0.5)