Fine-Tuning 模型微调

{"messages": [
  {"role": "system", "content": "你是云数据库技术支持，回答先给结论再给步骤，不确定就明说。"},
  {"role": "user", "content": "只读实例延迟突然变高，怎么排查？"},
  {"role": "assistant", "content": "先看主从复制延迟：控制台实例页的复制延迟曲线若超过 2 秒，进入第一步；若正常，转向负载分析……"}
]}
{"messages": [
  {"role": "system", "content": "你是云数据库技术支持，回答先给结论再给步骤，不确定就明说。"},
  {"role": "user", "content": "备份期间业务卡顿正常吗？"},
  {"role": "assistant", "content": "不正常。备份走快照机制，不应阻塞写入。若观察到卡顿，按以下顺序排查……"}
]}

from datasets import load_dataset
from peft import LoraConfig
from trl import SFTConfig, SFTTrainer

# 第五节沉淀好的 JSONL，已提前切成 train / test 两个文件
dataset = load_dataset("json", data_files={
    "train": "support_train.jsonl",
    "test": "support_test.jsonl",
})

peft_config = LoraConfig(
    r=16,
    lora_alpha=32,             # 有效缩放 = alpha / r = 2
    lora_dropout=0.05,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
    task_type="CAUSAL_LM",
)

args = SFTConfig(
    output_dir="sft-support-v1",
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=8,       # 有效 batch = 2 x 8 = 16
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    warmup_ratio=0.03,
    bf16=True,
    eval_strategy="steps",
    eval_steps=50,
    save_strategy="steps",
    save_steps=50,
    save_total_limit=3,
    logging_steps=10,
    max_seq_length=2048,
    load_best_model_at_end=True,         # 训练结束回滚到 eval 最优的 checkpoint
    metric_for_best_model="eval_loss",
)

trainer = SFTTrainer(
    model="Qwen/Qwen2.5-7B-Instruct",    # 基座：从 instruct 模型继续训，不从 base 起步
    args=args,
    train_dataset=dataset["train"],
    eval_dataset=dataset["test"],
    peft_config=peft_config,
)

trainer.train()
trainer.save_model("sft-support-v1/final")   # 保存的是 adapter，几十 MB

// 偏好数据：每行一个 prompt、一个好回答、一个坏回答
{"prompt": [{"role": "user", "content": "你们的价格为什么这么贵？"}],
 "chosen":   [{"role": "assistant", "content": "您说的对，单价确实高于入门套餐。性价比要看两点：一是自动扩缩容节省的运维人力，二是 SLA 赔付。给您算一笔账……"}],
 "rejected": [{"role": "assistant", "content": "不贵啊，我们的价格很便宜的，你可以去看看别家。"}]}

from trl import DPOConfig, DPOTrainer

args = DPOConfig(
    output_dir="dpo-support-v1",
    beta=0.1,                # 偏离参考模型的惩罚强度，越小越放得开
    learning_rate=5e-6,      # 远小于 SFT，DPO 是精修不是改造
    num_train_epochs=1,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=8,
    bf16=True,
    logging_steps=10,
)

trainer = DPOTrainer(
    model="sft-support-v1/final",        # SFT 产物作底座
    ref_model=None,                      # 不传则自动从底座克隆一份作参考
    args=args,
    train_dataset=pref_dataset,
    peft_config=peft_config,
)
trainer.train()

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-7B-Instruct")
model = PeftModel.from_pretrained(base, "sft-support-v1/final")

merged = model.merge_and_unload()          # W + BA 算进原权重，旁路消失
merged.save_pretrained("sft-support-v1/merged")

AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B-Instruct").save_pretrained(
    "sft-support-v1/merged")               # tokenizer 一并带过去

vllm serve sft-support-v1/merged --served-model-name support-7b

from langchain.chat_models import init_chat_model

llm = init_chat_model(
    "openai:support-7b",
    base_url="http://llm-gpu-host:8000/v1",
    api_key="EMPTY",                      # vLLM 默认不校验，填任意串
)