%%capture
# 該魔法命令用于在 Jupyter Notebook中隱藏輸出
# 避免輸出干擾執行結果，例如 `pip install` 過程中的信息


# 安裝 Unsloth 和 vLLM 這兩個庫
!pip install unsloth vllm


# 升級 Pillow（Python Imaging Library），確保使用最新版本
!pip install --upgrade pillow

from unsloth import FastLanguageModel, PatchFastRL  
# 從 unsloth 庫中導入 FastLanguageModel（加速的語言模型）和 PatchFastRL（加速補?。? 


PatchFastRL("GRPO", FastLanguageModel)  
# 對 FastLanguageModel 應用 "GRPO" 方案的補丁優化，以提高推理效率

from unsloth import is_bfloat16_supported  # 檢測是否支持 bfloat16（更節省顯存的數值格式）
import torch  # 導入 PyTorch 進行深度學習計算


# 設置模型參數
max_seq_length = 1024  # 最大序列長度，可以增大以支持更長的推理
lora_rank = 64  # LoRA 低秩適配矩陣的秩，較大值提高智能性但降低推理速度


# 加載 FastLanguageModel 預訓練模型
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="Qwen/Qwen2.5-3B-Instruct",  # 使用 Qwen2.5-3B-Instruct 作為基礎模型
    max_seq_length=max_seq_length,  # 設置最大序列長度
    load_in_4bit=True,  # 使用 4-bit 量化（減少顯存消耗），若為 False 則使用 16-bit LoRA
    fast_inference=True,  # 啟用 vLLM 加速推理（優化并行計算）
    max_lora_rank=lora_rank,  # 設置 LoRA 適配的秩
    gpu_memory_utilization=0.5,  # 限制 GPU 內存占用，降低該值可減少顯存溢出
)


# 為模型應用 PEFT（參數高效微調）和 LoRA（低秩適配）
model = FastLanguageModel.get_peft_model(
    model,
    r=lora_rank,  # LoRA 低秩矩陣的秩，可選 8, 16, 32, 64, 128
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",  # 量化關鍵模塊：QKV 投影和輸出投影
        "gate_proj", "up_proj", "down_proj",  # MLP 相關模塊
    ],  # 若顯存不足，可以移除 QKVO 相關模塊
    lora_alpha=lora_rank,  # LoRA 適配器的 alpha 參數，通常設為與 r 相同
    use_gradient_checkpointing="unsloth",  # 啟用梯度檢查點，減少顯存占用，適用于長上下文微調
    random_state=3407,  # 設置隨機種子，保證實驗可復現
)

# 輸出 
==((====))==  Unsloth 2025.2.12: Fast Qwen2 patching. Transformers: 4.48.3.
   \\   /|    GPU: Tesla T4. Max memory: 14.741 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.5.1+cu124. CUDA: 7.5. CUDA Toolkit: 12.4. Triton: 3.1.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.28.post3. FA2 = False]
 "-____-"     Free Apache license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
Unsloth: vLLM loading unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit with actual GPU utilization = 49.66%
Unsloth: Your GPU has CUDA compute capability 7.5 with VRAM = 14.74 GB.
Unsloth: Using conservativeness = 1.0. Chunked prefill tokens = 1024. Num Sequences = 192.
Unsloth: vLLM's KV Cache can use up to 4.9 GB. Also swap space = 2 GB.
WARNING 02-17 07:17:06 config.py:2386] Casting torch.bfloat16 to torch.float16.
INFO 02-17 07:17:19 config.py:542] This model supports multiple tasks: {'classify', 'generate', 'embed', 'reward', 'score'}. Defaulting to 'generate'.
Unsloth: vLLM Bitsandbytes config using kwargs = {'load_in_8bit': False, 'load_in_4bit': True, 'bnb_4bit_compute_dtype': 'float16', 'bnb_4bit_quant_storage': 'uint8', 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': True, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'llm_int8_skip_modules': ['lm_head', 'multi_modal_projector', 'merger', 'modality_projection', 'model.layers.2.mlp', 'model.layers.3.mlp', 'model.layers.30.mlp'], 'llm_int8_threshold': 6.0}
INFO 02-17 07:17:19 llm_engine.py:234] Initializing a V0 LLM engine (v0.7.2) with config: model='unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit', speculative_cnotallow=None, tokenizer='unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revisinotallow=None, override_neuron_cnotallow=None, tokenizer_revisinotallow=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=1024, download_dir=None, load_format=LoadFormat.BITSANDBYTES, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantizatinotallow=bitsandbytes, enforce_eager=False, kv_cache_dtype=auto,  device_cnotallow=cuda:0, decoding_cnotallow=DecodingConfig(guided_decoding_backend='xgrammar'), observability_cnotallow=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_cnotallow=None, compilation_cnotallow={"level":0,"splitting_ops":[],"compile_sizes":[],"cudagraph_capture_sizes":[192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":192}, use_cached_outputs=False, 
tokenizer_config.json:?100%
?7.36k/7.36k?[00:00<00:00,?451kB/s]
vocab.json:?100%
?2.78M/2.78M?[00:00<00:00,?8.77MB/s]
merges.txt:?100%
?1.67M/1.67M?[00:00<00:00,?6.92MB/s]
tokenizer.json:?100%
?11.4M/11.4M?[00:00<00:00,?42.1MB/s]
added_tokens.json:?100%
?605/605?[00:00<00:00,?37.9kB/s]
special_tokens_map.json:?100%
?614/614?[00:00<00:00,?54.5kB/s]
generation_config.json:?100%
?271/271?[00:00<00:00,?17.2kB/s]
INFO 02-17 07:17:23 cuda.py:179] Cannot use FlashAttention-2 backend for Volta and Turing GPUs.
INFO 02-17 07:17:23 cuda.py:227] Using XFormers backend.
INFO 02-17 07:17:24 model_runner.py:1110] Starting to load model unsloth/qwen2.5-3b-instruct-unsloth-bnb-4bit...
INFO 02-17 07:17:24 loader.py:1102] Loading weights with BitsAndBytes quantization.  May take a while ...
INFO 02-17 07:17:25 weight_utils.py:252] Using model weights format ['*.safetensors']
model.safetensors:?100%
?2.36G/2.36G?[00:15<00:00,?165MB/s]
Loading?safetensors?checkpoint?shards:?100%?Completed?|?1/1?[00:04<00:00,??4.67s/it]
Loading?safetensors?checkpoint?shards:?100%?Completed?|?1/1?[00:01<00:00,??1.17s/it]
INFO 02-17 07:17:48 model_runner.py:1115] Loading model weights took 2.2160 GB
INFO 02-17 07:17:48 punica_selector.py:18] Using PunicaWrapperGPU.
INFO 02-17 07:17:58 worker.py:267] Memory profiling takes 9.77 seconds
INFO 02-17 07:17:58 worker.py:267] the current vLLM instance can use total_gpu_memory (14.74GiB) x gpu_memory_utilization (0.50) = 7.32GiB
INFO 02-17 07:17:58 worker.py:267] model weights take 2.22GiB; non_torch_memory takes 0.05GiB; PyTorch activation peak memory takes 1.05GiB; the rest of the memory reserved for KV Cache is 4.01GiB.
INFO 02-17 07:17:58 executor_base.py:110] # CUDA blocks: 7300, # CPU blocks: 3640
INFO 02-17 07:17:58 executor_base.py:115] Maximum concurrency for 1024 tokens per request: 114.06x
INFO 02-17 07:18:01 model_runner.py:1434] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.
Capturing CUDA graph shapes: 100%|██████████| 27/27 [00:43<00:00,  1.60s/it]
INFO 02-17 07:18:44 model_runner.py:1562] Graph capturing finished in 43 secs, took 0.62 GiB
INFO 02-17 07:18:44 llm_engine.py:431] init engine (profile, create kv cache, warmup model) took 56.33 seconds
tokenizer_config.json:?100%
?7.36k/7.36k?[00:00<00:00,?650kB/s]
vocab.json:?100%
?2.78M/2.78M?[00:00<00:00,?14.2MB/s]
merges.txt:?100%
?1.67M/1.67M?[00:00<00:00,?26.5MB/s]
added_tokens.json:?100%
?605/605?[00:00<00:00,?45.0kB/s]
special_tokens_map.json:?100%
?614/614?[00:00<00:00,?32.3kB/s]
tokenizer.json:?100%
?11.4M/11.4M?[00:00<00:00,?40.2MB/s]
Unsloth 2025.2.12 patched 36 layers with 36 QKV layers, 36 O layers and 36 MLP layers.

import re
from datasets import load_dataset, Dataset
# Load and prep dataset
SYSTEM_PROMPT = """
Respond in the following format:
<reasoning>
...
</reasoning>
<answer>
...
</answer>
"""
def extract_xml_answer(text: str) -> str:
    """提取 <answer> 標簽內的內容"""
    match = re.search(r"<answer>(.*?)</answer>", text, re.DOTALL)
    return match.group(1).strip() if match else text.strip()
def get_medical_questions(split="train") -> Dataset:
    """加載 FreedomIntelligence/medical-o1-reasoning-SFT 數據集并格式化"""
    data = load_dataset("FreedomIntelligence/medical-o1-reasoning-SFT", 'zh')[split]  
    def format_example(x):
        xml_answer = f"""\
<reasoning>
{x['Complex_CoT'].strip()}
</reasoning>
<answer>
{x['Response'].strip()}
</answer>"""
        
        return {
            'prompt': [
                {'role': 'system', 'content': SYSTEM_PROMPT},
                {'role': 'user', 'content': x['Question']}
            ],
            'answer': extract_xml_answer(xml_answer)  # 確保解析正確答案
        }
    data = data.map(format_example)
    return data
# 加載數據集
dataset = get_medical_questions()
# 獎勵函數
def correctness_reward_func(prompts, completions, answer, **kwargs) -> list[float]:
    """檢查模型輸出的答案是否正確"""
    responses = [completion[0]['content'] for completion in completions]
    extracted_responses = [extract_xml_answer(r) for r in responses]
    return [2.0 if r == a else 0.0 for r, a in zip(extracted_responses, answer)]
def int_reward_func(completions, **kwargs) -> list[float]:
    """檢查答案是否為整數"""
    responses = [completion[0]['content'] for completion in completions]
    extracted_responses = [extract_xml_answer(r) for r in responses]
    return [0.5 if r.isdigit() else 0.0 for r in extracted_responses]
def strict_format_reward_func(completions, **kwargs) -> list[float]:
    """嚴格格式檢查：必須有換行符"""
    pattern = r"^<reasoning>\n.*?\n</reasoning>\n<answer>\n.*?\n</answer>\n$"
    responses = [completion[0]["content"] for completion in completions]
    return [0.5 if re.match(pattern, r) else 0.0 for r in responses]
def soft_format_reward_func(completions, **kwargs) -> list[float]:
    """寬松格式檢查：允許不嚴格換行"""
    pattern = r"<reasoning>.*?</reasoning>\s*<answer>.*?</answer>"
    responses = [completion[0]["content"] for completion in completions]
    return [0.5 if re.match(pattern, r) else 0.0 for r in responses]
def count_xml(text) -> float:
    """檢查 XML 結構完整性"""
    count = 0.0
    if text.count("<reasoning>\n") == 1:
        count += 0.125
    if text.count("\n</reasoning>\n") == 1:
        count += 0.125
    if text.count("\n<answer>\n") == 1:
        count += 0.125
        count -= len(text.split("\n</answer>\n")[-1]) * 0.001
    if text.count("\n</answer>") == 1:
        count += 0.125
        count -= (len(text.split("\n</answer>")[-1]) - 1) * 0.001
    return count
def xmlcount_reward_func(completions, **kwargs) -> list[float]:
    """計算 XML 結構完整性分數"""
    contents = [completion[0]["content"] for completion in completions]
    return [count_xml(c) for c in contents]

#輸出
medical_o1_sft_Chinese.json:?100%
?64.8M/64.8M?[00:00<00:00,?120MB/s]
Generating?train?split:?100%
?24772/24772?[00:01<00:00,?12662.63?examples/s]
Map:?100%
?24772/24772?[00:04<00:00,?5735.65?examples/s]

from trl import GRPOConfig, GRPOTrainer  # 導入 GRPO 訓練配置和訓練器


# 設置 GRPO（General Reinforcement Preference Optimization）的訓練參數
training_args = GRPOConfig(
    use_vllm=True,  # 啟用 vLLM 進行推理加速，提高訓練效率
    learning_rate=5e-6,  # 設置學習率，較小值防止梯度爆炸
    adam_beta1=0.9,  # AdamW 優化器的 beta1 參數（動量項）
    adam_beta2=0.99,  # AdamW 優化器的 beta2 參數（平方梯度平滑項）
    weight_decay=0.1,  # 權重衰減，防止過擬合
    warmup_ratio=0.1,  # 預熱步數比例，避免初始學習率過高導致的不穩定
    lr_scheduler_type="cosine",  # 采用余弦退火學習率調度
    optim="adamw_8bit",  # 使用 `8-bit AdamW` 優化器，節省顯存
    logging_steps=1,  # 每 1 步記錄日志
    bf16=is_bfloat16_supported(),  # 如果支持 bfloat16，則使用 bfloat16 精度
    fp16=not is_bfloat16_supported(),  # 如果不支持 bfloat16，則使用 fp16 精度
    per_device_train_batch_size=1,  # 每個設備的批量大小，顯存不足時設為 1
    gradient_accumulation_steps=1,  # 梯度累積步數，增加到 4 可以平滑訓練
    num_generations=8,  # 生成的樣本數，減少該值可降低顯存占用
    max_prompt_length=256,  # 最大輸入提示長度
    max_completion_length=200,  # 最大模型輸出長度
    # num_train_epochs=1,  # 訓練 1 輪，實際使用時可以調整
    max_steps=10,  # 訓練的最大步數（短時間測試用）正常為250
    save_steps=10,  # 每 10 步保存一次模型 正常為250
    max_grad_norm=0.1,  # 梯度裁剪，防止梯度爆炸
    report_to="none",  # 訓練日志不上傳到 W&B（Weights & Biases）
    output_dir="outputs",  # 訓練結果的保存目錄
)

trainer = GRPOTrainer(
    model=model,  # 使用 FastLanguageModel 加載的 Qwen2.5-3B-Instruct
    processing_class=tokenizer,  # 令牌化工具（Tokenizer）
    reward_funcs=[  # 定義獎勵函數
        xmlcount_reward_func,  # XML 結構完整性評分
        soft_format_reward_func,  # 寬松格式檢查
        strict_format_reward_func,  # 嚴格格式檢查
        int_reward_func,  # 評估答案是否為整數
        correctness_reward_func,  # 檢查答案是否正確
    ],
    args=training_args,  # 訓練參數配置
    train_dataset=dataset,  # 訓練數據集（醫學問答數據）
)


trainer.train()  # 運行 GRPO 強化學習訓練

==((====))==  Unsloth - 2x faster free finetuning | Num GPUs = 1
   \\   /|    Num examples = 24,772 | Num Epochs = 1
O^O/ \_/ \    Batch size per device = 8 | Gradient Accumulation steps = 1
\        /    Total batch size = 8 | Total steps = 10
 "-____-"     Number of trainable parameters = 119,734,272
 [10/10 03:46, Epoch 0/1]
Step
Training Loss
reward
reward_std
completion_length
kl
rewards / xmlcount_reward_func
rewards / soft_format_reward_func
rewards / strict_format_reward_func
rewards / int_reward_func
rewards / correctness_reward_func
....因輸出太長，此處省略
TrainOutput(global_step=10, training_loss=2.103237093251664e-05, metrics={'train_runtime': 254.9268, 'train_samples_per_second': 0.314, 'train_steps_per_second': 0.039, 'total_flos': 0.0, 'train_loss': 2.103237093251664e-05})

text = tokenizer.apply_chat_template([
    {"role" : "user", "content" : "一個10歲男孩在患膿皮癥后出現眼瞼水腫、尿少和高血壓等癥狀，作為醫生，你認為首選的治療藥物是什么?"
},
], tokenize = False, add_generation_prompt = True)
from vllm import SamplingParams
sampling_params = SamplingParams(
    temperature = 0.8,
    top_p = 0.95,
    max_tokens = 1024,
)
output = model.fast_generate(
    [text],
    sampling_params = sampling_params,
    lora_request = None,
)[0].outputs[0].text
output

# 回答
Processed prompts: 100%|██████████| 1/1 [00:08<00:00,  8.94s/it, est. speed input: 6.94 toks/s, output: 37.73 toks/s]
'
膿皮癥（也稱為膿皰瘡或膿皰?。┦且环N由細菌感染引起的皮膚感染，雖然膿皮癥與眼瞼水腫、尿少和高血壓等癥狀聯系不大，但是這些癥狀可能提示存在其他問題，如膿皮癥并發了其他并發癥，或者出現了膿皮癥以外的其他疾病。因此，首先需要通過詳細的病史詢問、體檢、實驗室檢查（如血液檢查、尿液檢查等）和影像學檢查（如必要時進行）來明確診斷。\n\n如果確診為膿皮癥，治療通常依賴抗生素。對于10歲男孩的膿皮癥，首選的治療藥物通常是：\n\n1. **青霉素類**：對于大多數膿皮癥患者，首選青霉素類抗生素治療（如阿莫西林），尤其是對于兒童而言。\n2. **大環內酯類**：如果患者對青霉素過敏，可以考慮使用大環內酯類抗生素（如紅霉素）。\n\n如果出現眼瞼水腫、尿少和高血壓等癥狀，可能需要更進一步的評估和治療，這可能包括：\n\n- **利尿劑**：用于減少體內液體積聚，減輕水腫。\n- **血管緊張素轉換酶抑制劑（ACEI）或血管緊張素受體拮抗劑（ARB）**：用于降低血壓，尤其是如果是因為腎臟受損導致的高血壓時。\n\n需要注意的是，這些癥狀可能提示存在并發癥或其它系統性疾病，因此，必須首先由專業的醫療人員進行評估和治療。請務必咨詢專業醫生，獲得針對具體情況的專業治療建議。在沒有得到專業醫生指導的情況下，不要自行給孩子使用任何藥物。

model.save_lora("grpo_saved_lora")

text = tokenizer.apply_chat_template([
    {"role" : "system", "content" : SYSTEM_PROMPT},
    {"role" : "user", "content" : "一個10歲男孩在患膿皮癥后出現眼瞼水腫、尿少和高血壓等癥狀，作為醫生，你認為首選的治療藥物是什么?"},
], tokenize = False, add_generation_prompt = True)
from vllm import SamplingParams
sampling_params = SamplingParams(
    temperature = 0.8,
    top_p = 0.95,
    max_tokens = 1024,
)
output = model.fast_generate(
    text,
    sampling_params = sampling_params,
    lora_request = model.load_lora("grpo_saved_lora"),
)[0].outputs[0].text
output

#回答
Processed prompts: 100%|██████████| 1/1 [00:09<00:00,  9.17s/it, est. speed input: 7.42 toks/s, output: 34.04 toks/s]
'
<reasoning>\n根據病史描述，這個10男孩出現眼瞼水腫、
尿少和高血壓的癥狀，而膿皮癥是引起這些癥狀的一個可能的原因，
但是最應該考慮的是水腫和高血壓可能與感染相關的系統性炎癥反應有關。
在這種情況下，首選的治療藥物需要考慮控制感染（膿皮癥）、
控制炎癥反應以及處理高血壓?？紤]到患兒的年齡，
首選的藥物應該是安全的并且可以兼顧上述三個問題。
\n\n1. 抗生素：膿皮癥是一種細菌感染，需要使用抗生素進行治療。對于10歲的小男孩，青霉素類藥物通常是首選，因為這類抗生素對兒童來說副作用較小。
\n\n2. 非甾體抗炎藥（NSAIDs）：這類藥物可以減輕炎癥反應，幫助控制膿皮癥。例如布洛芬，它相對安全，副作用較少，適用于兒童。
\n\n3. 抗高血壓藥物：如果伴有高血壓，可能需要使用抗高血壓藥物，例如ACE抑制劑或者鈣通道阻滯劑。然而，考慮到10歲兒童的特殊性，這些藥物的使用需要謹慎，并且需要兒科醫生的指導。
\n\n綜上所述，治療首選藥物應該是青霉素類抗生素來控制膿皮癥，并且根據病情可能還需要使用NSAIDs以及兒科醫生指導下可能需要的抗高血壓藥物。
\n...\n<answer>\n首選治療藥物是青霉素類抗生素來控制膿皮癥。其他可能需要的藥物包括NSAIDs和抗高血壓藥物，但是需要兒科醫生指導。