pip install --upgrade cleanlab-studio

from cleanlab_studio import Studio
studio = Studio("<CLEANLAB_API_KEY>") # 從上面獲取您的API密鑰
tlm = studio.TLM(options={"log": ["explanation"], "model": "gpt-4o"}) # GPT, Claude, etc
#設置提示
out = tlm.prompt("How many vowels are there in the word 'Abracadabra'.?")
#TLM響應包含實際輸出的“響應”、可信度評分和解釋
print(f"Model's response = {out['response']}")
print(f"Trustworthiness score = {out['trustworthiness_score']}")
print(f"Explanation = {out['log']['explanation']}")

Model's response = The word "Abracadabra" contains 6 vowels. The vowels are: A, a, a, a, a, and a.
Trustworthiness score = 0.6842228802750124
Explanation = This response is untrustworthy due to a lack of consistency in possible responses from the model. Here's one inconsistent alternate response that the model considered (which may not be accurate either):
5.

Model's response = Let me count the vowels in 'Abracadabra':
A-b-r-a-c-a-d-a-b-r-a

The vowels are: A, a, a, a, a

There are 5 vowels in the word 'Abracadabra'.
Trustworthiness score = 0.9378276048845285
Explanation = Did not find a reason to doubt trustworthiness.

python
from cleanlab_studio import Studio
import markdown
from IPython.core.display import display, Markdown

# 使用API密鑰初始化Cleanlab Studio
studio = Studio("<CLEANLAB_API_KEY>") #替換為您的實際API密鑰

# 要評估的模型列表
models = ["gpt-4o", "claude-3.5-sonnet-v2"]

# 定義提示
prompt_text = "Which one of 9.11 and 9.9 is bigger?"

# 遍歷每個模型并進行評估
for model in models:
 tlm = studio.TLM(options={"log": ["explanation"], "model": model})
 out = tlm.prompt(prompt_text)

 md_content = f"""
## 模型: {model}

**響應**: {out['response']}

**可信度評分**: {out['trustworthiness_score']}

**解釋**: {out['log']['explanation']}

---
"""
 display(Markdown(md_content))

import streamlit as st
from langchain_groq.chat_models import ChatGroq
import os
os.environ["GROQ_API_KEY"]=st.secrets["GROQ_API_KEY"]
#初始化Groq Llama即時模型
groq_llm = ChatGroq(model="deepseek-r1-distill-llama-70b", temperature=0.5)
prompt = "Which one of 9.11 and 9.9 is bigger?"
# Get the response from the model
response = groq_llm.invoke(prompt)
#初始化Cleanlab的studio
studio = Studio("226eeab91e944b23bd817a46dbe3c8ae") 
cleanlab_tlm = studio.TLM(optinotallow={"log": ["explanation"]}) #供解釋
#得到包含可信度得分和解釋的輸出
output = cleanlab_tlm.get_trustworthiness_score(prompt, respnotallow=response.content.strip())
md_content = f"""
## 模型: {model}
**Response:** {response.content.strip()}
**Trustworthiness Score:** {output['trustworthiness_score']}
**Explanation:** {output['log']['explanation']}
---
"""
display(Markdown(md_content))

pip install llama-parse llama-index-core llama-index-embeddings-huggingface 
llama-index-llms-cleanlab requests beautifulsoup4 pdfkit nest-asyncio

from llama_parse import LlamaParse
from llama_index.core import VectorStoreIndex
import requests
from bs4 import BeautifulSoup
import pdfkit
from llama_index.readers.docling import DoclingReader
from llama_index.core import Settings
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.llms.cleanlab import CleanlabTLM
from typing import Dict, List, ClassVar
from llama_index.core.instrumentation.events import BaseEvent
from llama_index.core.instrumentation.event_handlers import BaseEventHandler
from llama_index.core.instrumentation import get_dispatcher
from llama_index.core.instrumentation.events.llm import LLMCompletionEndEvent
import nest_asyncio
import os

options = {
 "model": "gpt-4o",
 "max_tokens": 512,
 "log": ["explanation"]
}
llm = CleanlabTLM(api_key="<CLEANLAB_API_KEY>", optinotallow=options) # 從https://cleanlab.ai/獲取您的免費API
Settings.llm = llm
Settings.embed_model = HuggingFaceEmbedding(
 model_name="BAAI/bge-small-en-v1.5"
)

# 可信度評分事件處理程序
class GetTrustworthinessScore(BaseEventHandler):
 events: ClassVar[List[BaseEvent]] = []
 trustworthiness_score: float = 0.0
 @classmethod
 def class_name(cls) -> str:
 return "GetTrustworthinessScore"
 def handle(self, event: BaseEvent) -> Dict:
 if isinstance(event, LLMCompletionEndEvent):
 self.trustworthiness_score = event.response.additional_kwargs.get("trustworthiness_score", 0.0)
 self.events.append(event)
 return {}

# 顯示LLM響應的輔助函數
def display_response(response):
 response_str = response.response
 trustworthiness_score = event_handler.trustworthiness_score
 print(f"Response: {response_str}")
 print(f"Trustworthiness score: {round(trustworthiness_score, 2)}")

##########################################
# 從多個URL生成PDF
##########################################
# 配置wkhtmltopdf路徑
wkhtml_path = r'C:\Program Files\wkhtmltopdf\bin\wkhtmltopdf.exe'
config = pdfkit.configuration(wkhtmltopdf=wkhtml_path)
# 定義URL和分配文檔名稱
urls = {
 "LLMs": "https://en.wikipedia.org/wiki/Large_language_model"
}
# 保存PDF的目錄
pdf_directory = "PDFs"
os.makedirs(pdf_directory, exist_ok=True)
pdf_paths = {}
for doc_name, url in urls.items():
 try:
 print(f"Processing {doc_name} from {url} ...")
 response = requests.get(url)
 soup = BeautifulSoup(response.text, "html.parser")
 main_content = soup.find("div", {"id": "mw-content-text"})
 if main_content is None:
 raise ValueError("Main content not found")
 # 將協議相對URL替換為絕對URL
 html_string = str(main_content).replace('src="http://', 'src="https://').replace('href="http://', 'href="https://')
 pdf_file_path = os.path.join(pdf_directory, f"{doc_name}.pdf")
 pdfkit.from_string(
 html_string,
 pdf_file_path,
 optinotallow={'encoding': 'UTF-8', 'quiet': ''},
 cnotallow=config
 )
 pdf_paths[doc_name] = pdf_file_path
 print(f"Saved PDF for {doc_name} at {pdf_file_path}")
 except Exception as e:
 print(f"Error processing {doc_name}: {e}")

##########################################
# 使用LlamaParse解析PDF并注入元數據
##########################################

# 定義解析指令（如果您的解析器支持）
parsing_instructions = """提取文檔的markdown格式內容。
按頁將文檔拆分為節點（例如）。
確保每個節點具有文檔名稱和頁碼的元數據。"""

# 創建LlamaParse實例
parser = LlamaParse(
 api_key="<LLAMACLOUD_API_KEY>", # 替換為您的實際密鑰
 parsing_instructinotallow=parsing_instructions,
 result_type="markdown",
 premium_mode=True,
 max_timeout=600
)
# 保存合并的Markdown文件的目錄（每個PDF一個）
output_md_dir = os.path.join(pdf_directory, "markdown_docs")
os.makedirs(output_md_dir, exist_ok=True)
# 列表，用于保存所有更新后的節點以供索引
all_nodes = []
for doc_name, pdf_path in pdf_paths.items():
 try:
 print(f"Parsing PDF for {doc_name} from {pdf_path} ...")
 nodes = parser.load_data(pdf_path) # 返回節點列表
 updated_nodes = []
 # 處理每個節點：更新元數據并在文本中注入引用標題。
 for i, node in enumerate(nodes, start=1):
 # 復制現有元數據（如果有），并添加我們自己的鍵。
 new_metadata = dict(node.metadata) if node.metadata else {}
 new_metadata["document_name"] = doc_name
 if "page_number" not in new_metadata:
 new_metadata["page_number"] = str(i)
 # 構建引用標題。
 citation_header = f"[{new_metadata['document_name']}, page {new_metadata['page_number']}]\n\n"
 # 在節點的文本前添加引用標題。
 updated_text = citation_header + node.text
 new_node = node.__class__(text=updated_text, metadata=new_metadata)
 updated_nodes.append(new_node)
 # 使用更新后的節點文本為文檔保存一個合并的Markdown文件。
 combined_texts = [node.text for node in updated_nodes]
 combined_md = "\n\n---\n\n".join(combined_texts)
 md_filename = f"{doc_name}.md"
 md_filepath = os.path.join(output_md_dir, md_filename)
 with open(md_filepath, "w", encoding="utf-8") as f:
 f.write(combined_md)
 print(f"Saved combined markdown for {doc_name} to {md_filepath}")
 # 將更新后的節點添加到全局列表以供索引。
 all_nodes.extend(updated_nodes)
 print(f"Parsed {len(updated_nodes)} nodes from {doc_name}.")
 except Exception as e:
 print(f"Error parsing {doc_name}: {e}")

##########################################

# 創建索引和查詢引擎

##########################################

# 從所有節點創建索引。

index = VectorStoreIndex.from_documents(documents=all_nodes)

# 定義一個自定義提示模板，強制包含引用。

prompt_template = """
你是一個具有主題專業知識的AI助手。
僅使用提供的上下文回答問題。
在必要時，以格式良好的Markdown格式回答，包含項目符號和章節。
如果提供的上下文不支持答案，請回復“我不知道。”
上下文：
{context_str}
問題：
{query_str}
答案：
"""
# 使用自定義提示創建查詢引擎。
query_engine = index.as_query_engine(similarity_top_k=3, llm=llm, prompt_template=prompt_template)
print("Combined index and query engine created successfully!")

query = "When is mixture of experts approach used?"
response = query_engine.query(query)
display_response(response)

query = "How do you compare Deepseek model with OpenAI's models?"
response = query_engine.query(query)
display_response(response)