Option Awards Stock Awards
 Name
 Option/ 
PSU/RSU 
...
...
... (truncated)
Shares, Units, or 
Other Rights That 
Have Not Vested 
($)(1)
 D. Ralston 1/20/2021 67,220(2) 3,498,129
 1/20/2021 33,610(3) 1,749,064
 2/3/2021 29,390(4) 1,529,456
 1/19/2022 64,587(5) 3,361,107
 1/19/2022 22,321(6) 1,161,585
 1/18/2023 41,668(7) 2,168,403
 1/18/2023 14,427(8) 750,781

Total Shareholder Return
 Indexed ($100)
 $50.0 \n$40.0 \n$30.0 \n$20.0 \n$10.0 \n$0.0-$10.0-$20.0
 2020 2021 2023 2022
 CEO CAP Avg. NEO CAP SLB TSR-$10.6
 $2.2 \n$32.4 \n$13.1 \n$39.4 \n$26.2 \n$17.3 \n$9.3 
 ...
 ...
 ... (truncated)
 $40.00 \n$20.00 \n$0.00
 OSX TSR
 CAP vs. Total Shareholder Return
 (SLB and OSX)
 (in millions of US dollars)
 Compensation Actually Paid

conda create -n multimodal_rag pythnotallow=3.11 
conda activate multimodal_rag

ultralytics 
git+https://github.com/illuin-tech/colpali 
groq 
pill 
pymilvus 
sentence-transformers 
uvicorn 
fastapi 
opencv-python 
pytesseract 
PyMuPDF 
pydantic 
chainlit 
pybase64 
huggingface_hub[hf_transfer]

pip install -r requirements.txt

# 下載YOLO DocLayNet用于布局檢測
HF_HUB_ENABLE_HF_TRANSFER=1 hf download hantian/yolo-doclaynet --local-dir pretrained_models/yolo-doclaynet

# 下載 ColQwen 2.5（用于 ColPALI）用于基于圖像的檢索
HF_HUB_ENABLE_HF_TRANSFER=1 hf download vidore/colqwen2.5-v0.2 --local-dir pretrained_models/colqwen2.5-v0.2

# 下載 Mixedbread Embed Large 模型用于基于文本的檢索
HF_HUB_ENABLE_HF_TRANSFER=1 hf download mixedbread-ai/mxbai-embed-large-v1 --local-dir pretrained_models/mxbai-embed-large-v1

import torch 
from Typing import Cast 
from Ultralytics import YOLO 
from transforms.utils.import_utils import is_flash_attn_2_available 
from colpali_engine.models.paligemma.colpali.processing_colpali import ColPaliProcessor 
from colpali_engine.models import ColQwen2_5, ColQwen2_5_Processor 
from sentence_transformers import SentenceTransformer 

# 定義設(shè)備
device = "cuda" if torch.cuda.is_available() else "cpu" 

# 定義知識庫源和目標(biāo)圖像目錄
document_source_dir = "document_sources"
 img_dir = "image_database"
 os.makedirs(img_dir, exist_ok= True ) # 確保目錄存在

# YOLO-12L-Doclaynet
 yolo_model = YOLO( "pretrained_models/yolo-doclaynet/yolov12l-doclaynet.pt" ) 
yolo_model = yolo_model.to(device) 

# ColQwen2.5-Colpali
 colpali_model = ColQwen2_5.from_pretrained( 
 "pretrained_models/colqwen2.5-v0.2" , 
 torch_dtype=torch.bfloat16, 
 device_map=device, # 或 "mps" 如果在 Apple Silicon 上
 attn_implementatinotallow= "flash_attention_2" 如果is_flash_attn_2_available() else None , 
 )。eval () 
colpali_processor = ColQwen2_5_Processor.from_pretrained( "pretrained_models/colqwen2.5-v0.2" ) 
processor = cast( 
 ColPaliProcessor, 
 colpali_processor) 

# Mxbai-embed-large-v1
 embed_model = SentenceTransformer( "pretrained_models/mxbai-embed-large-v1" ,device=device) 

# 定義實體顏色
ENTITIES_COLORS = { 
 "Picture" : ( 255 , 72 , 88 ), 
 "Table" : ( 128 , 0 , 128 ) 
} 

print ( "FINISH SETUP..." )

import matplotlib.pyplot as plt 
import cv2 

def display_img_array ( img_arr ): 
 image_rgb = cv2.cvtColor(img_arr, cv2.COLOR_BGR2RGB) 
 plt.figure(figsize=( 30 , 30 )) 
 plt.imshow(image_rgb) 
 plt.axis( 'off' ) 
 plt.show() 

def show_layout_detection ( detection_results, img_arr ): 
 for result indetection_results : 
 boxes = result.boxes # 獲取檢測框
 for box in boxes: 
 x, y, w, h = box.xywh[ 0 ] # 框坐標(biāo)（中心 x, y, 寬度, 高度）
 x, y, w, h = int (x), int (y), int (w), int (h) # 轉(zhuǎn)換為整數(shù)
 conf = box.conf.item() # 置信度得分
 cls = int (box.cls.item()) # 類 ID
 label = f" {yolo_model.model.names[cls]} {conf: .2 f} "
 color = ENTITIES_COLORS[yolo_model.model.names[cls]] # 獲取此類的顏色
 top_left = (x - w // 2 , y - h // 2 ) 
 bottom_right = (x + w // 2 , y + h // 2 ) # 特定類的彩色框
 cv2.rectangle(img_arr, top_left, bottom_right, color, 2 ) 
 cv2.putText(img_arr, label, (top_left[ 0 ], top_left[ 1 ] - 10 ), cv2.FONT_HERSHEY_SIMPLEX, 0.9 , color, 2 ) # 匹配文本顏色
 display_img_array(img_arr)

import fitz # PyMuPDF
import numpy as np
import cv2
import os

# 定義文件名和頁面
page_id = 38
 filename = "SLB-2023-Annual-Report.pdf" 

# 閱讀文檔
doc = fitz. open (os.path.join(document_source_dir,filename)) 
page = doc.load_page(page_id) 
pix = page.get_pixmap(dpi= 300 ) 
img_rgb = np.frombuffer(pix.samples, dtype=np.uint8).reshape((pix.height, pix.width, pix.n)) 
img_page = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR)

def layout_detection(img_doc):
 return yolo_model.predict(
 source=img_doc, 
 classes=[6,8],
 cnotallow=0.25, 
 iou=0.45)

layout_results = layout_detection(img_page)
show_layout_detection(layout_results, img_page)

def extract_and_masking_images(img_doc, layout_results):
 height, width, _ = img_doc.shape
 extracted_imgs = []
 for box in layout_results[0].boxes:
 x, y, w, h = map(int,box.xywh[0]) #矩形坐標(biāo)（中心x、y、寬度、高度）
 # 計算左上角(x_min, y_min)
 x_min = x - w // 2
 y_min = y - h // 2
 x_max = x_min + w
 y_max = y_min + h
 # 將坐標(biāo)夾緊到圖像邊界
 x_start = max(0, x_min)
 y_start = max(0, y_min)
 x_end = min(width, x_max)
 y_end = min(height, y_max)
 # 如果區(qū)域無效，則跳過
 if x_start >= x_end or y_start >= y_end:
 continue
 # 將圖像提取到extracted_imgs數(shù)組中
 extracted_imgs.append(img_doc[y_start:y_end, x_start:x_end].copy())
 #將區(qū)域設(shè)置為白色
 img_doc[y_start:y_end, x_start:x_end] = [255, 255, 255]
 return extracted_imgs, img_doc

extracted_imgs, img_page = extract_and_masking_images(img_page, layout_results)
display_img_array(img_page)

import os
import cv2

def save_img_files(extracted_imgs, filename, page_id):
 #目標(biāo)路徑
 save_path = os.path.join(img_dir, filename, f"page_{page_id}/")
 # 確保目錄存在
 os.makedirs(os.path.dirname(save_path), exist_ok=True)
 # 保存圖像
 for i in range(len(extracted_imgs)):
 cv2.imwrite(save_path+f"fig_{i}.jpg", extracted_imgs[i])

save_img_files(extracted_imgs, filename, page_id)

import pytesseract

text = pytesseract.image_to_string(img_page)
print(text)

Short-Term Cash Incentive Awards

We pay performance-based short-term (annual) cash incentives to
our executives to foster a results-driven, pay-for-performance culture,
and to align executives’ interests with those of our shareholders. STI
awards are earned according to the achievement of quantitative
Company financial and non-financial objectives, as well as strategic
objectives. Our Compensation Committee selects performance
measures that it believes support our strategy and strike a balance
between motivating our executives to increase near-term financial and
operating results and driving profitable long-term Company growth
and value for shareholders.

2022 STI Opportunity Mix

Compensation Discussion and Analysis

For 2023, 70% of our NEOs’ target STI opportunity was based on
achieving quantitative Company financial objectives, 10% was based
on achieving quantitative Company non-financial objectives, and 20%
was based on strategic personal objectives. The financial portion of
the target plan was evenly split between adjusted EBITDA and free
cash flow performance goals. The total maximum STI payout for 2023
was 200% of target—consistent with 2022—and the weighted payout
range for each metric as a percentage of target is reflected by the outer
bars in the 2023 STI Opportunity Mix chart below.

2023 STI Opportunity Mix

?>

?>

In January 2023, our Compensation Committee determined to leave the target STI opportunity for all NEOs unchanged from 2022, following
a review of market data indicating that our NEOs’ target STI opportunity (as a percentage of base salary) was competitively positioned. As a
result, the 2023 target STI opportunity for our CEO was 150% of his base salary and for our other NEOs it was 100% of base salary.

The following table reflects our NEOs’ full-year 2023 STI results, together with relevant weightings of the different components and payouts

under each component.

(1) Equals the sum of the financial, non-financial, and personal portions of the STI achieved, shown as a percentage of base salary.
(2) In January 2024, due to factors not contemplated in the 2023 forecast, our Compensation Committee applied a discretionary downward
adjustment to reduce all executive payouts by 5% under our 2023 STI plan.

2024 Proxy Statement

from pymilvus import MilvusClient, DataType
import numpy as np
import concurrent.futures
import os
import base64

class MilvusColbertRetriever:
 def __init__(self, milvus_client, collection_name, img_dir, dim=128):
 #使用Milvus客戶端、集合名稱和向量嵌入的維度初始化檢索器。
 # If the collection exists, load it.
 self.collection_name = collection_name
 self.client = milvus_client
 if self.client.has_collection(collection_name=self.collection_name):
 self.client.load_collection(collection_name)
 self.dim = dim
 self.img_dir = img_dir

 def create_collection(self):
 # 在Milvus中創(chuàng)建一個新的集合來存儲嵌入。
 #如果現(xiàn)有集合已存在，請刪除該集合，并為該集合定義架構(gòu)。
 if self.client.has_collection(collection_name=self.collection_name):
 self.client.drop_collection(collection_name=self.collection_name)
 schema = self.client.create_schema(auto_id=True, enable_dynamic_field=True)
 schema.add_field(field_name="pk", datatype=DataType.INT64, is_primary=True)
 schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self.dim)
 schema.add_field(field_name="seq_id", datatype=DataType.INT16)
 schema.add_field(field_name="doc_id", datatype=DataType.INT64)
 schema.add_field(field_name="doc", datatype=DataType.VARCHAR, max_length=65535)
 self.client.create_collection(collection_name=self.collection_name, schema=schema)

 def create_index(self):
 # 在向量字段上創(chuàng)建索引，以實現(xiàn)快速相似性搜索。
 # 在使用指定參數(shù)創(chuàng)建新索引之前，釋放并刪除任何現(xiàn)有索引。
 self.client.release_collection(collection_name=self.collection_name)
 self.client.drop_index(collection_name=self.collection_name, index_name="vector")
 index_params = self.client.prepare_index_params()
 index_params.add_index(
 field_name="vector",
 index_name="vector_index",
 index_type="IVF_FLAT", 
 metric_type="IP", 
 )
 self.client.create_index(collection_name=self.collection_name, index_params=index_params, sync=True)

 def search(self, data, topk):
 # 對集合執(zhí)行向量搜索，以找到前k個最相似的文檔。
 search_params = {"metric_type": "IP", "params": {}}
 results = self.client.search(
 self.collection_name,
 data,
 limit=int(50),
 output_fields=["vector", "seq_id", "doc_id","$meta"],
 search_params=search_params,
 )
 doc_meta = {}
 for r_id in range(len(results)):
 for r in range(len(results[r_id])):
 entity = results[r_id][r]["entity"]
 doc_id = entity["doc_id"]
 if doc_id not in doc_meta:
 doc_meta[doc_id] = {
 "page_id": entity["page_id"],
 "fig_id": entity["fig_id"],
 "filename": entity["filename"],
 }
 scores = []

 def rerank_single_doc(doc_id, data, client, collection_name):
 #通過檢索單個文檔的嵌入并計算其與查詢的相似度來對其重新排序。
 doc_colbert_vecs = client.query(
 collection_name=collection_name,
 filter=f"doc_id in [{doc_id}]",
 output_fields=["seq_id", "vector", "doc"],
 limit=1000,
 )
 doc_vecs = np.vstack(
 [doc_colbert_vecs[i]["vector"] for i in range(len(doc_colbert_vecs))]
 )
 score = np.dot(data, doc_vecs.T).max(1).sum()
 return (score, doc_id)

 with concurrent.futures.ThreadPoolExecutor(max_workers=300) as executor:
 futures = {
 executor.submit(
 rerank_single_doc, doc_id, data, self.client, self.collection_name
 ): doc_id
 for doc_id in doc_meta.keys()
 }
 for future in concurrent.futures.as_completed(futures):
 score, doc_id = future.result()
 meta = doc_meta[doc_id]
 img_path = os.path.join(self.img_dir, meta["filename"], f"page_{meta['page_id']}", f"fig_{meta['fig_id']}.jpg")
 with open(img_path, "rb") as f:
 img_base64 = base64.b64encode(f.read()).decode('utf-8')
 scores.append({
 "score":float(score), 
 "page_id": meta["page_id"],
 "fig_id": meta["fig_id"],
 "filename": meta["filename"],
 "content": img_base64})

 scores.sort(key=lambda x: x["score"], reverse=True)
 if len(scores) >= topk:
 return scores[:topk]
 else:
 return scores

 def insert(self, data):
 # 將文檔的ColBERT嵌入和元數(shù)據(jù)插入集合中。
 #將數(shù)據(jù)作為多個向量（每個序列一個）與相應(yīng)的元數(shù)據(jù)一起插入。
 colbert_vecs = [vec for vec in data["colbert_vecs"]]
 seq_length = len(colbert_vecs)
 self.client.insert(
 self.collection_name,
 [
 {
 "vector": colbert_vecs[i],
 "seq_id": i,
 "doc_id": data["doc_id"] ,
 "doc": "",
 "page_id": data["page_id"],
 "fig_id": data["fig_id"],
 "filename": data["filename"],
 }
 for i in range(seq_length)
 ],
 )

class MilvusBasicRetriever:
 def __init__(self, milvus_client, collection_name, dim=1024):
 # 使用Milvus客戶端、集合名稱和向量嵌入的維度初始化檢索器。
 #如果集合存在，則加載之。
 self.collection_name = collection_name
 self.client = milvus_client
 if self.client.has_collection(collection_name=self.collection_name):
 self.client.load_collection(collection_name)
 self.dim = dim

 def normalize(self, vec):
 #把向量規(guī)范化
 norm = np.linalg.norm(vec)
 if norm == 0:
 return vec
 return vec / norm

 def create_collection(self):
 # 在Milvus中創(chuàng)建一個新的集合來存儲嵌入。
 #如果現(xiàn)有集合已存在，請刪除該集合，并為該集合定義架構(gòu)。
 if self.client.has_collection(collection_name=self.collection_name):
 self.client.drop_collection(collection_name=self.collection_name)
 schema = self.client.create_schema(auto_id=True, enable_dynamic_field=True)
 schema.add_field(field_name="pk", datatype=DataType.INT64, is_primary=True)
 schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self.dim)
 schema.add_field(field_name="content", datatype=DataType.VARCHAR, max_length=65535)
 self.client.create_collection(collection_name=self.collection_name, schema=schema)

 def create_index(self):
 #在向量字段上創(chuàng)建索引，以實現(xiàn)快速相似性搜索。
 # 在使用指定參數(shù)創(chuàng)建新索引之前，釋放并刪除任何現(xiàn)有索引。
 self.client.release_collection(collection_name=self.collection_name)
 self.client.drop_index(collection_name=self.collection_name, index_name="vector")
 index_params = self.client.prepare_index_params()
 index_params.add_index(
 field_name="vector",
 index_name="vector_index",
 index_type="IVF_FLAT", # or any other index type you want
 metric_type="IP", # or the appropriate metric type
 )
 self.client.create_index(collection_name=self.collection_name, index_params=index_params, sync=True)

 def search(self, data, topk):
 #對集合執(zhí)行向量搜索，以找到前k個最相似的文檔。
 normalized_data = self.normalize(data)
 search_params = {"metric_type": "IP", "params": {}}
 results = self.client.search(
 self.collection_name,
 [normalized_data],
 limit=topk,
 output_fields=["vector", "content","$meta"],
 search_params=search_params,
 )
 return_arr = []
 for hit in results[0]:
 return_arr.append({
 "score":hit.distance,
 "page_id":hit["entity"]["page_id"],
 "filename":hit["entity"]["filename"],
 "content":hit["entity"]["content"]
 })
 return return_arr

 def insert(self, data):
 data["vector"] = self.normalize(np.array(data["vector"])).tolist()
 self.client.insert(
 self.collection_name,
 [data]
 )

client = MilvusClient("milvus_file.db")
colbert_retriever = MilvusColbertRetriever(collection_name="colbert", milvus_client=client,img_dir=img_dir)
basic_retriever = MilvusBasicRetriever(collection_name="basic", milvus_client=client)

colbert_retriever.create_collection() 
colbert_retriever.create_index() 
basic_retriever.create_collection() 
basic_retriever.create_index()

from colpali_engine.utils.torch_utils import ListDataset
from torch.utils.data import DataLoader
from typing import List
from tqdm import tqdm
from PIL import Image

def create_image_embedding_loader(extracted_imgs):
 images = [Image.fromarray(img_arr) for img_arr in extracted_imgs]
 dataloader_images = DataLoader(
 dataset=ListDataset[str](images),
 batch_size=1,
 shuffle=False,
 collate_fn=lambda x: processor.process_images(x),
 )
 ds: List[torch.Tensor] = []
 for batch_doc in tqdm(dataloader_images):
 with torch.no_grad():
 batch_doc = {k: v.to(colpali_model.device) for k, v in batch_doc.items()}
 embeddings_doc = colpali_model(**batch_doc)
 ds.extend(list(torch.unbind(embeddings_doc.to("cpu"))))
 return ds

embedding_loader = create_image_embedding_loader(extracted_imgs)
embedding_loader

[tensor([[-0.0055, 0.0991, -0.0903, ..., -0.0474, -0.0042, -0.1138],
 [-0.0067, 0.1064, -0.0488, ..., -0.0723, 0.0535, -0.0986],
 [-0.0200, 0.1113, -0.1084, ..., -0.0747, 0.0447, -0.0786],
 ...,
 [-0.0027, 0.0811, -0.1602, ..., 0.0354, -0.0112, -0.1670],
 [-0.0557, -0.1099, 0.0128, ..., 0.0203, -0.0728, -0.0688],
 [ 0.1025, 0.0145, -0.0420, ..., 0.0894, -0.0413, 0.1650]], dtype=torch.bfloat16),
 tensor([[-0.0055, 0.0991, -0.0903, ..., -0.0474, -0.0042, -0.1138],
 [-0.0067, 0.1064, -0.0488, ..., -0.0723, 0.0535, -0.0986],
 [-0.0200, 0.1113, -0.1084, ..., -0.0747, 0.0447, -0.0786],
 ...,
 [-0.0141, 0.0645, -0.1377, ..., 0.0430, -0.0061, -0.1338],
 [-0.0835, -0.1094, 0.0049, ..., 0.0211, -0.0608, -0.0645],
 [ 0.1396, 0.0549, -0.0669, ..., 0.0942, 0.0038, 0.1514]], dtype=torch.bfloat16),
 tensor([[-0.0053, 0.0996, -0.0894, ..., -0.0471, -0.0042, -0.1128],
 [-0.0068, 0.1060, -0.0491, ..., -0.0713, 0.0532, -0.0986],
 [-0.0204, 0.1118, -0.1089, ..., -0.0752, 0.0444, -0.0791],
 ...,
 [ 0.0330, 0.0398, -0.0505, ..., 0.0586, 0.0250, -0.1099],
 [-0.0508, -0.0981, -0.0126, ..., 0.0183, -0.0791, -0.0713],
 [ 0.1387, 0.0698, -0.0330, ..., 0.0238, 0.0923, 0.0337]], dtype=torch.bfloat16)]

import random

for i in range(len(extracted_imgs)):
 data = {
 "colbert_vecs": embedding_loader[i].float().numpy(),
 "doc_id": random.getrandbits(63),
 "page_id": page_id,
 "fig_id": i,
 "filename": filename,
 }
 colbert_retriever.insert(data)

data = {
 "vector": embed_model.encode(text),
 "content": text,
 "page_id": page_id,
 "filename": filename
}
basic_retriever.insert(data)

query = "O.Le Peuch Payout Results in percentage according to SLB Financial Objectives"
batch_query = colpali_processor.process_queries([query]).to(device)
embeddings_query = torch.unbind(colpali_model(**batch_query).to("cpu"))[0].float().numpy()
colbert_retriever_result = colbert_retriever.search(embeddings_query, topk=3)
colbert_retriever_result

[{ 'score' : 20.13466208751197, 
 'page_id' : 38, 
 'fig_id' : 1, 
 'filename' : 'SLB-2023-Annual-Report.pdf' , 
 'content' : '/9j/4AAQSkZJRgABAQAAAQABAAD/...' }, 
{ 'score' : 20.13466208751197, 
 'page_id' : 38, 
 'fig_id' : 1, 
 'filename' : 'SLB-2023-Annual-Report.pdf' , 
 'content' : '/9j/4AAQSkZJRgABAQAAAQABAAD/...' }, 
{ 'score' : 15.088707166083623，
 'page_id'：41，
 'fig_id'：1，
 'filename'：'SLB-2023-Annual-Report.pdf'，
 'content'：'/9j/4AAQSkZJRgABAQAAAQABAAD/...' }]

query = "Potential Payout as a % of Target Opportunity"
basic_retriever_result = basic_retriever.search(embed_model.encode(query), topk=3)
basic_retriever_result

[{'score': 0.6565427184104919,
 'page_id': 38,
 'filename': 'SLB-2023-Annual-Report.pdf',
 'content': 'Short-Term Cash Incentive Awards\n\nWe pay performance-based short-term (annual) cash incentives to\nour ... (truncated)'},
 {'score': 0.6533020734786987,
 'page_id': 40,
 'filename': 'SLB-2023-Annual-Report.pdf',
 'content': "Free Cash Flow Targets and Results\n\nIn January 2023, our Compensation Committee considered SLB's\n2022 ... (truncated)"},
 {'score': 0.6505128145217896,
 'page_id': 59,
 'filename': 'SLB-2023-Annual-Report.pdf',
 'content': "Executive Compensation Tables\n\nChange in Control\n\nUnder our omnibus incentive plans, in the event of ... (truncated)"}]

import os
from groq import Groq

# Groq API-Llama4
os.environ["GROQ_API_KEY"] = "<your-api-key>"
client_groq = Groq()

def url_conversion(img_base64):
 return f"data:image/jpeg;base64,{img_base64}"

def llama4_inference(messages, token=1024):
 completion = client_groq.chat.completions.create(
 model="meta-llama/llama-4-maverick-17b-128e-instruct",
 messages=messages,
 temperature=0.1,
 max_completion_tokens=token,
 top_p=1,
 stream=True,
 stop=None,
 )
 inference_result = ""
 for chunk in completion:
 chunk_inference = chunk.choices[0].delta.content or ""
 inference_result += chunk_inference
 text = inference_result
 return text

user_query = "I want to know the payout"
batch_query = colpali_processor.process_queries([user_query]).to(device)
embeddings_query = torch.unbind(colpali_model(**batch_query).to("cpu"))[0].float().numpy()
colbert_retriever_result = colbert_retriever.search(embeddings_query, topk=3)
basic_retriever_result = basic_retriever.search(embed_model.encode(user_query), topk=3)

system_instruction = """
You are a helpful assistant designed to answer user queries based on document-related content.

You will be provided with two types of context:
1. Text-based context — extracted textual content from documents.
2. Image-based context — visual content (e.g., figures, tables, or screenshots) extracted from documents.

Your tasks are:
- Analyze the user query and determine the appropriate response using the available context.
- Decide whether the answer requires information from the image-based context.

If the image context is necessary to answer the query:
- Set "need_image" to True.
- Set "image_index" to the appropriate index of the image used (e.g., 0 for the first image, 1 for the second, and so on).
- Include a clear explanation or reasoning in the response.

If the image context is **not** needed:
- Set "need_image" to False.
- Set "image_index" to -1.

All responses **must be returned in strict JSON format**:
{"response": <string>, "need_image": <true|false>, "image_index": <int>}

If you are unsure or cannot answer based on the given context, clearly state that you do not know.

Examples:
{"response": "The chart in image 1 shows the revenue trend.", "need_image": true, "image_index": 1}
{"response": "The policy details are outlined in the text section.", "need_image": false, "image_index": -1}
"""

#定義有效載荷內(nèi)容
payload_content = [{
 "type": "text",
 "text": f"User Query: {user_query}"
 }]

# 構(gòu)造正在檢索的圖像URL
for i in range(len(colbert_retriever_result)):
 img_payload = {
 "type": "image_url",
 "image_url": {"url":url_conversion(colbert_retriever_result[i]["content"])}
 }
 payload_content.append(img_payload)

# 構(gòu)建基于文本的上下文
for i in range(len(basic_retriever_result)):
 txt_payload = {
 "type": "text",
 "text": f"Text-based Context #{i+1}:\n{basic_retriever_result[i]['content']}"
 }
 payload_content.append(txt_payload)

#創(chuàng)建最終消息形式
messages = [
 {
 "role": "system",
 "content": system_instruction
 },
 {
 "role": "user",
 "content": payload_content
 }
]

[{'role': 'system',
 'content': '\nYou are a helpful assistant designed to answer user queries based on document-related content.\n\nYou will be provided with two types of context:\n1. Text-based ... (truncated)'},
 {'role': 'user',
 'content': [{'type': 'text',
 'text': 'User Query: I want to know the payout'},
 {'type': 'image_url',
 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/...'}},
 {'type': 'image_url',
 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/...'}},
 {'type': 'image_url',
 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/...'}},
 {'type': 'text',
 'text': 'Text-based Context #1:\nExecutive Compensation Tables\n\nSummary Compensation Table\n\nThe following table sets forth information regarding the total compensation ... (truncated)'},
 {'type': 'text',
 'text': 'Text-based Context #2:\nExecutive Compensation Tables\n\nGrants of Plan-Based Awards in 2023\n\nThe following table provides additional information regarding cash ... (truncated)'},
 {'type': 'text',
 'text': 'Text-based Context #3:\nPay vs. Performance Comparison\n\nPay vs. Performance Comparison\n\nAs discussed in the CD&A above, our Compensation Committee has implemented ... (truncated)'}]}

import json
import re 

chat_result = llama4_inference(messages) 
chat_result = re.findall( r'\{[^{}]+\}' , chat_result) 
chat_result = json.loads(chat_result[- 1 ]) 
chat_result

{'response': 'The payout varies based on the performance metric. For Relative TSR Percentile Rank, Delta ROCE, and FCF Margin, the payouts are illustrated in the provided graphs. For example, at a Relative TSR Percentile Rank of 60%, the payout is 60%; at a Delta ROCE of 0 bps, the payout is 100%; and at an FCF Margin of 10%, the payout is 100%.',
 'need_image': True,
 'image_index': 0}

if chat_result[ "need_image" ]: 
 img_content = colbert_retriever_result[chat_result[ 'image_index' ]][ 'content' ] 
else : 
 img_content = ""

 output_response = { 
 "response" :chat_result[ "response" ], 
 "need_image" :chat_result[ "need_image" ], 
 "img_base64" :img_content 
 } 
output_response

{'response': 'The payout varies based on the performance metric. For Relative TSR Percentile Rank, Delta ROCE, and FCF Margin, the payouts are illustrated in the provided graphs. For example, at a Relative TSR Percentile Rank of 60%, the payout is 60%; at a Delta ROCE of 0 bps, the payout is 100%; and at an FCF Margin of 10%, the payout is 100%.',
 'need_image': True,
 'img_base64': '/9j/4AAQSkZJRgABAQAAAQABAAD/...'}