!uv pip install -U -q google-genai ultralytics

import json
import cv2
import ultralytics
from google import genai
from google.genai import types
from PIL import Image
from ultralytics.utils.downloads import safe_download
from ultralytics.utils.plotting import Annotator, colors
ultralytics.checks()

# Replace "your_api_key" with your actual key
# Use GenerativeModel for newer versions of the library
# Initialize the Gemini client with your API key
client = genai.Client(api_key=”your_api_key”)

def inference(image, prompt, temp=0.5):
 """
 Performs inference using Google Gemini 2.5 Pro Experimental model.
 Args:
 image (str or genai.types.Blob): The image input, either as a base64-encoded string or Blob object.
 prompt (str): A text prompt to guide the model's response.
 temp (float, optional): Sampling temperature for response randomness. Default is 0.5.
 Returns:
 str: The text response generated by the Gemini model based on the prompt and image.
 """
 response = client.models.generate_content(
 model="gemini-2.5-pro-exp-03-25",
 cnotallow=[prompt, image], # Provide both the text prompt and image as input
 cnotallow=types.GenerateContentConfig(
 temperature=temp, # Controls creativity vs. determinism in output
 ),
 )
 return response.text # Return the generated textual response

def read_image(filename):
 image_name = safe_download(filename)
 # Read image with opencv
 image = cv2.cvtColor(cv2.imread(f"/content/{image_name}"), cv2.COLOR_BGR2RGB)
 # Extract width and height
 h, w = image.shape[:2]
 # # Read the image using OpenCV and convert it into the PIL format
 return Image.fromarray(image), w, h

def clean_results(results):
 """Clean the results for visualization."""
 return results.strip().removeprefix("```json").removesuffix("```").strip()

# Define the text prompt
prompt = """
Detect the 2d bounding boxes of objects in image.
"""
# Fixed, plotting function depends on this.
output_prompt = "Return just box_2d and labels, no additional text."
image, w, h = read_image("https://media-cldnry.s-nbcnews.com/image/upload/t_fit-1000w,f_auto,q_auto:best/newscms/2019_02/2706861/190107-messy-desk-stock-cs-910a.jpg") # Read img, extract width, height
results = inference(image, prompt + output_prompt) # Perform inference
cln_results = json.loads(clean_results(results)) # Clean results, list convert
annotator = Annotator(image) # initialize Ultralytics annotator
for idx, item in enumerate(cln_results):
 # By default, gemini model return output with y coordinates first.
 # Scale normalized box coordinates (0–1000) to image dimensions
 y1, x1, y2, x2 = item["box_2d"] # bbox post processing,
 y1 = y1 / 1000 * h
 x1 = x1 / 1000 * w
 y2 = y2 / 1000 * h
 x2 = x2 / 1000 * w
 if x1 > x2:
 x1, x2 = x2, x1 # Swap x-coordinates if needed
 if y1 > y2:
 y1, y2 = y2, y1 # Swap y-coordinates if needed
 annotator.box_label([x1, y1, x2, y2], label=item["label"], color=colors(idx, True))
Image.fromarray(annotator.result()) # display the output

# Define the text prompt
prompt = """
Detect the 2d bounding box around:
highlight the area of morning light +
PC on table
potted plant
coffee cup on table
"""
# Fixed, plotting function depends on this.
output_prompt = "Return just box_2d and labels, no additional text."
image, w, h = read_image("https://thumbs.dreamstime.com/b/modern-office-workspace-laptop-coffee-cup-cityscape-sunrise-sleek-desk-featuring-stationery-organized-neatly-city-345762953.jpg") # Read image and extract width, height
results = inference(image, prompt + output_prompt)
# Clean the results and load results in list format
cln_results = json.loads(clean_results(results))
annotator = Annotator(image) # initialize Ultralytics annotator
for idx, item in enumerate(cln_results):
 # By default, gemini model return output with y coordinates first.
 # Scale normalized box coordinates (0–1000) to image dimensions
 y1, x1, y2, x2 = item["box_2d"] # bbox post processing,
 y1 = y1 / 1000 * h
 x1 = x1 / 1000 * w
 y2 = y2 / 1000 * h
 x2 = x2 / 1000 * w
 if x1 > x2:
 x1, x2 = x2, x1 # Swap x-coordinates if needed
 if y1 > y2:
 y1, y2 = y2, y1 # Swap y-coordinates if needed
 annotator.box_label([x1, y1, x2, y2], label=item["label"], color=colors(idx, True))
Image.fromarray(annotator.result()) # display the output

# Define the text prompt
prompt = """
What's inside the image, generate a detailed captioning in the form of short
story, Make 4-5 lines and start each sentence on a new line.
"""
image, _, _ = read_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg") # Read image and extract width, height
plt.imshow(image)
plt.axis('off') # Hide axes
plt.show()
print(inference(image, prompt)) # Display the results

# Define the text prompt
prompt = """
Extract the text from the image
"""
# Fixed, plotting function depends on this.
output_prompt = """
Return just box_2d which will be location of detected text areas + label"""
image, w, h = read_image("https://cdn.mos.cms.futurecdn.net/4sUeciYBZHaLoMa5KiYw7h-1200-80.jpg") # Read image and extract width, height
results = inference(image, prompt + output_prompt)
# Clean the results and load results in list format
cln_results = json.loads(clean_results(results))
print()
annotator = Annotator(image) # initialize Ultralytics annotator
for idx, item in enumerate(cln_results):
 # By default, gemini model return output with y coordinates first.
 # Scale normalized box coordinates (0–1000) to image dimensions
 y1, x1, y2, x2 = item["box_2d"] # bbox post processing,
 y1 = y1 / 1000 * h
 x1 = x1 / 1000 * w
 y2 = y2 / 1000 * h
 x2 = x2 / 1000 * w
 if x1 > x2:
 x1, x2 = x2, x1 # Swap x-coordinates if needed
 if y1 > y2:
 y1, y2 = y2, y1 # Swap y-coordinates if needed
 annotator.box_label([x1, y1, x2, y2], label=item["label"], color=colors(idx, True))
Image.fromarray(annotator.result()) # display the output