Images in Conversations

import requests

catalog = requests.get("https://platform.ai.gloo.com/platform/v2/models", timeout=30).json()
READS_IMAGES = {m["id"] for m in catalog["data"] if m.get("supports_vision")}


def reads_images(model: str) -> bool:
    return model in READS_IMAGES

const catalog = await (await fetch("https://platform.ai.gloo.com/platform/v2/models")).json();
const READS_IMAGES = new Set<string>(
  catalog.data.filter((m: any) => m.supports_vision).map((m: any) => m.id),
);

function readsImages(model: string): boolean {
  return READS_IMAGES.has(model);
}

import hashlib
import os

import requests

API_URL = "https://platform.ai.gloo.com/ai/v2/direct/responses"
HEADERS = {"Authorization": f"Bearer {os.environ['GLOO_API_KEY']}", "Content-Type": "application/json"}
VISION_MODEL = "gloo-anthropic-claude-sonnet-5.5"
DESCRIBE = (
    "Describe this image for someone who can't see it. Copy any text in it exactly, "
    "and include every detail a follow-up question might need."
)


def ask(model: str, conversation: list) -> str:
    r = requests.post(API_URL, headers=HEADERS, json={
        "model": model, "input": conversation, "max_output_tokens": 16384,
    }, timeout=120)
    r.raise_for_status()
    return "".join(
        part.get("text", "")
        for item in r.json().get("output", [])
        if item.get("type") == "message"
        for part in item.get("content", [])
        if part.get("type") == "output_text"
    )


def describe(part: dict, descriptions: dict) -> str:
    key = hashlib.sha256(part["image_url"].encode()).hexdigest()
    if key not in descriptions:
        descriptions[key] = ask(VISION_MODEL, [
            {"role": "user", "content": [{"type": "input_text", "text": DESCRIBE}, part]},
        ])
    return descriptions[key]


def for_model(conversation: list, model: str, descriptions: dict) -> list:
    """The conversation as this model can take it: images replaced by text if it can't read them.

    descriptions belongs to this conversation: store it alongside the conversation's history.
    """
    if reads_images(model):
        return conversation
    prepared = []
    for item in conversation:
        content = item.get("content")
        if isinstance(content, list):
            content = [
                {"type": "input_text", "text": f"[Image: {describe(part, descriptions)}]"}
                if part.get("type") == "input_image" else part
                for part in content
            ]
            item = {**item, "content": content}
        prepared.append(item)
    return prepared


# descriptions = {}  # one per conversation, kept with its history
# answer = ask(model, for_model(conversation, model, descriptions))

import { createHash } from "node:crypto";

const API_URL = "https://platform.ai.gloo.com/ai/v2/direct/responses";
const HEADERS = {
  Authorization: `Bearer ${process.env.GLOO_API_KEY}`,
  "Content-Type": "application/json",
};
const VISION_MODEL = "gloo-anthropic-claude-sonnet-5.5";
const DESCRIBE =
  "Describe this image for someone who can't see it. Copy any text in it exactly, " +
  "and include every detail a follow-up question might need.";

async function ask(model: string, conversation: any[]): Promise<string> {
  const res = await fetch(API_URL, {
    method: "POST",
    headers: HEADERS,
    body: JSON.stringify({ model, input: conversation, max_output_tokens: 16384 }),
  });
  if (!res.ok) throw new Error(`Request failed: ${res.status} ${await res.text()}`);
  const body = await res.json();
  return (body.output ?? [])
    .filter((item: any) => item.type === "message")
    .flatMap((item: any) => item.content ?? [])
    .filter((part: any) => part.type === "output_text")
    .map((part: any) => part.text ?? "")
    .join("");
}

async function describe(part: any, descriptions: Map<string, string>): Promise<string> {
  const key = createHash("sha256").update(part.image_url).digest("hex");
  if (!descriptions.has(key)) {
    const text = await ask(VISION_MODEL, [
      { role: "user", content: [{ type: "input_text", text: DESCRIBE }, part] },
    ]);
    descriptions.set(key, text);
  }
  return descriptions.get(key)!;
}

/**
 * The conversation as this model can take it: images replaced by text if it can't read them.
 * `descriptions` belongs to this conversation: store it alongside the conversation's history.
 */
async function forModel(
  conversation: any[],
  model: string,
  descriptions: Map<string, string>,
): Promise<any[]> {
  if (readsImages(model)) return conversation;
  const prepared = [];
  for (const item of conversation) {
    if (!Array.isArray(item.content)) {
      prepared.push(item);
      continue;
    }
    const content = [];
    for (const part of item.content) {
      content.push(
        part.type === "input_image"
          ? { type: "input_text", text: `[Image: ${await describe(part, descriptions)}]` }
          : part,
      );
    }
    prepared.push({ ...item, content });
  }
  return prepared;
}

// const descriptions = new Map<string, string>(); // one per conversation, kept with its history
// const answer = await ask(model, await forModel(conversation, model, descriptions));