Vision input

from openai import OpenAI
import base64
import os

api_base = "http://localhost:8080/v1"
image_path = "image.jpg"  # insert path to image
model = "kimi-latest"  # select model

client = OpenAI(
    api_key=os.environ.get("PRIVATE_MODE_API_KEY"),
    base_url=api_base,
)


def encode_image_to_base64(image_path):
    with open(image_path, "rb") as image_file:
        return base64.b64encode(image_file.read()).decode("utf-8")


if not os.path.exists(image_path):
    print(f"Error: Image file not found at {image_path}")
    exit(1)

base64_image = encode_image_to_base64(image_path)

chat_response = client.chat.completions.create(
    model=model,
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "What's in this image?"},
                {
                    "type": "image_url",
                    "image_url": {"url": f"data:image/png;base64,{base64_image}"},
                },
            ],
        }
    ],
)

print("Chat completion output:", chat_response.choices[0].message.content)

from openai import OpenAI
import base64
import os
import re

prompt = "<|grounding|>Convert the document to markdown."
# prompt = "Free OCR."

client = OpenAI(
    api_key=os.environ.get("PRIVATE_MODE_API_KEY"),
    base_url="http://localhost:8080/v1",
)


with open("document.png", "rb") as image:
    base64_image = base64.b64encode(image.read()).decode("utf-8")

response = client.chat.completions.create(
    model="deepseek-ocr-2",
    messages=[
        {
            "role": "user",
            "content": [
                {
                    "type": "image_url",
                    "image_url": {"url": f"data:image/png;base64,{base64_image}"},
                },
                {"type": "text", "text": prompt},
            ],
        }
    ],
)

content = response.choices[0].message.content

print(content)

# The following part is only relevant for the grounding prompt. With "Free OCR."
# the response carries no annotations, so it needs no post-processing.

# Grounding output interleaves layout annotations of the form
# "<|ref|>LABEL<|/ref|><|det|>[[x1, y1, x2, y2]]<|/det|>" with the markdown.
# LABEL is a layout type such as title, text, table or image.
GROUNDING_RE = re.compile(
    r"<\|ref\|>(.*?)<\|/ref\|><\|det\|>(.*?)<\|/det\|>", re.DOTALL
)
RESIDUAL_TOKENS = ("<|grounding|>", "<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>")


def strip_bounding_boxes(text: str) -> str:
    """Remove the layout annotations, leaving clean markdown."""
    text = GROUNDING_RE.sub("", text)
    for token in RESIDUAL_TOKENS:
        text = text.replace(token, "")
    return text.strip()


def extract_bounding_boxes(text: str) -> list[tuple[str, str]]:
    """Return (label, coordinates) pairs, e.g. to highlight regions in the image."""
    return GROUNDING_RE.findall(text)


stripped_markdown_content = strip_bounding_boxes(content)
bounding_boxes = extract_bounding_boxes(content)
print(bounding_boxes)

with open("output.md", "w", encoding="utf-8") as f:
    f.write(stripped_markdown_content)