PDF → Markdown

curl https://api.brainiall.com/v1/document/pdf-to-markdown/base64 \
  -H "Authorization: Bearer $BRAINIALL_API_KEY" \
  -H "Content-Type: application/json" \
  -H "Accept: text/markdown, text/plain, application/json" \
  -d '{"pdf":"'"$(base64 < report.pdf | tr -d '\n')"'","page_range":"1-15"}'

変換してからチャンク分割し、各チャンクを自分の埋め込みインデックスへ渡します。

import os, json, base64, urllib.request

API = "https://api.brainiall.com/v1/document/pdf-to-markdown/base64"

def pdf_to_markdown(path, page_range="1-15"):
    with open(path, "rb") as handle:
        payload = json.dumps({
            "pdf": base64.b64encode(handle.read()).decode("ascii"),
            "page_range": page_range,
        }).encode("utf-8")
    req = urllib.request.Request(
        API,
        data=payload,
        headers={
            "Authorization": f"Bearer {os.environ['BRAINIALL_API_KEY']}",
            "Content-Type": "application/json",
            "Accept": "text/markdown, text/plain, application/json",
        },
        method="POST",
    )
    with urllib.request.urlopen(req) as resp:
        return resp.read().decode("utf-8")

def chunk_for_index(markdown, max_chars=1200):
    chunks, buf, size = [], [], 0
    for block in markdown.split("\n\n"):
        if size + len(block) > max_chars and buf:
            chunks.append("\n\n".join(buf))
            buf, size = [], 0
        buf.append(block)
        size += len(block)
    if buf:
        chunks.append("\n\n".join(buf))
    return chunks

# markdown = pdf_to_markdown("report.pdf")
# chunks = chunk_for_index(markdown)
# send each chunk to your embedding index