TUI Image Gen
This commit is contained in:
parent
2b034f8987
commit
abe64e46a4
@ -45,6 +45,15 @@ Kamu dapat bertindak sebagai coding agent yang membantu software engineering tas
|
||||
- Format yang didukung: jpg, jpeg, png, gif, webp, bmp.
|
||||
- Ukuran max 10MB.
|
||||
|
||||
## Image Generation
|
||||
|
||||
- Gunakan `generate_image` untuk membuat gambar dari teks prompt.
|
||||
- Simpan ke path yang diminta user (misal ~/Downloads).
|
||||
- Model default ada di config, cukup specify prompt dan output_path.
|
||||
- Support aspect ratio (1:1, 16:9, dll) dan resolution (512, 1K, 2K, 4K).
|
||||
- Support image-to-image: gunakan parameter `input_images` dengan list path/URL gambar referensi.
|
||||
- Untuk workflow berkelanjutan (modifikasi gambar sebelumnya), gunakan path file dari response sebelumnya sebagai `input_images`.
|
||||
|
||||
## Git Policy
|
||||
|
||||
- Kamu boleh menjalankan `git status`, `git diff`, `git log` secara bebas untuk inspeksi.
|
||||
|
||||
33
config.py
33
config.py
@ -126,6 +126,39 @@ TYPING_SPEED = float( _yaml_get("delay", "typing_speed", default="15.0" ) )
|
||||
TYPING_MAX = float( _yaml_get("delay", "typing_max", default="10.0" ) )
|
||||
|
||||
|
||||
# ─── Image Generation (OpenRouter) ──────────────────────────────────────────────
|
||||
|
||||
_imagegen_providers = _yaml_get("imagegen", "providers", default=[])
|
||||
IMAGEGEN_MODELS_ITEMS = []
|
||||
|
||||
for prov in _imagegen_providers:
|
||||
pname = prov.get("name", "")
|
||||
base_url = prov.get("base_url", "").rstrip("/")
|
||||
api_key = prov.get("api_key", "")
|
||||
models = prov.get("models", [])
|
||||
for m in models:
|
||||
model_name = m.get("name", "")
|
||||
is_default = m.get("default", False)
|
||||
IMAGEGEN_MODELS_ITEMS.append({
|
||||
"model" : model_name,
|
||||
"provider" : pname,
|
||||
"base_url" : base_url,
|
||||
"api_key" : api_key,
|
||||
"default" : is_default,
|
||||
})
|
||||
|
||||
IMAGEGEN_MODEL = ""
|
||||
IMAGEGEN_API_KEY = ""
|
||||
for item in IMAGEGEN_MODELS_ITEMS:
|
||||
if item.get("default"):
|
||||
IMAGEGEN_MODEL = item["model"]
|
||||
IMAGEGEN_API_KEY = item["api_key"]
|
||||
break
|
||||
if not IMAGEGEN_MODEL and IMAGEGEN_MODELS_ITEMS:
|
||||
IMAGEGEN_MODEL = IMAGEGEN_MODELS_ITEMS[0]["model"]
|
||||
IMAGEGEN_API_KEY = IMAGEGEN_MODELS_ITEMS[0]["api_key"]
|
||||
|
||||
|
||||
# ─── Character Loader ──────────────────────────────────────────────────────────
|
||||
|
||||
CHARACTERS_DIR = Path(__file__).resolve().parent / "agent" / "characters"
|
||||
|
||||
@ -5,7 +5,7 @@ import threading, time, signal
|
||||
|
||||
import config
|
||||
|
||||
from tools import coder, carrack, ragroleplay, vision
|
||||
from tools import coder, carrack, ragroleplay, vision, imagegen
|
||||
|
||||
from services.xmpp_client import XMPPClient
|
||||
from services.telegram_client import TelegramClient
|
||||
@ -37,6 +37,9 @@ tools_definition = [
|
||||
# Vision Tools
|
||||
gadget.tools_mapping( schema = vision.schema_describe_image, handler = vision.describe_image ),
|
||||
|
||||
# Image Generation Tools
|
||||
gadget.tools_mapping( schema = imagegen.schema_generate_image, handler = imagegen.generate_image ),
|
||||
|
||||
# Rag Roleplay Tools
|
||||
gadget.tools_mapping( schema = ragroleplay.schema_users_store, handler = ragroleplay.users_store ),
|
||||
gadget.tools_mapping( schema = ragroleplay.schema_user_load, handler = ragroleplay.user_load ), # s_load
|
||||
|
||||
@ -168,9 +168,11 @@ def _agent_loop(app):
|
||||
else:
|
||||
_add_msg(app, "tool", str(result), tool_call_id=tc["id"])
|
||||
else:
|
||||
if response.content:
|
||||
has_content = bool(response.content) or (isinstance(response.content, list) and len(response.content) > 0)
|
||||
if has_content:
|
||||
if isinstance(response.content, list):
|
||||
log(app, "ai", "[Image generated]")
|
||||
_add_msg(app, "assistant", response.content)
|
||||
# Placeholder sudah terupdate via streaming, jangan log lagi
|
||||
log(app, "sep", "")
|
||||
ntro.end(stamp)
|
||||
app.agent_done.set()
|
||||
|
||||
@ -72,9 +72,16 @@ def run_agent_loop(session, llm_client, TOOLS, TOOL_HANDLERS, max_iterations, on
|
||||
'content': str(result),
|
||||
})
|
||||
else:
|
||||
if response.content:
|
||||
print(f'[{_ts()}] Response generated ({len(response.content)} chars)', flush=True)
|
||||
has_content = bool(response.content) or (isinstance(response.content, list) and len(response.content) > 0)
|
||||
if has_content:
|
||||
session.messages.append({'role': 'assistant', 'content': response.content})
|
||||
if isinstance(response.content, list):
|
||||
text_parts = [b.get('text', '') for b in response.content
|
||||
if isinstance(b, dict) and b.get('type') == 'text']
|
||||
text = '\n'.join(text_parts) if text_parts else "[Image generated]"
|
||||
print(f'[{_ts()}] Response generated (multimodal)', flush=True)
|
||||
return text, session_ended
|
||||
print(f'[{_ts()}] Response generated ({len(response.content)} chars)', flush=True)
|
||||
return response.content, session_ended
|
||||
return None, False
|
||||
|
||||
|
||||
@ -51,6 +51,7 @@ class LLMClient:
|
||||
full_content = ""
|
||||
full_tool_calls = []
|
||||
reasoning_content = ""
|
||||
full_multimodal_content = []
|
||||
|
||||
try:
|
||||
self.cancel_requested = False
|
||||
@ -108,17 +109,28 @@ class LLMClient:
|
||||
if 'function' in tc and 'arguments' in tc['function']:
|
||||
full_tool_calls[idx]['function']['arguments'] += tc['function']['arguments']
|
||||
|
||||
# Stream content (text response)
|
||||
# Stream content (text response atau multimodal blocks)
|
||||
if 'content' in delta:
|
||||
chunk_text = delta['content'] or ""
|
||||
full_content += chunk_text
|
||||
|
||||
# Callback untuk streaming ke UI
|
||||
if on_stream_chunk and chunk_text:
|
||||
on_stream_chunk(chunk_text)
|
||||
chunk = delta['content']
|
||||
if isinstance(chunk, str):
|
||||
full_content += chunk or ""
|
||||
if on_stream_chunk and chunk:
|
||||
on_stream_chunk(chunk)
|
||||
elif isinstance(chunk, list):
|
||||
full_multimodal_content.extend(chunk)
|
||||
for block in chunk:
|
||||
if isinstance(block, dict) and block.get('type') == 'text':
|
||||
t = block.get('text', '')
|
||||
if t:
|
||||
full_content += t
|
||||
if on_stream_chunk:
|
||||
on_stream_chunk(t)
|
||||
|
||||
# Build final response
|
||||
message = {'content': full_content}
|
||||
if full_multimodal_content:
|
||||
message = {'content': full_multimodal_content}
|
||||
else:
|
||||
message = {'content': full_content}
|
||||
|
||||
if full_tool_calls:
|
||||
# Filter tool_calls yang valid (ada name dan arguments)
|
||||
|
||||
136
tools/imagegen.py
Normal file
136
tools/imagegen.py
Normal file
@ -0,0 +1,136 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import base64
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import config
|
||||
|
||||
schema_generate_image = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "generate_image",
|
||||
"description": "Generate an image from a text prompt using AI image generation. Saves the result to a file.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "Text description of the image to generate"
|
||||
},
|
||||
"output_path": {
|
||||
"type": "string",
|
||||
"description": "Directory or full path to save the image (e.g. ~/Downloads, ~/Downloads/myart.png)"
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "OpenRouter model slug (optional, uses default from config)"
|
||||
},
|
||||
"aspect_ratio": {
|
||||
"type": "string",
|
||||
"description": "Aspect ratio: 1:1, 16:9, 9:16, 4:3, 3:4, etc. (optional)"
|
||||
},
|
||||
"resolution": {
|
||||
"type": "string",
|
||||
"description": "Resolution tier: 512, 1K, 2K, 4K (optional)"
|
||||
},
|
||||
"input_images": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of image paths or URLs to use as reference for image-to-image generation (optional)"
|
||||
}
|
||||
},
|
||||
"required": ["prompt", "output_path"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def generate_image(prompt, output_path, model=None, aspect_ratio=None, resolution=None, input_images=None):
|
||||
if not config.IMAGEGEN_API_KEY:
|
||||
return "Error: No OpenRouter API key configured. Set imagegen.provider in config.yaml"
|
||||
if not config.IMAGEGEN_MODEL and not model:
|
||||
return "Error: No image model configured. Set imagegen.default_model in config.yaml or specify model in tool call"
|
||||
|
||||
model = model or config.IMAGEGEN_MODEL
|
||||
|
||||
payload = {"model": model, "prompt": prompt}
|
||||
if aspect_ratio:
|
||||
payload["aspect_ratio"] = aspect_ratio
|
||||
if resolution:
|
||||
payload["resolution"] = resolution
|
||||
|
||||
# Build input_references dari paths/URLs
|
||||
if input_images:
|
||||
references = []
|
||||
for img_ref in input_images:
|
||||
img_ref = img_ref.strip()
|
||||
if not img_ref:
|
||||
continue
|
||||
if img_ref.startswith("http://") or img_ref.startswith("https://"):
|
||||
references.append({"type": "image_url", "image_url": {"url": img_ref}})
|
||||
else:
|
||||
full = os.path.expanduser(img_ref)
|
||||
if not os.path.isfile(full):
|
||||
return f"Error: reference image not found: {full}"
|
||||
with open(full, "rb") as f:
|
||||
b64 = base64.b64encode(f.read()).decode("ascii")
|
||||
ref_ext = os.path.splitext(full)[1].lower()
|
||||
ref_mime = {".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".webp": "image/webp"}.get(ref_ext, "image/png")
|
||||
references.append({"type": "image_url", "image_url": {"url": f"data:{ref_mime};base64,{b64}"}})
|
||||
if references:
|
||||
payload["input_references"] = references
|
||||
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
"https://openrouter.ai/api/v1/images",
|
||||
data=data,
|
||||
method="POST",
|
||||
)
|
||||
req.add_header("Content-Type", "application/json")
|
||||
req.add_header("Authorization", f"Bearer {config.IMAGEGEN_API_KEY}")
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
result = json.loads(resp.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace")
|
||||
return f"Error: HTTP {e.code} - {body[:500]}"
|
||||
except Exception as e:
|
||||
return f"Error: {e}"
|
||||
|
||||
images = result.get("data", [])
|
||||
if not images:
|
||||
return f"Error: No images in response - {json.dumps(result)[:500]}"
|
||||
|
||||
img = images[0]
|
||||
img_bytes = base64.b64decode(img["b64_json"])
|
||||
|
||||
# Detect actual format from magic bytes
|
||||
if img_bytes[:8] == b"\x89PNG\r\n\x1a\n":
|
||||
ext = ".png"
|
||||
elif img_bytes[:3] == b"\xff\xd8\xff":
|
||||
ext = ".jpg"
|
||||
elif img_bytes[:4] == b"RIFF" and img_bytes[8:12] == b"WEBP":
|
||||
ext = ".webp"
|
||||
else:
|
||||
media_type = img.get("media_type", "image/png")
|
||||
ext_map = {"image/png": ".png", "image/jpeg": ".jpg", "image/webp": ".webp"}
|
||||
ext = ext_map.get(media_type, ".png")
|
||||
|
||||
output_path = os.path.expanduser(output_path)
|
||||
if os.path.isdir(output_path):
|
||||
filename = f"generated_{int(time.time())}{ext}"
|
||||
output_path = os.path.join(output_path, filename)
|
||||
elif not os.path.splitext(output_path)[1]:
|
||||
output_path += ext
|
||||
|
||||
parent = os.path.dirname(output_path)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(img_bytes)
|
||||
|
||||
cost = result.get("usage", {}).get("cost", "?")
|
||||
return f"Image saved to {output_path} ({len(img_bytes)} bytes, cost: ${cost})"
|
||||
Loading…
Reference in New Issue
Block a user