Compare commits
No commits in common. "2b034f8987960873c5d33bb2a7aaaedf1a788f45" and "9fe400517966828062da3c949830e3d2dafc5e1d" have entirely different histories.
2b034f8987
...
9fe4005179
@ -38,13 +38,6 @@ Kamu dapat bertindak sebagai coding agent yang membantu software engineering tas
|
|||||||
Contoh: "File config.py berisi konfigurasi LLM provider. Saya perlu menambahkan..."
|
Contoh: "File config.py berisi konfigurasi LLM provider. Saya perlu menambahkan..."
|
||||||
- Hindari hanya menyebut nama tool tanpa konteks — selalu sertakan alasan atau tujuan.
|
- Hindari hanya menyebut nama tool tanpa konteks — selalu sertakan alasan atau tujuan.
|
||||||
|
|
||||||
## Vision
|
|
||||||
|
|
||||||
- Gunakan `describe_image` untuk menganalisis gambar (screenshot, diagram, foto, dll).
|
|
||||||
- Path bisa absolut atau relatif terhadap workspace.
|
|
||||||
- Format yang didukung: jpg, jpeg, png, gif, webp, bmp.
|
|
||||||
- Ukuran max 10MB.
|
|
||||||
|
|
||||||
## Git Policy
|
## Git Policy
|
||||||
|
|
||||||
- Kamu boleh menjalankan `git status`, `git diff`, `git log` secara bebas untuk inspeksi.
|
- Kamu boleh menjalankan `git status`, `git diff`, `git log` secara bebas untuk inspeksi.
|
||||||
|
|||||||
@ -44,7 +44,6 @@ for prov in _providers:
|
|||||||
"base_url" : base_url,
|
"base_url" : base_url,
|
||||||
"api_key" : api_key,
|
"api_key" : api_key,
|
||||||
"default" : is_default,
|
"default" : is_default,
|
||||||
"vision" : m.get("vision", False),
|
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -5,7 +5,7 @@ import threading, time, signal
|
|||||||
|
|
||||||
import config
|
import config
|
||||||
|
|
||||||
from tools import coder, carrack, ragroleplay, vision
|
from tools import coder, carrack, ragroleplay
|
||||||
|
|
||||||
from services.xmpp_client import XMPPClient
|
from services.xmpp_client import XMPPClient
|
||||||
from services.telegram_client import TelegramClient
|
from services.telegram_client import TelegramClient
|
||||||
@ -34,9 +34,6 @@ tools_definition = [
|
|||||||
# Carrack Tools
|
# Carrack Tools
|
||||||
gadget.tools_mapping( schema = carrack.schema_sendhttprequest, handler = carrack.sendhttprequest ),
|
gadget.tools_mapping( schema = carrack.schema_sendhttprequest, handler = carrack.sendhttprequest ),
|
||||||
|
|
||||||
# Vision Tools
|
|
||||||
gadget.tools_mapping( schema = vision.schema_describe_image, handler = vision.describe_image ),
|
|
||||||
|
|
||||||
# Rag Roleplay Tools
|
# Rag Roleplay Tools
|
||||||
gadget.tools_mapping( schema = ragroleplay.schema_users_store, handler = ragroleplay.users_store ),
|
gadget.tools_mapping( schema = ragroleplay.schema_users_store, handler = ragroleplay.users_store ),
|
||||||
gadget.tools_mapping( schema = ragroleplay.schema_user_load, handler = ragroleplay.user_load ), # s_load
|
gadget.tools_mapping( schema = ragroleplay.schema_user_load, handler = ragroleplay.user_load ), # s_load
|
||||||
|
|||||||
@ -3,7 +3,6 @@ import threading
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import config
|
import config
|
||||||
from lib import ntro, agent_loop
|
from lib import ntro, agent_loop
|
||||||
from tools.vision import ImagePayload
|
|
||||||
|
|
||||||
|
|
||||||
def _add_msg(app, role, content, **kwargs):
|
def _add_msg(app, role, content, **kwargs):
|
||||||
@ -160,13 +159,7 @@ def _agent_loop(app):
|
|||||||
}))
|
}))
|
||||||
app.scroll = 999999
|
app.scroll = 999999
|
||||||
result = agent_loop.execute_tool(tc, app.TOOL_HANDLERS)
|
result = agent_loop.execute_tool(tc, app.TOOL_HANDLERS)
|
||||||
if isinstance(result, ImagePayload):
|
_add_msg(app, "tool", str(result), tool_call_id=tc["id"])
|
||||||
_add_msg(app, "tool", [
|
|
||||||
{"type": "text", "text": f"[Image loaded: {result.path}]"},
|
|
||||||
{"type": "image_url", "image_url": {"url": f"data:{result.mime};base64,{result.data}"}}
|
|
||||||
], tool_call_id=tc["id"])
|
|
||||||
else:
|
|
||||||
_add_msg(app, "tool", str(result), tool_call_id=tc["id"])
|
|
||||||
else:
|
else:
|
||||||
if response.content:
|
if response.content:
|
||||||
_add_msg(app, "assistant", response.content)
|
_add_msg(app, "assistant", response.content)
|
||||||
|
|||||||
@ -1,6 +1,5 @@
|
|||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from tools.vision import ImagePayload
|
|
||||||
|
|
||||||
def _ts():
|
def _ts():
|
||||||
return datetime.now().strftime('%H:%M:%S')
|
return datetime.now().strftime('%H:%M:%S')
|
||||||
@ -56,21 +55,11 @@ def run_agent_loop(session, llm_client, TOOLS, TOOL_HANDLERS, max_iterations, on
|
|||||||
if tc['function']['name'] == 'end_session':
|
if tc['function']['name'] == 'end_session':
|
||||||
session_ended = True
|
session_ended = True
|
||||||
result = execute_tool(tc, TOOL_HANDLERS)
|
result = execute_tool(tc, TOOL_HANDLERS)
|
||||||
if isinstance(result, ImagePayload):
|
session.messages.append({
|
||||||
session.messages.append({
|
'role': 'tool',
|
||||||
'role': 'tool',
|
'tool_call_id': tc['id'],
|
||||||
'tool_call_id': tc['id'],
|
'content': str(result),
|
||||||
'content': [
|
})
|
||||||
{"type": "text", "text": f"[Image loaded: {result.path}]"},
|
|
||||||
{"type": "image_url", "image_url": {"url": f"data:{result.mime};base64,{result.data}"}}
|
|
||||||
]
|
|
||||||
})
|
|
||||||
else:
|
|
||||||
session.messages.append({
|
|
||||||
'role': 'tool',
|
|
||||||
'tool_call_id': tc['id'],
|
|
||||||
'content': str(result),
|
|
||||||
})
|
|
||||||
else:
|
else:
|
||||||
if response.content:
|
if response.content:
|
||||||
print(f'[{_ts()}] Response generated ({len(response.content)} chars)', flush=True)
|
print(f'[{_ts()}] Response generated ({len(response.content)} chars)', flush=True)
|
||||||
|
|||||||
@ -10,8 +10,8 @@ def tool_schemas(tools_definition):
|
|||||||
def tool_handlers(tools_definition):
|
def tool_handlers(tools_definition):
|
||||||
return {t["name"]: t["handler"] for t in tools_definition}
|
return {t["name"]: t["handler"] for t in tools_definition}
|
||||||
|
|
||||||
def strip_thinking(text):
|
def strip_thinking(text: str) -> str:
|
||||||
if not text or not isinstance(text, str):
|
if not text:
|
||||||
return text
|
return text
|
||||||
# Strip XML-style thinking blocks (case-insensitive, DOTALL for multiline)
|
# Strip XML-style thinking blocks (case-insensitive, DOTALL for multiline)
|
||||||
text = re.sub(r'<think[^>]*>.*?</think>', '', text, flags=re.DOTALL | re.IGNORECASE)
|
text = re.sub(r'<think[^>]*>.*?</think>', '', text, flags=re.DOTALL | re.IGNORECASE)
|
||||||
|
|||||||
@ -5,4 +5,3 @@ tinydb>=4.8.0
|
|||||||
requests
|
requests
|
||||||
lancedb
|
lancedb
|
||||||
pyarrow
|
pyarrow
|
||||||
paramiko
|
|
||||||
|
|||||||
166
tools/vision.py
166
tools/vision.py
@ -1,166 +0,0 @@
|
|||||||
import os
|
|
||||||
import re
|
|
||||||
import io
|
|
||||||
import base64
|
|
||||||
import tempfile
|
|
||||||
import urllib.request
|
|
||||||
import ftplib
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
from tools.coder import get_current_workspace
|
|
||||||
|
|
||||||
MIME_MAP = {
|
|
||||||
".jpg": "image/jpeg",
|
|
||||||
".jpeg": "image/jpeg",
|
|
||||||
".png": "image/png",
|
|
||||||
".gif": "image/gif",
|
|
||||||
".webp": "image/webp",
|
|
||||||
".bmp": "image/bmp",
|
|
||||||
}
|
|
||||||
|
|
||||||
MIME_EXT_REVERSE = {v: k for k, v in MIME_MAP.items()}
|
|
||||||
MAX_SIZE = 10 * 1024 * 1024 # 10MB
|
|
||||||
|
|
||||||
|
|
||||||
class ImagePayload:
|
|
||||||
def __init__(self, data, mime, path):
|
|
||||||
self.data = data
|
|
||||||
self.mime = mime
|
|
||||||
self.path = path
|
|
||||||
|
|
||||||
|
|
||||||
schema_describe_image = {
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "describe_image",
|
|
||||||
"description": "Load an image and send it to the model for visual analysis. Supports local files, HTTP/HTTPS URLs, FTP, and SFTP paths.",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"path": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Image source. Local path, or URL (http/https/ftp/sftp://user:pass@host/path)"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required": ["path"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _detect_mime(path):
|
|
||||||
ext = os.path.splitext(path.split("?")[0])[1].lower()
|
|
||||||
return MIME_MAP.get(ext)
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_and_pack(raw, source_label):
|
|
||||||
if len(raw) > MAX_SIZE:
|
|
||||||
return f"Error: image too large ({len(raw) / 1024 / 1024:.1f}MB). Max: {MAX_SIZE / 1024 / 1024:.0f}MB"
|
|
||||||
|
|
||||||
mime = _detect_mime(source_label)
|
|
||||||
if not mime:
|
|
||||||
return f"Error: cannot detect image format from '{source_label}'. Supported: {', '.join(MIME_MAP.keys())}"
|
|
||||||
|
|
||||||
encoded = base64.b64encode(raw).decode("ascii")
|
|
||||||
return ImagePayload(data=encoded, mime=mime, path=source_label)
|
|
||||||
|
|
||||||
|
|
||||||
def _fetch_http(url):
|
|
||||||
req = urllib.request.Request(url, headers={"User-Agent": "Hendrik/1.0"})
|
|
||||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
||||||
raw = resp.read()
|
|
||||||
# Try Content-Type header for MIME detection
|
|
||||||
ct = resp.headers.get("Content-Type", "")
|
|
||||||
ct_mime = ct.split(";")[0].strip().lower() if ct else None
|
|
||||||
if ct_mime in MIME_EXT_REVERSE:
|
|
||||||
# Append extension based on Content-Type for _detect_mime
|
|
||||||
ext = MIME_EXT_REVERSE[ct_mime]
|
|
||||||
final_url = resp.url + ext
|
|
||||||
else:
|
|
||||||
final_url = resp.url
|
|
||||||
return raw, final_url
|
|
||||||
|
|
||||||
|
|
||||||
def _fetch_ftp(parsed):
|
|
||||||
host = parsed.hostname
|
|
||||||
port = parsed.port or 21
|
|
||||||
path = parsed.path.lstrip("/")
|
|
||||||
username = parsed.username or "anonymous"
|
|
||||||
password = parsed.password or ""
|
|
||||||
|
|
||||||
buf = io.BytesIO()
|
|
||||||
ftp = ftplib.FTP()
|
|
||||||
ftp.connect(host, port, timeout=30)
|
|
||||||
ftp.login(username, password)
|
|
||||||
ftp.retrbinary(f"RETR {path}", buf.write)
|
|
||||||
ftp.quit()
|
|
||||||
return buf.getvalue()
|
|
||||||
|
|
||||||
|
|
||||||
def _fetch_sftp(url, parsed):
|
|
||||||
try:
|
|
||||||
import paramiko
|
|
||||||
except ImportError:
|
|
||||||
return "Error: paramiko not installed. Install with: pip install paramiko"
|
|
||||||
|
|
||||||
host = parsed.hostname
|
|
||||||
port = parsed.port or 22
|
|
||||||
path = parsed.path
|
|
||||||
username = parsed.username or ""
|
|
||||||
password = parsed.password or ""
|
|
||||||
|
|
||||||
transport = paramiko.Transport((host, port))
|
|
||||||
transport.connect(username=username, password=password)
|
|
||||||
sftp = paramiko.SFTPClient.from_transport(transport)
|
|
||||||
|
|
||||||
with sftp.open(path, "rb") as f:
|
|
||||||
raw = f.read()
|
|
||||||
|
|
||||||
sftp.close()
|
|
||||||
transport.close()
|
|
||||||
return raw
|
|
||||||
|
|
||||||
|
|
||||||
def describe_image(path):
|
|
||||||
path = path.strip()
|
|
||||||
|
|
||||||
# ── HTTP / HTTPS ───────────────────────────────────────────────────
|
|
||||||
if re.match(r"https?://", path):
|
|
||||||
try:
|
|
||||||
raw, final_url = _fetch_http(path)
|
|
||||||
return _validate_and_pack(raw, final_url)
|
|
||||||
except Exception as e:
|
|
||||||
return f"Error fetching URL: {e}"
|
|
||||||
|
|
||||||
# ── FTP / FTPS ─────────────────────────────────────────────────────
|
|
||||||
if re.match(r"ftps?://", path):
|
|
||||||
parsed = urlparse(path)
|
|
||||||
try:
|
|
||||||
raw = _fetch_ftp(parsed)
|
|
||||||
return _validate_and_pack(raw, path)
|
|
||||||
except Exception as e:
|
|
||||||
return f"Error fetching FTP: {e}"
|
|
||||||
|
|
||||||
# ── SFTP ───────────────────────────────────────────────────────────
|
|
||||||
if path.startswith("sftp://"):
|
|
||||||
parsed = urlparse(path)
|
|
||||||
try:
|
|
||||||
raw = _fetch_sftp(path, parsed)
|
|
||||||
if isinstance(raw, str):
|
|
||||||
return raw # error message
|
|
||||||
return _validate_and_pack(raw, path)
|
|
||||||
except Exception as e:
|
|
||||||
return f"Error fetching SFTP: {e}"
|
|
||||||
|
|
||||||
# ── Local file ─────────────────────────────────────────────────────
|
|
||||||
full_path = os.path.join(get_current_workspace(), path) if not os.path.isabs(path) else path
|
|
||||||
|
|
||||||
if not os.path.isfile(full_path):
|
|
||||||
return f"Error: file not found: {full_path}"
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(full_path, "rb") as f:
|
|
||||||
raw = f.read()
|
|
||||||
except Exception as e:
|
|
||||||
return f"Error reading file: {e}"
|
|
||||||
|
|
||||||
return _validate_and_pack(raw, full_path)
|
|
||||||
Loading…
Reference in New Issue
Block a user