From de01227c5d539e7dcac9ada76b996f4220119c6f Mon Sep 17 00:00:00 2001 From: Dita Aji Pratama Date: Tue, 7 Jul 2026 05:10:52 +0700 Subject: [PATCH] Renaming and major change for tools preparation --- embedding.py | 6 - lib/ragroleplay.py | 542 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 542 insertions(+), 6 deletions(-) delete mode 100644 embedding.py create mode 100644 lib/ragroleplay.py diff --git a/embedding.py b/embedding.py deleted file mode 100644 index e3b1150..0000000 --- a/embedding.py +++ /dev/null @@ -1,6 +0,0 @@ -import requests - -def embed_text(url, model, text): - response = requests.post(url=url, json={"model": model, "input": text} ) - data = response.json() - return data["embeddings"][0] diff --git a/lib/ragroleplay.py b/lib/ragroleplay.py new file mode 100644 index 0000000..891191e --- /dev/null +++ b/lib/ragroleplay.py @@ -0,0 +1,542 @@ +import requests, gc, lancedb, pyarrow, uuid +from datetime import datetime + +def create_table(db_path, vector_size): + db = lancedb.connect(db_path) + + schema_user = pyarrow.schema([ + pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), + pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), + pyarrow.field('fullname', pyarrow.string(), metadata={'description': 'Fullname'}), + pyarrow.field('nickname', pyarrow.string(), metadata={'description': 'Nickname'}), + pyarrow.field('alias', pyarrow.string(), nullable=True, metadata={'description': 'Another call name'}), + pyarrow.field('persona', pyarrow.string(), nullable=True, metadata={'description': 'Persona'}), + pyarrow.field('telegram_id', pyarrow.string(), nullable=True, metadata={'description': 'Telegram ID'}), + pyarrow.field('telegram_username', pyarrow.string(), nullable=True, metadata={'description': 'Telegram username'}), + pyarrow.field('xampp_username', pyarrow.string(), nullable=True, metadata={'description': 'XAMPP username'}), + pyarrow.field('vector_fullname', pyarrow.list_(pyarrow.float32(), vector_size) metadata={'description': 'Vector data of fullname'}), + ]) + db.create_table("knowledge_user", schema=schema_user) + + schema_memories = pyarrow.schema([ + pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), + pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), + pyarrow.field('character', pyarrow.string(), metadata={'description': 'Active Character'}), + pyarrow.field('user', pyarrow.string(), metadata={'description': 'Active User'}), + pyarrow.field('relative_time', pyarrow.string(), metadata={'description': 'Explaining when it happen'}), + pyarrow.field('event', pyarrow.string(), metadata={'description': 'What event'}), + pyarrow.field('category', pyarrow.string(), metadata={'description': 'Event category'}), + pyarrow.field('detail', pyarrow.string(), metadata={'description': 'Event detail'}), + pyarrow.field('physical', pyarrow.string(), metadata={'description': 'Character physical state'}), + pyarrow.field('emotional', pyarrow.string(), metadata={'description': 'Character emotional state'}), + pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size) metadata={'description': 'Vector data of combined data'}), + ]) + db.create_table("knowledge_memories", schema=schema_memories) + + schema_scenario = pyarrow.schema([ + pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), + pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), + pyarrow.field('character', pyarrow.string(), metadata={'description': 'Active Character'}), + pyarrow.field('user', pyarrow.string(), metadata={'description': 'Active User'}), + pyarrow.field('scenario', pyarrow.string(), metadata={'description': 'Detailed scenario description'}), + pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of scenario'}), + ]) + db.create_table("knowledge_scenario", schema=schema_scenario) + + schema_todo = pyarrow.schema([ + pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), + pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), + pyarrow.field('keyword', pyarrow.string(), metadata={'description': 'Keywords to trigger to-do'}), + pyarrow.field('when', pyarrow.string(), metadata={'description': 'When'}), + pyarrow.field('do', pyarrow.string(), metadata={'description': 'Do'}), + pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of keyword+when'}), + ]) + db.create_table("knowledge_todo", schema=schema_todo) + + schema_outfit_set = pyarrow.schema([ + pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), + pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), + pyarrow.field('keyword', pyarrow.string(), metadata={'description': 'Keywords of outfit set'}), + pyarrow.field('when', pyarrow.string(), metadata={'description': 'When it better to use'}), + pyarrow.field('outfit', pyarrow.string(), metadata={'description': 'Outfit set description'}), + pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of keyword+when'}), + ]) + db.create_table("knowledge_outfit_set", schema=schema_outfit_set) + + schema_world = pyarrow.schema([ + pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), + pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), + pyarrow.field('category', pyarrow.string(), metadata={'description': 'Category of world'}), + pyarrow.field('location', pyarrow.string(), metadata={'description': 'Name of the place or location in the world'}), + pyarrow.field('description', pyarrow.string(), metadata={'description': 'Detailed world description'}), + pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of combined content'}), + ]) + db.create_table("knowledge_world", schema=schema_world) + + schema_object = pyarrow.schema([ + pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), + pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), + pyarrow.field('keyword', pyarrow.string(), metadata={'description': 'Keywords of object'}), + pyarrow.field('when', pyarrow.string(), metadata={'description': 'When it better to use'}), + pyarrow.field('where', pyarrow.string(), metadata={'description': 'Where the object can be found'}), + pyarrow.field('name', pyarrow.string(), metadata={'description': 'Object name'}), + pyarrow.field('description', pyarrow.string(), metadata={'description': 'Object description'}), + pyarrow.field('shape', pyarrow.string(), metadata={'description': 'Object shape'}), + pyarrow.field('usage', pyarrow.string(), metadata={'description': 'Object usage'}), + pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of keyword+when'}), + ]) + db.create_table("knowledge_object", schema=schema_object) + + # parameters: eat, drink, sleep + # Time: morning, afternoon, evening, night + + del db + gc.collect() + + +def embed_text(url, model, text): + response = requests.post(url=url, json={"model": model, "input": text} ) + data = response.json() + return data["embeddings"][0] + +def users_store(db_path, model_url, model_name, fullname, nickname, alias, persona, telegram_id, telegram_username, xampp_username): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_user") + vector = embed_text(model_url, model_name, fullname) + record = { + "id": str(uuid.uuid4()), + "timestamp": datetime.now(), + "fullname": fullname, + "nickname": nickname, + "alias": alias, + "persona": persona, + "telegram_id": telegram_id, + "telegram_username": telegram_username, + "xampp_username": xampp_username, + "vector_fullname": vector + } + + table.add([record]) + del table + del db + gc.collect() + return True + except Exception as e: + print(f"Error storing user: {e}") + return False + +def users_filter(db_path, query_name=None, unique_id=None, persona_keyword=None, user_id=None): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_user") + filters = [] + if user_id: + filters.append(f"id = '{user_id}'") + if unique_id: + filters.append(f"(telegram_id = '{unique_id}' OR telegram_username = '{unique_id}' OR xampp_username = '{unique_id}')") + if query_name: + filters.append(f"(fullname = '{query_name}' OR nickname = '{query_name}' OR alias = '{query_name}')") + if persona_keyword: + filters.append(f"persona LIKE '%{persona_keyword}%'") + + query = " AND ".join(filters) + results = table.search().where(query).to_list() if query else table.to_list() + + del table + del db + gc.collect() + return results + except Exception as e: + print(f"Error filtering users: {e}") + return [] + +def users_delete(db_path, user_id): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_user") + + table.delete(f"id = '{user_id}'") + + del table + del db + gc.collect() + return True + except Exception as e: + print(f"Error deleting user: {e}") + return False + +def users_update(db_path, model_url, model_name, user_id, **updates): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_user") + + current_record = table.search().where(f"id = '{user_id}'").to_list() + + if not current_record: + return False + + record = current_record[0] + needs_reembed = 'fullname' in updates + + for key, value in updates.items(): + if key in record: + record[key] = value + + if needs_reembed: + record['vector_fullname'] = embed_text(model_url, model_name, record['fullname']) + + table.upsert([record]) + + del table + del db + gc.collect() + return True + except Exception as e: + print(f"Error updating user: {e}") + return False + +def memories_store(db_path, model_url, model_name, character, user, relative_time, event, category, detail, physical, emotional): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_memories") + + context_text = f"Event: {event}\nDetail: {detail}\nCategory: {category}\nTime: {relative_time}" + vector = embed_text(model_url, model_name, context_text) + + record = { + "id": str(uuid.uuid4()), + "timestamp": datetime.now(), + "character": character, + "user": user, + "relative_time": relative_time, + "event": event, + "category": category, + "detail": detail, + "physical": physical, + "emotional": emotional, + "vector_context": vector + } + table.add([record]) + del table + del db + gc.collect() + return True + except Exception as e: + print(f"Error storing memory: {e}") + return False + +def memories_filter(db_path, category=None, character=None, user=None): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_memories") + query = "" + filters = [] + + if category: + filters.append(f"category = '{category}'") + if character: + filters.append(f"character = '{character}'") + if user: + filters.append(f"user = '{user}'") + if filters: + query = " AND ".join(filters) + + results = table.search().where(query).to_list() if query else table.to_list() + + del table + del db + gc.collect() + return results + except Exception as e: + print(f"Error filtering memories: {e}") + return [] + +def memories_summarize(db_path, prompt_text, model_url, model_name, search_limit=5): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_memories") + vector = embed_text(model_url, model_name, prompt_text) + results = table.search(vector, vector_column_name="vector_context").limit(search_limit).to_list() + + if not results: + return "Tidak ada memori relevan untuk diringkas." + + summaries = [] + for res in results: + summaries.append(f"[{res['category']}] {res['event']}: {res['detail']}") + + result_text = "Ringkasan Memori Relevan:\n" + "\n".join(summaries) + + del table + del db + gc.collect() + return result_text + except Exception as e: + print(f"Error summarizing memories: {e}") + return f"Gagal membuat ringkasan: {str(e)}" + +def memories_delete(db_path, memory_id): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_memories") + table.delete(f"id = '{memory_id}'") + del table + del db + gc.collect() + return True + except Exception as e: + print(f"Error deleting memory: {e}") + return False + +def memories_update(db_path, model_url, model_name, memory_id, **updates): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_memories") + current_record = table.search().where(f"id = '{memory_id}'").to_list() + if not current_record: + return False + + record = current_record[0] + affected_fields = ['event', 'detail', 'category', 'relative_time'] + needs_reembed = any(field in updates for field in affected_fields) + + for key, value in updates.items(): + if key in record: + record[key] = value + + if needs_reembed: + context_text = f"Event: {record['event']}\nDetail: {record['detail']}\nCategory: {record['category']}\nTime: {record['relative_time']}" + record['vector_context'] = embed_text(model_url, model_name, context_text) + + table.upsert([record]) + del table + del db + gc.collect() + return True + except Exception as e: + print(f"Error updating memory: {e}") + return False + +def objects_store(db_path, model_url, model_name, keyword, when, where, name, description, shape, usage): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_object") + context_text = f"Keyword: {keyword}\nWhen: {when}\nName: {name}" + vector = embed_text(model_url, model_name, context_text) + record = { + "id": str(uuid.uuid4()), + "timestamp": datetime.now(), + "keyword": keyword, "when": when, "where": where, "name": name, "description": description, "shape": shape, "usage": usage, + "vector_context": vector + } + table.add([record]) + del table; del db; gc.collect() + return True + except Exception as e: return False + +def objects_filter(db_path, keyword=None, object_id=None): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_object") + query = "" + if object_id: query = f"id = '{object_id}'" + elif keyword: query = f"keyword = '{keyword}'" + results = table.search().where(query).to_list() if query else table.to_list() + del table; del db; gc.collect() + return results + except Exception as e: return [] + +def objects_update(db_path, model_url, model_name, object_id, **updates): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_object") + current = table.search().where(f"id = '{object_id}'").to_list() + if not current: return False + record = current[0] + needs_reembed = any(f in updates for f in ['keyword', 'when', 'name']) + for k, v in updates.items(): + if k in record: record[k] = v + if needs_reembed: + record['vector_context'] = embed_text(model_url, model_name, f"Keyword: {record['keyword']}\nWhen: {record['when']}\nName: {record['name']}") + table.upsert([record]) + del table; del db; gc.collect() + return True + except Exception as e: return False + +def objects_delete(db_path, object_id): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_object") + table.delete(f"id = '{object_id}'") + del table; del db; gc.collect() + return True + except Exception as e: return False + +def outfits_store(db_path, model_url, model_name, keyword, when, outfit): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_outfit_set") + context_text = f"Keyword: {keyword}\nWhen: {when}" + vector = embed_text(model_url, model_name, context_text) + record = { + "id": str(uuid.uuid4()), + "timestamp": datetime.now(), + "keyword": keyword, "when": when, "outfit": outfit, + "vector_context": vector + } + table.add([record]) + del table; del db; gc.collect() + return True + except Exception as e: return False + +def outfits_filter(db_path, keyword=None, outfit_id=None): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_outfit_set") + query = "" + if outfit_id: query = f"id = '{outfit_id}'" + elif keyword: query = f"keyword = '{keyword}'" + results = table.search().where(query).to_list() if query else table.to_list() + del table; del db; gc.collect() + return results + except Exception as e: return [] + +def outfits_update(db_path, model_url, model_name, outfit_id, **updates): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_outfit_set") + current = table.search().where(f"id = '{outfit_id}'").to_list() + if not current: return False + record = current[0] + needs_reembed = any(f in updates for f in ['keyword', 'when']) + for k, v in updates.items(): + if k in record: record[k] = v + if needs_reembed: + record['vector_context'] = embed_text(model_url, model_name, f"Keyword: {record['keyword']}\nWhen: {record['when']}") + table.upsert([record]) + del table; del db; gc.collect() + return True + except Exception as e: return False + +def outfits_delete(db_path, outfit_id): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_outfit_set") + table.delete(f"id = '{outfit_id}'") + del table; del db; gc.collect() + return True + except Exception as e: return False + +def todos_store(db_path, model_url, model_name, keyword, when, do): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_todo") + context_text = f"Keyword: {keyword}\nWhen: {when}" + vector = embed_text(model_url, model_name, context_text) + record = { + "id": str(uuid.uuid4()), + "timestamp": datetime.now(), + "keyword": keyword, "when": when, "do": do, + "vector_context": vector + } + table.add([record]) + del table; del db; gc.collect() + return True + except Exception as e: return False + +def todos_filter(db_path, keyword=None, todo_id=None): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_todo") + query = "" + if todo_id: query = f"id = '{todo_id}'" + elif keyword: query = f"keyword = '{keyword}'" + results = table.search().where(query).to_list() if query else table.to_list() + del table; del db; gc.collect() + return results + except Exception as e: return [] + +def todos_update(db_path, model_url, model_name, todo_id, **updates): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_todo") + current = table.search().where(f"id = '{todo_id}'").to_list() + if not current: return False + record = current[0] + needs_reembed = any(f in updates for f in ['keyword', 'when']) + for k, v in updates.items(): + if k in record: record[k] = v + if needs_reembed: + record['vector_context'] = embed_text(model_url, model_name, f"Keyword: {record['keyword']}\nWhen: {record['when']}") + table.upsert([record]) + del table; del db; gc.collect() + return True + except Exception as e: return False + +def todos_delete(db_path, todo_id): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_todo") + table.delete(f"id = '{todo_id}'") + del table; del db; gc.collect() + return True + except Exception as e: return False + +def worlds_store(db_path, model_url, model_name, category, location, description): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_world") + context_text = f"Category: {category}\nLocation: {location}\nDescription: {description}" + vector = embed_text(model_url, model_name, context_text) + record = { + "id": str(uuid.uuid4()), + "timestamp": datetime.now(), + "category": category, "location": location, "description": description, + "vector_context": vector + } + table.add([record]) + del table; del db; gc.collect() + return True + except Exception as e: return False + +def worlds_filter(db_path, category=None, location=None, world_id=None): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_world") + filters = [] + if world_id: filters.append(f"id = '{world_id}'") + if category: filters.append(f"category = '{category}'") + if location: filters.append(f"location = '{location}'") + query = " AND ".join(filters) + results = table.search().where(query).to_list() if query else table.to_list() + del table; del db; gc.collect() + return results + except Exception as e: return [] + +def worlds_update(db_path, model_url, model_name, world_id, **updates): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_world") + current = table.search().where(f"id = '{world_id}'").to_list() + if not current: return False + record = current[0] + needs_reembed = any(f in updates for f in ['category', 'location', 'description']) + for k, v in updates.items(): + if k in record: record[k] = v + if needs_reembed: + record['vector_context'] = embed_text(model_url, model_name, f"Category: {record['category']}\nLocation: {record['location']}\nDescription: {record['description']}") + table.upsert([record]) + del table; del db; gc.collect() + return True + except Exception as e: return False + +def worlds_delete(db_path, world_id): + try: + db = lancedb.connect(db_path) + table = db.open_table("knowledge_world") + table.delete(f"id = '{world_id}'") + del table; del db; gc.collect() + return True + except Exception as e: return False +