import requests, gc, lancedb, pyarrow, uuid from datetime import datetime def create_table(db_path, vector_size): db = lancedb.connect(db_path) schema_user = pyarrow.schema([ pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), pyarrow.field('fullname', pyarrow.string(), metadata={'description': 'Fullname'}), pyarrow.field('nickname', pyarrow.string(), metadata={'description': 'Nickname'}), pyarrow.field('alias', pyarrow.string(), nullable=True, metadata={'description': 'Another call name'}), pyarrow.field('persona', pyarrow.string(), nullable=True, metadata={'description': 'Persona'}), pyarrow.field('telegram_id', pyarrow.string(), nullable=True, metadata={'description': 'Telegram ID'}), pyarrow.field('telegram_username', pyarrow.string(), nullable=True, metadata={'description': 'Telegram username'}), pyarrow.field('xampp_username', pyarrow.string(), nullable=True, metadata={'description': 'XAMPP username'}), pyarrow.field('vector_fullname', pyarrow.list_(pyarrow.float32(), vector_size) metadata={'description': 'Vector data of fullname'}), ]) db.create_table("knowledge_user", schema=schema_user) schema_memories = pyarrow.schema([ pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), pyarrow.field('character', pyarrow.string(), metadata={'description': 'Active Character'}), pyarrow.field('user', pyarrow.string(), metadata={'description': 'Active User'}), pyarrow.field('relative_time', pyarrow.string(), metadata={'description': 'Explaining when it happen'}), pyarrow.field('event', pyarrow.string(), metadata={'description': 'What event'}), pyarrow.field('category', pyarrow.string(), metadata={'description': 'Event category'}), pyarrow.field('detail', pyarrow.string(), metadata={'description': 'Event detail'}), pyarrow.field('physical', pyarrow.string(), metadata={'description': 'Character physical state'}), pyarrow.field('emotional', pyarrow.string(), metadata={'description': 'Character emotional state'}), pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size) metadata={'description': 'Vector data of combined data'}), ]) db.create_table("knowledge_memories", schema=schema_memories) schema_scenario = pyarrow.schema([ pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), pyarrow.field('character', pyarrow.string(), metadata={'description': 'Active Character'}), pyarrow.field('user', pyarrow.string(), metadata={'description': 'Active User'}), pyarrow.field('scenario', pyarrow.string(), metadata={'description': 'Detailed scenario description'}), pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of scenario'}), ]) db.create_table("knowledge_scenario", schema=schema_scenario) schema_todo = pyarrow.schema([ pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), pyarrow.field('keyword', pyarrow.string(), metadata={'description': 'Keywords to trigger to-do'}), pyarrow.field('when', pyarrow.string(), metadata={'description': 'When'}), pyarrow.field('do', pyarrow.string(), metadata={'description': 'Do'}), pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of keyword+when'}), ]) db.create_table("knowledge_todo", schema=schema_todo) schema_outfit_set = pyarrow.schema([ pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), pyarrow.field('keyword', pyarrow.string(), metadata={'description': 'Keywords of outfit set'}), pyarrow.field('when', pyarrow.string(), metadata={'description': 'When it better to use'}), pyarrow.field('outfit', pyarrow.string(), metadata={'description': 'Outfit set description'}), pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of keyword+when'}), ]) db.create_table("knowledge_outfit_set", schema=schema_outfit_set) schema_world = pyarrow.schema([ pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), pyarrow.field('category', pyarrow.string(), metadata={'description': 'Category of world'}), pyarrow.field('location', pyarrow.string(), metadata={'description': 'Name of the place or location in the world'}), pyarrow.field('description', pyarrow.string(), metadata={'description': 'Detailed world description'}), pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of combined content'}), ]) db.create_table("knowledge_world", schema=schema_world) schema_object = pyarrow.schema([ pyarrow.field('id', pyarrow.uuid(), metadata={'description': 'Unique identifier (UUID)'}), pyarrow.field('timestamp', pyarrow.timestamp('ms'), metadata={'description': 'When record created'}), pyarrow.field('keyword', pyarrow.string(), metadata={'description': 'Keywords of object'}), pyarrow.field('when', pyarrow.string(), metadata={'description': 'When it better to use'}), pyarrow.field('where', pyarrow.string(), metadata={'description': 'Where the object can be found'}), pyarrow.field('name', pyarrow.string(), metadata={'description': 'Object name'}), pyarrow.field('description', pyarrow.string(), metadata={'description': 'Object description'}), pyarrow.field('shape', pyarrow.string(), metadata={'description': 'Object shape'}), pyarrow.field('usage', pyarrow.string(), metadata={'description': 'Object usage'}), pyarrow.field('vector_context', pyarrow.list_(pyarrow.float32(), vector_size), metadata={'description': 'Vector data of keyword+when'}), ]) db.create_table("knowledge_object", schema=schema_object) # parameters: eat, drink, sleep # Time: morning, afternoon, evening, night del db gc.collect() def embed_text(url, model, text): response = requests.post(url=url, json={"model": model, "input": text} ) data = response.json() return data["embeddings"][0] def users_store(db_path, model_url, model_name, fullname, nickname, alias, persona, telegram_id, telegram_username, xampp_username): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_user") vector = embed_text(model_url, model_name, fullname) record = { "id": str(uuid.uuid4()), "timestamp": datetime.now(), "fullname": fullname, "nickname": nickname, "alias": alias, "persona": persona, "telegram_id": telegram_id, "telegram_username": telegram_username, "xampp_username": xampp_username, "vector_fullname": vector } table.add([record]) del table del db gc.collect() return True except Exception as e: print(f"Error storing user: {e}") return False def users_filter(db_path, query_name=None, unique_id=None, persona_keyword=None, user_id=None): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_user") filters = [] if user_id: filters.append(f"id = '{user_id}'") if unique_id: filters.append(f"(telegram_id = '{unique_id}' OR telegram_username = '{unique_id}' OR xampp_username = '{unique_id}')") if query_name: filters.append(f"(fullname = '{query_name}' OR nickname = '{query_name}' OR alias = '{query_name}')") if persona_keyword: filters.append(f"persona LIKE '%{persona_keyword}%'") query = " AND ".join(filters) results = table.search().where(query).to_list() if query else table.to_list() del table del db gc.collect() return results except Exception as e: print(f"Error filtering users: {e}") return [] def users_delete(db_path, user_id): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_user") table.delete(f"id = '{user_id}'") del table del db gc.collect() return True except Exception as e: print(f"Error deleting user: {e}") return False def users_update(db_path, model_url, model_name, user_id, **updates): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_user") current_record = table.search().where(f"id = '{user_id}'").to_list() if not current_record: return False record = current_record[0] needs_reembed = 'fullname' in updates for key, value in updates.items(): if key in record: record[key] = value if needs_reembed: record['vector_fullname'] = embed_text(model_url, model_name, record['fullname']) table.upsert([record]) del table del db gc.collect() return True except Exception as e: print(f"Error updating user: {e}") return False def memories_store(db_path, model_url, model_name, character, user, relative_time, event, category, detail, physical, emotional): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_memories") context_text = f"Event: {event}\nDetail: {detail}\nCategory: {category}\nTime: {relative_time}" vector = embed_text(model_url, model_name, context_text) record = { "id": str(uuid.uuid4()), "timestamp": datetime.now(), "character": character, "user": user, "relative_time": relative_time, "event": event, "category": category, "detail": detail, "physical": physical, "emotional": emotional, "vector_context": vector } table.add([record]) del table del db gc.collect() return True except Exception as e: print(f"Error storing memory: {e}") return False def memories_filter(db_path, category=None, character=None, user=None): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_memories") query = "" filters = [] if category: filters.append(f"category = '{category}'") if character: filters.append(f"character = '{character}'") if user: filters.append(f"user = '{user}'") if filters: query = " AND ".join(filters) results = table.search().where(query).to_list() if query else table.to_list() del table del db gc.collect() return results except Exception as e: print(f"Error filtering memories: {e}") return [] def memories_summarize(db_path, prompt_text, model_url, model_name, search_limit=5): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_memories") vector = embed_text(model_url, model_name, prompt_text) results = table.search(vector, vector_column_name="vector_context").limit(search_limit).to_list() if not results: return "Tidak ada memori relevan untuk diringkas." summaries = [] for res in results: summaries.append(f"[{res['category']}] {res['event']}: {res['detail']}") result_text = "Ringkasan Memori Relevan:\n" + "\n".join(summaries) del table del db gc.collect() return result_text except Exception as e: print(f"Error summarizing memories: {e}") return f"Gagal membuat ringkasan: {str(e)}" def memories_delete(db_path, memory_id): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_memories") table.delete(f"id = '{memory_id}'") del table del db gc.collect() return True except Exception as e: print(f"Error deleting memory: {e}") return False def memories_update(db_path, model_url, model_name, memory_id, **updates): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_memories") current_record = table.search().where(f"id = '{memory_id}'").to_list() if not current_record: return False record = current_record[0] affected_fields = ['event', 'detail', 'category', 'relative_time'] needs_reembed = any(field in updates for field in affected_fields) for key, value in updates.items(): if key in record: record[key] = value if needs_reembed: context_text = f"Event: {record['event']}\nDetail: {record['detail']}\nCategory: {record['category']}\nTime: {record['relative_time']}" record['vector_context'] = embed_text(model_url, model_name, context_text) table.upsert([record]) del table del db gc.collect() return True except Exception as e: print(f"Error updating memory: {e}") return False def objects_store(db_path, model_url, model_name, keyword, when, where, name, description, shape, usage): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_object") context_text = f"Keyword: {keyword}\nWhen: {when}\nName: {name}" vector = embed_text(model_url, model_name, context_text) record = { "id": str(uuid.uuid4()), "timestamp": datetime.now(), "keyword": keyword, "when": when, "where": where, "name": name, "description": description, "shape": shape, "usage": usage, "vector_context": vector } table.add([record]) del table; del db; gc.collect() return True except Exception as e: return False def objects_filter(db_path, keyword=None, object_id=None): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_object") query = "" if object_id: query = f"id = '{object_id}'" elif keyword: query = f"keyword = '{keyword}'" results = table.search().where(query).to_list() if query else table.to_list() del table; del db; gc.collect() return results except Exception as e: return [] def objects_update(db_path, model_url, model_name, object_id, **updates): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_object") current = table.search().where(f"id = '{object_id}'").to_list() if not current: return False record = current[0] needs_reembed = any(f in updates for f in ['keyword', 'when', 'name']) for k, v in updates.items(): if k in record: record[k] = v if needs_reembed: record['vector_context'] = embed_text(model_url, model_name, f"Keyword: {record['keyword']}\nWhen: {record['when']}\nName: {record['name']}") table.upsert([record]) del table; del db; gc.collect() return True except Exception as e: return False def objects_delete(db_path, object_id): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_object") table.delete(f"id = '{object_id}'") del table; del db; gc.collect() return True except Exception as e: return False def outfits_store(db_path, model_url, model_name, keyword, when, outfit): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_outfit_set") context_text = f"Keyword: {keyword}\nWhen: {when}" vector = embed_text(model_url, model_name, context_text) record = { "id": str(uuid.uuid4()), "timestamp": datetime.now(), "keyword": keyword, "when": when, "outfit": outfit, "vector_context": vector } table.add([record]) del table; del db; gc.collect() return True except Exception as e: return False def outfits_filter(db_path, keyword=None, outfit_id=None): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_outfit_set") query = "" if outfit_id: query = f"id = '{outfit_id}'" elif keyword: query = f"keyword = '{keyword}'" results = table.search().where(query).to_list() if query else table.to_list() del table; del db; gc.collect() return results except Exception as e: return [] def outfits_update(db_path, model_url, model_name, outfit_id, **updates): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_outfit_set") current = table.search().where(f"id = '{outfit_id}'").to_list() if not current: return False record = current[0] needs_reembed = any(f in updates for f in ['keyword', 'when']) for k, v in updates.items(): if k in record: record[k] = v if needs_reembed: record['vector_context'] = embed_text(model_url, model_name, f"Keyword: {record['keyword']}\nWhen: {record['when']}") table.upsert([record]) del table; del db; gc.collect() return True except Exception as e: return False def outfits_delete(db_path, outfit_id): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_outfit_set") table.delete(f"id = '{outfit_id}'") del table; del db; gc.collect() return True except Exception as e: return False def todos_store(db_path, model_url, model_name, keyword, when, do): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_todo") context_text = f"Keyword: {keyword}\nWhen: {when}" vector = embed_text(model_url, model_name, context_text) record = { "id": str(uuid.uuid4()), "timestamp": datetime.now(), "keyword": keyword, "when": when, "do": do, "vector_context": vector } table.add([record]) del table; del db; gc.collect() return True except Exception as e: return False def todos_filter(db_path, keyword=None, todo_id=None): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_todo") query = "" if todo_id: query = f"id = '{todo_id}'" elif keyword: query = f"keyword = '{keyword}'" results = table.search().where(query).to_list() if query else table.to_list() del table; del db; gc.collect() return results except Exception as e: return [] def todos_update(db_path, model_url, model_name, todo_id, **updates): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_todo") current = table.search().where(f"id = '{todo_id}'").to_list() if not current: return False record = current[0] needs_reembed = any(f in updates for f in ['keyword', 'when']) for k, v in updates.items(): if k in record: record[k] = v if needs_reembed: record['vector_context'] = embed_text(model_url, model_name, f"Keyword: {record['keyword']}\nWhen: {record['when']}") table.upsert([record]) del table; del db; gc.collect() return True except Exception as e: return False def todos_delete(db_path, todo_id): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_todo") table.delete(f"id = '{todo_id}'") del table; del db; gc.collect() return True except Exception as e: return False def worlds_store(db_path, model_url, model_name, category, location, description): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_world") context_text = f"Category: {category}\nLocation: {location}\nDescription: {description}" vector = embed_text(model_url, model_name, context_text) record = { "id": str(uuid.uuid4()), "timestamp": datetime.now(), "category": category, "location": location, "description": description, "vector_context": vector } table.add([record]) del table; del db; gc.collect() return True except Exception as e: return False def worlds_filter(db_path, category=None, location=None, world_id=None): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_world") filters = [] if world_id: filters.append(f"id = '{world_id}'") if category: filters.append(f"category = '{category}'") if location: filters.append(f"location = '{location}'") query = " AND ".join(filters) results = table.search().where(query).to_list() if query else table.to_list() del table; del db; gc.collect() return results except Exception as e: return [] def worlds_update(db_path, model_url, model_name, world_id, **updates): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_world") current = table.search().where(f"id = '{world_id}'").to_list() if not current: return False record = current[0] needs_reembed = any(f in updates for f in ['category', 'location', 'description']) for k, v in updates.items(): if k in record: record[k] = v if needs_reembed: record['vector_context'] = embed_text(model_url, model_name, f"Category: {record['category']}\nLocation: {record['location']}\nDescription: {record['description']}") table.upsert([record]) del table; del db; gc.collect() return True except Exception as e: return False def worlds_delete(db_path, world_id): try: db = lancedb.connect(db_path) table = db.open_table("knowledge_world") table.delete(f"id = '{world_id}'") del table; del db; gc.collect() return True except Exception as e: return False