#include "Random.hpp" #include "sqlite_modern_cpp/sqlite_modern_cpp.h" #include #include #include #include #include #include #include #include #include #include using namespace ChatFuse; using namespace QLog; class Cache { Logger logger; std::unordered_map cache; public: struct StoreResult { const Json::Value& data; enum { unchanged = 0b00, updated = 0b01, created = 0b11, }; uint8_t changed; }; struct Miss : public std::runtime_error { Miss() : std::runtime_error("Object cache miss") {} }; Cache() { logger.inst_ptr = this; } StoreResult store(Discord::Snowflake id, const Json::Value& object) { logger.log(Loglevel::verbose, "Stored "+id.str()+" in cache"); auto cache_hit = cache.find(id); if (cache_hit != cache.end()) [[likely]] { bool changed = cache_hit->second != object; if (changed) { cache_hit->second = object; } return {cache_hit->second, changed}; } else { return {cache.emplace(id, object).first->second, StoreResult::created}; } } StoreResult store(Discord::Snowflake id, Json::Value&& object) { logger.log(Loglevel::verbose, "Moved "+id.str()+" into cache"); auto cache_hit = cache.find(id); if (cache_hit != cache.end()) [[likely]] { bool changed = cache_hit->second != object; if (changed) { cache_hit->second = std::move(object); } return {cache_hit->second, changed}; } else { return {cache.emplace(id, std::move(object)).first->second, true}; } } StoreResult store(const Json::Value& object) { auto id = object["id"].asString(); return store(id, object); } StoreResult store(Json::Value&& object) { auto id = object["id"].asString(); return store(id, std::move(object)); } const Json::Value& fetch(Discord::Snowflake id) const { const auto& entry = cache.find(id); if (entry == cache.end()) [[unlikely]] { throw Miss(); } return entry->second; } const Json::Value& fetch(const std::string& id) const { return fetch(Discord::Snowflake(id)); } bool has(Discord::Snowflake id) const { return cache.find(id) != cache.end(); } }; static inline std::optional GetJSONAsOptionalString(const Json::Value& data) { return data.isString()?data.asString():std::optional(); } static inline std::optional GetJSONAsOptionalInt(const Json::Value& data) { return data.isString()?data.asInt():std::optional(); } static inline std::optional GetJSONAsOptionalBool(const Json::Value& data) { return data.isBool()?data.asBool():std::optional(); } static inline std::string GetJSONAsJSONData(const Json::Value& data) { return Json::writeString(Json::StreamWriterBuilder(), data); } static inline std::optional GetJSONAsOptionalJSONData(const Json::Value& data) { return ((!data.empty()&&!data.isNull())?GetJSONAsJSONData(data):std::optional()); } class MyClient final : public Discord::Client { sqlite::database db; Cache cache; Json::Value *me; static inline std::string getDatabaseFileName() { return "log-"+std::to_string(time(nullptr))+".sqlite3"; } public: MyClient(boost::asio::io_service& io, const ChatFuse::Discord::Settings& settings = {}) : ChatFuse::Discord::Client(io, settings), db(getDatabaseFileName()) { // Improve database performance db << "pragma journal_mode = WAL;"; db << "pragma synchronous = normal;"; db << "pragma temp_store = memory;"; // Create tables { db << "CREATE TABLE IF NOT EXISTS messages (" " id TEXT PRIMARY KEY NOT NULL," " type INTEGER NOT NULL," " channel_id TEXT NOT NULL," " author_id TEXT NOT NULL," " replied_to_id TEXT," " is_deleted INTEGER DEFAULT 0 NOT NULL," " is_edited INTEGER DEFAULT 0 NOT NULL," " creation_timestamp TEXT NOT NULL," " update_timestamp TEXT DEFAULT '0' NOT NULL," " deletion_timestamp TEXT DEFAULT '0' NOT NULL," " UNIQUE(id)" ");"; db << "CREATE TABLE IF NOT EXISTS message_contents (" " message_id TEXT NOT NULL," " is_initial INTEGER DEFAULT 1 NOT NULL," " timestamp TEXT NOT NULL," " content TEXT NOT NULL," " embeds TEXT" // Discord JSON embed object array ");"; db << "CREATE TABLE IF NOT EXISTS users (" " id TEXT NOT NULL," " is_initial INTEGER DEFAULT 1 NOT NULL," " timestamp TEXT NOT NULL," " full_username TEXT NOT NULL," // Username#Discriminator " avatar TEXT," " bio TEXT," " has_nitro INTEGER NOT NULL," " is_bot INTEGER NOT NULL" ");"; db << "CREATE TABLE IF NOT EXISTS user_presences (" " user_id TEXT NOT NULL," " is_initial INTEGER DEFAULT 1 NOT NULL," " timestamp TEXT NOT NULL," " status TEXT," // JSON array or null if not updated: [string:online/dnd/afk/offline, string:device] " activities TEXT" // Discord JSON Activity object list ");"; db << "CREATE TABLE IF NOT EXISTS members (" " user_id TEXT NOT NULL," " guild_id TEXT NOT NULL," " is_initial INTEGER DEFAULT 1 NOT NULL," " timestamp TEXT NOT NULL," " nickname TEXT," " avatar TEXT," " in_guild INTEGER NOT NULL" ");"; db << "CREATE TABLE IF NOT EXISTS member_voice_connections (" " channel_id TEXT," " user_id TEXT NOT NULL," " timestamp TEXT NOT NULL," " self_mute INTEGER NOT NULL," " self_deaf INTEGER NOT NULL," " self_video INTEGER NOT NULL," " self_stream INTEGER NOT NULL," " server_mute INTEGER NOT NULL," " server_deaf INTEGER NOT NULL" ");"; db << "CREATE TABLE IF NOT EXISTS channels (" " id TEXT NOT NULL," " guild_id TEXT," " parent_id TEXT," " timestamp TEXT NOT NULL," " is_initial INTEGER DEFAULT 1 NOT NULL," " type INTEGER NOT NULL," " name TEXT," " topic TEXT," " is_deleted INTEGER DEFAULT 0 NOT NULL" ");"; db << "CREATE TABLE IF NOT EXISTS guilds (" " id TEXT NOT NULL," " timestamp TEXT NOT NULL," " is_initial INTEGER DEFAULT 1 NOT NULL," " name TEXT NOT NULL," " owner_user_id TEXT NOT NULL," " in_guild INTEGER DEFAULT 1 NOT NULL" ");"; } } protected: /* * Note: On creation, update, and deletion, it is recommended to use * the functions starting with 'insert' as they do not store any * data in the cache. For creation, it is also recommended to * use the functions starting with 'process' as they not only store * data in the cache, but also automatically handle related members * (e.g. channels in a guild). The functions starting with 'update' * are preferred for both creation and update and store data in cache * as well. When accessing the cache, it is recommended to use the * functions starting with 'cache' rather than accessing it directly. */ /* * Guilds */ boost::asio::awaitable fetchAllGuilds(Json::Value&& unavailable_guild_array) { RandomGenerator rng; rng.seed(); // Fetch guilds and channels from API for (auto& incomplete_data : unavailable_guild_array) { Discord::Snowflake guild_id = incomplete_data["id"]; // Use cached guild if possible if (cache.has(guild_id)) continue; // In this case, we skip processing the guild, // since it's safe to assume it's already been cached // Make sure data is actually incomplete if ((incomplete_data["name"].isString() && incomplete_data["channels"].isArray()) || incomplete_data["properties"].isObject()) { // It's not, so we'll just use it as-is processGuild(std::move(incomplete_data)); continue; } // Fetch guild from API and process it processGuild(co_await api.call(boost::beast::http::verb::get, "/guilds/"+guild_id.str())); // Delay co_await asyncSleep(rng.getUInt(6000, 15000)); } } void fixGuild(Json::Value& data) { if (!settings.is_bot) { auto& props = data["properties"]; for (const auto& prop_name : props.getMemberNames()) { data[prop_name] = std::move(props[prop_name]); } data.removeMember("properties"); } } Cache::StoreResult processGuild(Json::Value&& data) { // Fix guild fixGuild(data); // Get guild ID const auto& id = data["id"]; // Insert into database insertGuildUpdate(data, !cache.has(id)); // Insert and store all channels for (auto& channel_data : data["channels"]) { // Make sure we haven't cached this channel yet if (cache.has(channel_data["id"])) continue; // Add guild_id to channel (it'll be missing) channel_data["guild_id"] = id; // Insert into database insertChannelUpdate(channel_data, true); // Store in cache cache.store(std::move(channel_data)); } // Process all members for (auto& member_data : data["members"]) { updateMember(std::move(member_data), id); } // Remove channels and members data.removeMember("channels"); data.removeMember("members"); // Store in cache and return cached guild return cache.store(std::move(data)); } void insertGuildUpdate(const Json::Value& data, bool is_initial, bool is_deleted = false) { db << "INSERT INTO guilds (id, timestamp, is_initial, name, owner_user_id, in_guild)" " VALUES (?, ?, ?, ?, ?, ? );" << data["id"].asString() << std::to_string(time(nullptr)) << is_initial << data["name"].asString() << data["owner_id"].asString() << !is_deleted; } void insertGuildDelete(Discord::Snowflake id) { auto cached_data = cache.fetch(id); insertGuildUpdate(cached_data, false, true); } /* * Members */ void updateMember(Json::Value&& data, Discord::Snowflake guild_id, Discord::Snowflake user_id = {}) { if (user_id.empty()) { // Get user const auto& user_data = data["user"]; if (!user_data.isObject()) { logger.log(Loglevel::error, "A processMember call has been aborted because the passed 'data' object doesn't contain a 'user' key."); return; } user_id = user_data["id"].asString(); // Insert and cache user auto changed = cache.store(user_data).changed; if (changed) { insertUserUpdate(user_data, changed==Cache::StoreResult::created); } } // Cache member auto changed = cacheStoreMember(std::move(data), user_id, guild_id).changed; // Insert member if (changed) { insertMemberUpdate(data, user_id, guild_id, changed == Cache::StoreResult::created); } } Cache::StoreResult cacheStoreMember(const Json::Value& data, Discord::Snowflake user_id, Discord::Snowflake guild_id) { return cache.store(guild_id.uint()|user_id.uint(), data); } const Json::Value& fetchFetchMember(Discord::Snowflake user_id, Discord::Snowflake guild_id) { return cache.fetch(guild_id.uint()|user_id.uint()); } void insertMemberUpdate(const Json::Value& data, Discord::Snowflake user_id, Discord::Snowflake guild_id, bool is_initial, bool is_removed = false) { db << "INSERT INTO members (user_id, guild_id, is_initial, timestamp, nickname, avatar, in_guild)" " VALUES (?, ?, ?, ?, ?, ?, ? );" << user_id.str() << guild_id.str() << is_initial << std::to_string(time(nullptr)) << GetJSONAsOptionalString(data["nick"]) << GetJSONAsOptionalString(data["avatar"]) << !is_removed; } void insertMemberDelete(Discord::Snowflake user_id, Discord::Snowflake guild_id) { insertMemberUpdate(fetchFetchMember(user_id, guild_id), user_id, guild_id, false, true); } /* * Users */ void insertUserUpdate(const Json::Value& data, bool is_initial) { db << "INSERT INTO users (id, is_initial, timestamp, full_username, avatar, bio, has_nitro, is_bot)" " VALUES (?, ?, ?, ?, ?, ?, ?, ? );" << data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["username"].asString()+'#'+data["discriminator"].asString() << GetJSONAsOptionalString(data["avatar"]) << GetJSONAsOptionalString(data["bio"]) << GetJSONAsOptionalInt(data["premium_type"]).value_or(0) << GetJSONAsOptionalBool(data["bot"]).value_or(false); } /* * Messages */ void processMessage(Json::Value&& data) { const auto& guild_id = data["guild_id"]; const auto& user_data = data["author"]; // Process member auto& member_data = data["member"]; if (member_data.isObject() && guild_id.isString()) { updateMember(std::move(member_data), guild_id, user_data["id"]); } // Insert and cache user const auto& [cached_user_data, changed] = cache.store(std::move(user_data)); if (changed) { insertUserUpdate(user_data, changed==Cache::StoreResult::created); } // Insert message into database insertMessageUpdate(data, true); } void insertMessageContent(const Json::Value& data, bool is_initial) { const auto& embeds = data["embeds"]; db << "INSERT INTO message_contents (message_id, is_initial, timestamp, content, embeds)" " VALUES (?, ?, ?, ?, ? );" << data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString() << GetJSONAsOptionalJSONData(embeds); } void insertMessageUpdate(const Json::Value& data, bool is_initial) { bool has_content = !data["content"].asString().empty() || data["embeds"].isArray(); if (is_initial) { int type = data["type"].asInt(); db << "INSERT INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)" " VALUES (?, ?, ?, ?, ?, ? );" << data["id"].asString() << type << data["channel_id"].asString() << data["author"]["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional()) << std::to_string(time(nullptr)); if (has_content) insertMessageContent(data, false); } else { db << "UPDATE messages " "SET is_edited = 1, update_timestamp = ? " "WHERE id = ?;" << std::to_string(time(nullptr)) << data["id"].asString(); if (has_content) insertMessageContent(data, false); } } void insertMessageDelete(const std::string& id) { db << "UPDATE messages " "SET is_deleted = 1, deletion_timestamp = ? " "WHERE id = ?;" << std::to_string(time(nullptr)) << id; } /* * Channels */ void insertChannelUpdate(const Json::Value& data, bool is_initial, bool is_deleted = false) { db << "INSERT INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic, is_deleted)" " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);" << data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"]) << std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"]) << GetJSONAsOptionalString(data["topic"]) << is_deleted; } void insertChannelDelete(Discord::Snowflake id) { auto cached_data = cache.fetch(id); insertChannelUpdate(cached_data, false, true); } /* * Member voice connections */ void updateMemberVoiceConnection(Json::Value&& data) { // Insert voice connection update insertMemberVoiceConnectionUpdate(data); // Update member updateMember(std::move(data["member"]), data["guild_id"]); } void insertMemberVoiceConnectionUpdate(const Json::Value& data) { db << "INSERT INTO member_voice_connections (channel_id, user_id, timestamp, self_mute, self_deaf, self_video, self_stream, server_mute, server_deaf)" " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ? );" << data["channel_id"].asString() << data["user_id"].asString() << std::to_string(time(nullptr)) << data["self_mute"].asBool() << data["self_deaf"].asBool() << data["self_video"].asBool() << GetJSONAsOptionalBool(data["self_stream"]).value_or(false) << data["mute"].asBool() << data["deaf"].asBool(); } /* * User presences */ void insertUserPresenceUpdate(const Json::Value& data, bool is_initial) { // Build status std::string status_str; bool is_online; { const auto& client_status = data["client_status"]; if (client_status.isObject() && !client_status.empty()) { const auto device = client_status.getMemberNames()[0]; status_str = "[\""+client_status[device].asString()+"\", \""+device+"\"]"; is_online = true; } else { status_str = R"(["offline", "none"])"; is_online = false; } } // Insert db << "INSERT INTO user_presences (user_id, is_initial, timestamp, status, activities)" " VALUES (?, ?, ?, ?, ? );" << data["user"]["id"].asString() << is_initial << std::to_string(time(nullptr)) << status_str << (is_online?GetJSONAsOptionalJSONData(data["activities"]):std::optional()); } /* * Intent handler */ virtual boost::asio::awaitable intentHandler(std::string intent, Json::Value data) override { if (intent == "READY") [[unlikely]] { // Get own user const auto& user = cache.store(std::move(data["user"])).data; logger.log(Loglevel::info, "Connected to Discord as: "+user["username"].asString()+'#'+user["discriminator"].asString()); // Get guilds co_await fetchAllGuilds(std::move(data["guilds"])); // Get users auto& users = data["users"]; if (users.isArray()) [[likely]] { for (const auto& user_data : users) { insertUserUpdate(user_data, true); cache.store(std::move(user_data)); } } } else if (intent == "GUILD_CREATE") [[unlikely]] { fixGuild(data); processGuild(std::move(data)); } else if (intent == "GUILD_UPDATE") [[unlikely]] { fixGuild(data); if (cache.store(data).changed) insertGuildUpdate(data, false); } else if (intent == "GUILD_DELETE") [[unlikely]] { fixGuild(data); insertGuildDelete(data["id"]); } else if (intent == "MESSAGE_CREATE") [[likely]] { processMessage(std::move(data)); } else if (intent == "MESSAGE_UPDATE") [[likely]] { insertMessageUpdate(data, false); } else if (intent == "MESSAGE_DELETE") { insertMessageDelete(data["id"].asString()); } else if (intent == "MESSAGE_DELETE_BULK") { for (const auto& id : data["ids"]) { insertMessageDelete(id.asString()); } } else if (intent == "CHANNEL_CREATE") [[unlikely]] { insertChannelUpdate(data, true); cache.store(std::move(data)); } else if (intent == "CHANNEL_UPDATE") [[unlikely]] { insertChannelUpdate(data, false); if (cache.store(data).changed) cache.store(std::move(data)); } else if (intent == "CHANNEL_DELETE") [[unlikely]] { const auto& id = data["id"]; if (id.isString()) { insertChannelDelete(id); } } else if (intent == "GUILD_MEMBER_ADD" || intent == "GUILD_MEMBER_UDATE") [[unlikely]] { auto guild_id = std::move(data["guild_id"]); updateMember(std::move(data), guild_id.asString()); } else if (intent == "GUILD_MEMBER_REMOVE") [[unlikely]] { const auto& user_data = data["user"]; if (user_data["username"].isString()) { cache.store(user_data); insertMemberDelete(user_data["id"], data["guild_id"]); } } else if (intent == "VOICE_STATE_UPDATE") { updateMemberVoiceConnection(std::move(data)); } else if (intent == "PRESENCE_UPDATE") { auto changed = cache.store(Discord::Snowflake(data["user_id"]).uint()|0x04e2e7ce00000000, data).changed; insertUserPresenceUpdate(data, changed==Cache::StoreResult::created); } else if (intent == "GUILD_BAN_ADD") [[unlikely]] { // Should we log bans? Let's just use it as a source of user data const auto& user_data = data["user"]; auto changed = cache.store(user_data).changed; if (changed) [[unlikely]] { insertUserUpdate(user_data, changed==Cache::StoreResult::created); } //TODO: Should we do insertMemberDelete here? } co_return; } }; int main(int argc, char **argv) { // Check args if (argc == 1) { std::cout << "Usage: " << argv[0] << " " << std::endl; return EXIT_FAILURE; } // Create io service boost::asio::io_service io; // Set up client auto client = std::make_shared(io, Discord::Settings{ .bot_token = argv[1], .intents = Discord::intents::all & ~Discord::intents::direct_messages, .is_bot = false }); client->detach(); // Erase bot token from argv (so it's no longer visible in /proc/{pid}/cmdline) memset(argv[1], 0, strlen(argv[1])); // Run!!! io.run(); return EXIT_SUCCESS; }