From 9d1de2610911e0daacae8a4a653a33a78a1a7990 Mon Sep 17 00:00:00 2001 From: niansa Date: Sun, 8 Jan 2023 21:40:43 +0100 Subject: [PATCH] Added user and member logging --- dcboost | 2 +- main.cpp | 316 +++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 238 insertions(+), 80 deletions(-) diff --git a/dcboost b/dcboost index b0e59f7..9c225be 160000 --- a/dcboost +++ b/dcboost @@ -1 +1 @@ -Subproject commit b0e59f70b30ca7ad8c4b95e71f6c798249b4ade8 +Subproject commit 9c225be66bdec7a509efb32f40b3d45ec592709b diff --git a/main.cpp b/main.cpp index deea5a3..8bbdaa9 100644 --- a/main.cpp +++ b/main.cpp @@ -17,36 +17,72 @@ using namespace QLog; class Cache { - struct Miss : public std::runtime_error { - Miss() : std::runtime_error("Object cache miss") {} - }; - Logger logger; std::unordered_map cache; public: + struct StoreRes { + const Json::Value& data; + enum { + unchanged = 0b00, + updated = 0b01, + created = 0b11, + }; + uint8_t changed; + }; + + struct Miss : public std::runtime_error { + Miss() : std::runtime_error("Object cache miss") {} + }; + Cache() { logger.inst_ptr = this; } - const Json::Value& store(const Json::Value& object) { - const auto& id_str = object["id"].asString(); - logger.log(Loglevel::verbose, "Stored "+id_str+" in cache"); - return cache.insert_or_assign(std::move(id_str), object).first->second; + StoreRes store(Discord::Snowflake id, const Json::Value& object) { + logger.log(Loglevel::verbose, "Stored "+id.str()+" in cache"); + auto cache_hit = cache.find(id); + if (cache_hit != cache.end()) { + bool changed = cache_hit->second != object; + if (changed) { + cache_hit->second = object; + } + return {cache_hit->second, changed}; + } else { + return {cache.emplace(id, object).first->second, StoreRes::created}; + } } - const Json::Value& store(Json::Value&& object) { - const auto& id_str = object["id"].asString(); - logger.log(Loglevel::verbose, "Moved "+id_str+" into cache"); - return cache.insert_or_assign(std::move(id_str), std::move(object)).first->second; + StoreRes store(Discord::Snowflake id, Json::Value&& object) { + logger.log(Loglevel::verbose, "Moved "+id.str()+" into cache"); + auto cache_hit = cache.find(id); + if (cache_hit != cache.end()) { + bool changed = cache_hit->second != object; + if (changed) { + cache_hit->second = std::move(object); + } + return {cache_hit->second, changed}; + } else { + return {cache.emplace(id, std::move(object)).first->second, true}; + } + } + StoreRes store(const Json::Value& object) { + auto id = object["id"].asString(); + return store(id, object); + } + StoreRes store(Json::Value&& object) { + auto id = object["id"].asString(); + return store(id, std::move(object)); } const Json::Value& fetch(Discord::Snowflake id) const { const auto& entry = cache.find(id); if (entry == cache.end()) { - logger.log(Loglevel::warn, "Missing "+id.str()); throw Miss(); } return entry->second; } + const Json::Value& fetch(const std::string& id) const { + return fetch(Discord::Snowflake(id)); + } bool has(Discord::Snowflake id) const { return cache.find(id) != cache.end(); } @@ -56,6 +92,10 @@ public: static inline std::optional GetJSONAsOptionalString(const Json::Value& data) { return data.isString()?data.asString():std::optional(); } +static inline std::optional GetJSONAsOptionalInt(const Json::Value& data) { + return data.isString()?data.asInt():std::optional(); +} + class MyClient final : public Discord::Client { sqlite::database db; @@ -85,6 +125,7 @@ public: " is_edited INTEGER DEFAULT 0 NOT NULL," " creation_timestamp TEXT NOT NULL," " update_timestamp TEXT DEFAULT '0' NOT NULL," + " deletion_timestamp TEXT DEFAULT '0' NOT NULL," " UNIQUE(id)" ");"; db << "CREATE TABLE IF NOT EXISTS message_contents (" @@ -92,7 +133,7 @@ public: " is_initial INTEGER DEFAULT 1 NOT NULL," " timestamp TEXT NOT NULL," " content TEXT NOT NULL," - " embed TEXT" // Discord JSON embed object + " embeds TEXT" // Discord JSON embed object array ");"; db << "CREATE TABLE IF NOT EXISTS users (" " id TEXT NOT NULL," @@ -117,8 +158,8 @@ public: " is_initial INTEGER DEFAULT 1 NOT NULL," " timestamp TEXT NOT NULL," " nickname TEXT," - " in_guild INTEGER NOT NULL," - " is_untracked INTEGER NOT NULL" + " avatar TEXT," + " in_guild INTEGER NOT NULL" ");"; db << "CREATE TABLE IF NOT EXISTS member_voice_connections (" " channel_id TEXT NOT NULL," @@ -139,7 +180,8 @@ public: " is_initial INTEGER DEFAULT 1 NOT NULL," " type INTEGER NOT NULL," " name TEXT," - " topic TEXT" + " topic TEXT," + " is_deleted INTEGER DEFAULT 0 NOT NULL" ");"; db << "CREATE TABLE IF NOT EXISTS guilds (" " id TEXT NOT NULL," @@ -153,43 +195,37 @@ public: } protected: + /* + * Note: On creation, update, and deletion, it is recommended to use + * the functions starting with 'insert' as they do not store any + * data in the cache. For creation, it is also recommended to + * use the functions starting with 'process' as they not only store + * data in the cache, but also automatically handle related members + * (e.g. channels in a guild). The functions starting with 'update' + * are preferred for both creation and update and store data in cache + * as well. When accessing the cache, it is recommended to use the + * functions starting with 'cache' rather than accessing it directly. + */ + /* * Guilds */ - void processGuild(const Json::Value& data) { - const auto& id = data["id"]; - - // Insert into database - insertGuildUpdate(data, !cache.has(id)); - - // Store in cache - cache.store(data); - - // Insert and store all channels - for (auto channel_data : data["channels"]) { - // Add guild_id to channel (it'll be missing) - channel_data["guild_id"] = id; - - // Insert into database - insertChannelUpdate(channel_data, !cache.has(channel_data["id"])); - - // Store in cache - cache.store(std::move(channel_data)); - } - } - boost::asio::awaitable fetchAllGuilds(const Json::Value& unavailable_guild_array) { + boost::asio::awaitable fetchAllGuilds(Json::Value&& unavailable_guild_array) { RandomGenerator rng; rng.seed(); - for (const auto& incomplete_data : unavailable_guild_array) { + + // Fetch guilds and channels from API + for (auto& incomplete_data : unavailable_guild_array) { Discord::Snowflake guild_id = incomplete_data["id"]; - // Make sure guild isn't cached yet - if (cache.has(guild_id)) continue; + // Use cached guild if possible + if (cache.has(guild_id)) continue; // In this case, we skip processing the guild, + // since it's safe to assume it's already been cached // Make sure data is actually incomplete if (incomplete_data["name"].isString() && incomplete_data["channels"].isArray()) { - // We'll just use it as-is - processGuild(incomplete_data); + // It's not, so we'll just use it as-is + processGuild(std::move(incomplete_data)); continue; } @@ -200,78 +236,184 @@ protected: co_await asyncSleep(rng.getUInt(6000, 15000)); } } - void insertGuildUpdate(const Json::Value& data, bool is_initial) { - db << "INSERT OR IGNORE INTO guilds (id, timestamp, is_initial, name, owner_user_id)" - " VALUES (?, ?, ?, ?, ? );" - << data["id"].asString() << std::to_string(time(nullptr)) << is_initial << data["name"].asString() << data["owner_id"].asString(); - cache.store(data); + Cache::StoreRes processGuild(Json::Value&& data) { + const auto& id = data["id"]; + + // Insert into database + insertGuildUpdate(data, !cache.has(id)); + + // Insert and store all channels + for (auto& channel_data : data["channels"]) { + // Make sure we haven't cached this channel yet + if (cache.has(channel_data["id"])) continue; + + // Add guild_id to channel (it'll be missing) + channel_data["guild_id"] = id; + + // Insert into database + insertChannelUpdate(channel_data, true); + + // Store in cache + cache.store(std::move(channel_data)); + } + + // Process all members + for (auto& member_data : data["members"]) { + updateMember(std::move(member_data), id); + } + + // Remove channels and members + data.removeMember("channels"); + data.removeMember("members"); + + // Store in cache and return cached guild + return cache.store(std::move(data)); + } + void insertGuildUpdate(const Json::Value& data, bool is_initial, bool is_deleted = false) { + db << "INSERT INTO guilds (id, timestamp, is_initial, name, owner_user_id, in_guild)" + " VALUES (?, ?, ?, ?, ?, ? );" + << data["id"].asString() << std::to_string(time(nullptr)) << is_initial << data["name"].asString() + << data["owner_id"].asString() << !is_deleted; } void insertGuildDelete(const Json::Value& data) { auto cached_data = cache.fetch(data["id"]); - db << "INSERT OR IGNORE INTO guilds (id, timestamp, is_initial, name, owner_user_id, in_guild)" - " VALUES (?, ?, 0, ?, ?, 0 );" - << data["id"].asString() << std::to_string(time(nullptr)) << cached_data["name"].asString() << cached_data["owner_id"].asString(); + insertGuildUpdate(cached_data, false, true); + } + + /* + * Members + */ + void updateMember(Json::Value&& data, Discord::Snowflake guild_id, Discord::Snowflake user_id = {}) { + if (user_id.empty()) { + // Get user + const auto& user_data = data["user"]; + if (!user_data.isObject()) { + logger.log(Loglevel::error, "A processMember call has been aborted because the passed 'data' object doesn't contain a 'user' key."); + return; + } + user_id = user_data["id"].asString(); + + // Insert and cache user + auto changed = cache.store(user_data).changed; + if (changed) { + insertUserUpdate(user_data, (changed&Cache::StoreRes::created)?true:false); + } + } + + // Cache member + auto changed = cacheStoreMember(std::move(data), user_id, guild_id).changed; + + // Insert member + if (changed) { + insertMemberUpdate(data, user_id, guild_id, !cache.has(data["id"])); + } + } + Cache::StoreRes cacheStoreMember(const Json::Value& data, Discord::Snowflake user_id, Discord::Snowflake guild_id) { + return cache.store(guild_id.uint()|user_id.uint(), data); + } + const Json::Value& fetchFetchMember(Discord::Snowflake user_id, Discord::Snowflake guild_id) { + return cache.fetch(guild_id.uint()|user_id.uint()); + } + void insertMemberUpdate(const Json::Value& data, Discord::Snowflake user_id, Discord::Snowflake guild_id, bool is_initial, bool is_removed = false) { + db << "INSERT INTO members (user_id, guild_id, is_initial, timestamp, nickname, avatar, in_guild)" + " VALUES (?, ?, ?, ?, ?, ?, ? );" + << user_id.str() << guild_id.str() << is_initial << std::to_string(time(nullptr)) << GetJSONAsOptionalString(data["nick"]) << GetJSONAsOptionalString(data["avatar"]) << !is_removed; + } + void insertMemberDelete(Discord::Snowflake user_id, Discord::Snowflake guild_id) { + insertMemberUpdate(fetchFetchMember(user_id, guild_id), user_id, guild_id, false, true); + } + + /* + * Users + */ + void insertUserUpdate(const Json::Value& data, bool is_initial) { + db << "INSERT INTO users (id, is_initial, timestamp, full_username, avatar, bio, has_nitro, is_bot)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ? );" + << data["id"].asString() << is_initial << std::to_string(time(nullptr)) + << data["username"].asString()+'#'+data["discriminator"].asString() << GetJSONAsOptionalString(data["avatar"]) + << GetJSONAsOptionalString(data["bio"]) << GetJSONAsOptionalInt(data["has_nitro"]).value_or(0) + << GetJSONAsOptionalInt(data["bot"]).value_or(false); } /* * Messages */ + void processMessage(Json::Value&& data) { + const auto& guild_id = data["guild_id"]; + const auto& user = data["author"]; + + // Process member + auto& member_data = data["member"]; + if (member_data.isObject() && guild_id.isString()) { + updateMember(std::move(member_data), guild_id, user["id"]); + } + + // Insert message into database + insertMessageContent(data, true); + } void insertMessageContent(const Json::Value& data, bool is_initial) { - const auto& embed = data["embed"]; - db << "INSERT OR IGNORE INTO message_contents (message_id, is_initial, timestamp, content, embed)" - " VALUES (?, ?, ?, ?, ? );" - << data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString() << (embed.isObject()?embed.toStyledString():std::optional()); + const auto& embeds = data["embeds"]; + db << "INSERT INTO message_contents (message_id, is_initial, timestamp, content, embeds)" + " VALUES (?, ?, ?, ?, ? );" + << data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString() + << (embeds.isArray()?Json::writeString(Json::StreamWriterBuilder(), embeds):std::optional()); } void insertMessageUpdate(const Json::Value& data, bool is_initial) { + bool has_content = !data["content"].asString().empty() || data["embeds"].isArray(); if (is_initial) { const auto& author = data["author"]; int type = data["type"].asInt(); - db << "INSERT OR IGNORE INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)" - " VALUES (?, ?, ?, ?, ?, ? );" - << data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional()) << std::to_string(time(nullptr)); - if (!data["content"].asString().empty()) insertMessageContent(data, true); + db << "INSERT INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)" + " VALUES (?, ?, ?, ?, ?, ? );" + << data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() + << (type==19?data["referenced_message"]["id"].asString():std::optional()) << std::to_string(time(nullptr)); cache.store(author); + if (has_content) insertMessageContent(data, true); } else { - if (!data["content"].asString().empty()) { - db << "UPDATE messages " - "SET is_edited = 1, update_timestamp = ? " - "WHERE id = ?;" - << std::to_string(time(nullptr)) << data["id"].asString(); - insertMessageContent(data, true); - } + db << "UPDATE messages " + "SET is_edited = 1, update_timestamp = ? " + "WHERE id = ?;" + << std::to_string(time(nullptr)) << data["id"].asString(); + if (has_content) insertMessageContent(data, false); } } void insertMessageDelete(const Json::Value& data) { db << "UPDATE messages " - "SET is_deleted = 1 " + "SET is_deleted = 1, deletion_timestamp = ? " "WHERE id = ?;" - << data["id"].asString(); + << std::to_string(time(nullptr)) << data["id"].asString(); } /* * Channels */ - void insertChannelUpdate(const Json::Value& data, bool is_initial) { - db << "INSERT OR IGNORE INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic)" - " VALUES (?, ?, ?, ?, ?, ?, ?, ? );" - << data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"]) << std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"]) << GetJSONAsOptionalString(data["topic"]); - cache.store(data); + void insertChannelUpdate(const Json::Value& data, bool is_initial, bool is_deleted = false) { + db << "INSERT INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic, is_deleted)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);" + << data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"]) + << std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"]) + << GetJSONAsOptionalString(data["topic"]) << is_deleted; + } + void insertChannelDelete(const Json::Value& data) { + auto cached_data = cache.fetch(data["id"]); + insertChannelUpdate(cached_data, false, true); } /* * Intent handler */ - virtual boost::asio::awaitable intentHandler(const std::string& intent, const Json::Value& data) override { + virtual boost::asio::awaitable intentHandler(std::string intent, Json::Value data) override { if (intent == "READY") [[unlikely]] { - const auto& user = cache.store(data["user"]); + const auto& user = cache.store(std::move(data["user"])).data; logger.log(Loglevel::info, "Connected to Discord as: "+user["username"].asString()+'#'+user["discriminator"].asString()); - settings.is_bot = user["bot"].asBool(); - co_await fetchAllGuilds(data["guilds"]); + settings.is_bot = GetJSONAsOptionalInt(user["bot"]).value_or(false); + co_await fetchAllGuilds(std::move(data["guilds"])); } else if (intent == "GUILD_CREATE") [[unlikely]] { - insertGuildUpdate(data, true); + processGuild(std::move(data)); } else if (intent == "GUILD_UPDATE") [[unlikely]] { insertGuildUpdate(data, false); + cache.store(std::move(data)); } else if (intent == "GUILD_DELETE") [[unlikely]] { insertGuildDelete(data); } @@ -282,6 +424,22 @@ protected: } else if (intent == "MESSAGE_DELETE") { insertMessageDelete(data); } + else if (intent == "CHANNEL_CREATE") [[unlikely]] { + insertChannelUpdate(data, true); + cache.store(std::move(data)); + } else if (intent == "CHANNEL_UPDATE") [[unlikely]] { + insertChannelUpdate(data, false); + cache.store(std::move(data)); + } else if (intent == "CHANNEL_DELETE") [[unlikely]] { + insertChannelDelete(data); + } + else if (intent == "GUILD_MEMBER_ADD" || intent == "GUILD_MEMBER_UDATE") [[unlikely]] { + auto guild_id = std::move(data["guild_id"]); + updateMember(std::move(data), guild_id.asString()); + } else if (intent == "GUILD_MEMBER_REMOVE") [[unlikely]] { + const auto& user_data = cache.store(data["user"]).data; + insertMemberDelete(user_data["id"], data["guild_id"]); + } co_return; } };