Added user and member logging

dcboost

@@ -1 +1 @@
-Subproject commit b0e59f70b30ca7ad8c4b95e71f6c798249b4ade8
+Subproject commit 9c225be66bdec7a509efb32f40b3d45ec592709b

main.cpp

@@ -17,36 +17,72 @@ using namespace QLog;
 
 
 class Cache {
-    struct Miss : public std::runtime_error {
-        Miss() : std::runtime_error("Object cache miss") {}
-    };
-
     Logger<Cache> logger;
     std::unordered_map<Discord::Snowflake, Json::Value> cache;
 
 public:
+    struct StoreRes {
+        const Json::Value& data;
+        enum {
+            unchanged = 0b00,
+            updated = 0b01,
+            created = 0b11,
+        };
+        uint8_t changed;
+    };
+
+    struct Miss : public std::runtime_error {
+        Miss() : std::runtime_error("Object cache miss") {}
+    };
+
     Cache() {
         logger.inst_ptr = this;
     }
 
-    const Json::Value& store(const Json::Value& object) {
+    StoreRes store(Discord::Snowflake id, const Json::Value& object) {
-        const auto& id_str = object["id"].asString();
+        logger.log(Loglevel::verbose, "Stored "+id.str()+" in cache");
-        logger.log(Loglevel::verbose, "Stored "+id_str+" in cache");
+        auto cache_hit = cache.find(id);
-        return cache.insert_or_assign(std::move(id_str), object).first->second;
+        if (cache_hit != cache.end()) {
+            bool changed = cache_hit->second != object;
+            if (changed) {
+                cache_hit->second = object;
+            }
+            return {cache_hit->second, changed};
+        } else {
+            return {cache.emplace(id, object).first->second, StoreRes::created};
+        }
     }
-    const Json::Value& store(Json::Value&& object) {
+    StoreRes store(Discord::Snowflake id, Json::Value&& object) {
-        const auto& id_str = object["id"].asString();
+        logger.log(Loglevel::verbose, "Moved "+id.str()+" into cache");
-        logger.log(Loglevel::verbose, "Moved "+id_str+" into cache");
+        auto cache_hit = cache.find(id);
-        return cache.insert_or_assign(std::move(id_str), std::move(object)).first->second;
+        if (cache_hit != cache.end()) {
+            bool changed = cache_hit->second != object;
+            if (changed) {
+                cache_hit->second = std::move(object);
+            }
+            return {cache_hit->second, changed};
+        } else {
+            return {cache.emplace(id, std::move(object)).first->second, true};
+        }
+    }
+    StoreRes store(const Json::Value& object) {
+        auto id = object["id"].asString();
+        return store(id, object);
+    }
+    StoreRes store(Json::Value&& object) {
+        auto id = object["id"].asString();
+        return store(id, std::move(object));
     }
     const Json::Value& fetch(Discord::Snowflake id) const {
         const auto& entry = cache.find(id);
         if (entry == cache.end()) {
-            logger.log(Loglevel::warn, "Missing "+id.str());
             throw Miss();
         }
         return entry->second;
     }
+    const Json::Value& fetch(const std::string& id) const {
+        return fetch(Discord::Snowflake(id));
+    }
     bool has(Discord::Snowflake id) const {
         return cache.find(id) != cache.end();
     }
@@ -56,6 +92,10 @@ public:
 static inline std::optional<std::string> GetJSONAsOptionalString(const Json::Value& data) {
     return data.isString()?data.asString():std::optional<std::string>();
 }
+static inline std::optional<int> GetJSONAsOptionalInt(const Json::Value& data) {
+    return data.isString()?data.asInt():std::optional<int>();
+}
+
 
 class MyClient final : public Discord::Client {
     sqlite::database db;
@@ -85,6 +125,7 @@ public:
                   "    is_edited INTEGER DEFAULT 0 NOT NULL,"
                   "    creation_timestamp TEXT NOT NULL,"
                   "    update_timestamp TEXT DEFAULT '0' NOT NULL,"
+                  "    deletion_timestamp TEXT DEFAULT '0' NOT NULL,"
                   "    UNIQUE(id)"
                   ");";
             db << "CREATE TABLE IF NOT EXISTS message_contents ("
@@ -92,7 +133,7 @@ public:
                   "    is_initial INTEGER DEFAULT 1 NOT NULL,"
                   "    timestamp TEXT NOT NULL,"
                   "    content TEXT NOT NULL,"
-                  "    embed TEXT" // Discord JSON embed object
+                  "    embeds TEXT" // Discord JSON embed object array
                   ");";
             db << "CREATE TABLE IF NOT EXISTS users ("
                   "    id TEXT NOT NULL,"
@@ -117,8 +158,8 @@ public:
                   "    is_initial INTEGER DEFAULT 1 NOT NULL,"
                   "    timestamp TEXT NOT NULL,"
                   "    nickname TEXT,"
-                  "    in_guild INTEGER NOT NULL,"
+                  "    avatar TEXT,"
-                  "    is_untracked INTEGER NOT NULL"
+                  "    in_guild INTEGER NOT NULL"
                   ");";
             db << "CREATE TABLE IF NOT EXISTS member_voice_connections ("
                   "    channel_id TEXT NOT NULL,"
@@ -139,7 +180,8 @@ public:
                   "    is_initial INTEGER DEFAULT 1 NOT NULL,"
                   "    type INTEGER NOT NULL,"
                   "    name TEXT,"
-                  "    topic TEXT"
+                  "    topic TEXT,"
+                  "    is_deleted INTEGER DEFAULT 0 NOT NULL"
                   ");";
             db << "CREATE TABLE IF NOT EXISTS guilds ("
                   "    id TEXT NOT NULL,"
@@ -153,43 +195,37 @@ public:
     }
 
 protected:
+    /*
+     * Note: On creation, update, and deletion, it is recommended to use
+     *       the functions starting with 'insert' as they do not store any
+     *       data in the cache. For creation, it is also recommended to
+     *       use the functions starting with 'process' as they not only store
+     *       data in the cache, but also automatically handle related members
+     *       (e.g. channels in a guild). The functions starting with 'update'
+     *       are preferred for both creation and update and store data in cache
+     *       as well. When accessing the cache, it is recommended to use the
+     *       functions starting with 'cache' rather than accessing it directly.
+     */
+
     /*
      * Guilds
      */
-    void processGuild(const Json::Value& data) {
+    boost::asio::awaitable<void> fetchAllGuilds(Json::Value&& unavailable_guild_array) {
-        const auto& id = data["id"];
-
-        // Insert into database
-        insertGuildUpdate(data, !cache.has(id));
-
-        // Store in cache
-        cache.store(data);
-
-        // Insert and store all channels
-        for (auto channel_data : data["channels"]) {
-            // Add guild_id to channel (it'll be missing)
-            channel_data["guild_id"] = id;
-
-            // Insert into database
-            insertChannelUpdate(channel_data, !cache.has(channel_data["id"]));
-
-            // Store in cache
-            cache.store(std::move(channel_data));
-        }
-    }
-    boost::asio::awaitable<void> fetchAllGuilds(const Json::Value& unavailable_guild_array) {
         RandomGenerator rng;
         rng.seed();
-        for (const auto& incomplete_data : unavailable_guild_array) {
+
+        // Fetch guilds and channels from API
+        for (auto& incomplete_data : unavailable_guild_array) {
             Discord::Snowflake guild_id = incomplete_data["id"];
 
-            // Make sure guild isn't cached yet
+            // Use cached guild if possible
-            if (cache.has(guild_id)) continue;
+            if (cache.has(guild_id)) continue; // In this case, we skip processing the guild,
+                                               // since it's safe to assume it's already been cached
 
             // Make sure data is actually incomplete
             if (incomplete_data["name"].isString() && incomplete_data["channels"].isArray()) {
-                // We'll just use it as-is
+                // It's not, so we'll just use it as-is
-                processGuild(incomplete_data);
+                processGuild(std::move(incomplete_data));
                 continue;
             }
 
@@ -200,78 +236,184 @@ protected:
             co_await asyncSleep(rng.getUInt(6000, 15000));
         }
     }
-    void insertGuildUpdate(const Json::Value& data, bool is_initial) {
+    Cache::StoreRes processGuild(Json::Value&& data) {
-        db << "INSERT OR IGNORE INTO guilds (id, timestamp, is_initial, name, owner_user_id)"
+        const auto& id = data["id"];
-              "                      VALUES (?,  ?,         ?,          ?,    ?            );"
+
-           << data["id"].asString() << std::to_string(time(nullptr)) << is_initial << data["name"].asString() << data["owner_id"].asString();
+        // Insert into database
-        cache.store(data);
+        insertGuildUpdate(data, !cache.has(id));
+
+        // Insert and store all channels
+        for (auto& channel_data : data["channels"]) {
+            // Make sure we haven't cached this channel yet
+            if (cache.has(channel_data["id"])) continue;
+
+            // Add guild_id to channel (it'll be missing)
+            channel_data["guild_id"] = id;
+
+            // Insert into database
+            insertChannelUpdate(channel_data, true);
+
+            // Store in cache
+            cache.store(std::move(channel_data));
+        }
+
+        // Process all members
+        for (auto& member_data : data["members"]) {
+            updateMember(std::move(member_data), id);
+        }
+
+        // Remove channels and members
+        data.removeMember("channels");
+        data.removeMember("members");
+
+        // Store in cache and return cached guild
+        return cache.store(std::move(data));
+    }
+    void insertGuildUpdate(const Json::Value& data, bool is_initial, bool is_deleted = false) {
+        db << "INSERT INTO guilds (id, timestamp, is_initial, name, owner_user_id, in_guild)"
+              "            VALUES (?,  ?,         ?,          ?,    ?,             ?       );"
+           << data["id"].asString() << std::to_string(time(nullptr)) << is_initial << data["name"].asString()
+           << data["owner_id"].asString() << !is_deleted;
     }
     void insertGuildDelete(const Json::Value& data) {
         auto cached_data = cache.fetch(data["id"]);
-        db << "INSERT OR IGNORE INTO guilds (id, timestamp, is_initial, name, owner_user_id, in_guild)"
+        insertGuildUpdate(cached_data, false, true);
-              "                      VALUES (?,  ?,         0,          ?,    ?,             0       );"
+    }
-           << data["id"].asString() << std::to_string(time(nullptr)) << cached_data["name"].asString() << cached_data["owner_id"].asString();
+
+    /*
+     * Members
+     */
+    void updateMember(Json::Value&& data, Discord::Snowflake guild_id, Discord::Snowflake user_id = {}) {
+        if (user_id.empty()) {
+            // Get user
+            const auto& user_data = data["user"];
+            if (!user_data.isObject()) {
+                logger.log(Loglevel::error, "A processMember call has been aborted because the passed 'data' object doesn't contain a 'user' key.");
+                return;
+            }
+            user_id = user_data["id"].asString();
+
+            // Insert and cache user
+            auto changed = cache.store(user_data).changed;
+            if (changed) {
+                insertUserUpdate(user_data, (changed&Cache::StoreRes::created)?true:false);
+            }
+        }
+
+        // Cache member
+        auto changed = cacheStoreMember(std::move(data), user_id, guild_id).changed;
+
+        // Insert member
+        if (changed) {
+            insertMemberUpdate(data, user_id, guild_id, !cache.has(data["id"]));
+        }
+    }
+    Cache::StoreRes cacheStoreMember(const Json::Value& data, Discord::Snowflake user_id, Discord::Snowflake guild_id) {
+        return cache.store(guild_id.uint()|user_id.uint(), data);
+    }
+    const Json::Value& fetchFetchMember(Discord::Snowflake user_id, Discord::Snowflake guild_id) {
+        return cache.fetch(guild_id.uint()|user_id.uint());
+    }
+    void insertMemberUpdate(const Json::Value& data, Discord::Snowflake user_id, Discord::Snowflake guild_id, bool is_initial, bool is_removed = false) {
+        db << "INSERT INTO members (user_id, guild_id, is_initial, timestamp, nickname, avatar, in_guild)"
+              "             VALUES (?,       ?,        ?,          ?,         ?,        ?,      ?       );"
+           << user_id.str() << guild_id.str() << is_initial << std::to_string(time(nullptr)) << GetJSONAsOptionalString(data["nick"]) << GetJSONAsOptionalString(data["avatar"]) << !is_removed;
+    }
+    void insertMemberDelete(Discord::Snowflake user_id, Discord::Snowflake guild_id) {
+        insertMemberUpdate(fetchFetchMember(user_id, guild_id), user_id, guild_id, false, true);
+    }
+
+    /*
+     * Users
+     */
+    void insertUserUpdate(const Json::Value& data, bool is_initial) {
+        db << "INSERT INTO users (id, is_initial, timestamp, full_username, avatar, bio, has_nitro, is_bot)"
+              "           VALUES (?,  ?,          ?,         ?,             ?,      ?,   ?,         ?     );"
+           << data["id"].asString() << is_initial << std::to_string(time(nullptr))
+           << data["username"].asString()+'#'+data["discriminator"].asString() << GetJSONAsOptionalString(data["avatar"])
+           << GetJSONAsOptionalString(data["bio"]) << GetJSONAsOptionalInt(data["has_nitro"]).value_or(0)
+           << GetJSONAsOptionalInt(data["bot"]).value_or(false);
     }
 
     /*
      * Messages
      */
+    void processMessage(Json::Value&& data) {
+        const auto& guild_id = data["guild_id"];
+        const auto& user = data["author"];
+
+        // Process member
+        auto& member_data = data["member"];
+        if (member_data.isObject() && guild_id.isString()) {
+            updateMember(std::move(member_data), guild_id, user["id"]);
+        }
+
+        // Insert message into database
+        insertMessageContent(data, true);
+    }
     void insertMessageContent(const Json::Value& data, bool is_initial) {
-        const auto& embed = data["embed"];
+        const auto& embeds = data["embeds"];
-        db << "INSERT OR IGNORE INTO message_contents (message_id, is_initial, timestamp, content, embed)"
+        db << "INSERT INTO message_contents (message_id, is_initial, timestamp, content, embeds)"
-              "                                VALUES (?,          ?,          ?,         ?,       ?    );"
+              "                      VALUES (?,          ?,          ?,         ?,       ?    );"
-           << data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString() << (embed.isObject()?embed.toStyledString():std::optional<std::string>());
+           << data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString()
+           << (embeds.isArray()?Json::writeString(Json::StreamWriterBuilder(), embeds):std::optional<std::string>());
     }
     void insertMessageUpdate(const Json::Value& data, bool is_initial) {
+        bool has_content = !data["content"].asString().empty() || data["embeds"].isArray();
         if (is_initial) {
             const auto& author = data["author"];
             int type = data["type"].asInt();
-            db << "INSERT OR IGNORE INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)"
+            db << "INSERT  INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)"
-                  "                        VALUES (?,  ?,    ?,          ?,         ?,             ?                 );"
+                  "               VALUES (?,  ?,    ?,          ?,         ?,             ?                 );"
-               << data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional<std::string>()) << std::to_string(time(nullptr));
+               << data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString()
-            if (!data["content"].asString().empty()) insertMessageContent(data, true);
+               << (type==19?data["referenced_message"]["id"].asString():std::optional<std::string>()) << std::to_string(time(nullptr));
             cache.store(author);
+            if (has_content) insertMessageContent(data, true);
         } else {
-            if (!data["content"].asString().empty()) {
+            db << "UPDATE messages "
-                db << "UPDATE messages "
+                  "SET is_edited = 1, update_timestamp = ? "
-                      "SET is_edited = 1, update_timestamp = ? "
+                  "WHERE id = ?;"
-                      "WHERE id = ?;"
+               << std::to_string(time(nullptr)) << data["id"].asString();
-                   << std::to_string(time(nullptr)) << data["id"].asString();
+            if (has_content) insertMessageContent(data, false);
-                insertMessageContent(data, true);
-            }
         }
     }
     void insertMessageDelete(const Json::Value& data) {
         db << "UPDATE messages "
-              "SET is_deleted = 1 "
+              "SET is_deleted = 1, deletion_timestamp = ? "
               "WHERE id = ?;"
-           << data["id"].asString();
+           << std::to_string(time(nullptr)) << data["id"].asString();
     }
 
     /*
      * Channels
      */
-    void insertChannelUpdate(const Json::Value& data, bool is_initial) {
+    void insertChannelUpdate(const Json::Value& data, bool is_initial, bool is_deleted = false) {
-        db << "INSERT OR IGNORE INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic)"
+        db << "INSERT INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic, is_deleted)"
-              "                        VALUES (?,  ?,        ?,         ?,         ?,          ?,    ?,    ?    );"
+              "              VALUES (?,  ?,        ?,         ?,         ?,          ?,    ?,    ?,     ?);"
-           << data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"]) << std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"]) << GetJSONAsOptionalString(data["topic"]);
+           << data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"])
-        cache.store(data);
+           << std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"])
+           << GetJSONAsOptionalString(data["topic"]) << is_deleted;
+    }
+    void insertChannelDelete(const Json::Value& data) {
+        auto cached_data = cache.fetch(data["id"]);
+        insertChannelUpdate(cached_data, false, true);
     }
 
     /*
      * Intent handler
      */
-    virtual boost::asio::awaitable<void> intentHandler(const std::string& intent, const Json::Value& data) override {
+    virtual boost::asio::awaitable<void> intentHandler(std::string intent, Json::Value data) override {
         if (intent == "READY") [[unlikely]] {
-            const auto& user = cache.store(data["user"]);
+            const auto& user = cache.store(std::move(data["user"])).data;
             logger.log(Loglevel::info, "Connected to Discord as: "+user["username"].asString()+'#'+user["discriminator"].asString());
-            settings.is_bot = user["bot"].asBool();
+            settings.is_bot = GetJSONAsOptionalInt(user["bot"]).value_or(false);
-            co_await fetchAllGuilds(data["guilds"]);
+            co_await fetchAllGuilds(std::move(data["guilds"]));
         }
         else if (intent == "GUILD_CREATE") [[unlikely]] {
-            insertGuildUpdate(data, true);
+            processGuild(std::move(data));
         } else if (intent == "GUILD_UPDATE") [[unlikely]] {
             insertGuildUpdate(data, false);
+            cache.store(std::move(data));
         } else if (intent == "GUILD_DELETE") [[unlikely]] {
             insertGuildDelete(data);
         }
@@ -282,6 +424,22 @@ protected:
         } else if (intent == "MESSAGE_DELETE") {
             insertMessageDelete(data);
         }
+        else if (intent == "CHANNEL_CREATE") [[unlikely]] {
+            insertChannelUpdate(data, true);
+            cache.store(std::move(data));
+        } else if (intent == "CHANNEL_UPDATE") [[unlikely]] {
+            insertChannelUpdate(data, false);
+            cache.store(std::move(data));
+        } else if (intent == "CHANNEL_DELETE") [[unlikely]] {
+            insertChannelDelete(data);
+        }
+        else if (intent == "GUILD_MEMBER_ADD" || intent == "GUILD_MEMBER_UDATE") [[unlikely]] {
+            auto guild_id = std::move(data["guild_id"]);
+            updateMember(std::move(data), guild_id.asString());
+        } else if (intent == "GUILD_MEMBER_REMOVE") [[unlikely]] {
+            const auto& user_data = cache.store(data["user"]).data;
+            insertMemberDelete(user_data["id"], data["guild_id"]);
+        }
         co_return;
     }
 };