Imroved database performance and added channel logging

dcboost

@@ -1 +1 @@
-Subproject commit 2f77d0a9c0248a496be051cbe29f41d8091741bc
+Subproject commit b0e59f70b30ca7ad8c4b95e71f6c798249b4ade8

main.cpp

@@ -34,6 +34,11 @@ public:
         logger.log(Loglevel::verbose, "Stored "+id_str+" in cache");
         return cache.insert_or_assign(std::move(id_str), object).first->second;
     }
+    const Json::Value& store(Json::Value&& object) {
+        const auto& id_str = object["id"].asString();
+        logger.log(Loglevel::verbose, "Moved "+id_str+" into cache");
+        return cache.insert_or_assign(std::move(id_str), std::move(object)).first->second;
+    }
     const Json::Value& fetch(Discord::Snowflake id) const {
         const auto& entry = cache.find(id);
         if (entry == cache.end()) {
@@ -48,6 +53,10 @@ public:
 };
 
 
+static inline std::optional<std::string> GetJSONAsOptionalString(const Json::Value& data) {
+    return data.isString()?data.asString():std::optional<std::string>();
+}
+
 class MyClient final : public Discord::Client {
     sqlite::database db;
     Cache cache;
@@ -59,6 +68,11 @@ public:
         : ChatFuse::Discord::Client(io, settings),
           db("log.sqlite3")
     {
+        // Improve database performance
+        db << "pragma journal_mode = WAL;";
+        db << "pragma synchronous = normal;";
+        db << "pragma temp_store = memory;";
+
         // Create tables
         {
             db << "CREATE TABLE IF NOT EXISTS messages ("
@@ -119,14 +133,13 @@ public:
                   ");";
             db << "CREATE TABLE IF NOT EXISTS channels ("
                   "    id TEXT NOT NULL,"
-                  "    guild_id TEXT NOT NULL,"
+                  "    guild_id TEXT,"
-                  "    category_channel_id TEXT NOT NULL,"
+                  "    parent_id TEXT,"
                   "    timestamp TEXT NOT NULL,"
                   "    is_initial INTEGER DEFAULT 1 NOT NULL,"
                   "    type INTEGER NOT NULL,"
-                  "    name TEXT NOT NULL,"
+                  "    name TEXT,"
-                  "    topic TEXT,"
+                  "    topic TEXT"
-                  "    has_access INTEGER NOT NULL"
                   ");";
             db << "CREATE TABLE IF NOT EXISTS guilds ("
                   "    id TEXT NOT NULL,"
@@ -140,6 +153,30 @@ public:
     }
 
 protected:
+    /*
+     * Guilds
+     */
+    void processGuild(const Json::Value& data) {
+        const auto& id = data["id"];
+
+        // Insert into database
+        insertGuildUpdate(data, !cache.has(id));
+
+        // Store in cache
+        cache.store(data);
+
+        // Insert and store all channels
+        for (auto channel_data : data["channels"]) {
+            // Add guild_id to channel (it'll be missing)
+            channel_data["guild_id"] = id;
+
+            // Insert into database
+            insertChannelUpdate(channel_data, !cache.has(channel_data["id"]));
+
+            // Store in cache
+            cache.store(std::move(channel_data));
+        }
+    }
     boost::asio::awaitable<void> fetchAllGuilds(const Json::Value& unavailable_guild_array) {
         RandomGenerator rng;
         rng.seed();
@@ -150,25 +187,19 @@ protected:
             if (cache.has(guild_id)) continue;
 
             // Make sure data is actually incomplete
-            if (incomplete_data["name"].isString()) {
+            if (incomplete_data["name"].isString() && incomplete_data["channels"].isArray()) {
                 // We'll just use it as-is
-                cache.store(incomplete_data);
+                processGuild(incomplete_data);
-                insertGuildUpdate(incomplete_data, true);
                 continue;
             }
 
-            // Fetch guild from API and store it in cache
+            // Fetch guild from API and process it
-            auto data = co_await api.call(boost::beast::http::verb::get, "/guilds/"+guild_id.str());
+            processGuild(co_await api.call(boost::beast::http::verb::get, "/guilds/"+guild_id.str()));
-            cache.store(data);
-
-            // Insert guild into database
-            insertGuildUpdate(data, true);
 
             // Delay
             co_await asyncSleep(rng.getUInt(6000, 15000));
         }
     }
-
     void insertGuildUpdate(const Json::Value& data, bool is_initial) {
         db << "INSERT OR IGNORE INTO guilds (id, timestamp, is_initial, name, owner_user_id)"
               "                      VALUES (?,  ?,         ?,          ?,    ?            );"
@@ -182,31 +213,31 @@ protected:
            << data["id"].asString() << std::to_string(time(nullptr)) << cached_data["name"].asString() << cached_data["owner_id"].asString();
     }
 
+    /*
+     * Messages
+     */
     void insertMessageContent(const Json::Value& data, bool is_initial) {
-        auto created_time = Discord::Utilities::TimestampToTimeT(data["timestamp"].asString());
-        auto edited_time = Discord::Utilities::TimestampToTimeT(data["edited_timestamp"].asString());
         const auto& embed = data["embed"];
         db << "INSERT OR IGNORE INTO message_contents (message_id, is_initial, timestamp, content, embed)"
               "                                VALUES (?,          ?,          ?,         ?,       ?    );"
-           << data["id"].asString() << is_initial << std::to_string(edited_time?edited_time:created_time) << data["content"].asString() << (embed.isObject()?embed.toStyledString():std::optional<std::string>());
+           << data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString() << (embed.isObject()?embed.toStyledString():std::optional<std::string>());
     }
     void insertMessageUpdate(const Json::Value& data, bool is_initial) {
         if (is_initial) {
             const auto& author = data["author"];
             int type = data["type"].asInt();
-            auto created_time = Discord::Utilities::TimestampToTimeT(data["edited_timestamp"].asString());
             db << "INSERT OR IGNORE INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)"
                   "                        VALUES (?,  ?,    ?,          ?,         ?,             ?                 );"
-               << data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional<std::string>()) << created_time;
+               << data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional<std::string>()) << std::to_string(time(nullptr));
             if (!data["content"].asString().empty()) insertMessageContent(data, true);
             cache.store(author);
         } else {
             if (!data["content"].asString().empty()) {
-                insertMessageContent(data, true);
                 db << "UPDATE messages "
-                      "SET is_edited = 1 "
+                      "SET is_edited = 1, update_timestamp = ? "
                       "WHERE id = ?;"
-                   << data["id"].asString();
+                   << std::to_string(time(nullptr)) << data["id"].asString();
+                insertMessageContent(data, true);
             }
         }
     }
@@ -217,6 +248,19 @@ protected:
            << data["id"].asString();
     }
 
+    /*
+     * Channels
+     */
+    void insertChannelUpdate(const Json::Value& data, bool is_initial) {
+        db << "INSERT OR IGNORE INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic)"
+              "                        VALUES (?,  ?,        ?,         ?,         ?,          ?,    ?,    ?    );"
+           << data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"]) << std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"]) << GetJSONAsOptionalString(data["topic"]);
+        cache.store(data);
+    }
+
+    /*
+     * Intent handler
+     */
     virtual boost::asio::awaitable<void> intentHandler(const std::string& intent, const Json::Value& data) override {
         if (intent == "READY") [[unlikely]] {
             const auto& user = cache.store(data["user"]);