diff --git a/dcboost b/dcboost index 2f77d0a..b0e59f7 160000 --- a/dcboost +++ b/dcboost @@ -1 +1 @@ -Subproject commit 2f77d0a9c0248a496be051cbe29f41d8091741bc +Subproject commit b0e59f70b30ca7ad8c4b95e71f6c798249b4ade8 diff --git a/main.cpp b/main.cpp index 1be9eac..deea5a3 100644 --- a/main.cpp +++ b/main.cpp @@ -34,6 +34,11 @@ public: logger.log(Loglevel::verbose, "Stored "+id_str+" in cache"); return cache.insert_or_assign(std::move(id_str), object).first->second; } + const Json::Value& store(Json::Value&& object) { + const auto& id_str = object["id"].asString(); + logger.log(Loglevel::verbose, "Moved "+id_str+" into cache"); + return cache.insert_or_assign(std::move(id_str), std::move(object)).first->second; + } const Json::Value& fetch(Discord::Snowflake id) const { const auto& entry = cache.find(id); if (entry == cache.end()) { @@ -48,6 +53,10 @@ public: }; +static inline std::optional GetJSONAsOptionalString(const Json::Value& data) { + return data.isString()?data.asString():std::optional(); +} + class MyClient final : public Discord::Client { sqlite::database db; Cache cache; @@ -59,6 +68,11 @@ public: : ChatFuse::Discord::Client(io, settings), db("log.sqlite3") { + // Improve database performance + db << "pragma journal_mode = WAL;"; + db << "pragma synchronous = normal;"; + db << "pragma temp_store = memory;"; + // Create tables { db << "CREATE TABLE IF NOT EXISTS messages (" @@ -119,14 +133,13 @@ public: ");"; db << "CREATE TABLE IF NOT EXISTS channels (" " id TEXT NOT NULL," - " guild_id TEXT NOT NULL," - " category_channel_id TEXT NOT NULL," + " guild_id TEXT," + " parent_id TEXT," " timestamp TEXT NOT NULL," " is_initial INTEGER DEFAULT 1 NOT NULL," " type INTEGER NOT NULL," - " name TEXT NOT NULL," - " topic TEXT," - " has_access INTEGER NOT NULL" + " name TEXT," + " topic TEXT" ");"; db << "CREATE TABLE IF NOT EXISTS guilds (" " id TEXT NOT NULL," @@ -140,6 +153,30 @@ public: } protected: + /* + * Guilds + */ + void processGuild(const Json::Value& data) { + const auto& id = data["id"]; + + // Insert into database + insertGuildUpdate(data, !cache.has(id)); + + // Store in cache + cache.store(data); + + // Insert and store all channels + for (auto channel_data : data["channels"]) { + // Add guild_id to channel (it'll be missing) + channel_data["guild_id"] = id; + + // Insert into database + insertChannelUpdate(channel_data, !cache.has(channel_data["id"])); + + // Store in cache + cache.store(std::move(channel_data)); + } + } boost::asio::awaitable fetchAllGuilds(const Json::Value& unavailable_guild_array) { RandomGenerator rng; rng.seed(); @@ -150,25 +187,19 @@ protected: if (cache.has(guild_id)) continue; // Make sure data is actually incomplete - if (incomplete_data["name"].isString()) { + if (incomplete_data["name"].isString() && incomplete_data["channels"].isArray()) { // We'll just use it as-is - cache.store(incomplete_data); - insertGuildUpdate(incomplete_data, true); + processGuild(incomplete_data); continue; } - // Fetch guild from API and store it in cache - auto data = co_await api.call(boost::beast::http::verb::get, "/guilds/"+guild_id.str()); - cache.store(data); - - // Insert guild into database - insertGuildUpdate(data, true); + // Fetch guild from API and process it + processGuild(co_await api.call(boost::beast::http::verb::get, "/guilds/"+guild_id.str())); // Delay co_await asyncSleep(rng.getUInt(6000, 15000)); } } - void insertGuildUpdate(const Json::Value& data, bool is_initial) { db << "INSERT OR IGNORE INTO guilds (id, timestamp, is_initial, name, owner_user_id)" " VALUES (?, ?, ?, ?, ? );" @@ -182,31 +213,31 @@ protected: << data["id"].asString() << std::to_string(time(nullptr)) << cached_data["name"].asString() << cached_data["owner_id"].asString(); } + /* + * Messages + */ void insertMessageContent(const Json::Value& data, bool is_initial) { - auto created_time = Discord::Utilities::TimestampToTimeT(data["timestamp"].asString()); - auto edited_time = Discord::Utilities::TimestampToTimeT(data["edited_timestamp"].asString()); const auto& embed = data["embed"]; db << "INSERT OR IGNORE INTO message_contents (message_id, is_initial, timestamp, content, embed)" " VALUES (?, ?, ?, ?, ? );" - << data["id"].asString() << is_initial << std::to_string(edited_time?edited_time:created_time) << data["content"].asString() << (embed.isObject()?embed.toStyledString():std::optional()); + << data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString() << (embed.isObject()?embed.toStyledString():std::optional()); } void insertMessageUpdate(const Json::Value& data, bool is_initial) { if (is_initial) { const auto& author = data["author"]; int type = data["type"].asInt(); - auto created_time = Discord::Utilities::TimestampToTimeT(data["edited_timestamp"].asString()); db << "INSERT OR IGNORE INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)" " VALUES (?, ?, ?, ?, ?, ? );" - << data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional()) << created_time; + << data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional()) << std::to_string(time(nullptr)); if (!data["content"].asString().empty()) insertMessageContent(data, true); cache.store(author); } else { if (!data["content"].asString().empty()) { - insertMessageContent(data, true); db << "UPDATE messages " - "SET is_edited = 1 " + "SET is_edited = 1, update_timestamp = ? " "WHERE id = ?;" - << data["id"].asString(); + << std::to_string(time(nullptr)) << data["id"].asString(); + insertMessageContent(data, true); } } } @@ -217,6 +248,19 @@ protected: << data["id"].asString(); } + /* + * Channels + */ + void insertChannelUpdate(const Json::Value& data, bool is_initial) { + db << "INSERT OR IGNORE INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ? );" + << data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"]) << std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"]) << GetJSONAsOptionalString(data["topic"]); + cache.store(data); + } + + /* + * Intent handler + */ virtual boost::asio::awaitable intentHandler(const std::string& intent, const Json::Value& data) override { if (intent == "READY") [[unlikely]] { const auto& user = cache.store(data["user"]);