mirror of
https://gitlab.com/niansa/dpplogger.git
synced 2025-03-06 20:48:29 +01:00
Imroved database performance and added channel logging
This commit is contained in:
niansa
parent
812b1d99f4
commit
39c79cb42b
2 changed files with 68 additions and 24 deletions
2
dcboost
2
dcboost
|
|
@ -1 +1 @@
|
|||
Subproject commit 2f77d0a9c0248a496be051cbe29f41d8091741bc
|
||||
Subproject commit b0e59f70b30ca7ad8c4b95e71f6c798249b4ade8
|
||||
90
main.cpp
90
main.cpp
|
|
@ -34,6 +34,11 @@ public:
|
|||
logger.log(Loglevel::verbose, "Stored "+id_str+" in cache");
|
||||
return cache.insert_or_assign(std::move(id_str), object).first->second;
|
||||
}
|
||||
const Json::Value& store(Json::Value&& object) {
|
||||
const auto& id_str = object["id"].asString();
|
||||
logger.log(Loglevel::verbose, "Moved "+id_str+" into cache");
|
||||
return cache.insert_or_assign(std::move(id_str), std::move(object)).first->second;
|
||||
}
|
||||
const Json::Value& fetch(Discord::Snowflake id) const {
|
||||
const auto& entry = cache.find(id);
|
||||
if (entry == cache.end()) {
|
||||
|
|
@ -48,6 +53,10 @@ public:
|
|||
};
|
||||
|
||||
|
||||
static inline std::optional<std::string> GetJSONAsOptionalString(const Json::Value& data) {
|
||||
return data.isString()?data.asString():std::optional<std::string>();
|
||||
}
|
||||
|
||||
class MyClient final : public Discord::Client {
|
||||
sqlite::database db;
|
||||
Cache cache;
|
||||
|
|
@ -59,6 +68,11 @@ public:
|
|||
: ChatFuse::Discord::Client(io, settings),
|
||||
db("log.sqlite3")
|
||||
{
|
||||
// Improve database performance
|
||||
db << "pragma journal_mode = WAL;";
|
||||
db << "pragma synchronous = normal;";
|
||||
db << "pragma temp_store = memory;";
|
||||
|
||||
// Create tables
|
||||
{
|
||||
db << "CREATE TABLE IF NOT EXISTS messages ("
|
||||
|
|
@ -119,14 +133,13 @@ public:
|
|||
");";
|
||||
db << "CREATE TABLE IF NOT EXISTS channels ("
|
||||
" id TEXT NOT NULL,"
|
||||
" guild_id TEXT NOT NULL,"
|
||||
" category_channel_id TEXT NOT NULL,"
|
||||
" guild_id TEXT,"
|
||||
" parent_id TEXT,"
|
||||
" timestamp TEXT NOT NULL,"
|
||||
" is_initial INTEGER DEFAULT 1 NOT NULL,"
|
||||
" type INTEGER NOT NULL,"
|
||||
" name TEXT NOT NULL,"
|
||||
" topic TEXT,"
|
||||
" has_access INTEGER NOT NULL"
|
||||
" name TEXT,"
|
||||
" topic TEXT"
|
||||
");";
|
||||
db << "CREATE TABLE IF NOT EXISTS guilds ("
|
||||
" id TEXT NOT NULL,"
|
||||
|
|
@ -140,6 +153,30 @@ public:
|
|||
}
|
||||
|
||||
protected:
|
||||
/*
|
||||
* Guilds
|
||||
*/
|
||||
void processGuild(const Json::Value& data) {
|
||||
const auto& id = data["id"];
|
||||
|
||||
// Insert into database
|
||||
insertGuildUpdate(data, !cache.has(id));
|
||||
|
||||
// Store in cache
|
||||
cache.store(data);
|
||||
|
||||
// Insert and store all channels
|
||||
for (auto channel_data : data["channels"]) {
|
||||
// Add guild_id to channel (it'll be missing)
|
||||
channel_data["guild_id"] = id;
|
||||
|
||||
// Insert into database
|
||||
insertChannelUpdate(channel_data, !cache.has(channel_data["id"]));
|
||||
|
||||
// Store in cache
|
||||
cache.store(std::move(channel_data));
|
||||
}
|
||||
}
|
||||
boost::asio::awaitable<void> fetchAllGuilds(const Json::Value& unavailable_guild_array) {
|
||||
RandomGenerator rng;
|
||||
rng.seed();
|
||||
|
|
@ -150,25 +187,19 @@ protected:
|
|||
if (cache.has(guild_id)) continue;
|
||||
|
||||
// Make sure data is actually incomplete
|
||||
if (incomplete_data["name"].isString()) {
|
||||
if (incomplete_data["name"].isString() && incomplete_data["channels"].isArray()) {
|
||||
// We'll just use it as-is
|
||||
cache.store(incomplete_data);
|
||||
insertGuildUpdate(incomplete_data, true);
|
||||
processGuild(incomplete_data);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fetch guild from API and store it in cache
|
||||
auto data = co_await api.call(boost::beast::http::verb::get, "/guilds/"+guild_id.str());
|
||||
cache.store(data);
|
||||
|
||||
// Insert guild into database
|
||||
insertGuildUpdate(data, true);
|
||||
// Fetch guild from API and process it
|
||||
processGuild(co_await api.call(boost::beast::http::verb::get, "/guilds/"+guild_id.str()));
|
||||
|
||||
// Delay
|
||||
co_await asyncSleep(rng.getUInt(6000, 15000));
|
||||
}
|
||||
}
|
||||
|
||||
void insertGuildUpdate(const Json::Value& data, bool is_initial) {
|
||||
db << "INSERT OR IGNORE INTO guilds (id, timestamp, is_initial, name, owner_user_id)"
|
||||
" VALUES (?, ?, ?, ?, ? );"
|
||||
|
|
@ -182,31 +213,31 @@ protected:
|
|||
<< data["id"].asString() << std::to_string(time(nullptr)) << cached_data["name"].asString() << cached_data["owner_id"].asString();
|
||||
}
|
||||
|
||||
/*
|
||||
* Messages
|
||||
*/
|
||||
void insertMessageContent(const Json::Value& data, bool is_initial) {
|
||||
auto created_time = Discord::Utilities::TimestampToTimeT(data["timestamp"].asString());
|
||||
auto edited_time = Discord::Utilities::TimestampToTimeT(data["edited_timestamp"].asString());
|
||||
const auto& embed = data["embed"];
|
||||
db << "INSERT OR IGNORE INTO message_contents (message_id, is_initial, timestamp, content, embed)"
|
||||
" VALUES (?, ?, ?, ?, ? );"
|
||||
<< data["id"].asString() << is_initial << std::to_string(edited_time?edited_time:created_time) << data["content"].asString() << (embed.isObject()?embed.toStyledString():std::optional<std::string>());
|
||||
<< data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString() << (embed.isObject()?embed.toStyledString():std::optional<std::string>());
|
||||
}
|
||||
void insertMessageUpdate(const Json::Value& data, bool is_initial) {
|
||||
if (is_initial) {
|
||||
const auto& author = data["author"];
|
||||
int type = data["type"].asInt();
|
||||
auto created_time = Discord::Utilities::TimestampToTimeT(data["edited_timestamp"].asString());
|
||||
db << "INSERT OR IGNORE INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)"
|
||||
" VALUES (?, ?, ?, ?, ?, ? );"
|
||||
<< data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional<std::string>()) << created_time;
|
||||
<< data["id"].asString() << type << data["channel_id"].asString() << author["id"].asString() << (type==19?data["referenced_message"]["id"].asString():std::optional<std::string>()) << std::to_string(time(nullptr));
|
||||
if (!data["content"].asString().empty()) insertMessageContent(data, true);
|
||||
cache.store(author);
|
||||
} else {
|
||||
if (!data["content"].asString().empty()) {
|
||||
insertMessageContent(data, true);
|
||||
db << "UPDATE messages "
|
||||
"SET is_edited = 1 "
|
||||
"SET is_edited = 1, update_timestamp = ? "
|
||||
"WHERE id = ?;"
|
||||
<< data["id"].asString();
|
||||
<< std::to_string(time(nullptr)) << data["id"].asString();
|
||||
insertMessageContent(data, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -217,6 +248,19 @@ protected:
|
|||
<< data["id"].asString();
|
||||
}
|
||||
|
||||
/*
|
||||
* Channels
|
||||
*/
|
||||
void insertChannelUpdate(const Json::Value& data, bool is_initial) {
|
||||
db << "INSERT OR IGNORE INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?, ?, ? );"
|
||||
<< data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"]) << std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"]) << GetJSONAsOptionalString(data["topic"]);
|
||||
cache.store(data);
|
||||
}
|
||||
|
||||
/*
|
||||
* Intent handler
|
||||
*/
|
||||
virtual boost::asio::awaitable<void> intentHandler(const std::string& intent, const Json::Value& data) override {
|
||||
if (intent == "READY") [[unlikely]] {
|
||||
const auto& user = cache.store(data["user"]);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue