1
0
Fork 0
mirror of https://gitlab.com/niansa/dpplogger.git synced 2025-03-06 20:48:29 +01:00
dpplogger/main.cpp
2023-01-20 21:58:32 +01:00

585 lines
24 KiB
C++

#include "Random.hpp"
#include "sqlite_modern_cpp/sqlite_modern_cpp.h"
#include <iostream>
#include <unordered_map>
#include <optional>
#include <chrono>
#include <memory>
#include <stdexcept>
#include <boost/asio/io_service.hpp>
#include <dcboost/discord.hpp>
#include <dcboost/snowflake.hpp>
#include <fstream>
using namespace ChatFuse;
using namespace QLog;
class Cache {
Logger<Cache> logger;
std::unordered_map<Discord::Snowflake, Json::Value> cache;
public:
struct StoreResult {
const Json::Value& data;
enum {
unchanged = 0b00,
updated = 0b01,
created = 0b11,
};
uint8_t changed;
};
struct Miss : public std::runtime_error {
Miss() : std::runtime_error("Object cache miss") {}
};
Cache() {
logger.inst_ptr = this;
}
StoreResult store(Discord::Snowflake id, const Json::Value& object) {
logger.log(Loglevel::verbose, "Stored "+id.str()+" in cache");
auto cache_hit = cache.find(id);
if (cache_hit != cache.end()) [[likely]] {
bool changed = cache_hit->second != object;
if (changed) {
cache_hit->second = object;
}
return {cache_hit->second, changed};
} else {
return {cache.emplace(id, object).first->second, StoreResult::created};
}
}
StoreResult store(Discord::Snowflake id, Json::Value&& object) {
logger.log(Loglevel::verbose, "Moved "+id.str()+" into cache");
auto cache_hit = cache.find(id);
if (cache_hit != cache.end()) [[likely]] {
bool changed = cache_hit->second != object;
if (changed) {
cache_hit->second = std::move(object);
}
return {cache_hit->second, changed};
} else {
return {cache.emplace(id, std::move(object)).first->second, true};
}
}
StoreResult store(const Json::Value& object) {
auto id = object["id"].asString();
return store(id, object);
}
StoreResult store(Json::Value&& object) {
auto id = object["id"].asString();
return store(id, std::move(object));
}
const Json::Value& fetch(Discord::Snowflake id) const {
const auto& entry = cache.find(id);
if (entry == cache.end()) [[unlikely]] {
throw Miss();
}
return entry->second;
}
const Json::Value& fetch(const std::string& id) const {
return fetch(Discord::Snowflake(id));
}
bool has(Discord::Snowflake id) const {
return cache.find(id) != cache.end();
}
};
static inline std::optional<std::string> GetJSONAsOptionalString(const Json::Value& data) {
return data.isString()?data.asString():std::optional<std::string>();
}
static inline std::optional<int> GetJSONAsOptionalInt(const Json::Value& data) {
return data.isString()?data.asInt():std::optional<int>();
}
static inline std::optional<bool> GetJSONAsOptionalBool(const Json::Value& data) {
return data.isBool()?data.asBool():std::optional<bool>();
}
static inline std::string GetJSONAsJSONData(const Json::Value& data) {
return Json::writeString(Json::StreamWriterBuilder(), data);
}
static inline std::optional<std::string> GetJSONAsOptionalJSONData(const Json::Value& data) {
return ((!data.empty()&&!data.isNull())?GetJSONAsJSONData(data):std::optional<std::string>());
}
class MyClient final : public Discord::Client {
sqlite::database db;
Cache cache;
Json::Value *me;
static inline std::string getDatabaseFileName() {
return "log-"+std::to_string(time(nullptr))+".sqlite3";
}
public:
MyClient(boost::asio::io_service& io, const ChatFuse::Discord::Settings& settings = {})
: ChatFuse::Discord::Client(io, settings),
db(getDatabaseFileName())
{
// Improve database performance
db << "pragma journal_mode = WAL;";
db << "pragma synchronous = normal;";
db << "pragma temp_store = memory;";
// Create tables
{
db << "CREATE TABLE IF NOT EXISTS messages ("
" id TEXT PRIMARY KEY NOT NULL,"
" type INTEGER NOT NULL,"
" channel_id TEXT NOT NULL,"
" author_id TEXT NOT NULL,"
" replied_to_id TEXT,"
" is_deleted INTEGER DEFAULT 0 NOT NULL,"
" is_edited INTEGER DEFAULT 0 NOT NULL,"
" creation_timestamp TEXT NOT NULL,"
" update_timestamp TEXT DEFAULT '0' NOT NULL,"
" deletion_timestamp TEXT DEFAULT '0' NOT NULL,"
" UNIQUE(id)"
");";
db << "CREATE TABLE IF NOT EXISTS message_contents ("
" message_id TEXT NOT NULL,"
" is_initial INTEGER DEFAULT 1 NOT NULL,"
" timestamp TEXT NOT NULL,"
" content TEXT NOT NULL,"
" embeds TEXT" // Discord JSON embed object array
");";
db << "CREATE TABLE IF NOT EXISTS users ("
" id TEXT NOT NULL,"
" is_initial INTEGER DEFAULT 1 NOT NULL,"
" timestamp TEXT NOT NULL,"
" full_username TEXT NOT NULL," // Username#Discriminator
" avatar TEXT,"
" bio TEXT,"
" has_nitro INTEGER NOT NULL,"
" is_bot INTEGER NOT NULL"
");";
db << "CREATE TABLE IF NOT EXISTS user_presences ("
" user_id TEXT NOT NULL,"
" is_initial INTEGER DEFAULT 1 NOT NULL,"
" timestamp TEXT NOT NULL,"
" status TEXT," // JSON array or null if not updated: [string:online/dnd/afk/offline, string:device]
" activities TEXT" // Discord JSON Activity object list
");";
db << "CREATE TABLE IF NOT EXISTS members ("
" user_id TEXT NOT NULL,"
" guild_id TEXT NOT NULL,"
" is_initial INTEGER DEFAULT 1 NOT NULL,"
" timestamp TEXT NOT NULL,"
" nickname TEXT,"
" avatar TEXT,"
" in_guild INTEGER NOT NULL"
");";
db << "CREATE TABLE IF NOT EXISTS member_voice_connections ("
" channel_id TEXT,"
" user_id TEXT NOT NULL,"
" timestamp TEXT NOT NULL,"
" self_mute INTEGER NOT NULL,"
" self_deaf INTEGER NOT NULL,"
" self_video INTEGER NOT NULL,"
" self_stream INTEGER NOT NULL,"
" server_mute INTEGER NOT NULL,"
" server_deaf INTEGER NOT NULL"
");";
db << "CREATE TABLE IF NOT EXISTS channels ("
" id TEXT NOT NULL,"
" guild_id TEXT,"
" parent_id TEXT,"
" timestamp TEXT NOT NULL,"
" is_initial INTEGER DEFAULT 1 NOT NULL,"
" type INTEGER NOT NULL,"
" name TEXT,"
" topic TEXT,"
" is_deleted INTEGER DEFAULT 0 NOT NULL"
");";
db << "CREATE TABLE IF NOT EXISTS guilds ("
" id TEXT NOT NULL,"
" timestamp TEXT NOT NULL,"
" is_initial INTEGER DEFAULT 1 NOT NULL,"
" name TEXT NOT NULL,"
" owner_user_id TEXT NOT NULL,"
" in_guild INTEGER DEFAULT 1 NOT NULL"
");";
}
}
protected:
/*
* Note: On creation, update, and deletion, it is recommended to use
* the functions starting with 'insert' as they do not store any
* data in the cache. For creation, it is also recommended to
* use the functions starting with 'process' as they not only store
* data in the cache, but also automatically handle related members
* (e.g. channels in a guild). The functions starting with 'update'
* are preferred for both creation and update and store data in cache
* as well. When accessing the cache, it is recommended to use the
* functions starting with 'cache' rather than accessing it directly.
*/
/*
* Guilds
*/
boost::asio::awaitable<void> fetchAllGuilds(Json::Value&& unavailable_guild_array) {
RandomGenerator rng;
rng.seed();
// Fetch guilds and channels from API
for (auto& incomplete_data : unavailable_guild_array) {
Discord::Snowflake guild_id = incomplete_data["id"];
// Use cached guild if possible
if (cache.has(guild_id)) continue; // In this case, we skip processing the guild,
// since it's safe to assume it's already been cached
// Make sure data is actually incomplete
if ((incomplete_data["name"].isString() && incomplete_data["channels"].isArray()) || incomplete_data["properties"].isObject()) {
// It's not, so we'll just use it as-is
processGuild(std::move(incomplete_data));
continue;
}
// Fetch guild from API and process it
processGuild(co_await api.call(boost::beast::http::verb::get, "/guilds/"+guild_id.str()));
// Delay
co_await asyncSleep(rng.getUInt(6000, 15000));
}
}
void fixGuild(Json::Value& data) {
if (!settings.is_bot) {
auto& props = data["properties"];
for (const auto& prop_name : props.getMemberNames()) {
data[prop_name] = std::move(props[prop_name]);
}
data.removeMember("properties");
}
}
Cache::StoreResult processGuild(Json::Value&& data) {
// Fix guild
fixGuild(data);
// Get guild ID
const auto& id = data["id"];
// Insert into database
insertGuildUpdate(data, !cache.has(id));
// Insert and store all channels
for (auto& channel_data : data["channels"]) {
// Make sure we haven't cached this channel yet
if (cache.has(channel_data["id"])) continue;
// Add guild_id to channel (it'll be missing)
channel_data["guild_id"] = id;
// Insert into database
insertChannelUpdate(channel_data, true);
// Store in cache
cache.store(std::move(channel_data));
}
// Process all members
for (auto& member_data : data["members"]) {
updateMember(std::move(member_data), id);
}
// Remove channels and members
data.removeMember("channels");
data.removeMember("members");
// Store in cache and return cached guild
return cache.store(std::move(data));
}
void insertGuildUpdate(const Json::Value& data, bool is_initial, bool is_deleted = false) {
db << "INSERT INTO guilds (id, timestamp, is_initial, name, owner_user_id, in_guild)"
" VALUES (?, ?, ?, ?, ?, ? );"
<< data["id"].asString() << std::to_string(time(nullptr)) << is_initial << data["name"].asString()
<< data["owner_id"].asString() << !is_deleted;
}
void insertGuildDelete(Discord::Snowflake id) {
auto cached_data = cache.fetch(id);
insertGuildUpdate(cached_data, false, true);
}
/*
* Members
*/
void updateMember(Json::Value&& data, Discord::Snowflake guild_id, Discord::Snowflake user_id = {}) {
if (user_id.empty()) {
// Get user
const auto& user_data = data["user"];
if (!user_data.isObject()) {
logger.log(Loglevel::error, "A processMember call has been aborted because the passed 'data' object doesn't contain a 'user' key.");
return;
}
user_id = user_data["id"].asString();
// Insert and cache user
auto changed = cache.store(user_data).changed;
if (changed) {
insertUserUpdate(user_data, (changed&Cache::StoreResult::created)?true:false);
}
}
// Cache member
auto changed = cacheStoreMember(std::move(data), user_id, guild_id).changed;
// Insert member
if (changed) {
insertMemberUpdate(data, user_id, guild_id, changed == Cache::StoreResult::created);
}
}
Cache::StoreResult cacheStoreMember(const Json::Value& data, Discord::Snowflake user_id, Discord::Snowflake guild_id) {
return cache.store(guild_id.uint()|user_id.uint(), data);
}
const Json::Value& fetchFetchMember(Discord::Snowflake user_id, Discord::Snowflake guild_id) {
return cache.fetch(guild_id.uint()|user_id.uint());
}
void insertMemberUpdate(const Json::Value& data, Discord::Snowflake user_id, Discord::Snowflake guild_id, bool is_initial, bool is_removed = false) {
db << "INSERT INTO members (user_id, guild_id, is_initial, timestamp, nickname, avatar, in_guild)"
" VALUES (?, ?, ?, ?, ?, ?, ? );"
<< user_id.str() << guild_id.str() << is_initial << std::to_string(time(nullptr)) << GetJSONAsOptionalString(data["nick"]) << GetJSONAsOptionalString(data["avatar"]) << !is_removed;
}
void insertMemberDelete(Discord::Snowflake user_id, Discord::Snowflake guild_id) {
insertMemberUpdate(fetchFetchMember(user_id, guild_id), user_id, guild_id, false, true);
}
/*
* Users
*/
void insertUserUpdate(const Json::Value& data, bool is_initial) {
db << "INSERT INTO users (id, is_initial, timestamp, full_username, avatar, bio, has_nitro, is_bot)"
" VALUES (?, ?, ?, ?, ?, ?, ?, ? );"
<< data["id"].asString() << is_initial << std::to_string(time(nullptr))
<< data["username"].asString()+'#'+data["discriminator"].asString() << GetJSONAsOptionalString(data["avatar"])
<< GetJSONAsOptionalString(data["bio"]) << GetJSONAsOptionalInt(data["premium_type"]).value_or(0)
<< GetJSONAsOptionalBool(data["bot"]).value_or(false);
}
/*
* Messages
*/
void processMessage(Json::Value&& data) {
const auto& guild_id = data["guild_id"];
const auto& user_data = data["author"];
// Process member
auto& member_data = data["member"];
if (member_data.isObject() && guild_id.isString()) {
updateMember(std::move(member_data), guild_id, user_data["id"]);
}
// Insert and cache user
const auto& [cached_user_data, changed] = cache.store(std::move(user_data));
if (changed) {
insertUserUpdate(user_data, (changed&Cache::StoreResult::created)?true:false);
}
// Insert message into database
insertMessageUpdate(data, true);
}
void insertMessageContent(const Json::Value& data, bool is_initial) {
const auto& embeds = data["embeds"];
db << "INSERT INTO message_contents (message_id, is_initial, timestamp, content, embeds)"
" VALUES (?, ?, ?, ?, ? );"
<< data["id"].asString() << is_initial << std::to_string(time(nullptr)) << data["content"].asString()
<< GetJSONAsOptionalJSONData(embeds);
}
void insertMessageUpdate(const Json::Value& data, bool is_initial) {
bool has_content = !data["content"].asString().empty() || data["embeds"].isArray();
if (is_initial) {
int type = data["type"].asInt();
db << "INSERT INTO messages (id, type, channel_id, author_id, replied_to_id, creation_timestamp)"
" VALUES (?, ?, ?, ?, ?, ? );"
<< data["id"].asString() << type << data["channel_id"].asString() << data["author"]["id"].asString()
<< (type==19?data["referenced_message"]["id"].asString():std::optional<std::string>()) << std::to_string(time(nullptr));
if (has_content) insertMessageContent(data, false);
} else {
db << "UPDATE messages "
"SET is_edited = 1, update_timestamp = ? "
"WHERE id = ?;"
<< std::to_string(time(nullptr)) << data["id"].asString();
if (has_content) insertMessageContent(data, false);
}
}
void insertMessageDelete(const std::string& id) {
db << "UPDATE messages "
"SET is_deleted = 1, deletion_timestamp = ? "
"WHERE id = ?;"
<< std::to_string(time(nullptr)) << id;
}
/*
* Channels
*/
void insertChannelUpdate(const Json::Value& data, bool is_initial, bool is_deleted = false) {
db << "INSERT INTO channels (id, guild_id, parent_id, timestamp, is_initial, type, name, topic, is_deleted)"
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);"
<< data["id"].asString() << GetJSONAsOptionalString(data["guild_id"])<< GetJSONAsOptionalString(data["parent_id"])
<< std::to_string(time(nullptr)) << is_initial << data["type"].asInt() << GetJSONAsOptionalString(data["name"])
<< GetJSONAsOptionalString(data["topic"]) << is_deleted;
}
void insertChannelDelete(Discord::Snowflake id) {
auto cached_data = cache.fetch(id);
insertChannelUpdate(cached_data, false, true);
}
/*
* Member voice connections
*/
void updateMemberVoiceConnection(Json::Value&& data) {
// Insert voice connection update
insertMemberVoiceConnectionUpdate(data);
// Update member
updateMember(std::move(data["member"]), data["guild_id"]);
}
void insertMemberVoiceConnectionUpdate(const Json::Value& data) {
db << "INSERT INTO member_voice_connections (channel_id, user_id, timestamp, self_mute, self_deaf, self_video, self_stream, server_mute, server_deaf)"
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ? );"
<< data["channel_id"].asString() << data["user_id"].asString() << std::to_string(time(nullptr)) << data["self_mute"].asBool()
<< data["self_deaf"].asBool() << data["self_video"].asBool() << GetJSONAsOptionalBool(data["self_stream"]).value_or(false)
<< data["mute"].asBool() << data["deaf"].asBool();
}
/*
* User presences
*/
void insertUserPresenceUpdate(const Json::Value& data, bool is_initial) {
// Build status
std::string status_str;
bool is_online;
{
const auto& client_status = data["client_status"];
if (client_status.isObject() && !client_status.empty()) {
const auto device = client_status.getMemberNames()[0];
status_str = "[\""+client_status[device].asString()+"\", \""+device+"\"]";
is_online = true;
} else {
status_str = R"(["offline", "none"])";
is_online = false;
}
}
// Insert
db << "INSERT INTO user_presences (user_id, is_initial, timestamp, status, activities)"
" VALUES (?, ?, ?, ?, ? );"
<< data["user"]["id"].asString() << is_initial << std::to_string(time(nullptr)) << status_str << (is_online?GetJSONAsOptionalJSONData(data["activities"]):std::optional<std::string>());
}
/*
* Intent handler
*/
virtual boost::asio::awaitable<void> intentHandler(std::string intent, Json::Value data) override {
if (intent == "READY") [[unlikely]] {
// Get own user
const auto& user = cache.store(std::move(data["user"])).data;
logger.log(Loglevel::info, "Connected to Discord as: "+user["username"].asString()+'#'+user["discriminator"].asString());
// Get guilds
co_await fetchAllGuilds(std::move(data["guilds"]));
// Get users
auto& users = data["users"];
if (users.isArray()) [[likely]] {
for (const auto& user_data : users) {
insertUserUpdate(user_data, true);
cache.store(std::move(user_data));
}
}
}
else if (intent == "GUILD_CREATE") [[unlikely]] {
fixGuild(data);
processGuild(std::move(data));
} else if (intent == "GUILD_UPDATE") [[unlikely]] {
fixGuild(data);
if (cache.store(data).changed) insertGuildUpdate(data, false);
} else if (intent == "GUILD_DELETE") [[unlikely]] {
fixGuild(data);
insertGuildDelete(data["id"]);
}
else if (intent == "MESSAGE_CREATE") [[likely]] {
processMessage(std::move(data));
} else if (intent == "MESSAGE_UPDATE") [[likely]] {
insertMessageUpdate(data, false);
} else if (intent == "MESSAGE_DELETE") {
insertMessageDelete(data["id"].asString());
} else if (intent == "MESSAGE_DELETE_BULK") {
for (const auto& id : data["ids"]) {
insertMessageDelete(id.asString());
}
}
else if (intent == "CHANNEL_CREATE") [[unlikely]] {
insertChannelUpdate(data, true);
cache.store(std::move(data));
} else if (intent == "CHANNEL_UPDATE") [[unlikely]] {
insertChannelUpdate(data, false);
if (cache.store(data).changed) cache.store(std::move(data));
} else if (intent == "CHANNEL_DELETE") [[unlikely]] {
const auto& guild_id = data["guild_id"];
if (guild_id.isString()) {
insertChannelDelete(guild_id);
}
}
else if (intent == "GUILD_MEMBER_ADD" || intent == "GUILD_MEMBER_UDATE") [[unlikely]] {
auto guild_id = std::move(data["guild_id"]);
updateMember(std::move(data), guild_id.asString());
} else if (intent == "GUILD_MEMBER_REMOVE") [[unlikely]] {
const auto& user_data = data["user"];
if (user_data["username"].isString()) {
cache.store(user_data);
insertMemberDelete(user_data["id"], data["guild_id"]);
}
}
else if (intent == "VOICE_STATE_UPDATE") {
updateMemberVoiceConnection(std::move(data));
}
else if (intent == "PRESENCE_UPDATE") {
auto changed = cache.store(Discord::Snowflake(data["user_id"]).uint()|0x04e2e7ce00000000, data).changed;
insertUserPresenceUpdate(data, changed==Cache::StoreResult::created);
}
else if (intent == "GUILD_BAN_ADD") [[unlikely]] {
// Should we log bans? Let's just use it as a source of user data
const auto& user_data = data["user"];
auto changed = cache.store(user_data).changed;
if (changed) [[unlikely]] {
insertUserUpdate(user_data, (changed&Cache::StoreResult::created)?true:false);
}
//TODO: Should we do insertMemberDelete here?
}
co_return;
}
};
int main(int argc, char **argv) {
// Check args
if (argc == 1) {
std::cout << "Usage: " << argv[0] << " <token>" << std::endl;
return EXIT_FAILURE;
}
// Create io service
boost::asio::io_service io;
// Set up client
auto client = std::make_shared<MyClient>(io, Discord::Settings{
.bot_token = argv[1],
.intents = Discord::intents::all & ~Discord::intents::direct_messages,
.is_bot = false
});
client->detach();
// Erase bot token from argv (so it's no longer visible in /proc/{pid}/cmdline)
memset(argv[1], 0, strlen(argv[1]));
// Run!!!
io.run();
return EXIT_SUCCESS;
}