Large 0.3 changes
CI and release / Detect release commit (push) Successful in 27s
CI and release / Run tests (push) Successful in 4m14s
CI and release / Build and publish container (push) Successful in 9m2s
CI and release / Create release (push) Skipped

- Split the gigantic mess that web.rs was into seperate files
- Split the app into two clear functions: web and bot
- Add privacy policy
- Add privacy
- Make the site respect GDPR and Discord ToS
- Add basic API
- Add timezone detection and automatic time changes
This commit is contained in:
Elias Wendland committed 2026-08-31 17:10:02 +02:00
1 parent 96a8c9ea8c
commit 126b9e9e9f
23 files changed
+3876 -2767

No files matched your search

+26
View File
@@ -0,0 +1,26 @@
use crate::bot::messaging::{edit_response_message, trace_message};
use crate::{Context, Error};
use tracing::debug;
/// Get a link to the git repo
#[poise::command(slash_command, prefix_command)]
pub async fn git(ctx: Context<'_>) -> Result<(), Error> {
debug!(
"git command ran by {} in {}",
ctx.author().name,
ctx.channel_id().get()
);
let response = ctx.say("Processing...").await?;
let msg = "Git repo: https://git.ewenlau.net/ewenlau/tg-archive-bot\nIf you'd like to contribute, contact <@1389325880853270569> to get an account.";
edit_response_message(&response, ctx, msg, true).await?;
trace_message(
msg,
ctx.channel_id().get().to_string(),
ctx.guild_id().unwrap().get().to_string(),
)
.await;
Ok(())
}
+9
View File
@@ -0,0 +1,9 @@
pub mod git;
pub mod ping;
pub mod stats;
pub mod version;
pub use git::git;
pub use ping::ping;
pub use stats::stats;
pub use version::version;
+37
View File
@@ -0,0 +1,37 @@
use crate::bot::messaging::trace_message;
use crate::{Context, Error};
use poise::serenity_prelude as sere;
use tracing::{debug, trace};
/// Get the bot's current ping (back and forth)
#[poise::command(slash_command, prefix_command)]
pub async fn ping(ctx: Context<'_>) -> Result<(), Error> {
trace!(
"ping command called by user {} in guild {}",
ctx.author().id.get(),
ctx.guild_id().unwrap().get()
);
trace!("Calculating ping");
let command_time_in_microseconds = ctx.created_at().timestamp_micros();
trace!("Saved command time in microseconds");
let current_time_in_microseconds = sere::Timestamp::now().timestamp_micros();
trace!("Saved current time in microseconds");
let ping = (current_time_in_microseconds - command_time_in_microseconds) / 500;
trace!("Calculated ping: {}", ping);
// We divide by 1000 since we get the time in microseconds, and then multiply by 2 to get the roundtrip time
// This is arguably not the best way to calculate ping, since it assumes perfect clock accuracy, but I'm lazy
let msg = format!("Current ping: {} ms", ping);
trace_message(
&msg,
ctx.channel_id().to_string(),
ctx.guild_id().unwrap().to_string(),
)
.await;
ctx.say(msg).await?;
debug!(
"Ping command performed for user {} with ping {}",
ctx.author().name,
ping
);
Ok(())
}
+36
View File
@@ -0,0 +1,36 @@
use crate::bot::messaging::{trace_message, edit_response_message};
use crate::{Context, Error, archive_stats};
use tracing::{debug, trace};
#[doc = "Show archive statistics"]
#[poise::command(slash_command, prefix_command)]
pub async fn stats(ctx: Context<'_>) -> Result<(), Error> {
trace!(
"stats command called by user {} in guild {}",
ctx.author().id.get(),
ctx.guild_id().unwrap().get()
);
let response = ctx.say("Processing...").await?;
let stats = archive_stats::load(&ctx.data().pool).await?;
let msg = format!(
"# Archive Statistics\n## Messages\nArchived messages: {}\nArchived users: {}\nChannels: {}\nServers: {}\n## Storage usage\nArchived data: {}\nMessage content: {}\nAttachments: {}",
stats.messages,
stats.users,
stats.channels,
stats.servers,
archive_stats::format_bytes(stats.total_storage),
archive_stats::format_bytes(stats.message_storage),
archive_stats::format_bytes(stats.attachment_storage),
);
edit_response_message(&response, ctx, &msg, true).await?;
trace_message(
&msg,
ctx.channel_id().to_string(),
ctx.guild_id().unwrap().to_string(),
)
.await;
debug!("Stats command performed for user {}", ctx.author().name);
Ok(())
}
+30
View File
@@ -0,0 +1,30 @@
use crate::bot::messaging::{trace_message, edit_response_message};
use crate::{Context, Error};
use tracing::{debug, trace};
const VERSION: &str = include_str!(concat!(env!("OUT_DIR"), "/version.txt"));
/// Get the bot's current version
#[poise::command(slash_command, prefix_command)]
pub async fn version(ctx: Context<'_>) -> Result<(), Error> {
trace!(
"version command called by user {} in guild {}",
ctx.author().id.get(),
ctx.guild_id().unwrap().get()
);
trace!("Loading embedded version information");
let response = ctx.say("Processing...").await?;
let msg = format!("Current version:\n{VERSION}");
trace_message(
&msg,
ctx.channel_id().to_string(),
ctx.guild_id().unwrap().to_string(),
)
.await;
edit_response_message(&response, ctx, &msg, true).await?;
debug!(
"Version command performed for user {} with version information",
ctx.author().name
);
Ok(())
}
+103
View File
@@ -0,0 +1,103 @@
use crate::{Data, Error};
use poise::serenity_prelude as sere;
use tracing::{debug, trace};
pub async fn event_handler(
ctx: &sere::Context,
event: &sere::FullEvent,
_framework: poise::FrameworkContext<'_, Data, Error>,
user_data: &Data,
) -> Result<(), Error> {
match event {
sere::FullEvent::Message { new_message } => {
super::message_archive::record_message(ctx, new_message, &user_data.pool).await?;
}
sere::FullEvent::MessageUpdate {
old_if_available,
new,
event,
} => {
let old_guild_id = old_if_available
.as_ref()
.and_then(|message| message.guild_id);
let mut message = if let Some(new_message) = new {
new_message.clone()
} else if let Some(old_message) = old_if_available {
let mut updated_message = old_message.clone();
event.apply_to_message(&mut updated_message);
updated_message
} else {
debug!(
"Updated message {} was not cached, fetching it from Discord",
event.id.get()
);
event.channel_id.message(ctx, event.id).await?
};
message.guild_id =
updated_message_guild_id(message.guild_id, event.guild_id, old_guild_id);
if message.guild_id.is_none() {
debug!(
"Updated message {} had no server ID in the gateway event, fetching it from Discord",
event.id.get()
);
message = event.channel_id.message(ctx, event.id).await?;
}
super::message_archive::record_message_edit(ctx, &message, &user_data.pool).await?;
}
sere::FullEvent::MessageDelete {
deleted_message_id, ..
} => {
trace!(
"Message {} was deleted from Discord, retaining archived copy",
deleted_message_id.get()
);
}
sere::FullEvent::MessageDeleteBulk {
multiple_deleted_messages_ids,
..
} => {
trace!(
"{} messages were deleted from Discord, retaining archived copies",
multiple_deleted_messages_ids.len()
);
}
_ => {}
}
Ok(())
}
fn updated_message_guild_id(
message_guild_id: Option<sere::GuildId>,
event_guild_id: Option<sere::GuildId>,
old_guild_id: Option<sere::GuildId>,
) -> Option<sere::GuildId> {
message_guild_id.or(event_guild_id).or(old_guild_id)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retains_the_cached_server_when_an_update_omits_it() {
let guild_id = sere::GuildId::new(123);
assert_eq!(
updated_message_guild_id(None, None, Some(guild_id)),
Some(guild_id)
);
}
#[test]
fn prefers_the_updated_message_server() {
let message_guild_id = sere::GuildId::new(123);
let old_guild_id = sere::GuildId::new(456);
assert_eq!(
updated_message_guild_id(Some(message_guild_id), None, Some(old_guild_id)),
Some(message_guild_id)
);
}
}
+381
View File
@@ -0,0 +1,381 @@
use crate::Error;
use poise::serenity_prelude as sere;
use tracing::{debug, trace};
pub async fn record_message(
ctx: &sere::Context,
message: &sere::Message,
pool: &sqlx::PgPool,
) -> Result<(), Error> {
record_message_version(ctx, message, pool, false).await
}
pub async fn record_message_edit(
ctx: &sere::Context,
message: &sere::Message,
pool: &sqlx::PgPool,
) -> Result<(), Error> {
record_message_version(ctx, message, pool, true).await
}
async fn record_message_version(
ctx: &sere::Context,
message: &sere::Message,
pool: &sqlx::PgPool,
is_edit: bool,
) -> Result<(), Error> {
let Some(guild_id) = message.guild_id else {
trace!("Ignoring message {} without a server", message.id.get());
return Ok(());
};
trace!(
"Archiving message {} from channel {} in guild {}",
message.id.get(),
message.channel_id.get(),
guild_id.get()
);
let guild = guild_id.to_partial_guild(ctx).await?;
let channel = message
.channel_id
.to_channel(ctx)
.await?
.guild()
.ok_or_else(|| std::io::Error::other("Guild message belonged to a private channel"))?;
let mut downloaded_attachments = Vec::with_capacity(message.attachments.len());
for attachment in &message.attachments {
trace!(
"Downloading attachment {} from message {}",
attachment.id.get(),
message.id.get()
);
downloaded_attachments.push((attachment, attachment.download().await?));
}
let guild_id = guild_id.get() as i64;
let channel_id = message.channel_id.get() as i64;
let message_id = message.id.get() as i64;
let author_id = message.author.id.get() as i64;
let timestamp = message.timestamp.to_string();
let mut transaction = pool.begin().await?;
sqlx::query(
"INSERT INTO guilds (guild_id, guild_name, guild_icon_url)
VALUES ($1, $2, $3)
ON CONFLICT (guild_id) DO UPDATE SET
guild_name = EXCLUDED.guild_name,
guild_icon_url = EXCLUDED.guild_icon_url;",
)
.bind(guild_id)
.bind(&guild.name)
.bind(guild.icon_url())
.execute(&mut *transaction)
.await?;
sqlx::query(
"INSERT INTO guild_history (guild_id, guild_name, guild_icon_url)
VALUES ($1, $2, $3)
ON CONFLICT (guild_id, guild_name, guild_icon_url) DO UPDATE SET
last_seen_at = NOW();",
)
.bind(guild_id)
.bind(&guild.name)
.bind(guild.icon_url())
.execute(&mut *transaction)
.await?;
sqlx::query(
"INSERT INTO channels (guild_id, channel_id, channel_name)
VALUES ($1, $2, $3)
ON CONFLICT (channel_id) DO UPDATE SET
guild_id = EXCLUDED.guild_id,
channel_name = EXCLUDED.channel_name;",
)
.bind(guild_id)
.bind(channel_id)
.bind(&channel.name)
.execute(&mut *transaction)
.await?;
sqlx::query(
"INSERT INTO channel_history (channel_id, channel_name)
VALUES ($1, $2)
ON CONFLICT (channel_id, channel_name) DO UPDATE SET
last_seen_at = NOW();",
)
.bind(channel_id)
.bind(&channel.name)
.execute(&mut *transaction)
.await?;
sqlx::query(
"INSERT INTO discord_users (discord_id, discord_username)
VALUES ($1, $2)
ON CONFLICT (discord_id) DO UPDATE SET
discord_username = EXCLUDED.discord_username;",
)
.bind(author_id)
.bind(&message.author.name)
.execute(&mut *transaction)
.await?;
sqlx::query(
"INSERT INTO discord_user_history (discord_id, discord_username, discord_avatar_url)
VALUES ($1, $2, $3)
ON CONFLICT (discord_id, discord_username, discord_avatar_url) DO UPDATE SET
last_seen_at = NOW();",
)
.bind(author_id)
.bind(&message.author.name)
.bind(message.author.avatar_url())
.execute(&mut *transaction)
.await?;
sqlx::query(
"INSERT INTO guild_users (guild_id, discord_id)
VALUES ($1, $2)
ON CONFLICT (guild_id, discord_id) DO UPDATE SET
last_seen_at = NOW();",
)
.bind(guild_id)
.bind(author_id)
.execute(&mut *transaction)
.await?;
for role in guild.roles.values() {
sqlx::query(
"INSERT INTO discord_roles (guild_id, discord_id, discord_role_name)
VALUES ($1, $2, $3)
ON CONFLICT (guild_id, discord_id) DO UPDATE SET
discord_role_name = EXCLUDED.discord_role_name;",
)
.bind(guild_id)
.bind(role.id.get() as i64)
.bind(&role.name)
.execute(&mut *transaction)
.await?;
}
sqlx::query(
"INSERT INTO messages (
guild_id, channel_id, message_id, author_id, author_username, content, timestamp
)
VALUES ($1, $2, $3, $4, $5, $6, $7::timestamptz)
ON CONFLICT (message_id) DO UPDATE SET
guild_id = EXCLUDED.guild_id,
channel_id = EXCLUDED.channel_id,
author_id = EXCLUDED.author_id,
author_username = EXCLUDED.author_username,
content = EXCLUDED.content,
timestamp = EXCLUDED.timestamp;",
)
.bind(guild_id)
.bind(channel_id)
.bind(message_id)
.bind(author_id)
.bind(&message.author.name)
.bind(&message.content)
.bind(timestamp)
.execute(&mut *transaction)
.await?;
let message_version = if is_edit {
sqlx::query_scalar::<_, i64>(
"SELECT COALESCE(MAX(version), 0) + 1
FROM message_versions
WHERE message_id = $1;",
)
.bind(message_id)
.fetch_one(&mut *transaction)
.await?
} else {
1
};
sqlx::query(
"INSERT INTO message_versions (message_id, version, content)
VALUES ($1, $2, $3)
ON CONFLICT (message_id, version) DO UPDATE SET
content = EXCLUDED.content;",
)
.bind(message_id)
.bind(message_version)
.bind(&message.content)
.execute(&mut *transaction)
.await?;
for (attachment, data) in downloaded_attachments {
sqlx::query(
"INSERT INTO attachments (
attachment_id, message_id, message_version, filename, description, content_type,
size, width, height, data
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (message_id, message_version, attachment_id) DO UPDATE SET
filename = EXCLUDED.filename,
description = EXCLUDED.description,
content_type = EXCLUDED.content_type,
size = EXCLUDED.size,
width = EXCLUDED.width,
height = EXCLUDED.height,
data = EXCLUDED.data;",
)
.bind(attachment.id.get() as i64)
.bind(message_id)
.bind(message_version)
.bind(&attachment.filename)
.bind(&attachment.description)
.bind(&attachment.content_type)
.bind(i64::from(attachment.size))
.bind(optional_u32_to_i32(attachment.width)?)
.bind(optional_u32_to_i32(attachment.height)?)
.bind(data)
.execute(&mut *transaction)
.await?;
}
for (embed_index, embed) in message.embeds.iter().enumerate() {
let embed_index = i32::try_from(embed_index)?;
let embed_timestamp = embed.timestamp.map(|timestamp| timestamp.to_string());
sqlx::query(
"INSERT INTO embeds (
message_id, message_version, embed_index, embed_type, title, description, url,
timestamp, color,
footer_text, footer_icon_url, image_url, image_proxy_url, image_width,
image_height, thumbnail_url, thumbnail_proxy_url, thumbnail_width,
thumbnail_height, video_url, video_proxy_url, video_width, video_height,
provider_name, provider_url, author_name, author_url, author_icon_url,
author_proxy_icon_url
)
VALUES (
$1, $2, $3, $4, $5, $6, $7, $8::timestamptz, $9, $10, $11, $12, $13, $14,
$15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29
);",
)
.bind(message_id)
.bind(message_version)
.bind(embed_index)
.bind(&embed.kind)
.bind(&embed.title)
.bind(&embed.description)
.bind(&embed.url)
.bind(embed_timestamp)
.bind(embed.colour.map(|colour| colour.0 as i32))
.bind(embed.footer.as_ref().map(|footer| &footer.text))
.bind(
embed
.footer
.as_ref()
.and_then(|footer| footer.icon_url.as_ref()),
)
.bind(embed.image.as_ref().map(|image| &image.url))
.bind(
embed
.image
.as_ref()
.and_then(|image| image.proxy_url.as_ref()),
)
.bind(optional_u32_to_i32(
embed.image.as_ref().and_then(|image| image.width),
)?)
.bind(optional_u32_to_i32(
embed.image.as_ref().and_then(|image| image.height),
)?)
.bind(embed.thumbnail.as_ref().map(|thumbnail| &thumbnail.url))
.bind(
embed
.thumbnail
.as_ref()
.and_then(|thumbnail| thumbnail.proxy_url.as_ref()),
)
.bind(optional_u32_to_i32(
embed
.thumbnail
.as_ref()
.and_then(|thumbnail| thumbnail.width),
)?)
.bind(optional_u32_to_i32(
embed
.thumbnail
.as_ref()
.and_then(|thumbnail| thumbnail.height),
)?)
.bind(embed.video.as_ref().map(|video| &video.url))
.bind(
embed
.video
.as_ref()
.and_then(|video| video.proxy_url.as_ref()),
)
.bind(optional_u32_to_i32(
embed.video.as_ref().and_then(|video| video.width),
)?)
.bind(optional_u32_to_i32(
embed.video.as_ref().and_then(|video| video.height),
)?)
.bind(
embed
.provider
.as_ref()
.and_then(|provider| provider.name.as_ref()),
)
.bind(
embed
.provider
.as_ref()
.and_then(|provider| provider.url.as_ref()),
)
.bind(embed.author.as_ref().map(|author| &author.name))
.bind(embed.author.as_ref().and_then(|author| author.url.as_ref()))
.bind(
embed
.author
.as_ref()
.and_then(|author| author.icon_url.as_ref()),
)
.bind(
embed
.author
.as_ref()
.and_then(|author| author.proxy_icon_url.as_ref()),
)
.execute(&mut *transaction)
.await?;
for (field_index, field) in embed.fields.iter().enumerate() {
sqlx::query(
"INSERT INTO embed_fields (embed_uuid, field_index, name, value, inline)
SELECT uuid, $4, $5, $6, $7
FROM embeds
WHERE message_id = $1 AND message_version = $2 AND embed_index = $3;",
)
.bind(message_id)
.bind(message_version)
.bind(embed_index)
.bind(i32::try_from(field_index)?)
.bind(&field.name)
.bind(&field.value)
.bind(field.inline)
.execute(&mut *transaction)
.await?;
}
}
transaction.commit().await?;
debug!(
"Archived version {} of message {} with {} attachments and {} embeds",
message_version,
message.id.get(),
message.attachments.len(),
message.embeds.len()
);
Ok(())
}
fn optional_u32_to_i32(value: Option<u32>) -> Result<Option<i32>, std::num::TryFromIntError> {
value.map(i32::try_from).transpose()
}
+34
View File
@@ -0,0 +1,34 @@
use crate::{Context, Error};
use tracing::trace;
pub async fn trace_message(msg: &str, channel: String, guild: String) {
trace!(
"Saying \"{}\" in channel {} in guild {}",
msg, channel, guild
);
}
pub async fn edit_response_message<'a>(
response_message: &poise::ReplyHandle<'a>,
ctx: Context<'_>,
content: &str,
silent: bool,
) -> Result<(), Error> {
if silent {
response_message
.edit(
ctx,
poise::CreateReply::default()
.content(content)
.allowed_mentions(
poise::serenity_prelude::CreateAllowedMentions::new().empty_users(),
),
)
.await?;
} else {
response_message
.edit(ctx, poise::CreateReply::default().content(content))
.await?;
}
Ok(())
}
+4
View File
@@ -0,0 +1,4 @@
pub mod commands;
pub mod event_handler;
pub mod message_archive;
pub mod messaging;