Compare commits
No commits in common. "7f6db2de403ac3f869220f555eb1c9b72dc74efd" and "2ab33b82d03fd8089f71e25b52287f12be17ab83" have entirely different histories.
7f6db2de40
...
2ab33b82d0
|
@ -491,7 +491,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "datatrash"
|
||||
version = "2.3.6"
|
||||
version = "2.3.5"
|
||||
dependencies = [
|
||||
"actix-files",
|
||||
"actix-governor",
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "datatrash"
|
||||
version = "2.3.6"
|
||||
version = "2.3.5"
|
||||
authors = ["neri"]
|
||||
edition = "2021"
|
||||
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
use futures_util::TryStreamExt;
|
||||
use sqlx::{postgres::PgPool, Row};
|
||||
use std::cmp::max;
|
||||
use std::error::Error;
|
||||
use std::fmt::Display;
|
||||
use std::path::{Path, PathBuf};
|
||||
use time::ext::NumericalStdDuration;
|
||||
use time::OffsetDateTime;
|
||||
|
@ -10,27 +8,26 @@ use tokio::fs;
|
|||
use tokio::sync::mpsc::Receiver;
|
||||
use tokio::time::timeout;
|
||||
|
||||
pub(crate) async fn delete_old_files(
|
||||
mut receiver: Receiver<()>,
|
||||
db: PgPool,
|
||||
files_dir: PathBuf,
|
||||
) -> Result<(), DeletionError> {
|
||||
pub(crate) async fn delete_old_files(mut receiver: Receiver<()>, db: PgPool, files_dir: PathBuf) {
|
||||
loop {
|
||||
wait_for_file_expiry(&mut receiver, &db).await?;
|
||||
wait_for_file_expiry(&mut receiver, &db).await;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut rows = sqlx::query("SELECT file_id FROM files WHERE files.valid_till < $1")
|
||||
.bind(now)
|
||||
.fetch(&db);
|
||||
while let Some(row) = rows.try_next().await? {
|
||||
while let Some(row) = rows.try_next().await.expect("could not load expired files") {
|
||||
let file_id: String = row.try_get("file_id").expect("we selected this column");
|
||||
delete_content(&file_id, &files_dir).await?
|
||||
delete_content(&file_id, &files_dir)
|
||||
.await
|
||||
.expect("could not delete file");
|
||||
}
|
||||
|
||||
sqlx::query("DELETE FROM files WHERE valid_till < $1")
|
||||
.bind(now)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
.await
|
||||
.expect("could not delete expired files from database");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -49,24 +46,19 @@ pub(crate) async fn delete_by_id(
|
|||
|
||||
async fn delete_content(file_id: &str, files_dir: &Path) -> Result<(), std::io::Error> {
|
||||
let path = files_dir.join(file_id);
|
||||
if fs::try_exists(&path).await? {
|
||||
fs::remove_file(&path).await?;
|
||||
log::info!("deleted file {}", file_id);
|
||||
} else {
|
||||
log::warn!("expiring file {} was missing from the filesystem", file_id);
|
||||
if fs::remove_file(&path).await.is_ok() {
|
||||
log::info!("delete file {}", file_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_file_expiry(
|
||||
receiver: &mut Receiver<()>,
|
||||
db: &PgPool,
|
||||
) -> Result<(), DeletionError> {
|
||||
async fn wait_for_file_expiry(receiver: &mut Receiver<()>, db: &PgPool) {
|
||||
let valid_till: (Option<OffsetDateTime>,) =
|
||||
sqlx::query_as("SELECT MIN(valid_till) as min from files")
|
||||
.fetch_one(db)
|
||||
.await?;
|
||||
let next_timeout = match valid_till.0 {
|
||||
.await
|
||||
.expect("could not fetch expiring files from database");
|
||||
let next_timeout: std::time::Duration = match valid_till.0 {
|
||||
Some(valid_till) => (max(
|
||||
0,
|
||||
valid_till.unix_timestamp() - OffsetDateTime::now_utc().unix_timestamp(),
|
||||
|
@ -75,41 +67,4 @@ async fn wait_for_file_expiry(
|
|||
None => 1_u64.std_days(),
|
||||
};
|
||||
let _ = timeout(next_timeout, receiver.recv()).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum DeletionError {
|
||||
Db(sqlx::Error),
|
||||
Fs(std::io::Error),
|
||||
}
|
||||
|
||||
impl From<sqlx::Error> for DeletionError {
|
||||
fn from(value: sqlx::Error) -> Self {
|
||||
DeletionError::Db(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for DeletionError {
|
||||
fn from(value: std::io::Error) -> Self {
|
||||
DeletionError::Fs(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for DeletionError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
DeletionError::Db(_) => write!(f, "Failed to fetch expired files from database"),
|
||||
DeletionError::Fs(_) => write!(f, "Failed to delete file from filesystem"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for DeletionError {
|
||||
fn source(&self) -> Option<&(dyn Error + 'static)> {
|
||||
match self {
|
||||
DeletionError::Db(err) => Some(err),
|
||||
DeletionError::Fs(err) => Some(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -123,11 +123,9 @@ async fn main() -> std::io::Result<()> {
|
|||
.bind(bind_address)?
|
||||
.run();
|
||||
|
||||
// exit when http_server exits OR when deleter errors
|
||||
// exit when http_server exits OR when deleter panics
|
||||
tokio::select! {
|
||||
result = http_server => result,
|
||||
result = deleter => {
|
||||
result?.map(|_| unreachable!("deletion runs infinitely")).expect("deletion may not fail")
|
||||
},
|
||||
_ = deleter => panic!("deleter never returns")
|
||||
}
|
||||
}
|
||||
|
|
|
@ -9,7 +9,7 @@ use actix_web::http::header::LOCATION;
|
|||
use actix_web::{error, web, Error, HttpRequest, HttpResponse};
|
||||
use rand::{distributions::Slice, Rng};
|
||||
use sqlx::postgres::PgPool;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs::{self, OpenOptions};
|
||||
use tokio::sync::mpsc::Sender;
|
||||
|
||||
|
@ -46,7 +46,7 @@ pub async fn upload(
|
|||
)
|
||||
});
|
||||
|
||||
insert_file_metadata(&file_id, file_name, &file_path, &upload_config, db).await?;
|
||||
insert_file_metadata(&file_id, file_name, &upload_config, db).await?;
|
||||
|
||||
log::info!(
|
||||
"{} create new file {} (valid_till: {}, content_type: {}, delete_on_download: {})",
|
||||
|
@ -69,7 +69,6 @@ pub async fn upload(
|
|||
async fn insert_file_metadata(
|
||||
file_id: &String,
|
||||
file_name: String,
|
||||
file_path: &Path,
|
||||
upload_config: &UploadConfig,
|
||||
db: web::Data<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> Result<(), Error> {
|
||||
|
@ -86,8 +85,7 @@ async fn insert_file_metadata(
|
|||
.await;
|
||||
if let Err(db_err) = db_insert {
|
||||
log::error!("could not insert into datebase {:?}", db_err);
|
||||
|
||||
if let Err(file_err) = fs::remove_file(file_path).await {
|
||||
if let Err(file_err) = fs::remove_file(file_name).await {
|
||||
log::error!("could not remove file {:?}", file_err);
|
||||
}
|
||||
return Err(error::ErrorInternalServerError(
|
||||
|
|
Loading…
Reference in New Issue