From f80035ac826568c057d70810f52e99d77aa23828 Mon Sep 17 00:00:00 2001 From: neri Date: Sun, 21 Aug 2022 18:44:12 +0200 Subject: [PATCH] add rate limiting for download --- Cargo.lock | 170 +++++++++++++++++++++++++++++++++++++++++----- Cargo.toml | 2 + README.md | 16 +++-- src/config.rs | 19 ++++++ src/download.rs | 2 +- src/main.rs | 31 ++++++++- src/rate_limit.rs | 51 ++++++++++++++ src/upload.rs | 4 +- 8 files changed, 266 insertions(+), 29 deletions(-) create mode 100644 src/rate_limit.rs diff --git a/Cargo.lock b/Cargo.lock index c207cbe..efa81ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,6 +42,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "actix-governor" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a09c3caabdac53c829ad01be8b0251428ec3eb87521367d4b900befd820056" +dependencies = [ + "actix-http", + "actix-web", + "futures", + "governor", +] + [[package]] name = "actix-http" version = "3.0.4" @@ -413,15 +425,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "dashmap" +version = "5.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" +dependencies = [ + "cfg-if", + "hashbrown 0.12.3", + "lock_api", + "parking_lot_core 0.9.3", +] + [[package]] name = "datatrash" version = "1.1.2" dependencies = [ "actix-files", + "actix-governor", "actix-multipart", "actix-web", "env_logger", "futures-util", + "governor", "htmlescape", "log", "mime", @@ -564,6 +590,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.21" @@ -580,6 +621,17 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" +[[package]] +name = "futures-executor" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-intrusive" version = "0.4.0" @@ -591,6 +643,12 @@ dependencies = [ "parking_lot 0.11.2", ] +[[package]] +name = "futures-io" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93a66fc6d035a26a3ae255a6d2bca35eda63ae4c5512bef54449113f7a1228e5" + [[package]] name = "futures-macro" version = "0.3.21" @@ -614,16 +672,25 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -650,6 +717,23 @@ dependencies = [ "wasi 0.10.2+wasi-snapshot-preview1", ] +[[package]] +name = "governor" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19775995ee20209163239355bc3ad2f33f83da35d9ef72dea26e5af753552c87" +dependencies = [ + "dashmap", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot 0.12.0", + "quanta", + "rand", + "smallvec", +] + [[package]] name = "h2" version = "0.3.13" @@ -678,13 +762,19 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashlink" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" dependencies = [ - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -788,7 +878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -888,6 +978,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "matches" version = "0.1.9" @@ -963,6 +1062,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + [[package]] name = "nom" version = "7.1.1" @@ -973,6 +1078,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + [[package]] name = "ntapi" version = "0.3.7" @@ -1034,7 +1145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api", - "parking_lot_core 0.9.2", + "parking_lot_core 0.9.3", ] [[package]] @@ -1053,9 +1164,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", "libc", @@ -1119,6 +1230,22 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "quanta" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" +dependencies = [ + "crossbeam-utils", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.2+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.18" @@ -1158,6 +1285,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "10.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aa2540135b6a94f74c7bc90ad4b794f822026a894f3d7bcd185c100d13d4ad6" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.2.13" @@ -2034,9 +2170,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -2047,33 +2183,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index 9bdf289..633441e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,3 +30,5 @@ urlencoding = "2.1.0" tree_magic_mini = { version = "3.0.3", features = ["with-gpl-data"] } mime = "0.3.16" url = "2.2.2" +actix-governor = "0.3.1" +governor = "0.4.2" diff --git a/README.md b/README.md index 001d710..22abbd4 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,16 @@ To run the software directly, use the compiling instructions below. ### General configuration -| environment variable | default value | -| -------------------- | -------------- | -| STATIC_DIR | ./static | -| FILES_DIR | ./files | -| UPLOAD_MAX_BYTES | 8388608 (8MiB) | -| BIND_ADDRESS | 0.0.0.0:8000 | +| environment variable | default value | description | +| --------------------- | -------------- | ---------------------------------------------- | +| STATIC_DIR | ./static | directory to generate "static" files into | +| FILES_DIR | ./files | directory to save uploaded files into | +| UPLOAD_MAX_BYTES | 8388608 (8MiB) | maximum size for uploaded files | +| BIND_ADDRESS | 0.0.0.0:8000 | address to bind the server to | +| RATE_LIMIT | true | whether download rate should be limited | +| RATE_LIMIT_PROXIED | false | whether rate limit should read x-forwarded-for | +| RATE_LIMIT_PER_SECOND | 60 | seconds to wait between requests | +| RATE_LIMIT_BURST | 1440 | allowed request burst | ### Database configuration diff --git a/src/config.rs b/src/config.rs index 5ec9bd6..a7325a2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,6 +10,10 @@ pub struct Config { pub files_dir: PathBuf, pub max_file_size: Option, pub no_auth_limits: Option, + pub enable_rate_limit: bool, + pub proxied: bool, + pub rate_limit_per_second: u64, + pub rate_limit_burst: u32, } #[derive(Clone)] @@ -36,11 +40,26 @@ pub async fn get_config() -> Config { let no_auth_limits = get_no_auth_limits(); + let enable_rate_limit = matches!(env::var("RATE_LIMIT").as_deref(), Ok("true") | Err(_)); + let proxied = env::var("PROXIED").as_deref() == Ok("true"); + let rate_limit_per_second = env::var("RATE_LIMIT_PER_SECOND") + .ok() + .and_then(|rate_limit| rate_limit.parse().ok()) + .unwrap_or(60); + let rate_limit_burst = env::var("RATE_LIMIT_BURST") + .ok() + .and_then(|rate_limit| rate_limit.parse().ok()) + .unwrap_or(1440); + Config { static_dir, files_dir, max_file_size, no_auth_limits, + enable_rate_limit, + proxied, + rate_limit_per_second, + rate_limit_burst, } } diff --git a/src/download.rs b/src/download.rs index e766bd3..be5c904 100644 --- a/src/download.rs +++ b/src/download.rs @@ -173,7 +173,7 @@ fn add_headers(req: &HttpRequest, download: bool, response: &mut HttpResponse) { if !download && sec_fetch_mode.is_some() && sec_fetch_mode != Some("navigate") { response.headers_mut().insert( CONTENT_TYPE, - HeaderValue::from_str(&APPLICATION_OCTET_STREAM.to_string()) + HeaderValue::from_str(APPLICATION_OCTET_STREAM.as_ref()) .expect("mime type can be encoded to header value"), ); response diff --git a/src/main.rs b/src/main.rs index 41f07a2..68994e4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,10 +3,13 @@ mod db; mod deleter; mod download; mod multipart; +mod rate_limit; mod template; mod upload; +use crate::rate_limit::ForwardedPeerIpKeyExtractor; use actix_files::Files; +use actix_governor::{Governor, GovernorConfigBuilder}; use actix_web::{ http::header::{HeaderName, CONTENT_SECURITY_POLICY}, middleware::{self, DefaultHeaders, Logger}, @@ -52,9 +55,19 @@ async fn main() -> std::io::Result<()> { template::write_prefillable_templates(&config).await; let config = Data::new(config); + let governor_conf = GovernorConfigBuilder::default() + .per_second(config.rate_limit_per_second) + .burst_size(config.rate_limit_burst) + .key_extractor(ForwardedPeerIpKeyExtractor { + proxied: config.proxied, + }) + .use_headers() + .finish() + .unwrap(); + HttpServer::new({ move || { - App::new() + let app = App::new() .wrap(Logger::new(r#"%{r}a "%r" =%s %bbytes %Tsec"#)) .wrap(DefaultHeaders::new().add(DEFAULT_CSP)) .wrap(middleware::Compress::default()) @@ -68,7 +81,19 @@ async fn main() -> std::io::Result<()> { .route(web::get().to(upload::uploaded)), ) .service(Files::new("/static", "static").disable_content_disposition()) - .service( + .default_service(web::route().to(not_found)); + if config.enable_rate_limit { + app.service( + web::resource([ + "/{id:[a-z0-9]{5}}", + "/{id:[a-z0-9]{5}}/", + "/{id:[a-z0-9]{5}}/{name}", + ]) + .wrap(Governor::new(&governor_conf)) + .route(web::get().to(download::download)), + ) + } else { + app.service( web::resource([ "/{id:[a-z0-9]{5}}", "/{id:[a-z0-9]{5}}/", @@ -76,7 +101,7 @@ async fn main() -> std::io::Result<()> { ]) .route(web::get().to(download::download)), ) - .default_service(web::route().to(not_found)) + } } }) .bind(bind_address)? diff --git a/src/rate_limit.rs b/src/rate_limit.rs new file mode 100644 index 0000000..f62a781 --- /dev/null +++ b/src/rate_limit.rs @@ -0,0 +1,51 @@ +use actix_governor::KeyExtractor; +use actix_governor::PeerIpKeyExtractor; +use actix_web::{dev::ServiceRequest, http::header::ContentType}; +use governor::clock::{Clock, DefaultClock, QuantaInstant}; +use governor::NotUntil; +use std::net::IpAddr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ForwardedPeerIpKeyExtractor { + pub proxied: bool, +} + +impl KeyExtractor for ForwardedPeerIpKeyExtractor { + type Key = IpAddr; + type KeyExtractionError = &'static str; + + #[cfg(feature = "log")] + fn name(&self) -> &'static str { + "Forwarded peer IP" + } + + fn extract(&self, req: &ServiceRequest) -> Result { + let forwarded_for = req.headers().get("x-forwarded-for"); + if !self.proxied && forwarded_for.is_some() { + let forwarded_for = forwarded_for + .unwrap() + .to_str() + .map_err(|_| "x-forwarded-for contains invalid header value")?; + forwarded_for + .parse::() + .map_err(|_| "x-forwarded-for contains invalid ip adress") + } else { + PeerIpKeyExtractor.extract(req) + } + } + + fn response_error_content(&self, negative: &NotUntil) -> (String, ContentType) { + let wait_time = negative + .wait_time_from(DefaultClock::default().now()) + .as_secs(); + ( + format!("too many requests, retry in {}s", wait_time), + ContentType::plaintext(), + ) + } + + #[cfg(feature = "log")] + fn key_name(&self, key: &Self::Key) -> Option { + Some(key.to_string()) + } +} diff --git a/src/upload.rs b/src/upload.rs index 5878359..cfe860f 100644 --- a/src/upload.rs +++ b/src/upload.rs @@ -17,8 +17,8 @@ const UPLOAD_HTML: &str = include_str!("../template/upload.html"); const UPLOAD_SHORT_HTML: &str = include_str!("../template/upload-short.html"); const ID_CHARS: &[char] = &[ - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u', + 'v', 'w', 'x', 'y', 'z', '1', '2', '3', '4', '5', '6', '7', '8', '9', ]; pub async fn index(config: web::Data) -> Result {