add rate limiting for download

This commit is contained in:
neri 2022-08-21 18:44:12 +02:00
parent e4ff237905
commit f80035ac82
8 changed files with 266 additions and 29 deletions

170
Cargo.lock generated
View File

@ -42,6 +42,18 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "actix-governor"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69a09c3caabdac53c829ad01be8b0251428ec3eb87521367d4b900befd820056"
dependencies = [
"actix-http",
"actix-web",
"futures",
"governor",
]
[[package]]
name = "actix-http"
version = "3.0.4"
@ -413,15 +425,29 @@ dependencies = [
"typenum",
]
[[package]]
name = "dashmap"
version = "5.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f"
dependencies = [
"cfg-if",
"hashbrown 0.12.3",
"lock_api",
"parking_lot_core 0.9.3",
]
[[package]]
name = "datatrash"
version = "1.1.2"
dependencies = [
"actix-files",
"actix-governor",
"actix-multipart",
"actix-web",
"env_logger",
"futures-util",
"governor",
"htmlescape",
"log",
"mime",
@ -564,6 +590,21 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "futures"
version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.21"
@ -580,6 +621,17 @@ version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
[[package]]
name = "futures-executor"
version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-intrusive"
version = "0.4.0"
@ -591,6 +643,12 @@ dependencies = [
"parking_lot 0.11.2",
]
[[package]]
name = "futures-io"
version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93a66fc6d035a26a3ae255a6d2bca35eda63ae4c5512bef54449113f7a1228e5"
[[package]]
name = "futures-macro"
version = "0.3.21"
@ -614,16 +672,25 @@ version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a"
[[package]]
name = "futures-timer"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
[[package]]
name = "futures-util"
version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
@ -650,6 +717,23 @@ dependencies = [
"wasi 0.10.2+wasi-snapshot-preview1",
]
[[package]]
name = "governor"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19775995ee20209163239355bc3ad2f33f83da35d9ef72dea26e5af753552c87"
dependencies = [
"dashmap",
"futures",
"futures-timer",
"no-std-compat",
"nonzero_ext",
"parking_lot 0.12.0",
"quanta",
"rand",
"smallvec",
]
[[package]]
name = "h2"
version = "0.3.13"
@ -678,13 +762,19 @@ dependencies = [
"ahash",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashlink"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
dependencies = [
"hashbrown",
"hashbrown 0.11.2",
]
[[package]]
@ -788,7 +878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
dependencies = [
"autocfg",
"hashbrown",
"hashbrown 0.11.2",
]
[[package]]
@ -888,6 +978,15 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "mach"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
dependencies = [
"libc",
]
[[package]]
name = "matches"
version = "0.1.9"
@ -963,6 +1062,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "no-std-compat"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
[[package]]
name = "nom"
version = "7.1.1"
@ -973,6 +1078,12 @@ dependencies = [
"minimal-lexical",
]
[[package]]
name = "nonzero_ext"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
[[package]]
name = "ntapi"
version = "0.3.7"
@ -1034,7 +1145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
dependencies = [
"lock_api",
"parking_lot_core 0.9.2",
"parking_lot_core 0.9.3",
]
[[package]]
@ -1053,9 +1164,9 @@ dependencies = [
[[package]]
name = "parking_lot_core"
version = "0.9.2"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37"
checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
dependencies = [
"cfg-if",
"libc",
@ -1119,6 +1230,22 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "quanta"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8"
dependencies = [
"crossbeam-utils",
"libc",
"mach",
"once_cell",
"raw-cpuid",
"wasi 0.10.2+wasi-snapshot-preview1",
"web-sys",
"winapi",
]
[[package]]
name = "quote"
version = "1.0.18"
@ -1158,6 +1285,15 @@ dependencies = [
"getrandom",
]
[[package]]
name = "raw-cpuid"
version = "10.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6aa2540135b6a94f74c7bc90ad4b794f822026a894f3d7bcd185c100d13d4ad6"
dependencies = [
"bitflags",
]
[[package]]
name = "redox_syscall"
version = "0.2.13"
@ -2034,9 +2170,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-sys"
version = "0.34.0"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825"
checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
dependencies = [
"windows_aarch64_msvc",
"windows_i686_gnu",
@ -2047,33 +2183,33 @@ dependencies = [
[[package]]
name = "windows_aarch64_msvc"
version = "0.34.0"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d"
checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
[[package]]
name = "windows_i686_gnu"
version = "0.34.0"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed"
checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
[[package]]
name = "windows_i686_msvc"
version = "0.34.0"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956"
checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
[[package]]
name = "windows_x86_64_gnu"
version = "0.34.0"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4"
checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.34.0"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9"
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
[[package]]
name = "zstd"

View File

@ -30,3 +30,5 @@ urlencoding = "2.1.0"
tree_magic_mini = { version = "3.0.3", features = ["with-gpl-data"] }
mime = "0.3.16"
url = "2.2.2"
actix-governor = "0.3.1"
governor = "0.4.2"

View File

@ -18,12 +18,16 @@ To run the software directly, use the compiling instructions below.
### General configuration
| environment variable | default value |
| -------------------- | -------------- |
| STATIC_DIR | ./static |
| FILES_DIR | ./files |
| UPLOAD_MAX_BYTES | 8388608 (8MiB) |
| BIND_ADDRESS | 0.0.0.0:8000 |
| environment variable | default value | description |
| --------------------- | -------------- | ---------------------------------------------- |
| STATIC_DIR | ./static | directory to generate "static" files into |
| FILES_DIR | ./files | directory to save uploaded files into |
| UPLOAD_MAX_BYTES | 8388608 (8MiB) | maximum size for uploaded files |
| BIND_ADDRESS | 0.0.0.0:8000 | address to bind the server to |
| RATE_LIMIT | true | whether download rate should be limited |
| RATE_LIMIT_PROXIED | false | whether rate limit should read x-forwarded-for |
| RATE_LIMIT_PER_SECOND | 60 | seconds to wait between requests |
| RATE_LIMIT_BURST | 1440 | allowed request burst |
### Database configuration

View File

@ -10,6 +10,10 @@ pub struct Config {
pub files_dir: PathBuf,
pub max_file_size: Option<u64>,
pub no_auth_limits: Option<NoAuthLimits>,
pub enable_rate_limit: bool,
pub proxied: bool,
pub rate_limit_per_second: u64,
pub rate_limit_burst: u32,
}
#[derive(Clone)]
@ -36,11 +40,26 @@ pub async fn get_config() -> Config {
let no_auth_limits = get_no_auth_limits();
let enable_rate_limit = matches!(env::var("RATE_LIMIT").as_deref(), Ok("true") | Err(_));
let proxied = env::var("PROXIED").as_deref() == Ok("true");
let rate_limit_per_second = env::var("RATE_LIMIT_PER_SECOND")
.ok()
.and_then(|rate_limit| rate_limit.parse().ok())
.unwrap_or(60);
let rate_limit_burst = env::var("RATE_LIMIT_BURST")
.ok()
.and_then(|rate_limit| rate_limit.parse().ok())
.unwrap_or(1440);
Config {
static_dir,
files_dir,
max_file_size,
no_auth_limits,
enable_rate_limit,
proxied,
rate_limit_per_second,
rate_limit_burst,
}
}

View File

@ -173,7 +173,7 @@ fn add_headers(req: &HttpRequest, download: bool, response: &mut HttpResponse) {
if !download && sec_fetch_mode.is_some() && sec_fetch_mode != Some("navigate") {
response.headers_mut().insert(
CONTENT_TYPE,
HeaderValue::from_str(&APPLICATION_OCTET_STREAM.to_string())
HeaderValue::from_str(APPLICATION_OCTET_STREAM.as_ref())
.expect("mime type can be encoded to header value"),
);
response

View File

@ -3,10 +3,13 @@ mod db;
mod deleter;
mod download;
mod multipart;
mod rate_limit;
mod template;
mod upload;
use crate::rate_limit::ForwardedPeerIpKeyExtractor;
use actix_files::Files;
use actix_governor::{Governor, GovernorConfigBuilder};
use actix_web::{
http::header::{HeaderName, CONTENT_SECURITY_POLICY},
middleware::{self, DefaultHeaders, Logger},
@ -52,9 +55,19 @@ async fn main() -> std::io::Result<()> {
template::write_prefillable_templates(&config).await;
let config = Data::new(config);
let governor_conf = GovernorConfigBuilder::default()
.per_second(config.rate_limit_per_second)
.burst_size(config.rate_limit_burst)
.key_extractor(ForwardedPeerIpKeyExtractor {
proxied: config.proxied,
})
.use_headers()
.finish()
.unwrap();
HttpServer::new({
move || {
App::new()
let app = App::new()
.wrap(Logger::new(r#"%{r}a "%r" =%s %bbytes %Tsec"#))
.wrap(DefaultHeaders::new().add(DEFAULT_CSP))
.wrap(middleware::Compress::default())
@ -68,7 +81,19 @@ async fn main() -> std::io::Result<()> {
.route(web::get().to(upload::uploaded)),
)
.service(Files::new("/static", "static").disable_content_disposition())
.service(
.default_service(web::route().to(not_found));
if config.enable_rate_limit {
app.service(
web::resource([
"/{id:[a-z0-9]{5}}",
"/{id:[a-z0-9]{5}}/",
"/{id:[a-z0-9]{5}}/{name}",
])
.wrap(Governor::new(&governor_conf))
.route(web::get().to(download::download)),
)
} else {
app.service(
web::resource([
"/{id:[a-z0-9]{5}}",
"/{id:[a-z0-9]{5}}/",
@ -76,7 +101,7 @@ async fn main() -> std::io::Result<()> {
])
.route(web::get().to(download::download)),
)
.default_service(web::route().to(not_found))
}
}
})
.bind(bind_address)?

51
src/rate_limit.rs Normal file
View File

@ -0,0 +1,51 @@
use actix_governor::KeyExtractor;
use actix_governor::PeerIpKeyExtractor;
use actix_web::{dev::ServiceRequest, http::header::ContentType};
use governor::clock::{Clock, DefaultClock, QuantaInstant};
use governor::NotUntil;
use std::net::IpAddr;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ForwardedPeerIpKeyExtractor {
pub proxied: bool,
}
impl KeyExtractor for ForwardedPeerIpKeyExtractor {
type Key = IpAddr;
type KeyExtractionError = &'static str;
#[cfg(feature = "log")]
fn name(&self) -> &'static str {
"Forwarded peer IP"
}
fn extract(&self, req: &ServiceRequest) -> Result<Self::Key, Self::KeyExtractionError> {
let forwarded_for = req.headers().get("x-forwarded-for");
if !self.proxied && forwarded_for.is_some() {
let forwarded_for = forwarded_for
.unwrap()
.to_str()
.map_err(|_| "x-forwarded-for contains invalid header value")?;
forwarded_for
.parse::<IpAddr>()
.map_err(|_| "x-forwarded-for contains invalid ip adress")
} else {
PeerIpKeyExtractor.extract(req)
}
}
fn response_error_content(&self, negative: &NotUntil<QuantaInstant>) -> (String, ContentType) {
let wait_time = negative
.wait_time_from(DefaultClock::default().now())
.as_secs();
(
format!("too many requests, retry in {}s", wait_time),
ContentType::plaintext(),
)
}
#[cfg(feature = "log")]
fn key_name(&self, key: &Self::Key) -> Option<String> {
Some(key.to_string())
}
}

View File

@ -17,8 +17,8 @@ const UPLOAD_HTML: &str = include_str!("../template/upload.html");
const UPLOAD_SHORT_HTML: &str = include_str!("../template/upload-short.html");
const ID_CHARS: &[char] = &[
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u', 'v',
'w', 'x', 'y', 'z', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u',
'v', 'w', 'x', 'y', 'z', '1', '2', '3', '4', '5', '6', '7', '8', '9',
];
pub async fn index(config: web::Data<Config>) -> Result<NamedFile, Error> {