ps2.live/agg-population
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swap to sqlite

Browse source
This commit is contained in:
41666 2023-06-09 09:10:19 -04:00
parent 315a05d59a
commit 5fef497bfc
4 changed files with 213 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

284
Cargo.lock generated
View file

@ -10,16 +10,28 @@ dependencies = [
"bincode",
"chrono",
"openssl",
"r2d2",
"r2d2_sqlite",
"reqwest",
"serde",
"serde_json",
"sled",
"tokio",
"tower-http",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "ahash"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
dependencies = [
"cfg-if",
"once_cell",
"version_check",
]
[[package]]
name = "android-tzdata"
version = "0.1.1"
@ -60,7 +72,7 @@ checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39"
dependencies = [
"async-trait",
"axum-core",
"bitflags",
"bitflags 1.3.2",
"bytes",
"futures-util",
"http",
@ -122,18 +134,18 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84"
[[package]]
name = "bumpalo"
version = "3.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "1.4.0"
@ -184,37 +196,6 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
dependencies = [
"cfg-if",
]
[[package]]
name = "encoding_rs"
version = "0.8.32"
@ -245,6 +226,18 @@ dependencies = [
"libc",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastrand"
version = "1.9.0"
@ -284,16 +277,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "futures-channel"
version = "0.3.28"
@ -334,12 +317,14 @@ dependencies = [
]
[[package]]
name = "fxhash"
version = "0.2.1"
name = "getrandom"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"byteorder",
"cfg-if",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
@ -367,6 +352,24 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
dependencies = [
"ahash",
]
[[package]]
name = "hashlink"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0761a1b9491c4f2e3d66aa0f62d0fba0af9a0e2852e4d48ea506632a4b56e6aa"
dependencies = [
"hashbrown 0.13.2",
]
[[package]]
name = "hermit-abi"
version = "0.2.6"
@ -512,7 +515,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown",
"hashbrown 0.12.3",
]
[[package]]
@ -568,6 +571,16 @@ version = "0.2.144"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1"
[[package]]
name = "libsqlite3-sys"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"
dependencies = [
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.3.8"
@ -611,15 +624,6 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memoffset"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.17"
@ -696,7 +700,7 @@ version = "0.10.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12df40a956736488b7b44fe79fe12d4f245bb5b3f5a1f6095e499760015be392"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cfg-if",
"foreign-types",
"libc",
@ -750,17 +754,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
@ -768,21 +761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core 0.9.7",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
"parking_lot_core",
]
[[package]]
@ -842,6 +821,12 @@ version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "ppv-lite86"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "proc-macro2"
version = "1.0.59"
@ -860,13 +845,65 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r2d2"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
dependencies = [
"log",
"parking_lot",
"scheduled-thread-pool",
]
[[package]]
name = "r2d2_sqlite"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99f31323d6161385f385046738df520e0e8694fa74852d35891fc0be08348ddc"
dependencies = [
"r2d2",
"rusqlite",
"uuid",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
"bitflags 1.3.2",
]
[[package]]
@ -875,7 +912,7 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags",
"bitflags 1.3.2",
]
[[package]]
@ -965,13 +1002,27 @@ dependencies = [
"winapi",
]
[[package]]
name = "rusqlite"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"
dependencies = [
"bitflags 2.3.1",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
]
[[package]]
name = "rustix"
version = "0.37.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"errno",
"io-lifetimes",
"libc",
@ -1031,6 +1082,15 @@ dependencies = [
"windows-sys 0.42.0",
]
[[package]]
name = "scheduled-thread-pool"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
dependencies = [
"parking_lot",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
@ -1053,7 +1113,7 @@ version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"core-foundation",
"core-foundation-sys",
"libc",
@ -1149,22 +1209,6 @@ dependencies = [
"autocfg",
]
[[package]]
name = "sled"
version = "0.34.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
dependencies = [
"crc32fast",
"crossbeam-epoch",
"crossbeam-utils",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot 0.11.2",
]
[[package]]
name = "smallvec"
version = "1.10.0"
@ -1264,7 +1308,7 @@ dependencies = [
"libc",
"mio",
"num_cpus",
"parking_lot 0.12.1",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@ -1339,7 +1383,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"bytes",
"futures-core",
"futures-util",
@ -1471,6 +1515,16 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "uuid"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2"
dependencies = [
"getrandom",
"rand",
]
[[package]]
name = "valuable"
version = "0.1.0"
@ -1483,6 +1537,12 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "want"
version = "0.3.0"

5
Cargo.toml
View file

@ -12,9 +12,10 @@ serde_json = "1.0.96"
tokio = { version = "1.0", features = ["full"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
sled = "0.34"
reqwest = { version = "0.11", features = ["rustls-tls-webpki-roots", "rustls", "json"] }
openssl = { version = "0.10", features = ["vendored"] }
bincode = "1.3"
chrono = { version = "0.4", features = ["serde"] }
tower-http = { version = "0.4", features = ["trace"] }
tower-http = { version = "0.4", features = ["trace"] }
r2d2_sqlite = "0.22"
r2d2 = "0.8"

39
src/handlers.rs
View file

@ -6,13 +6,19 @@ use axum::{
extract::{Path, State},
Json,
};
use r2d2_sqlite::{rusqlite::params, SqliteConnectionManager};
use tokio::task::JoinSet;
pub async fn get_one_world(State(db): State<sled::Db>, Path(world): Path<i32>) -> Json<Response> {
pub async fn get_one_world(
State(db): State<r2d2::Pool<SqliteConnectionManager>>,
Path(world): Path<i32>,
) -> Json<Response> {
Json(get_world(db, world, false).await)
}
pub async fn get_all_worlds(State(db): State<sled::Db>) -> Json<Vec<Response>> {
pub async fn get_all_worlds(
State(db): State<r2d2::Pool<SqliteConnectionManager>>,
) -> Json<Vec<Response>> {
let mut set = JoinSet::new();
let mut worlds = vec![Response::default(); 8];
@ -29,7 +35,11 @@ pub async fn get_all_worlds(State(db): State<sled::Db>) -> Json<Vec<Response>> {
Json(worlds)
}
pub async fn get_world(db: sled::Db, world: i32, skip_cache: bool) -> Response {
pub async fn get_world(
db: r2d2::Pool<SqliteConnectionManager>,
world: i32,
skip_cache: bool,
) -> Response {
if !skip_cache {
if let Ok(data) = world_from_cache(db.clone(), world) {
return data;
@ -74,14 +84,16 @@ pub async fn get_world(db: sled::Db, world: i32, skip_cache: bool) -> Response {
response
}
fn world_from_cache(db: sled::Db, world: i32) -> Result<Response, ()> {
let key = format!("world:{}", world);
let value = match db.get(key) {
Ok(Some(value)) => value,
_ => return Err(()),
};
fn world_from_cache(db: r2d2::Pool<SqliteConnectionManager>, world: i32) -> Result<Response, ()> {
let db = db.get().unwrap();
let mut query = db.prepare("SELECT data FROM worlds WHERE id = ?").unwrap();
let value: Result<Vec<u8>, _> = query.query_row(params![world], |r| r.get(0));
match bincode::deserialize::<Response>(&value) {
if value.is_err() {
return Err(());
}
match bincode::deserialize::<Response>(value.unwrap().as_slice()) {
Ok(response) => {
if response.cached_at + chrono::Duration::minutes(5) < chrono::Utc::now() {
return Err(());
@ -92,8 +104,9 @@ fn world_from_cache(db: sled::Db, world: i32) -> Result<Response, ()> {
}
}
fn world_to_cache(db: sled::Db, world: i32, response: &Response) {
let key = format!("world:{}", world);
fn world_to_cache(db: r2d2::Pool<SqliteConnectionManager>, world: i32, response: &Response) {
let value = bincode::serialize(response).unwrap();
db.insert(key, value).unwrap();
let db = db.get().unwrap();
let mut query = db.prepare("INSERT INTO worlds (id, data) VALUES (?, ?) ON CONFLICT DO UPDATE SET data=excluded.data").unwrap();
query.execute(params![world, value]).unwrap();
}

15
src/main.rs
View file

@ -1,5 +1,6 @@
use crate::handlers::{get_all_worlds, get_one_world, get_world};
use axum::{response::Html, routing::get, Router};
use r2d2_sqlite::{rusqlite::params, SqliteConnectionManager};
use std::net::SocketAddr;
use tokio::task::JoinSet;
use tower_http::trace::TraceLayer;
@ -14,7 +15,15 @@ async fn main() {
.with_env_filter("tower_http=trace")
.init();
let db = sled::open("/tmp/agg-population").expect("open");
let sqlite_manager = SqliteConnectionManager::memory();
let pool = r2d2::Pool::new(sqlite_manager).unwrap();
pool.get()
.unwrap()
.execute(
"CREATE TABLE worlds (id INTEGER NOT NULL PRIMARY KEY, data BLOB);",
params![],
)
.unwrap();
let app = Router::new()
.route("/", get(root))
@ -22,13 +31,13 @@ async fn main() {
.route("/population/all", get(get_all_worlds))
.route("/population/:world", get(get_one_world))
.layer(TraceLayer::new_for_http())
.with_state(db.clone());
.with_state(pool.clone());
tokio::spawn(async move {
loop {
let mut set = JoinSet::new();
for world in vec![1, 10, 13, 17, 19, 40, 1000, 2000] {
set.spawn(get_world(db.clone(), world, true));
set.spawn(get_world(pool.clone(), world, true));
}
while let Some(_) = set.join_next().await {}