From 143ec0cd3b172240a222e784497d3f71deafc7b6 Mon Sep 17 00:00:00 2001 From: Katalina Okano Date: Fri, 2 Jun 2023 02:14:00 -0400 Subject: [PATCH] completely restructure database --- Cargo.lock | 42 ++++++++++++++-------------- services/api/src/classes.rs | 2 +- services/api/src/population.rs | 4 +-- services/api/src/vehicles.rs | 4 +-- services/tasks/src/main.rs | 9 +++++- services/tasks/src/migrations.rs | 47 ++++++++++++++++++++------------ services/websocket/src/main.rs | 34 +++++++++++++++++++++-- 7 files changed, 96 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03a9d84..1a3f3ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2434,13 +2434,13 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm 0.42.0", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", - "windows_x86_64_gnullvm 0.42.0", - "windows_x86_64_msvc 0.42.0", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -2469,9 +2469,9 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" @@ -2487,9 +2487,9 @@ checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" @@ -2505,9 +2505,9 @@ checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" @@ -2523,9 +2523,9 @@ checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" @@ -2541,9 +2541,9 @@ checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" @@ -2553,9 +2553,9 @@ checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" @@ -2571,9 +2571,9 @@ checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" diff --git a/services/api/src/classes.rs b/services/api/src/classes.rs index 40b59dd..f278c1b 100644 --- a/services/api/src/classes.rs +++ b/services/api/src/classes.rs @@ -16,7 +16,7 @@ impl Class { let pool = ctx.data::>().unwrap(); let sql = format!( - "SELECT count(distinct character_id) FROM players WHERE time > now() - interval '15 minutes' AND class_id = $1 {};", + "SELECT count(*) FROM players WHERE last_updated > now() - interval '15 minutes' AND class_name = $1 {};", filters.sql(), ); diff --git a/services/api/src/population.rs b/services/api/src/population.rs index 0b4ee92..237e93b 100644 --- a/services/api/src/population.rs +++ b/services/api/src/population.rs @@ -23,7 +23,7 @@ impl Population { let pool = ctx.data::>().unwrap(); let sql = format!( - "SELECT count(distinct character_id) FROM players WHERE time > now() - interval '15 minutes' AND faction_id = $1 {};", + "SELECT count(*) FROM players WHERE last_updated > now() - interval '15 minutes' AND faction_id = $1 {};", self.filters.sql(), ); @@ -46,7 +46,7 @@ impl Population { let pool = ctx.data::>().unwrap(); let sql = format!( - "SELECT count(distinct character_id) FROM players WHERE time > now() - interval '15 minutes' {};", + "SELECT count(*) FROM players WHERE last_updated > now() - interval '15 minutes' {};", self.filters.sql(), ); diff --git a/services/api/src/vehicles.rs b/services/api/src/vehicles.rs index bcb3442..601e30f 100644 --- a/services/api/src/vehicles.rs +++ b/services/api/src/vehicles.rs @@ -16,7 +16,7 @@ impl Vehicle { let pool = ctx.data::>().unwrap(); let sql = format!( - "SELECT count(distinct character_id) FROM players WHERE time > now() - interval '15 minutes' AND vehicle_id = $1 {};", + "SELECT count(*) FROM vehicles WHERE last_updated > now() - interval '15 minutes' AND vehicle_name = $1 {};", filters.sql(), ); @@ -89,7 +89,7 @@ impl Vehicles { let pool = ctx.data::>().unwrap(); let sql = format!( - "SELECT count(distinct character_id) FROM players WHERE time > now() - interval '15 minutes' AND vehicle_id != 'unknown' {};", + "SELECT count(*) FROM vehicles WHERE last_updated > now() - interval '15 minutes' {};", self.filters.sql(), ); diff --git a/services/tasks/src/main.rs b/services/tasks/src/main.rs index 0abf2de..420fc4f 100644 --- a/services/tasks/src/main.rs +++ b/services/tasks/src/main.rs @@ -18,13 +18,20 @@ async fn cmd_prune() { println!("Pruning old data..."); let pool = PG.get().await; - let rows = query("DELETE FROM players WHERE time < NOW() - INTERVAL '15 minutes';") + let rows = query("DELETE FROM players WHERE last_updated < NOW() - INTERVAL '15 minutes';") .execute(pool) .await .unwrap() .rows_affected(); println!("Deleted {} rows of old player data", rows); + let rows = query("DELETE FROM vehicles WHERE last_updated < NOW() - INTERVAL '15 minutes';") + .execute(pool) + .await + .unwrap() + .rows_affected(); + println!("Deleted {} rows of old vehicle data", rows); + let rows = query("DELETE FROM analytics WHERE time < NOW() - INTERVAL '1 day';") .execute(pool) .await diff --git a/services/tasks/src/migrations.rs b/services/tasks/src/migrations.rs index da53f7c..846687b 100644 --- a/services/tasks/src/migrations.rs +++ b/services/tasks/src/migrations.rs @@ -4,7 +4,7 @@ use sqlx::{query, Row}; pub async fn cmd_migrate() { println!("Migrating database..."); - tokio::join!(migrate_players(), migrate_analytics()); + tokio::join!(migrate_players(), migrate_vehicles(), migrate_analytics()); } async fn migrate_players() { @@ -21,35 +21,48 @@ async fn migrate_players() { println!("PLAYERS => CREATE TABLE players"); query( "CREATE TABLE players ( - character_id TEXT NOT NULL, - time TIMESTAMPTZ NOT NULL, + character_id TEXT NOT NULL PRIMARY KEY, + last_updated TIMESTAMPTZ NOT NULL, world_id INT NOT NULL, faction_id INT NOT NULL, zone_id INT NOT NULL, - class_id TEXT NOT NULL, - vehicle_id TEXT NOT NULL + class_name TEXT NOT NULL );", ) .execute(pool) .await .unwrap(); - println!("PLAYERS => create_hypertable"); + println!("PLAYERS => done!"); +} + +async fn migrate_vehicles() { + let pool = PG.get().await; + + println!("-> Migrating vehicles"); + + println!("VEHICLES => DROP TABLE IF EXISTS vehicles"); + query("DROP TABLE IF EXISTS vehicles") + .execute(pool) + .await + .unwrap(); + + println!("VEHICLES => CREATE TABLE vehicles"); query( - "SELECT create_hypertable('players', 'time', - chunk_time_interval => INTERVAL '5 minutes', if_not_exists => TRUE);", + "CREATE TABLE vehicles ( + character_id TEXT NOT NULL PRIMARY KEY, + last_updated TIMESTAMPTZ NOT NULL, + world_id INT NOT NULL, + faction_id INT NOT NULL, + zone_id INT NOT NULL, + vehicle_name TEXT NOT NULL + );", ) .execute(pool) .await .unwrap(); - println!("PLAYERS => add_retention_policy"); - query("SELECT add_retention_policy('players', INTERVAL '15 minutes');") - .execute(pool) - .await - .unwrap(); - - println!("PLAYERS => done!"); + println!("VEHICLES => done!"); } async fn migrate_analytics() { @@ -88,11 +101,11 @@ async fn migrate_analytics() { pub async fn is_migrated() -> bool { let pool = PG.get().await; - let tables: i64 = query("SELECT count(1) FROM pg_tables WHERE schemaname = 'public' AND tablename IN ('players', 'analytics');") + let tables: i64 = query("SELECT count(1) FROM pg_tables WHERE schemaname = 'public' AND tablename IN ('players', 'vehicles', 'analytics');") .fetch_one(pool) .await .unwrap() .get(0); - tables == 2 + tables == 3 } diff --git a/services/websocket/src/main.rs b/services/websocket/src/main.rs index 541fad6..094d298 100644 --- a/services/websocket/src/main.rs +++ b/services/websocket/src/main.rs @@ -107,16 +107,46 @@ async fn track_pop(pop_event: PopEvent) { translators::vehicle_to_name(vehicle_id.as_str()) }; - query("INSERT INTO players (time, character_id, world_id, faction_id, zone_id, class_id, vehicle_id) VALUES (now(), $1, $2, $3, $4, $5, $6);") + query( + " + INSERT INTO players (last_updated, character_id, world_id, faction_id, zone_id, class_name) + VALUES (now(), $1, $2, $3, $4, $5) + ON CONFLICT (character_id) DO UPDATE SET + last_updated = EXCLUDED.last_updated, + world_id = EXCLUDED.world_id, + faction_id = EXCLUDED.faction_id, + zone_id = EXCLUDED.zone_id, + class_name = EXCLUDED.class_name + ;", + ) + .bind(character_id.clone()) + .bind(world_id) + .bind(team_id) + .bind(zone_id) + .bind(class_name) + .execute(pool) + .await + .unwrap(); + + if vehicle_name != "unknown" { + query("INSERT INTO vehicles (last_updated, character_id, world_id, faction_id, zone_id, vehicle_name) + VALUES (now(), $1, $2, $3, $4, $5) + ON CONFLICT (character_id) DO UPDATE SET + last_updated = EXCLUDED.last_updated, + world_id = EXCLUDED.world_id, + faction_id = EXCLUDED.faction_id, + zone_id = EXCLUDED.zone_id, + vehicle_name = EXCLUDED.vehicle_name + ;") .bind(character_id) .bind(world_id) .bind(team_id) .bind(zone_id) - .bind(class_name) .bind(vehicle_name) .execute(pool) .await .unwrap(); + } } async fn track_analytics(analytics_event: AnalyticsEvent) {