feat: add new features

This commit is contained in:
Anna 2023-10-13 03:35:06 -04:00
parent a0a8ea43cd
commit a5b49f1b84
Signed by: anna
GPG Key ID: D0943384CD9F87D1
15 changed files with 346 additions and 125 deletions

View File

@ -1 +1 @@
DATABASE_URL=sqlite://./database.sqlite
DATABASE_URL=postgres://postgres:owo@127.0.0.1:21984/postgres

1
server/.gitignore vendored
View File

@ -2,3 +2,4 @@
/database.sqlite
/database.sqlite-shm
/database.sqlite-wal
/config.toml

117
server/Cargo.lock generated
View File

@ -29,6 +29,21 @@ dependencies = [
"version_check",
]
[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-stdlib"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "allocator-api2"
version = "0.2.16"
@ -62,6 +77,7 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c"
dependencies = [
"brotli",
"flate2",
"futures-core",
"memchr",
@ -197,6 +213,27 @@ dependencies = [
"generic-array",
]
[[package]]
name = "brotli"
version = "3.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]]
name = "bumpalo"
version = "3.14.0"
@ -310,6 +347,12 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crunchy"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
[[package]]
name = "crypto-common"
version = "0.1.6"
@ -555,6 +598,16 @@ version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
[[package]]
name = "half"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872"
dependencies = [
"cfg-if",
"crunchy",
]
[[package]]
name = "hashbrown"
version = "0.14.1"
@ -1064,6 +1117,7 @@ dependencies = [
"bytes",
"chrono",
"data-encoding",
"half",
"rand",
"rmp-serde",
"serde",
@ -1072,6 +1126,7 @@ dependencies = [
"siphasher",
"sqlx",
"tokio",
"toml",
"tower",
"tower-http",
]
@ -1270,6 +1325,15 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186"
dependencies = [
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@ -1304,6 +1368,15 @@ dependencies = [
"digest",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
dependencies = [
"libc",
]
[[package]]
name = "signature"
version = "2.1.0"
@ -1695,6 +1768,7 @@ dependencies = [
"mio",
"num_cpus",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.5.4",
"tokio-macros",
"windows-sys",
@ -1735,6 +1809,40 @@ dependencies = [
"tokio",
]
[[package]]
name = "toml"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit",
]
[[package]]
name = "toml_datetime"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338"
dependencies = [
"indexmap",
"serde",
"serde_spanned",
"toml_datetime",
"winnow",
]
[[package]]
name = "tower"
version = "0.4.13"
@ -2057,6 +2165,15 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "winnow"
version = "0.5.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907"
dependencies = [
"memchr",
]
[[package]]
name = "zeroize"
version = "1.6.0"

View File

@ -11,13 +11,15 @@ axum = "0.6"
bytes = "1"
chrono = { version = "0.4", features = ["serde"] }
data-encoding = "2"
half = "2"
rand = "0.8"
rmp-serde = "1"
serde = { version = "1", features = ["derive"] }
serde_bytes = "0.11"
serde_path_to_error = "0.1"
siphasher = "1"
sqlx = { version = "0.7", features = ["runtime-tokio", "sqlite", "chrono"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
sqlx = { version = "0.7", features = ["runtime-tokio", "postgres", "chrono"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal"] }
toml = "0.8"
tower = "0.4"
tower-http = { version = "0.4", features = ["compression-zstd", "compression-gzip", "decompression-gzip", "cors"] }
tower-http = { version = "0.4", features = ["compression-full", "decompression-gzip", "cors"] }

3
server/build.rs Normal file
View File

@ -0,0 +1,3 @@
fn main() {
println!("cargo:rerun-if-changed=migrations");
}

View File

@ -0,0 +1,6 @@
[database]
host = 'localhost'
port = 5432
name = 'megamappingway'
username = 'postgres'
password = 'changeme'

View File

@ -1,6 +1,6 @@
create table players
(
hash blob not null primary key,
hash bytea not null primary key,
world int not null,
timestamp timestamp with time zone not null,
territory int not null,
@ -9,7 +9,7 @@ create table players
y float not null,
z float not null,
w float not null,
customize blob not null,
customize bytea not null,
level smallint not null,
job int not null,
free_company text,

View File

@ -0,0 +1 @@
drop index players_current_world_idx;

View File

@ -0,0 +1 @@
create index players_current_world_idx on players (current_world);

View File

@ -0,0 +1,2 @@
alter table players
drop column party_id;

View File

@ -0,0 +1,2 @@
alter table players
add column party_id bigint;

View File

@ -0,0 +1,2 @@
alter table players
add column free_company text;

View File

@ -0,0 +1,2 @@
alter table players
drop column free_company;

15
server/src/config.rs Normal file
View File

@ -0,0 +1,15 @@
use serde::Deserialize;
#[derive(Deserialize)]
pub struct Config {
pub database: Database,
}
#[derive(Deserialize)]
pub struct Database {
pub host: String,
pub port: u16,
pub name: String,
pub username: String,
pub password: String,
}

View File

@ -1,3 +1,4 @@
use std::collections::HashMap;
use std::ops::{Deref, DerefMut};
use std::sync::Arc;
use std::time::Duration;
@ -6,7 +7,7 @@ use anyhow::Result;
use axum::{async_trait, BoxError, Router, Server};
use axum::body::HttpBody;
use axum::error_handling::HandleErrorLayer;
use axum::extract::{FromRequest, Path, State};
use axum::extract::{FromRequest, Path, Query, State};
use axum::extract::rejection::BytesRejection;
use axum::http::{HeaderValue, Request, StatusCode};
#[cfg(not(debug_assertions))]
@ -14,31 +15,58 @@ use axum::http::Method;
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use bytes::Bytes;
use half::f16;
use rand::Rng;
use serde::{Deserialize, Serialize};
use serde::de::DeserializeOwned;
use siphasher::sip::SipHasher;
use sqlx::SqlitePool;
use sqlx::PgPool;
use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
use tokio::sync::RwLock;
use tower::ServiceBuilder;
use tower_http::compression::CompressionLayer;
use tower_http::CompressionLevel;
use tower_http::cors::CorsLayer;
use tower_http::decompression::RequestDecompressionLayer;
use crate::config::Config;
mod config;
static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!();
type Populations = HashMap<u32, HashMap<u32, i64>>;
struct AppState {
salt: String,
pool: Arc<SqlitePool>,
pool: Arc<PgPool>,
hasher: SipHasher,
populations: Arc<RwLock<Populations>>,
}
#[tokio::main]
async fn main() -> Result<()> {
let pool = SqlitePool::connect("./database.sqlite").await?;
let config: Config = {
let t = tokio::fs::read_to_string("./config.toml").await?;
toml::from_str(&t)?
};
let pool = PgPoolOptions::new()
.max_connections(50)
.connect_with(
PgConnectOptions::new()
.host(&config.database.host)
.port(config.database.port)
.database(&config.database.name)
.username(&config.database.username)
.password(&config.database.password)
)
.await?;
MIGRATOR.run(&pool).await?;
let pool = Arc::new(pool);
// spawn old info deletion task
{
let pool = Arc::clone(&pool);
tokio::task::spawn(async move {
@ -48,8 +76,8 @@ async fn main() -> Result<()> {
println!("deleting old records");
let result = sqlx::query!(
// language=sqlite
"delete from players where unixepoch('now') - unixepoch(timestamp) > 3600",
// language=postgresql
"delete from players where current_timestamp - timestamp > interval '1 hour'",
)
.execute(&*pool)
.await;
@ -62,6 +90,43 @@ async fn main() -> Result<()> {
});
}
// spawn population task
let populations_cache = Arc::new(RwLock::new(HashMap::default()));
{
let pool = Arc::clone(&pool);
let populations_cache = Arc::clone(&populations_cache);
tokio::task::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
let result = sqlx::query!(
// language=postgresql
r#"select territory, current_world, coalesce(count(*), 0) as "count!" from players where current_timestamp - timestamp < interval '30 seconds' group by territory, current_world"#
)
.fetch_all(&*pool)
.await;
let result = match result {
Ok(r) => r,
Err(e) => {
eprintln!("could not calculate populations: {e:#}");
continue;
}
};
let mut output: Populations = HashMap::with_capacity(result.len());
for record in result {
output.entry(record.territory as u32)
.or_default()
.insert(record.current_world as u32, record.count);
}
*populations_cache.write().await = output;
}
});
}
let state = Arc::new(AppState {
pool,
salt: data_encoding::BASE64_NOPAD.encode(&{
@ -70,6 +135,7 @@ async fn main() -> Result<()> {
bytes
}),
hasher: SipHasher::new(),
populations: populations_cache,
});
#[cfg(not(debug_assertions))]
@ -84,11 +150,10 @@ async fn main() -> Result<()> {
let app = Router::new()
.route("/:territory", get(territory))
.route("/:territory/:world", get(territory_world))
.route("/upload", post(upload))
.with_state(state)
.layer(cors)
.layer(CompressionLayer::new())
.layer(CompressionLayer::new().quality(CompressionLevel::Best))
.layer(
ServiceBuilder::new()
.layer(HandleErrorLayer::new(|_: BoxError| async move {
@ -97,20 +162,38 @@ async fn main() -> Result<()> {
.layer(RequestDecompressionLayer::new())
);
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
tokio::task::spawn(async move {
tokio::signal::ctrl_c().await.ok();
println!("caught ctrl-c, shutting down");
shutdown_tx.send(()).ok();
});
Server::bind(&"127.0.0.1:30888".parse()?)
.serve(app.into_make_service())
.with_graceful_shutdown(async {
shutdown_rx.await.ok();
})
.await?;
Ok(())
}
#[derive(Deserialize, Default)]
#[serde(default)]
struct TerritoryQuery {
world: Option<u32>,
party: Option<u32>,
}
async fn territory(
State(state): State<Arc<AppState>>,
Path(territory): Path<u32>,
) -> Result<MsgPack<Vec<AnonymousPlayerInfo>>, AppError>
Query(query): Query<TerritoryQuery>,
) -> Result<MsgPack<QueryResponse<Vec<AnonymousPlayerInfo>>>, AppError>
{
let info = sqlx::query_as!(
AnonymousPlayerInfoInternal,
// language=sqlite
// language=postgresql
r#"
select hash,
world,
@ -121,71 +204,44 @@ async fn territory(
customize,
level,
job,
free_company,
current_hp,
max_hp,
coalesce(unixepoch('now') - unixepoch(timestamp), 30) as age
party_id,
coalesce(extract('epoch' from current_timestamp - timestamp), 30)::bigint as "age!"
from players
where territory = ?
and age < 30
where territory = $1
and ($2 or current_world = $3)
and current_timestamp - timestamp < interval '30 seconds'
"#,
territory,
territory as i64,
query.world.is_none(),
query.world.map(|x| x as i32),
)
.fetch_all(&*state.pool)
.await?;
let info = info.into_iter()
let parties = get_parties(&state, territory, &info);
let mut info: Vec<_> = info.into_iter()
.filter(|player| match query.party {
None => true,
x => player.party_hash(&state.hasher, &state.salt, territory) == x,
})
.map(|player| AnonymousPlayerInfo::new_from(player, &state.hasher, &state.salt, territory))
.collect();
info.sort_unstable_by_key(|p| p.territory_unique_id);
Ok(MsgPack(info))
}
async fn territory_world(
State(state): State<Arc<AppState>>,
Path((territory, world)): Path<(u32, u32)>,
) -> Result<MsgPack<Vec<AnonymousPlayerInfo>>, AppError>
{
let info = sqlx::query_as!(
AnonymousPlayerInfoInternal,
// language=sqlite
r#"
select hash,
world,
x,
y,
z,
w,
customize,
level,
job,
free_company,
current_hp,
max_hp,
coalesce(unixepoch('now') - unixepoch(timestamp), 30) as age
from players
where territory = ?
and current_world = ?
and age < 30
"#,
territory,
world,
)
.fetch_all(&*state.pool)
.await?;
let info = info.into_iter()
.map(|player| AnonymousPlayerInfo::new_from(player, &state.hasher, &state.salt, territory))
.collect();
Ok(MsgPack(info))
Ok(MsgPack(QueryResponse {
populations: state.populations.read().await.clone(),
parties,
data: info,
}))
}
async fn upload(
state: State<Arc<AppState>>,
data: MsgPack<Update>,
) -> Result<(), AppError> {
if data.version != 2 {
if data.version != 3 {
return Err(anyhow::anyhow!("invalid update request version").into());
}
@ -196,61 +252,43 @@ async fn upload(
continue;
}
let fc = match &player.free_company {
Some(x) if x.trim() == "" => None,
Some(x) => Some(x),
None => None,
};
sqlx::query!(
// language=sqlite
// language=postgresql
"
insert into players (hash, world, timestamp, territory, current_world, x, y, z, w, customize, level, job, free_company,
current_hp,
max_hp)
values (?, ?, current_timestamp, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
insert into players (hash, world, timestamp, territory, current_world,
x, y, z, w, customize,
level, job, current_hp, max_hp, party_id)
values ($1, $2, current_timestamp, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
on conflict (hash) do update set timestamp = current_timestamp,
territory = ?,
current_world = ?,
x = ?,
y = ?,
z = ?,
w = ?,
customize = ?,
level = ?,
job = ?,
free_company = ?,
current_hp = ?,
max_hp = ?
world = $2,
territory = $3,
current_world = $4,
x = $5,
y = $6,
z = $7,
w = $8,
customize = $9,
level = $10,
job = $11,
current_hp = $12,
max_hp = $13,
party_id = $14
",
player.hash,
player.world,
data.territory,
data.world,
player.world as i64,
data.territory as i64,
data.world as i64,
player.x,
player.y,
player.z,
player.w,
player.customize,
player.level,
player.job,
fc,
player.current_hp,
player.max_hp,
data.territory,
data.world,
player.x,
player.y,
player.z,
player.w,
player.customize,
player.level,
player.job,
fc,
player.current_hp,
player.max_hp,
player.level as i64,
player.job as i64,
player.current_hp as i64,
player.max_hp as i64,
player.party_id.map(|id| id as i64),
)
.execute(&mut *t)
.await?;
@ -260,6 +298,16 @@ async fn upload(
Ok(())
}
fn get_parties(state: &AppState, territory: u32, info: &[AnonymousPlayerInfoInternal]) -> Vec<u32> {
let mut parties: Vec<u32> = info.iter()
.flat_map(|player| player.party_hash(&state.hasher, &state.salt, territory))
.collect();
parties.sort_unstable();
parties.dedup();
parties
}
#[derive(Deserialize)]
struct Update {
version: u8,
@ -281,9 +329,9 @@ struct PlayerInfo {
customize: Vec<u8>,
level: u8,
job: u32,
free_company: Option<String>,
current_hp: u32,
max_hp: u32,
party_id: Option<u64>,
}
impl PlayerInfo {
@ -291,21 +339,26 @@ impl PlayerInfo {
!self.hash.is_empty()
&& self.world != 65535
&& self.level != 0
&& self.customize.len() >= 26
}
}
#[derive(Serialize)]
struct QueryResponse<T> {
populations: Populations,
parties: Vec<u32>,
data: T,
}
#[derive(Serialize)]
struct AnonymousPlayerInfo {
x: f64,
y: f64,
z: f64,
w: f64,
#[serde(with = "serde_bytes")]
floats: Vec<u8>,
gender: u8,
race: u8,
level: u8,
job: u8,
hp_percent: f64,
age: i8,
age: u8,
territory_unique_id: u64,
}
@ -313,18 +366,26 @@ impl AnonymousPlayerInfo {
fn new_from(value: AnonymousPlayerInfoInternal, hasher: &SipHasher, salt: &str, territory: u32) -> Self {
let customize = value.customize();
let x = f16::from_f64(value.x).to_le_bytes();
let y = f16::from_f64(value.y).to_le_bytes();
let z = f16::from_f64(value.z).to_le_bytes();
let w = f16::from_f64(value.w).to_le_bytes();
let hp = f16::from_f64(value.current_hp as f64 / value.max_hp as f64).to_le_bytes();
let floats = x.into_iter()
.chain(y)
.chain(z)
.chain(w)
.chain(hp)
.collect();
Self {
territory_unique_id: value.gen_hash(hasher, salt, territory),
x: value.x,
y: value.y,
z: value.z,
w: value.w,
floats,
gender: customize.gender,
race: customize.race,
level: value.level as u8,
job: value.job as u8,
hp_percent: value.current_hp as f64 / value.max_hp as f64,
age: value.age as i8,
age: value.age.max(0) as u8,
}
}
}
@ -408,18 +469,16 @@ struct AnonymousPlayerInfoInternal {
customize: Vec<u8>,
level: i64,
job: i64,
free_company: Option<String>,
current_hp: i64,
max_hp: i64,
party_id: Option<i64>,
age: i64,
}
impl AnonymousPlayerInfoInternal {
pub fn gen_hash(&self, hasher: &SipHasher, salt: &str, territory: u32) -> u64 {
hasher.hash(format!(
"{}-{}-{}",
salt,
territory,
"{salt}-{territory}-{}",
data_encoding::HEXLOWER.encode(&self.hash),
).as_bytes())
}
@ -427,6 +486,14 @@ impl AnonymousPlayerInfoInternal {
pub fn customize(&self) -> Customize {
Customize::new(&self.customize).unwrap_or_default()
}
pub fn party_hash(&self, hasher: &SipHasher, salt: &str, territory: u32) -> Option<u32> {
let party_id = self.party_id?;
Some(hasher.hash(
format!("{}-{territory}-{:x}", salt, party_id).as_bytes()
) as u32)
}
}
// Make our own error that wraps `anyhow::Error`.