From 3ce0c3dc5091a3bf05684be19215ea54800d161f Mon Sep 17 00:00:00 2001 From: Anna Date: Sat, 27 Aug 2022 15:55:47 -0400 Subject: [PATCH] fix: properly filter ExtraChat channels --- ChatTwo/Configuration.cs | 13 +++--- ChatTwo/Message.cs | 36 ++++++++++++++++ ChatTwo/Store.cs | 93 +++++++++++++++++++++++++++++++++------- 3 files changed, 119 insertions(+), 23 deletions(-) diff --git a/ChatTwo/Configuration.cs b/ChatTwo/Configuration.cs index dcc42ac..d267d58 100755 --- a/ChatTwo/Configuration.cs +++ b/ChatTwo/Configuration.cs @@ -2,7 +2,6 @@ using ChatTwo.Code; using ChatTwo.Resources; using ChatTwo.Ui; using Dalamud.Configuration; -using Dalamud.Game.Text.SeStringHandling.Payloads; using Dalamud.Logging; using ImGuiNET; @@ -50,6 +49,8 @@ internal class Configuration : IPluginConfiguration { public Dictionary ChatColours = new(); public List Tabs = new(); + public uint DatabaseMigration; + internal void UpdateFrom(Configuration other) { this.HideChat = other.HideChat; this.HideDuringCutscenes = other.HideDuringCutscenes; @@ -84,6 +85,7 @@ internal class Configuration : IPluginConfiguration { this.WindowAlpha = other.WindowAlpha; this.ChatColours = other.ChatColours.ToDictionary(entry => entry.Key, entry => entry.Value); this.Tabs = other.Tabs.Select(t => t.Clone()).ToList(); + this.DatabaseMigration = other.DatabaseMigration; } public void Migrate() { @@ -183,13 +185,8 @@ internal class Tab { } internal bool Matches(Message message) { - if (message.ContentSource.Payloads.Count > 0 && message.ContentSource.Payloads[0] is RawPayload raw) { - // this does an encode and clone every time it's accessed, so cache - var data = raw.Data; - if (data[1] == 0x27 && data[2] == 18 && data[3] == 0x20) { - var extraChatChannel = new Guid(data[4..^1]); - return this.ExtraChatAll || this.ExtraChatChannels.Contains(extraChatChannel); - } + if (message.ExtraChatChannel != Guid.Empty) { + return this.ExtraChatAll || this.ExtraChatChannels.Contains(message.ExtraChatChannel); } return message.Code.Type.IsGm() diff --git a/ChatTwo/Message.cs b/ChatTwo/Message.cs index 46ececf..bcba4ce 100755 --- a/ChatTwo/Message.cs +++ b/ChatTwo/Message.cs @@ -1,5 +1,6 @@ using ChatTwo.Code; using Dalamud.Game.Text.SeStringHandling; +using Dalamud.Game.Text.SeStringHandling.Payloads; using LiteDB; namespace ChatTwo; @@ -60,6 +61,7 @@ internal class Message { internal SeString ContentSource { get; } internal SortCode SortCode { get; } + internal Guid ExtraChatChannel { get; } internal int Hash { get; } @@ -72,6 +74,7 @@ internal class Message { this.SenderSource = senderSource; this.ContentSource = contentSource; this.SortCode = new SortCode(this.Code.Type, this.Code.Source); + this.ExtraChatChannel = this.ExtractExtraChatChannel(); this.Hash = this.GenerateHash(); foreach (var chunk in sender.Concat(content)) { @@ -90,6 +93,26 @@ internal class Message { this.SenderSource = BsonMapper.Global.Deserialize(senderSource); this.ContentSource = BsonMapper.Global.Deserialize(contentSource); this.SortCode = BsonMapper.Global.ToObject(sortCode); + this.ExtraChatChannel = this.ExtractExtraChatChannel(); + this.Hash = this.GenerateHash(); + + foreach (var chunk in this.Sender.Concat(this.Content)) { + chunk.Message = this; + } + } + + internal Message(ObjectId id, ulong receiver, ulong contentId, DateTime date, BsonDocument code, BsonArray sender, BsonArray content, BsonValue senderSource, BsonValue contentSource, BsonDocument sortCode, BsonValue extraChatChannel) { + this.Id = id; + this.Receiver = receiver; + this.ContentId = contentId; + this.Date = date; + this.Code = BsonMapper.Global.ToObject(code); + this.Sender = BsonMapper.Global.Deserialize>(sender); + this.Content = BsonMapper.Global.Deserialize>(content); + this.SenderSource = BsonMapper.Global.Deserialize(senderSource); + this.ContentSource = BsonMapper.Global.Deserialize(contentSource); + this.SortCode = BsonMapper.Global.ToObject(sortCode); + this.ExtraChatChannel = BsonMapper.Global.Deserialize(extraChatChannel); this.Hash = this.GenerateHash(); foreach (var chunk in this.Sender.Concat(this.Content)) { @@ -99,7 +122,20 @@ internal class Message { private int GenerateHash() { return this.SortCode.GetHashCode() + ^ this.ExtraChatChannel.GetHashCode() ^ string.Join("", this.Sender.Select(c => c.StringValue())).GetHashCode() ^ string.Join("", this.Content.Select(c => c.StringValue())).GetHashCode(); } + + private Guid ExtractExtraChatChannel() { + if (this.ContentSource.Payloads.Count > 0 && this.ContentSource.Payloads[0] is RawPayload raw) { + // this does an encode and clone every time it's accessed, so cache + var data = raw.Data; + if (data[1] == 0x27 && data[2] == 18 && data[3] == 0x20) { + return new Guid(data[4..^1]); + } + } + + return Guid.Empty; + } } diff --git a/ChatTwo/Store.cs b/ChatTwo/Store.cs index 313eda6..ecd5a97 100755 --- a/ChatTwo/Store.cs +++ b/ChatTwo/Store.cs @@ -5,6 +5,7 @@ using ChatTwo.Util; using Dalamud.Game; using Dalamud.Game.Text; using Dalamud.Game.Text.SeStringHandling; +using Dalamud.Logging; using LiteDB; using Lumina.Excel.GeneratedSheets; @@ -39,20 +40,40 @@ internal class Store : IDisposable { TrimWhitespace = false, // EnumAsInteger = true, }; - BsonMapper.Global.Entity() - .Id(msg => msg.Id) - .Ctor(doc => new Message( - doc["_id"].AsObjectId, - (ulong) doc["Receiver"].AsInt64, - (ulong) doc["ContentId"].AsInt64, - DateTime.UnixEpoch.AddMilliseconds(doc["Date"].AsInt64), - doc["Code"].AsDocument, - doc["Sender"].AsArray, - doc["Content"].AsArray, - doc["SenderSource"], - doc["ContentSource"], - doc["SortCode"].AsDocument - )); + + if (this.Plugin.Config.DatabaseMigration == 0) { + BsonMapper.Global.Entity() + .Id(msg => msg.Id) + .Ctor(doc => new Message( + doc["_id"].AsObjectId, + (ulong) doc["Receiver"].AsInt64, + (ulong) doc["ContentId"].AsInt64, + DateTime.UnixEpoch.AddMilliseconds(doc["Date"].AsInt64), + doc["Code"].AsDocument, + doc["Sender"].AsArray, + doc["Content"].AsArray, + doc["SenderSource"], + doc["ContentSource"], + doc["SortCode"].AsDocument + )); + } else { + BsonMapper.Global.Entity() + .Id(msg => msg.Id) + .Ctor(doc => new Message( + doc["_id"].AsObjectId, + (ulong) doc["Receiver"].AsInt64, + (ulong) doc["ContentId"].AsInt64, + DateTime.UnixEpoch.AddMilliseconds(doc["Date"].AsInt64), + doc["Code"].AsDocument, + doc["Sender"].AsArray, + doc["Content"].AsArray, + doc["SenderSource"], + doc["ContentSource"], + doc["SortCode"].AsDocument, + doc["ExtraChatChannel"] + )); + } + BsonMapper.Global.RegisterType( payload => { switch (payload) { @@ -116,6 +137,48 @@ internal class Store : IDisposable { this.Database = this.Connect(); this.Messages.EnsureIndex(msg => msg.Date); this.Messages.EnsureIndex(msg => msg.SortCode); + this.Messages.EnsureIndex(msg => msg.ExtraChatChannel); + + // re-save all messages, which will add the ExtraChat channel + if (this.Plugin.Config.DatabaseMigration == 0) { + var total = (float) this.Messages.LongCount() / 10_000.0; + var rounds = (long) Math.Ceiling(total); + var lastId = ObjectId.Empty; + for (var i = 0; i < rounds; i++) { + PluginLog.Log($"Update round {i}/{rounds}"); + var messages = this.Messages.Query() + .OrderBy(msg => msg.Id) + .Where(msg => msg.Id > lastId) + .Limit(10_000) + .ToArray(); + + foreach (var message in messages) { + this.Messages.Update(message); + lastId = message.Id; + } + } + + this.Database.Checkpoint(); + + this.Plugin.Config.DatabaseMigration = 1; + this.Plugin.SaveConfig(); + + BsonMapper.Global.Entity() + .Id(msg => msg.Id) + .Ctor(doc => new Message( + doc["_id"].AsObjectId, + (ulong) doc["Receiver"].AsInt64, + (ulong) doc["ContentId"].AsInt64, + DateTime.UnixEpoch.AddMilliseconds(doc["Date"].AsInt64), + doc["Code"].AsDocument, + doc["Sender"].AsArray, + doc["Content"].AsArray, + doc["SenderSource"], + doc["ContentSource"], + doc["SortCode"].AsDocument, + doc["ExtraChatChannel"] + )); + } this.Plugin.ChatGui.ChatMessageUnhandled += this.ChatMessage; this.Plugin.Framework.Update += this.GetMessageInfo; @@ -218,7 +281,7 @@ internal class Store : IDisposable { var query = this.Messages .Query() .OrderByDescending(msg => msg.Date) - .Where(msg => sortCodes.Contains(msg.SortCode)) + .Where(msg => sortCodes.Contains(msg.SortCode) || msg.ExtraChatChannel != Guid.Empty) .Where(msg => msg.Receiver == this.CurrentContentId); if (!this.Plugin.Config.FilterIncludePreviousSessions) { query = query.Where(msg => msg.Date >= this.Plugin.GameStarted);