anna
/
NoSoliciting
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: add machine learning mode

This commit is contained in:
Anna 2020-12-20 21:49:10 -05:00
parent bbfa04f4d8
commit 76462ff628
26 changed files with 1141 additions and 369 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
NoSoliciting.Classifier/Classifier.cs Normal file
View File

@ -0,0 +1,35 @@
using System;
using System.IO;
using System.Threading.Channels;
using Microsoft.ML;
namespace NoSoliciting.Classifier {
public class Classifier : IDisposable {
private string ConfigPath { get; }
private MLContext Context { get; }
private ITransformer Model { get; }
private DataViewSchema Schema { get; }
private PredictionEngine<MessageData, MessagePrediction> PredictionEngine { get; }
public Classifier(string configPath) {
this.ConfigPath = configPath;
this.Context = new MLContext();
this.Model = this.Context.Model.Load(Path.Combine(this.ConfigPath, "model.zip"), out var schema);
this.Schema = schema;
this.PredictionEngine = this.Context.Model.CreatePredictionEngine<MessageData, MessagePrediction>(this.Model, this.Schema);
}
public string Classify(ushort channel, string message) {
var data = new MessageData(channel, message);
var pred = this.PredictionEngine.Predict(data);
return pred.Category;
}
public void Dispose() {
this.PredictionEngine.Dispose();
}
}
}

16
NoSoliciting.Classifier/FodyWeavers.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<Weavers xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="FodyWeavers.xsd">
<Costura>
<ExcludeAssemblies>
Costura
</ExcludeAssemblies>
<Unmanaged64Assemblies>
CpuMathNative
LdaNative
</Unmanaged64Assemblies>
<PreloadOrder>
CpuMathNative
LdaNative
</PreloadOrder>
</Costura>
</Weavers>

24
NoSoliciting.Classifier/Models.cs Normal file
View File

@ -0,0 +1,24 @@
using Microsoft.ML.Data;
namespace NoSoliciting.Classifier {
public class MessageData {
public string? Category { get; }
public uint Channel { get; }
public string Message { get; }
public MessageData(uint channel, string message) {
this.Channel = channel;
this.Message = message;
}
}
public class MessagePrediction {
[ColumnName("PredictedLabel")]
public string Category { get; set; } = null!;
[ColumnName("Score")]
public float[] Probabilities { get; set; } = null!;
}
}

28
NoSoliciting.Classifier/NoSoliciting.Classifier.csproj Normal file
View File

@ -0,0 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net48</TargetFramework>
<Nullable>enable</Nullable>
<LangVersion>8</LangVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Costura.Fody" Version="4.1.0">
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Fody" Version="6.3.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.ML" Version="1.5.4" />
</ItemGroup>
</Project>

15
NoSoliciting.Classifier/app.config Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<runtime>
<assemblyBinding>
<dependentAssembly>
<assemblyIdentity name="System.Numerics.Vectors"
culture="neutral"
publicKeyToken="b03f5f7f11d50a3a" />
<bindingRedirect oldVersion="4.1.3.0"
newVersion="4.1.4.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

6
NoSoliciting.sln
View File

@ -9,6 +9,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NoSoliciting.Tests", "NoSoliciting.Tests\NoSoliciting.Tests.csproj", "{1962D91F-543A-4214-88FD-788BB7ACECE3}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NoSoliciting.Classifier", "NoSoliciting.Classifier\NoSoliciting.Classifier.csproj", "{F5F1671E-8DC4-47E4-AD3E-55DA5BDF1B93}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -23,6 +25,10 @@ Global
{1962D91F-543A-4214-88FD-788BB7ACECE3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1962D91F-543A-4214-88FD-788BB7ACECE3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1962D91F-543A-4214-88FD-788BB7ACECE3}.Release|Any CPU.Build.0 = Release|Any CPU
{F5F1671E-8DC4-47E4-AD3E-55DA5BDF1B93}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F5F1671E-8DC4-47E4-AD3E-55DA5BDF1B93}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F5F1671E-8DC4-47E4-AD3E-55DA5BDF1B93}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F5F1671E-8DC4-47E4-AD3E-55DA5BDF1B93}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

80
NoSoliciting/Definitions.cs
View File

@ -1,7 +1,4 @@
using Dalamud.Game.Chat;
using Dalamud.Plugin;
using NoSoliciting.Properties;
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@ -9,6 +6,9 @@ using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Dalamud.Game.Chat;
using Dalamud.Plugin;
using NoSoliciting.Properties;
using YamlDotNet.Core;
using YamlDotNet.Core.Events;
using YamlDotNet.Serialization;
@ -16,25 +16,24 @@ using YamlDotNet.Serialization.NamingConventions;
namespace NoSoliciting {
public class Definitions {
public static string LastError { get; private set; } = null;
public static DateTime? LastUpdate { get; set; } = null;
public static string? LastError { get; private set; }
public static DateTime? LastUpdate { get; set; }
private const string URL = "https://git.sr.ht/~jkcclemens/NoSoliciting/blob/master/NoSoliciting/definitions.yaml";
private const string Url = "https://git.sr.ht/~jkcclemens/NoSoliciting/blob/master/NoSoliciting/definitions.yaml";
public uint Version { get; private set; }
public Uri ReportUrl { get; private set; }
[YamlIgnore]
public int Count { get => this.Chat.Count + this.PartyFinder.Count + this.Global.Count; }
public Dictionary<string, Definition> Chat { get; private set; }
public Dictionary<string, Definition> PartyFinder { get; private set; }
public Dictionary<string, Definition> Global { get; private set; }
public static async Task<Definitions> UpdateAndCache(Plugin plugin) {
#if DEBUG
#if DEBUG
return LoadDefaults();
#endif
#endif
Definitions defs = null;
Definitions? defs = null;
var download = await Download().ConfigureAwait(true);
if (download != null) {
@ -43,7 +42,7 @@ namespace NoSoliciting {
try {
UpdateCache(plugin, download.Item2);
} catch (IOException e) {
PluginLog.Log($"Could not update cache.");
PluginLog.Log("Could not update cache.");
PluginLog.Log(e.ToString());
}
}
@ -60,21 +59,12 @@ namespace NoSoliciting {
return de.Deserialize<Definitions>(text);
}
private static string PluginFolder(Plugin plugin) {
return Path.Combine(new string[] {
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
"XIVLauncher",
"pluginConfigs",
plugin.Name,
});
}
private static async Task<Definitions> CacheOrDefault(Plugin plugin) {
if (plugin == null) {
throw new ArgumentNullException(nameof(plugin), "Plugin cannot be null");
}
var pluginFolder = PluginFolder(plugin);
var pluginFolder = Util.PluginFolder(plugin);
var cachedPath = Path.Combine(pluginFolder, "definitions.yaml");
if (!File.Exists(cachedPath)) {
@ -92,30 +82,30 @@ namespace NoSoliciting {
PluginLog.Log($"Could not load cached definitions: {e}. Loading defaults.");
}
LoadDefaults:
LoadDefaults:
return LoadDefaults();
}
private static Definitions LoadDefaults() {
return Load(Resources.default_definitions);
return Load(Resources.DefaultDefinitions);
}
private static async Task<Tuple<Definitions, string>> Download() {
private static async Task<Tuple<Definitions, string>?> Download() {
try {
using var client = new WebClient();
var text = await client.DownloadStringTaskAsync(URL).ConfigureAwait(true);
var text = await client.DownloadStringTaskAsync(Url).ConfigureAwait(true);
LastError = null;
return Tuple.Create(Load(text), text);
} catch (Exception e) when (e is WebException || e is YamlException) {
PluginLog.Log($"Could not download newest definitions.");
PluginLog.Log("Could not download newest definitions.");
PluginLog.Log(e.ToString());
LastError = e.Message;
return null;
}
}
private static async void UpdateCache(Plugin plugin, string defs) {
var pluginFolder = PluginFolder(plugin);
private static async void UpdateCache(IDalamudPlugin plugin, string defs) {
var pluginFolder = Util.PluginFolder(plugin);
Directory.CreateDirectory(pluginFolder);
var cachePath = Path.Combine(pluginFolder, "definitions.yaml");
@ -148,6 +138,7 @@ namespace NoSoliciting {
if (!plugin.Config.FilterStatus.TryGetValue(chat.Id, out _)) {
plugin.Config.FilterStatus[chat.Id] = chat.Default;
}
if (!plugin.Config.FilterStatus.TryGetValue(pf.Id, out _)) {
plugin.Config.FilterStatus[pf.Id] = pf.Default;
}
@ -158,25 +149,26 @@ namespace NoSoliciting {
}
public class Definition {
private bool initialised = false;
private bool _initialised;
[YamlIgnore]
public string Id { get; private set; }
public List<List<Matcher>> RequiredMatchers { get; private set; } = new List<List<Matcher>>();
public List<List<Matcher>> LikelyMatchers { get; private set; } = new List<List<Matcher>>();
public int LikelihoodThreshold { get; private set; } = 0;
public bool IgnoreCase { get; private set; } = false;
public int LikelihoodThreshold { get; private set; }
public bool IgnoreCase { get; private set; }
public bool Normalise { get; private set; } = true;
public List<XivChatType> Channels { get; private set; } = new List<XivChatType>();
public OptionNames Option { get; private set; }
public bool Default { get; private set; } = false;
public bool Default { get; private set; }
public void Initialise(string id) {
if (this.initialised) {
if (this._initialised) {
return;
}
this.initialised = true;
this._initialised = true;
this.Id = id ?? throw new ArgumentNullException(nameof(id), "string cannot be null");
@ -217,13 +209,7 @@ namespace NoSoliciting {
}
// calculate likelihood
var likelihood = 0;
foreach (var matchers in this.LikelyMatchers) {
if (matchers.Any(matcher => matcher.Matches(text))) {
likelihood += 1;
}
}
var likelihood = this.LikelyMatchers.Count(matchers => matchers.Any(matcher => matcher.Matches(text)));
// matches only if likelihood is greater than or equal the threshold
return likelihood >= this.LikelihoodThreshold;
@ -244,8 +230,8 @@ namespace NoSoliciting {
}
public class Matcher {
private string substring;
private Regex regex;
private string? substring;
private Regex? regex;
public Matcher(string substring) {
this.substring = substring ?? throw new ArgumentNullException(nameof(substring), "string cannot be null");
@ -261,7 +247,7 @@ namespace NoSoliciting {
}
if (this.regex != null) {
this.regex = new Regex(this.regex.ToString(), regex.Options | RegexOptions.IgnoreCase);
this.regex = new Regex(this.regex.ToString(), this.regex.Options | RegexOptions.IgnoreCase);
}
}
@ -271,7 +257,7 @@ namespace NoSoliciting {
}
if (this.substring != null) {
return text.Contains(substring);
return text.Contains(this.substring);
}
if (this.regex != null) {

2
NoSoliciting/Filter.Chat.cs
View File

@ -9,7 +9,7 @@ namespace NoSoliciting {
throw new ArgumentNullException(nameof(config), "PluginConfiguration cannot be null");
}
if (!config.AdvancedMode || !config.CustomChatFilter) {
if (!config.CustomChatFilter) {
return false;
}

2
NoSoliciting/Filter.PartyFinder.cs
View File

@ -9,7 +9,7 @@ namespace NoSoliciting {
throw new ArgumentNullException(nameof(config), "PluginConfiguration cannot be null");
}
if (!config.AdvancedMode || !config.CustomPFFilter) {
if (!config.CustomPFFilter) {
return false;
}

390
NoSoliciting/Filter.cs
View File

@ -4,49 +4,196 @@ using Dalamud.Hooking;
using Dalamud.Plugin;
using System;
using System.Runtime.InteropServices;
using NoSoliciting.Ml;
namespace NoSoliciting {
public partial class Filter : IDisposable {
private readonly Plugin plugin;
private bool clearOnNext = false;
private const uint MinWords = 4;
private delegate void HandlePFPacketDelegate(IntPtr param_1, IntPtr param_2);
private readonly Hook<HandlePFPacketDelegate> handlePacketHook;
public static readonly ChatType[] FilteredChatTypes = {
ChatType.Say,
ChatType.Yell,
ChatType.Shout,
ChatType.TellIncoming,
ChatType.Party,
ChatType.CrossParty,
ChatType.Alliance,
ChatType.FreeCompany,
ChatType.PvpTeam,
ChatType.CrossLinkshell1,
ChatType.CrossLinkshell2,
ChatType.CrossLinkshell3,
ChatType.CrossLinkshell4,
ChatType.CrossLinkshell5,
ChatType.CrossLinkshell6,
ChatType.CrossLinkshell7,
ChatType.CrossLinkshell8,
ChatType.Linkshell1,
ChatType.Linkshell2,
ChatType.Linkshell3,
ChatType.Linkshell4,
ChatType.Linkshell5,
ChatType.Linkshell6,
ChatType.Linkshell7,
ChatType.Linkshell8,
ChatType.NoviceNetwork,
};
private delegate long HandlePFSummary2Delegate(long param_1, long param_2, byte param_3);
private readonly Hook<HandlePFSummary2Delegate> handleSummaryHook;
private Plugin Plugin { get; }
private bool _clearOnNext;
private bool disposedValue;
private delegate void HandlePfPacketDelegate(IntPtr param1, IntPtr param2);
private readonly Hook<HandlePfPacketDelegate>? _handlePacketHook;
private delegate IntPtr HandlePfSummaryDelegate(IntPtr param1, IntPtr param2, byte param3);
private readonly Hook<HandlePfSummaryDelegate>? _handleSummaryHook;
private bool _disposedValue;
public Filter(Plugin plugin) {
this.plugin = plugin ?? throw new ArgumentNullException(nameof(plugin), "Plugin cannot be null");
this.Plugin = plugin ?? throw new ArgumentNullException(nameof(plugin), "Plugin cannot be null");
var listingPtr = this.plugin.Interface.TargetModuleScanner.ScanText("40 53 41 57 48 83 EC 28 48 8B D9");
var summaryPtr = this.plugin.Interface.TargetModuleScanner.ScanText("48 89 5C 24 ?? 48 89 74 24 ?? 57 48 83 EC 20 48 8B FA 48 8B F1 45 84 C0 74 ?? 0F B7 0A");
if (listingPtr == IntPtr.Zero || summaryPtr == IntPtr.Zero) {
PluginLog.Log("Party Finder filtering disabled because hook could not be created.");
return;
}
var listingPtr = this.Plugin.Interface.TargetModuleScanner.ScanText("40 53 41 57 48 83 EC 28 48 8B D9");
var summaryPtr = this.Plugin.Interface.TargetModuleScanner.ScanText("48 89 5C 24 ?? 48 89 74 24 ?? 57 48 83 EC 20 48 8B FA 48 8B F1 45 84 C0 74 ?? 0F B7 0A");
this.handlePacketHook = new Hook<HandlePFPacketDelegate>(listingPtr, new HandlePFPacketDelegate(this.HandlePFPacket));
this.handlePacketHook.Enable();
this._handlePacketHook = new Hook<HandlePfPacketDelegate>(listingPtr, new HandlePfPacketDelegate(this.TransformPfPacket));
this._handlePacketHook.Enable();
this.handleSummaryHook = new Hook<HandlePFSummary2Delegate>(summaryPtr, new HandlePFSummary2Delegate(this.HandleSummary));
this.handleSummaryHook.Enable();
this._handleSummaryHook = new Hook<HandlePfSummaryDelegate>(summaryPtr, new HandlePfSummaryDelegate(this.HandleSummary));
this._handleSummaryHook.Enable();
}
private void HandlePFPacket(IntPtr param_1, IntPtr param_2) {
if (this.plugin.Definitions == null) {
this.handlePacketHook.Original(param_1, param_2);
public void OnChat(XivChatType type, uint senderId, ref SeString sender, ref SeString message, ref bool isHandled) {
isHandled = isHandled || this.FilterMessage(type, sender, message);
}
private void TransformPfPacket(IntPtr param1, IntPtr data) {
if (data == IntPtr.Zero) {
goto Return;
}
if (this.Plugin.MlReady) {
this.MlTransformPfPacket(data);
} else if (this.Plugin.DefsReady) {
this.DefsTransformPfPacket(data);
}
Return:
this._handlePacketHook!.Original(param1, data);
}
private bool FilterMessage(XivChatType type, SeString sender, SeString message) {
if (message == null) {
throw new ArgumentNullException(nameof(message), "SeString cannot be null");
}
if (this.Plugin.MlReady) {
return this.MlFilterMessage(type, sender, message);
}
return this.Plugin.DefsReady && this.DefsFilterMessage(type, sender, message);
}
private bool MlFilterMessage(XivChatType type, SeString sender, SeString message) {
if (this.Plugin.MlFilter == null) {
return false;
}
var chatType = ChatTypeExt.FromDalamud(type);
// NOTE: don't filter on user-controlled chat types here because custom filters are supposed to check all
// messages except battle messages
if (chatType.IsBattle()) {
return false;
}
var text = message.TextValue;
// only look at messages >= min words
if (text.Split(' ').Length < MinWords) {
return false;
}
string? reason = null;
// step 1. classify the message using the model
var category = this.Plugin.MlFilter.ClassifyMessage((ushort) chatType, text);
// step 1a. only filter if configured to act on this channel
var filter = category != MessageCategory.Normal
&& this.Plugin.Config.MlEnabledOn(category, chatType)
&& SetReason(out reason, category.Name());
// step 2. check for custom filters if enabled
filter = filter || this.Plugin.Config.CustomChatFilter
&& Chat.MatchesCustomFilters(text, this.Plugin.Config)
&& SetReason(out reason, "custom");
this.Plugin.AddMessageHistory(new Message(
this.Plugin.MlFilter.Version,
ChatTypeExt.FromDalamud(type),
sender,
message,
reason
));
if (filter && this.Plugin.Config.LogFilteredChat) {
PluginLog.Log($"Filtered chat message ({reason}): {text}");
}
return filter;
}
private bool DefsFilterMessage(XivChatType type, SeString sender, SeString message) {
if (this.Plugin.Definitions == null || ChatTypeExt.FromDalamud(type).IsBattle()) {
return false;
}
var text = message.TextValue;
string? reason = null;
var filter = false;
foreach (var def in this.Plugin.Definitions.Chat.Values) {
filter = filter || this.Plugin.Config.FilterStatus.TryGetValue(def.Id, out var enabled)
&& enabled
&& def.Matches(type, text)
&& SetReason(out reason, def.Id);
}
// check for custom filters if enabled
filter = filter || this.Plugin.Config.CustomChatFilter
&& Chat.MatchesCustomFilters(text, this.Plugin.Config)
&& SetReason(out reason, "custom");
this.Plugin.AddMessageHistory(new Message(
this.Plugin.Definitions.Version,
ChatTypeExt.FromDalamud(type),
sender,
message,
reason
));
if (filter && this.Plugin.Config.LogFilteredChat) {
PluginLog.Log($"Filtered chat message ({reason}): {text}");
}
return filter;
}
private void MlTransformPfPacket(IntPtr data) {
if (this.Plugin.MlFilter == null) {
return;
}
if (this.clearOnNext) {
this.plugin.ClearPartyFinderHistory();
this.clearOnNext = false;
if (this._clearOnNext) {
this.Plugin.ClearPartyFinderHistory();
this._clearOnNext = false;
}
var dataPtr = param_2 + 0x10;
var dataPtr = data + 0x10;
// parse the packet into a struct
var packet = Marshal.PtrToStructure<PfPacket>(dataPtr);
@ -61,31 +208,39 @@ namespace NoSoliciting {
var desc = listing.Description();
string reason = null;
var filter = false;
filter = filter || (this.plugin.Config.FilterHugeItemLevelPFs
&& listing.minimumItemLevel > FilterUtil.MaxItemLevelAttainable(this.plugin.Interface.Data)
&& SetReason(ref reason, "ilvl"));
foreach (var def in this.plugin.Definitions.PartyFinder.Values) {
filter = filter || (this.plugin.Config.FilterStatus.TryGetValue(def.Id, out var enabled)
&& enabled
&& def.Matches(XivChatType.None, desc)
&& SetReason(ref reason, def.Id));
// only look at pfs >= min words
if (desc.Split(' ').Length < MinWords) {
continue;
}
// check for custom filters if enabled
filter = filter || (this.plugin.Config.CustomPFFilter
&& PartyFinder.MatchesCustomFilters(desc, this.plugin.Config)
&& SetReason(ref reason, "custom"));
string? reason = null;
this.plugin.AddPartyFinderHistory(new Message(
defs: this.plugin.Definitions,
type: ChatType.None,
sender: listing.Name(),
content: listing.Description(),
reason: reason
// step 1. check if pf has an item level that's too high
var filter = this.Plugin.Config.FilterHugeItemLevelPFs
&& listing.minimumItemLevel > FilterUtil.MaxItemLevelAttainable(this.Plugin.Interface.Data)
&& SetReason(out reason, "ilvl");
// step 2. check custom filters
filter = filter || this.Plugin.Config.CustomPFFilter
&& PartyFinder.MatchesCustomFilters(desc, this.Plugin.Config)
&& SetReason(out reason, "custom");
if (!filter) {
// step 3. check the model's prediction
var category = this.Plugin.MlFilter.ClassifyMessage((ushort) ChatType.None, desc);
// step 3a. filter the message if configured to do so
filter = category != MessageCategory.Normal
&& this.Plugin.Config.MlEnabledOn(category, ChatType.None)
&& SetReason(out reason, category.Name());
}
this.Plugin.AddPartyFinderHistory(new Message(
this.Plugin.MlFilter.Version,
ChatType.None,
listing.Name(),
listing.Description(),
reason
));
if (!filter) {
@ -95,7 +250,9 @@ namespace NoSoliciting {
// replace the listing with an empty one
packet.listings[i] = new PfListing();
PluginLog.Log($"Filtered PF listing from {listing.Name()} ({reason}): {listing.Description()}");
if (this.Plugin.Config.LogFilteredPfs) {
PluginLog.Log($"Filtered PF listing from {listing.Name()} ({reason}): {listing.Description()}");
}
}
// get some memory for writing to
@ -111,79 +268,114 @@ namespace NoSoliciting {
// free memory
pinnedArray.Free();
// call original function
this.handlePacketHook.Original(param_1, param_2);
}
private long HandleSummary(long param_1, long param_2, byte param_3) {
this.clearOnNext = true;
return this.handleSummaryHook.Original(param_1, param_2, param_3);
}
[System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "fulfilling a delegate")]
public void OnChat(XivChatType type, uint senderId, ref SeString sender, ref SeString message, ref bool isHandled) {
if (message == null) {
throw new ArgumentNullException(nameof(message), "SeString cannot be null");
}
if (this.plugin.Definitions == null || ChatTypeExt.FromDalamud(type).IsBattle()) {
private void DefsTransformPfPacket(IntPtr data) {
if (this.Plugin.Definitions == null) {
return;
}
var text = message.TextValue;
string reason = null;
var filter = false;
foreach (var def in this.plugin.Definitions.Chat.Values) {
filter = filter || (this.plugin.Config.FilterStatus.TryGetValue(def.Id, out var enabled)
&& enabled
&& def.Matches(type, text)
&& SetReason(ref reason, def.Id));
if (this._clearOnNext) {
this.Plugin.ClearPartyFinderHistory();
this._clearOnNext = false;
}
// check for custom filters if enabled
filter = filter || (this.plugin.Config.CustomChatFilter
&& Chat.MatchesCustomFilters(text, this.plugin.Config)
&& SetReason(ref reason, "custom"));
var dataPtr = data + 0x10;
this.plugin.AddMessageHistory(new Message(
defs: this.plugin.Definitions,
type: ChatTypeExt.FromDalamud(type),
sender: sender,
content: message,
reason: reason
));
// parse the packet into a struct
var packet = Marshal.PtrToStructure<PfPacket>(dataPtr);
if (!filter) {
return;
for (var i = 0; i < packet.listings.Length; i++) {
var listing = packet.listings[i];
// only look at listings that aren't null
if (listing.IsNull()) {
continue;
}
var desc = listing.Description();
string? reason = null;
var filter = false;
filter = filter || this.Plugin.Config.FilterHugeItemLevelPFs
&& listing.minimumItemLevel > FilterUtil.MaxItemLevelAttainable(this.Plugin.Interface.Data)
&& SetReason(out reason, "ilvl");
foreach (var def in this.Plugin.Definitions.PartyFinder.Values) {
filter = filter || this.Plugin.Config.FilterStatus.TryGetValue(def.Id, out var enabled)
&& enabled
&& def.Matches(XivChatType.None, desc)
&& SetReason(out reason, def.Id);
}
// check for custom filters if enabled
filter = filter || this.Plugin.Config.CustomPFFilter
&& PartyFinder.MatchesCustomFilters(desc, this.Plugin.Config)
&& SetReason(out reason, "custom");
this.Plugin.AddPartyFinderHistory(new Message(
this.Plugin.Definitions.Version,
ChatType.None,
listing.Name(),
listing.Description(),
reason
));
if (!filter) {
continue;
}
// replace the listing with an empty one
packet.listings[i] = new PfListing();
if (this.Plugin.Config.LogFilteredPfs) {
PluginLog.Log($"Filtered PF listing from {listing.Name()} ({reason}): {listing.Description()}");
}
}
PluginLog.Log($"Filtered chat message ({reason}): {text}");
isHandled = true;
// get some memory for writing to
var newPacket = new byte[PacketInfo.PacketSize];
var pinnedArray = GCHandle.Alloc(newPacket, GCHandleType.Pinned);
var pointer = pinnedArray.AddrOfPinnedObject();
// write our struct into the memory (doing this directly crashes the game)
Marshal.StructureToPtr(packet, pointer, false);
// copy our new memory over the game's
Marshal.Copy(newPacket, 0, dataPtr, PacketInfo.PacketSize);
// free memory
pinnedArray.Free();
}
private static bool SetReason(ref string reason, string value) {
private IntPtr HandleSummary(IntPtr param1, IntPtr param2, byte param3) {
this._clearOnNext = true;
return this._handleSummaryHook!.Original(param1, param2, param3);
}
private static bool SetReason(out string reason, string value) {
reason = value;
return true;
}
protected virtual void Dispose(bool disposing) {
if (!disposedValue) {
if (disposing) {
this.handlePacketHook?.Dispose();
this.handleSummaryHook?.Dispose();
}
disposedValue = true;
private void Dispose(bool disposing) {
if (this._disposedValue) {
return;
}
if (disposing) {
this._handlePacketHook?.Dispose();
this._handleSummaryHook?.Dispose();
}
this._disposedValue = true;
}
public void Dispose() {
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: true);
this.Dispose(true);
GC.SuppressFinalize(this);
}
}

16
NoSoliciting/GlobalSuppressions.cs
View File

@ -1,16 +0,0 @@
// This file is used by Code Analysis to maintain SuppressMessage
// attributes that are applied to this project.
// Project-level suppressions either have no target or are given
// a specific target and scoped to a namespace, type, member, etc.
using System.Diagnostics.CodeAnalysis;
[assembly: SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Scope = "type", Target = "~T:NoSoliciting.PFPacket")]
[assembly: SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Scope = "type", Target = "~T:NoSoliciting.PFListing")]
[assembly: SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Scope = "member", Target = "~F:NoSoliciting.PFPacket.padding1")]
[assembly: SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Scope = "member", Target = "~F:NoSoliciting.PFPacket.listings")]
[assembly: SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Scope = "member", Target = "~F:NoSoliciting.PFListing.header")]
[assembly: SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Scope = "module")]
[assembly: SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "no", Scope = "module")]
[assembly: SuppressMessage("Design", "CA1034:Nested types should not be visible", Scope = "module")]
[assembly: SuppressMessage("Design", "CA1724", Scope = "module")]

20
NoSoliciting/ILRepack.targets Normal file
View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="utf-8" ?>
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Target Name="ILRepacker" AfterTargets="Build" Condition="'$(Configuration)' == 'RELEASE'">
<ItemGroup>
<!-- Include="$(OutputPath)\NoSoliciting.dll;$(OutputPath)\System.*.dll;$(OutputPath)\Microsoft.*.dll;$(OutputPath)\YamlDotNet.dll;$(OutputPath)\Newtonsoft.Json.dll" -->
<InputAssemblies
Include="$(OutputPath)\NoSoliciting.dll;$(OutputPath)\NoSoliciting.Classifier.dll;$(OutputPath)\YamlDotNet.dll;$(OutputPath)\Newtonsoft.Json.dll"
Exclude="$(OutputPath)\*Native.dll"/>
</ItemGroup>
<ILRepack
Parallel="true"
Internalize="true"
InputAssemblies="@(InputAssemblies)"
TargetKind="Dll"
TargetPlatformVersion="v4"
LibraryPath="$(OutputPath)"
OutputFile="$(OutputPath)\$(AssemblyName).dll"/>
</Target>
</Project>

28
NoSoliciting/Message.cs
View File

@ -9,21 +9,17 @@ using System.Linq;
namespace NoSoliciting {
public class Message {
public Guid Id { get; private set; }
public uint DefinitionsVersion { get; private set; }
public DateTime Timestamp { get; private set; }
public ChatType ChatType { get; private set; }
public SeString Sender { get; private set; }
public SeString Content { get; private set; }
public string FilterReason { get; private set; }
public Message(Definitions defs, ChatType type, SeString sender, SeString content, string reason) {
if (defs == null) {
throw new ArgumentNullException(nameof(defs), "Definitions cannot be null");
}
public Guid Id { get; }
public uint DefinitionsVersion { get; }
public DateTime Timestamp { get; }
public ChatType ChatType { get; }
public SeString Sender { get; }
public SeString Content { get; }
public string? FilterReason { get; }
public Message(uint defsVersion, ChatType type, SeString sender, SeString content, string? reason) {
this.Id = Guid.NewGuid();
this.DefinitionsVersion = defs.Version;
this.DefinitionsVersion = defsVersion;
this.Timestamp = DateTime.Now;
this.ChatType = type;
this.Sender = sender;
@ -31,8 +27,8 @@ namespace NoSoliciting {
this.FilterReason = reason;
}
public Message(Definitions defs, ChatType type, string sender, string content, string reason) : this(
defs,
public Message(uint defsVersion, ChatType type, string sender, string content, string? reason) : this(
defsVersion,
type,
new SeString(new Payload[] { new TextPayload(sender) }),
new SeString(new Payload[] { new TextPayload(content) }),
@ -50,7 +46,7 @@ namespace NoSoliciting {
// and I don't want to write a custom converter to overwrite their stupiditiy
public List<byte> Sender { get; set; }
public List<byte> Content { get; set; }
public string Reason { get; set; }
public string? Reason { get; set; }
}
public string ToJson() {

6
NoSoliciting/Ml/Manifest.cs
View File

@ -1,5 +1,9 @@
namespace NoSoliciting.Ml {
using System;
namespace NoSoliciting.Ml {
public class Manifest {
public uint Version { get; set; }
public Uri ModelUrl { get; set; }
public Uri ReportUrl { get; set; }
}
}

159
NoSoliciting/Ml/MlFilter.cs
View File

@ -1,5 +1,156 @@
namespace NoSoliciting.Ml {
public class MlFilter {
using System;
using System.IO;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Dalamud.Plugin;
using Microsoft.ML;
using NoSoliciting.Properties;
using YamlDotNet.Core;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace NoSoliciting.Ml {
public class MlFilter : IDisposable {
public static string? LastError { get; private set; }
private const string ManifestName = "manifest.yaml";
private const string ModelName = "model.zip";
private const string Url = "http://localhost:8000/manifest.yaml";
public uint Version { get; }
private MLContext Context { get; }
private ITransformer Model { get; }
private DataViewSchema Schema { get; }
private PredictionEngine<MessageData, MessagePrediction> PredictionEngine { get; }
private MlFilter(uint version, MLContext context, ITransformer model, DataViewSchema schema) {
this.Version = version;
this.Context = context;
this.Model = model;
this.Schema = schema;
this.PredictionEngine = this.Context.Model.CreatePredictionEngine<MessageData, MessagePrediction>(this.Model, this.Schema);
}
public MessageCategory ClassifyMessage(ushort channel, string message) {
var data = new MessageData(channel, message);
var pred = this.PredictionEngine.Predict(data);
var category = MessageCategoryExt.FromString(pred.Category);
if (category != null) {
return (MessageCategory) category;
}
PluginLog.LogWarning($"Unknown message category: {pred.Category}");
return MessageCategory.Normal;
}
public static async Task<MlFilter?> Load(Plugin plugin) {
var manifest = await DownloadManifest();
if (manifest == null) {
return null;
}
byte[]? data = null;
var localManifest = LoadCachedManifest(plugin);
if (localManifest != null && localManifest.Version == manifest.Item1.Version) {
try {
data = File.ReadAllBytes(CachedFilePath(plugin, ModelName));
} catch (IOException) {
// ignored
}
}
data ??= await DownloadModel(manifest.Item1.ModelUrl);
if (data == null) {
return null;
}
UpdateCachedFile(plugin, ModelName, data);
UpdateCachedFile(plugin, ManifestName, Encoding.UTF8.GetBytes(manifest.Item2));
var context = new MLContext();
using var stream = new MemoryStream(data);
var model = context.Model.Load(stream, out var schema);
return new MlFilter(manifest.Item1.Version, context, model, schema);
}
private static async Task<byte[]?> DownloadModel(Uri url) {
try {
using var client = new WebClient();
var data = await client.DownloadDataTaskAsync(url);
return data;
} catch (WebException e) {
PluginLog.LogError("Could not download newest model.");
PluginLog.LogError(e.ToString());
LastError = e.Message;
return null;
}
}
private static string CachedFilePath(IDalamudPlugin plugin, string name) {
var pluginFolder = Util.PluginFolder(plugin);
Directory.CreateDirectory(pluginFolder);
return Path.Combine(pluginFolder, name);
}
private static async void UpdateCachedFile(IDalamudPlugin plugin, string name, byte[] data) {
var cachePath = CachedFilePath(plugin, name);
var file = File.OpenWrite(cachePath);
await file.WriteAsync(data, 0, data.Length);
await file.FlushAsync();
file.Dispose();
}
private static async Task<Tuple<Manifest, string>?> DownloadManifest() {
try {
using var client = new WebClient();
var data = await client.DownloadStringTaskAsync(Url);
LastError = null;
return Tuple.Create(LoadYaml<Manifest>(data), data);
} catch (Exception e) when (e is WebException || e is YamlException) {
PluginLog.LogError("Could not download newest model manifest.");
PluginLog.LogError(e.ToString());
LastError = e.Message;
return null;
}
}
private static Manifest? LoadCachedManifest(IDalamudPlugin plugin) {
var manifestPath = CachedFilePath(plugin, ManifestName);
if (!File.Exists(manifestPath)) {
return null;
}
string data;
try {
data = File.ReadAllText(manifestPath);
} catch (IOException) {
return null;
}
try {
return LoadYaml<Manifest>(data);
} catch (YamlException) {
return null;
}
}
private static T LoadYaml<T>(string data) {
var de = new DeserializerBuilder()
.WithNamingConvention(UnderscoredNamingConvention.Instance)
.IgnoreUnmatchedProperties()
.Build();
return de.Deserialize<T>(data);
}
public void Dispose() {
this.PredictionEngine.Dispose();
}
}
}
}

94
NoSoliciting/Ml/Models.cs Normal file
View File

@ -0,0 +1,94 @@
using System;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.ML.Data;
namespace NoSoliciting.Ml {
public class MessageData {
private static readonly Regex WardRegex = new Regex(@"w.{0,2}\d", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex PlotRegex = new Regex(@"p.{0,2}\d", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly string[] PlotWords = {
"plot",
"apartment",
"apt",
};
private static readonly Regex NumbersRegex = new Regex(@"\d{1,2}.{0,2}\d{1,2}", RegexOptions.Compiled);
private static readonly string[] TradeWords = {
"B>",
"S>",
"buy",
"sell",
};
public string? Category { get; }
public uint Channel { get; }
public string Message { get; }
public bool PartyFinder => this.Channel == 0;
public bool Shout => this.Channel == 11 || this.Channel == 30;
public bool ContainsWard => this.Message.ContainsIgnoreCase("ward") || WardRegex.IsMatch(this.Message);
public bool ContainsPlot => PlotWords.Any(word => this.Message.ContainsIgnoreCase(word)) || PlotRegex.IsMatch(this.Message);
public bool ContainsHousingNumbers => NumbersRegex.IsMatch(this.Message);
public bool ContainsTradeWords => TradeWords.Any(word => this.Message.ContainsIgnoreCase(word));
public MessageData(uint channel, string message) {
this.Channel = channel;
this.Message = message;
}
}
public class MessagePrediction {
[ColumnName("PredictedLabel")]
public string Category { get; set; } = null!;
[ColumnName("Score")]
public float[] Probabilities { get; set; } = null!;
}
public enum MessageCategory {
Trade,
FreeCompany,
Normal,
Phishing,
RmtContent,
RmtGil,
Roleplaying,
Static,
}
public static class MessageCategoryExt {
public static MessageCategory? FromString(string? category) => category switch {
"TRADE" => MessageCategory.Trade,
"FC" => MessageCategory.FreeCompany,
"NORMAL" => MessageCategory.Normal,
"PHISH" => MessageCategory.Phishing,
"RMT_C" => MessageCategory.RmtContent,
"RMT_G" => MessageCategory.RmtGil,
"RP" => MessageCategory.Roleplaying,
"STATIC" => MessageCategory.Static,
_ => null,
};
public static string Name(this MessageCategory category) => category switch {
MessageCategory.Trade => "Trades",
MessageCategory.FreeCompany => "Free Company ads",
MessageCategory.Normal => "Normal messages",
MessageCategory.Phishing => "Phishing messages",
MessageCategory.RmtContent => "RMT (content)",
MessageCategory.RmtGil => "RMT (gil)",
MessageCategory.Roleplaying => "Roleplaying",
MessageCategory.Static => "Static recruitment",
_ => throw new ArgumentException("Invalid category", nameof(category)),
};
}
}

29
NoSoliciting/NoSoliciting.csproj
View File

@ -3,31 +3,54 @@
<PropertyGroup>
<LangVersion>8</LangVersion>
<Nullable>enable</Nullable>
<AssemblyVersion>1.4.7</AssemblyVersion>
<FileVersion>1.4.7</FileVersion>
<AssemblyVersion>1.5.0</AssemblyVersion>
<FileVersion>1.5.0</FileVersion>
<TargetFramework>net48</TargetFramework>
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<ItemGroup>
<Reference Include="Dalamud, Version=5.2.0.5, Culture=neutral, PublicKeyToken=null">
<Reference Include="Dalamud, Version=5.2.1.0, Culture=neutral, PublicKeyToken=null">
<HintPath>$(AppData)\XIVLauncher\addon\Hooks\Dalamud.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="ImGui.NET, Version=1.72.0.0, Culture=neutral, PublicKeyToken=null">
<HintPath>$(AppData)\XIVLauncher\addon\Hooks\ImGui.NET.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="ImGuiScene, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null">
<HintPath>$(AppData)\XIVLauncher\addon\Hooks\ImGuiScene.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="Lumina, Version=2.0.0.0, Culture=neutral, PublicKeyToken=null">
<HintPath>$(AppData)\XIVLauncher\addon\Hooks\Lumina.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="Lumina.Excel, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null">
<HintPath>$(AppData)\XIVLauncher\addon\Hooks\Lumina.Excel.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed">
<HintPath>$(AppData)\XIVLauncher\addon\Hooks\Newtonsoft.Json.dll</HintPath>
<Private>False</Private>
</Reference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.ML" Version="1.5.4" />
<PackageReference Include="YamlDotNet" Version="9.1.0" />
<PackageReference Include="ILRepack.Lib.MSBuild.Task" Version="2.0.18.2" />
</ItemGroup>
<ItemGroup>
<Compile Update="Properties\Resources.Designer.cs">
<DesignTime>True</DesignTime>
<AutoGen>True</AutoGen>
<DependentUpon>Resources.resx</DependentUpon>
</Compile>
</ItemGroup>
<ItemGroup>
<EmbeddedResource Update="Properties\Resources.resx">
<Generator>ResXFileCodeGenerator</Generator>
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
</EmbeddedResource>
</ItemGroup>
</Project>

2
NoSoliciting/NoSoliciting.json
View File

@ -3,7 +3,7 @@
"Name": "NoSoliciting",
"Description": "Hides RMT in chat and the party finder. /prmt",
"InternalName": "NoSoliciting",
"AssemblyVersion": "1.4.7",
"AssemblyVersion": "1.5.0",
"RepoUrl": "https://git.sr.ht/~jkcclemens/NoSoliciting",
"ApplicableVersion": "any",
"DalamudApiLevel": 2

106
NoSoliciting/Plugin.cs
View File

@ -2,54 +2,85 @@
using Dalamud.Plugin;
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Threading.Tasks;
using NoSoliciting.Ml;
namespace NoSoliciting {
public partial class Plugin : IDalamudPlugin {
private bool disposedValue;
public class Plugin : IDalamudPlugin {
private bool _disposedValue;
public string Name => "NoSoliciting";
private PluginUi ui;
private Filter filter;
private PluginUi Ui { get; set; } = null!;
private Filter Filter { get; set; } = null!;
public DalamudPluginInterface Interface { get; private set; }
public PluginConfiguration Config { get; private set; }
public Definitions Definitions { get; private set; }
public DalamudPluginInterface Interface { get; private set; } = null!;
public PluginConfiguration Config { get; private set; } = null!;
public Definitions? Definitions { get; private set; }
public MlFilter? MlFilter { get; set; }
public bool MlReady => this.Config.UseMachineLearning && this.MlFilter != null;
public bool DefsReady => !this.Config.UseMachineLearning && this.Definitions != null;
private readonly List<Message> messageHistory = new List<Message>();
public IEnumerable<Message> MessageHistory { get => this.messageHistory; }
private readonly List<Message> _messageHistory = new List<Message>();
public IEnumerable<Message> MessageHistory => this._messageHistory;
private readonly List<Message> partyFinderHistory = new List<Message>();
public IEnumerable<Message> PartyFinderHistory { get => this.partyFinderHistory; }
private readonly List<Message> _partyFinderHistory = new List<Message>();
public IEnumerable<Message> PartyFinderHistory => this._partyFinderHistory;
// ReSharper disable once MemberCanBePrivate.Global
// ReSharper disable once AutoPropertyCanBeMadeGetOnly.Local
public string AssemblyLocation { get; private set; } = Assembly.GetExecutingAssembly().Location;
public void Initialize(DalamudPluginInterface pluginInterface) {
// NOTE: THE SECRET IS TO DOWNGRADE System.Numerics.Vectors THAT'S INCLUDED WITH DALAMUD
// CRY
string path = Environment.GetEnvironmentVariable("PATH")!;
string newPath = Path.GetDirectoryName(this.AssemblyLocation)!;
Environment.SetEnvironmentVariable("PATH", $"{path};{newPath}");
this.Interface = pluginInterface ?? throw new ArgumentNullException(nameof(pluginInterface), "DalamudPluginInterface cannot be null");
this.ui = new PluginUi(this);
this.Ui = new PluginUi(this);
this.Config = this.Interface.GetPluginConfig() as PluginConfiguration ?? new PluginConfiguration();
this.Config.Initialise(this.Interface);
this.UpdateDefinitions();
this.filter = new Filter(this);
this.Filter = new Filter(this);
if (this.Config.UseMachineLearning) {
this.InitialiseMachineLearning();
}
// pre-compute the max ilvl to prevent stutter
Task.Run(async () => {
while (!this.Interface.Data.IsDataReady) {
await Task.Delay(1_000).ConfigureAwait(true);
}
FilterUtil.MaxItemLevelAttainable(this.Interface.Data);
});
this.Interface.Framework.Gui.Chat.OnCheckMessageHandled += this.filter.OnChat;
this.Interface.UiBuilder.OnBuildUi += this.ui.Draw;
this.Interface.UiBuilder.OnOpenConfigUi += this.ui.OpenSettings;
this.Interface.CommandManager.AddHandler("/prmt", new CommandInfo(OnCommand) {
this.Interface.Framework.Gui.Chat.OnCheckMessageHandled += this.Filter.OnChat;
this.Interface.UiBuilder.OnBuildUi += this.Ui.Draw;
this.Interface.UiBuilder.OnOpenConfigUi += this.Ui.OpenSettings;
this.Interface.CommandManager.AddHandler("/prmt", new CommandInfo(this.OnCommand) {
HelpMessage = "Opens the NoSoliciting configuration",
});
}
internal void InitialiseMachineLearning() {
if (this.MlFilter != null) {
return;
}
Task.Run(async () => { this.MlFilter = await MlFilter.Load(this); })
.ContinueWith(_ => PluginLog.Log("Machine learning model loaded"));
}
internal void UpdateDefinitions() {
Task.Run(async () => {
var defs = await Definitions.UpdateAndCache(this).ConfigureAwait(true);
@ -62,45 +93,46 @@ namespace NoSoliciting {
});
}
public void OnCommand(string command, string args) {
this.ui.OpenSettings(null, null);
private void OnCommand(string command, string args) {
this.Ui.OpenSettings(null, null);
}
public void AddMessageHistory(Message message) {
this.messageHistory.Insert(0, message);
this._messageHistory.Insert(0, message);
while (this.messageHistory.Count > 250) {
this.messageHistory.RemoveAt(this.messageHistory.Count - 1);
while (this._messageHistory.Count > 250) {
this._messageHistory.RemoveAt(this._messageHistory.Count - 1);
}
}
public void ClearPartyFinderHistory() {
this.partyFinderHistory.Clear();
this._partyFinderHistory.Clear();
}
public void AddPartyFinderHistory(Message message) {
this.partyFinderHistory.Add(message);
this._partyFinderHistory.Add(message);
}
protected virtual void Dispose(bool disposing) {
if (!this.disposedValue) {
if (disposing) {
this.filter.Dispose();
this.Interface.Framework.Gui.Chat.OnCheckMessageHandled -= this.filter.OnChat;
this.Interface.UiBuilder.OnBuildUi -= this.ui.Draw;
this.Interface.UiBuilder.OnOpenConfigUi -= this.ui.OpenSettings;
this.Interface.CommandManager.RemoveHandler("/prmt");
}
// TODO: free unmanaged resources (unmanaged objects) and override finalizer
// TODO: set large fields to null
disposedValue = true;
if (this._disposedValue) {
return;
}
if (disposing) {
this.Filter.Dispose();
this.MlFilter?.Dispose();
this.Interface.Framework.Gui.Chat.OnCheckMessageHandled -= this.Filter.OnChat;
this.Interface.UiBuilder.OnBuildUi -= this.Ui.Draw;
this.Interface.UiBuilder.OnOpenConfigUi -= this.Ui.OpenSettings;
this.Interface.CommandManager.RemoveHandler("/prmt");
}
this._disposedValue = true;
}
public void Dispose() {
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: true);
this.Dispose(true);
GC.SuppressFinalize(this);
}
}

75
NoSoliciting/PluginConfiguration.cs
View File

@ -5,10 +5,13 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using NoSoliciting.Ml;
namespace NoSoliciting {
[Serializable]
public class PluginConfiguration : IPluginConfiguration {
public static readonly PluginConfiguration Default = new PluginConfiguration();
[NonSerialized]
private DalamudPluginInterface pi;
@ -16,33 +19,77 @@ namespace NoSoliciting {
[Obsolete("Use FilterStatus")]
public bool FilterChat { get; set; } = true;
[Obsolete("Use FilterStatus")]
public bool FilterFCRecruitments { get; set; } = false;
[Obsolete("Use FilterStatus")]
public bool FilterChatRPAds { get; set; } = false;
[Obsolete("Use FilterStatus")]
public bool FilterPartyFinder { get; set; } = true;
[Obsolete("Use FilterStatus")]
public bool FilterPartyFinderRPAds { get; set; } = false;
public Dictionary<string, bool> FilterStatus { get; private set; } = new Dictionary<string, bool>();
public bool AdvancedMode { get; set; } = false;
public bool AdvancedMode { get; set; }
public bool CustomChatFilter { get; set; } = false;
public bool CustomChatFilter { get; set; }
public List<string> ChatSubstrings { get; } = new List<string>();
public List<string> ChatRegexes { get; } = new List<string>();
[JsonIgnore]
public List<Regex> CompiledChatRegexes { get; private set; } = new List<Regex>();
public bool CustomPFFilter { get; set; } = false;
public bool CustomPFFilter { get; set; }
public List<string> PFSubstrings { get; } = new List<string>();
public List<string> PFRegexes { get; } = new List<string>();
[JsonIgnore]
public List<Regex> CompiledPFRegexes { get; private set; } = new List<Regex>();
public bool FilterHugeItemLevelPFs { get; set; } = false;
public bool FilterHugeItemLevelPFs { get; set; }
public bool UseMachineLearning { get; set; }
public HashSet<MessageCategory> BasicMlFilters { get; set; } = new HashSet<MessageCategory> {
MessageCategory.RmtGil,
MessageCategory.RmtContent,
MessageCategory.Phishing,
};
public Dictionary<MessageCategory, HashSet<ChatType>> MlFilters { get; set; } = new Dictionary<MessageCategory, HashSet<ChatType>> {
[MessageCategory.RmtGil] = new HashSet<ChatType> {
ChatType.Say,
},
[MessageCategory.RmtContent] = new HashSet<ChatType> {
ChatType.None,
},
[MessageCategory.Phishing] = new HashSet<ChatType> {
ChatType.TellIncoming,
},
[MessageCategory.Roleplaying] = new HashSet<ChatType> {
ChatType.None,
ChatType.Shout,
ChatType.Yell,
},
[MessageCategory.FreeCompany] = new HashSet<ChatType> {
ChatType.None,
ChatType.Shout,
ChatType.Yell,
ChatType.TellIncoming,
},
[MessageCategory.Static] = new HashSet<ChatType> {
ChatType.None,
},
[MessageCategory.Trade] = new HashSet<ChatType> {
ChatType.None,
},
};
public bool LogFilteredPfs { get; set; } = true;
public bool LogFilteredChat { get; set; } = true;
public void Initialise(DalamudPluginInterface pi) {
this.pi = pi ?? throw new ArgumentNullException(nameof(pi), "DalamudPluginInterface cannot be null");
@ -61,5 +108,25 @@ namespace NoSoliciting {
.Select(reg => new Regex(reg, RegexOptions.Compiled))
.ToList();
}
internal bool MlEnabledOn(MessageCategory category, ChatType chatType) {
HashSet<ChatType> filtered;
if (this.AdvancedMode) {
if (!this.MlFilters.TryGetValue(category, out filtered)) {
return false;
}
} else {
if (!this.BasicMlFilters.Contains(category)) {
return false;
}
if (!Default.MlFilters.TryGetValue(category, out filtered)) {
return false;
}
}
return filtered.Contains(chatType);
}
}
}

249
NoSoliciting/PluginUi.cs
View File

@ -11,6 +11,7 @@ using System.Net;
using System.Numerics;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using NoSoliciting.Ml;
namespace NoSoliciting {
public class PluginUi {
@ -36,7 +37,7 @@ namespace NoSoliciting {
this.Plugin = plugin ?? throw new ArgumentNullException(nameof(plugin), "Plugin cannot be null");
}
public void OpenSettings(object sender, EventArgs e) {
public void OpenSettings(object? sender, EventArgs? e) {
this.ShowSettings = true;
}
@ -62,22 +63,43 @@ namespace NoSoliciting {
return;
}
if (this.Plugin.Config.AdvancedMode) {
this.DrawAdvancedSettings();
} else {
this.DrawBasicSettings();
}
var modes = new[] {
"Definition matchers (default)",
"Machine learning (experimental)",
};
var modeIndex = this.Plugin.Config.UseMachineLearning ? 1 : 0;
if (ImGui.Combo("Filter mode", ref modeIndex, modes, modes.Length)) {
this.Plugin.Config.UseMachineLearning = modeIndex == 1;
this.Plugin.Config.Save();
ImGui.Separator();
if (this.Plugin.Config.UseMachineLearning) {
this.Plugin.InitialiseMachineLearning();
}
}
var advanced = this.Plugin.Config.AdvancedMode;
if (ImGui.Checkbox("Advanced mode", ref advanced)) {
this.Plugin.Config.AdvancedMode = advanced;
this.Plugin.Config.Save();
this._resizeWindow = true;
}
ImGui.SameLine();
ImGui.Spacing();
if (!ImGui.BeginTabBar("##nosoliciting-tabs")) {
return;
}
if (this.Plugin.Config.UseMachineLearning) {
this.DrawMachineLearningConfig();
} else {
this.DrawDefinitionsConfig();
}
this.DrawOtherFilters();
ImGui.EndTabBar();
ImGui.Separator();
if (ImGui.Button("Show reporting window")) {
this.ShowReporting = true;
@ -86,70 +108,78 @@ namespace NoSoliciting {
ImGui.End();
}
private void DrawBasicSettings() {
private void DrawOtherFilters() {
if (!ImGui.BeginTabItem("Other filters")) {
return;
}
if (ImGui.CollapsingHeader("Chat filters")) {
var customChat = this.Plugin.Config.CustomChatFilter;
if (ImGui.Checkbox("Enable custom chat filters", ref customChat)) {
this.Plugin.Config.CustomChatFilter = customChat;
this.Plugin.Config.Save();
}
if (this.Plugin.Config.CustomChatFilter) {
var substrings = this.Plugin.Config.ChatSubstrings;
var regexes = this.Plugin.Config.ChatRegexes;
this.DrawCustom("chat", ref substrings, ref regexes);
}
}
if (ImGui.CollapsingHeader("Party Finder filters")) {
var filterHugeItemLevelPFs = this.Plugin.Config.FilterHugeItemLevelPFs;
// ReSharper disable once InvertIf
if (ImGui.Checkbox("Filter PFs with item level above maximum", ref filterHugeItemLevelPFs)) {
this.Plugin.Config.FilterHugeItemLevelPFs = filterHugeItemLevelPFs;
this.Plugin.Config.Save();
}
var customPf = this.Plugin.Config.CustomPFFilter;
if (ImGui.Checkbox("Enable custom Party Finder filters", ref customPf)) {
this.Plugin.Config.CustomPFFilter = customPf;
this.Plugin.Config.Save();
}
if (this.Plugin.Config.CustomPFFilter) {
var substrings = this.Plugin.Config.PFSubstrings;
var regexes = this.Plugin.Config.PFRegexes;
this.DrawCustom("pf", ref substrings, ref regexes);
}
}
ImGui.EndTabItem();
}
private void DrawDefsBasicSettings() {
if (this.Plugin.Definitions == null) {
return;
}
if (!ImGui.BeginTabItem("Filters")) {
return;
}
this.DrawCheckboxes(this.Plugin.Definitions.Chat.Values, true, "chat");
ImGui.Separator();
this.DrawCheckboxes(this.Plugin.Definitions.PartyFinder.Values, true, "Party Finder");
var filterHugeItemLevelPFs = this.Plugin.Config.FilterHugeItemLevelPFs;
// ReSharper disable once InvertIf
if (ImGui.Checkbox("Filter PFs with item level above maximum", ref filterHugeItemLevelPFs)) {
this.Plugin.Config.FilterHugeItemLevelPFs = filterHugeItemLevelPFs;
this.Plugin.Config.Save();
}
ImGui.EndTabItem();
}
private void DrawAdvancedSettings() {
if (!ImGui.BeginTabBar("##nosoliciting-tabs")) {
return;
}
private void DrawDefsAdvancedSettings() {
if (this.Plugin.Definitions != null) {
if (ImGui.BeginTabItem("Chat")) {
this.DrawCheckboxes(this.Plugin.Definitions.Chat.Values, false, "chat");
var customChat = this.Plugin.Config.CustomChatFilter;
if (ImGui.Checkbox("Enable custom chat filters", ref customChat)) {
this.Plugin.Config.CustomChatFilter = customChat;
this.Plugin.Config.Save();
}
if (this.Plugin.Config.CustomChatFilter) {
var substrings = this.Plugin.Config.ChatSubstrings;
var regexes = this.Plugin.Config.ChatRegexes;
this.DrawCustom("chat", ref substrings, ref regexes);
}
ImGui.EndTabItem();
}
if (ImGui.BeginTabItem("Party Finder")) {
this.DrawCheckboxes(this.Plugin.Definitions.PartyFinder.Values, false, "Party Finder");
var filterHugeItemLevelPFs = this.Plugin.Config.FilterHugeItemLevelPFs;
if (ImGui.Checkbox("Enable built-in maximum item level filter", ref filterHugeItemLevelPFs)) {
this.Plugin.Config.FilterHugeItemLevelPFs = filterHugeItemLevelPFs;
this.Plugin.Config.Save();
}
var customPf = this.Plugin.Config.CustomPFFilter;
if (ImGui.Checkbox("Enable custom Party Finder filters", ref customPf)) {
this.Plugin.Config.CustomPFFilter = customPf;
this.Plugin.Config.Save();
}
if (this.Plugin.Config.CustomPFFilter) {
var substrings = this.Plugin.Config.PFSubstrings;
var regexes = this.Plugin.Config.PFRegexes;
this.DrawCustom("pf", ref substrings, ref regexes);
}
ImGui.EndTabItem();
}
}
@ -171,9 +201,112 @@ namespace NoSoliciting {
if (ImGui.Button("Update definitions")) {
this.Plugin.UpdateDefinitions();
}
ImGui.EndTabItem();
}
}
private void DrawMachineLearningConfig() {
if (this.Plugin.Config.AdvancedMode) {
this.DrawAdvancedMachineLearningConfig();
} else {
this.DrawBasicMachineLearningConfig();
}
ImGui.EndTabBar();
if (ImGui.BeginTabItem("Model")) {
ImGui.TextUnformatted($"Version: {this.Plugin.MlFilter?.Version}");
var lastError = MlFilter.LastError;
if (lastError != null) {
ImGui.TextUnformatted($"Last error: {lastError}");
}
if (ImGui.Button("Update model")) {
this.Plugin.MlFilter = null;
this.Plugin.InitialiseMachineLearning();
}
ImGui.EndTabItem();
}
}
private void DrawBasicMachineLearningConfig() {
if (!ImGui.BeginTabItem("Filters")) {
return;
}
foreach (var category in (MessageCategory[]) Enum.GetValues(typeof(MessageCategory))) {
if (category == MessageCategory.Normal) {
continue;
}
var check = this.Plugin.Config.BasicMlFilters.Contains(category);
if (ImGui.Checkbox(category.Name(), ref check)) {
if (check) {
this.Plugin.Config.BasicMlFilters.Add(category);
} else {
this.Plugin.Config.BasicMlFilters.Remove(category);
}
this.Plugin.Config.Save();
}
}
ImGui.EndTabItem();
}
private void DrawAdvancedMachineLearningConfig() {
if (!ImGui.BeginTabItem("Filters")) {
return;
}
foreach (var category in (MessageCategory[]) Enum.GetValues(typeof(MessageCategory))) {
if (category == MessageCategory.Normal) {
continue;
}
if (!ImGui.CollapsingHeader(category.Name())) {
continue;
}
if (!this.Plugin.Config.MlFilters.ContainsKey(category)) {
this.Plugin.Config.MlFilters[category] = new HashSet<ChatType>();
}
var types = this.Plugin.Config.MlFilters[category];
void DrawTypes(ChatType type) {
var name = type == ChatType.None ? "Party Finder" : type.ToString();
var check = types.Contains(type);
if (!ImGui.Checkbox(name, ref check)) {
return;
}
if (check) {
types.Add(type);
} else {
types.Remove(type);
}
this.Plugin.Config.Save();
}
DrawTypes(ChatType.None);
foreach (var type in Filter.FilteredChatTypes) {
DrawTypes(type);
}
}
ImGui.EndTabItem();
}
private void DrawDefinitionsConfig() {
if (this.Plugin.Config.AdvancedMode) {
this.DrawDefsAdvancedSettings();
} else {
this.DrawDefsBasicSettings();
}
}
private void DrawCustom(string name, ref List<string> substrings, ref List<string> regexes) {
@ -393,11 +526,9 @@ namespace NoSoliciting {
ImGui.Text("Reporting this message will let the developer know that you think this message was incorrectly classified.");
if (message.FilterReason != null) {
ImGui.Text("Specifically, this message WAS filtered but shouldn't have been.");
} else {
ImGui.Text("Specifically, this message WAS NOT filtered but should have been.");
}
ImGui.Text(message.FilterReason != null
? "Specifically, this message WAS filtered but shouldn't have been."
: "Specifically, this message WAS NOT filtered but should have been.");
ImGui.Separator();
@ -410,11 +541,11 @@ namespace NoSoliciting {
} else {
if (ImGui.Button("Report")) {
Task.Run(async () => {
string resp = null;
string? resp = null;
try {
using var client = new WebClient();
this.LastReportStatus = ReportStatus.InProgress;
resp = await client.UploadStringTaskAsync(this.Plugin.Definitions.ReportUrl, message.ToJson()).ConfigureAwait(true);
resp = await client.UploadStringTaskAsync(this.Plugin.Definitions!.ReportUrl, message.ToJson()).ConfigureAwait(true);
} catch (Exception) {
// ignored
}

38
NoSoliciting/Properties/AssemblyInfo.cs
View File

@ -1,38 +0,0 @@
using System.Resources;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("NoSoliciting")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("NoSoliciting")]
[assembly: AssemblyCopyright("Copyright © 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("e4c12987-9064-4788-8783-be418b2c0142")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.4.7")]
[assembly: AssemblyFileVersion("1.4.7")]
[assembly: NeutralResourcesLanguage("en-GB")]

6
NoSoliciting/Properties/Resources.Designer.cs generated
View File

@ -19,7 +19,7 @@ namespace NoSoliciting.Properties {
// class via a tool like ResGen or Visual Studio.
// To add or remove a member, edit your .ResX file then rerun ResGen
// with the /str option, or rebuild your VS project.
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")]
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")]
[global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
[global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
internal class Resources {
@ -75,9 +75,9 @@ namespace NoSoliciting.Properties {
///# Each subsection is a separate built-in filter that can be toggled on
///# and off. The option shown in the UI is defined [rest of string was truncated]&quot;;.
/// </summary>
internal static string default_definitions {
internal static string DefaultDefinitions {
get {
return ResourceManager.GetString("default_definitions", resourceCulture);
return ResourceManager.GetString("DefaultDefinitions", resourceCulture);
}
}
}

59
NoSoliciting/Properties/Resources.resx
View File

@ -1,17 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
<!--
Microsoft ResX Schema
Version 2.0
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.
Example:
... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
@ -26,36 +26,36 @@
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>
There are any number of "resheader" rows that contain simple
There are any number of "resheader" rows that contain simple
name/value pairs.
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.
mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
@ -118,8 +118,7 @@
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<assembly alias="System.Windows.Forms" name="System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" />
<data name="default_definitions" type="System.Resources.ResXFileRef, System.Windows.Forms">
<data name="DefaultDefinitions" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\definitions.yaml;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
<comment>The default definitions to be used as the last-resort fallback. Automatically included from current definitions at release.</comment>
</data>
</root>
</root>

16
NoSoliciting/Util.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.IO;
using Dalamud.Plugin;
namespace NoSoliciting {
public static class Util {
public static string PluginFolder(IDalamudPlugin plugin) {
return Path.Combine(new[] {
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
"XIVLauncher",
"pluginConfigs",
plugin.Name,
});
}
}
}

9
NoSoliciting/packages.config
View File

@ -1,9 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.CodeAnalysis.FxCopAnalyzers" version="2.9.6" targetFramework="net48" developmentDependency="true" />
<package id="Microsoft.CodeAnalysis.VersionCheckAnalyzer" version="2.9.6" targetFramework="net48" developmentDependency="true" />
<package id="Microsoft.CodeQuality.Analyzers" version="2.9.6" targetFramework="net48" developmentDependency="true" />
<package id="Microsoft.NetCore.Analyzers" version="2.9.6" targetFramework="net48" developmentDependency="true" />
<package id="Microsoft.NetFramework.Analyzers" version="2.9.6" targetFramework="net48" developmentDependency="true" />
<package id="YamlDotNet" version="8.1.2" targetFramework="net48" />
</packages>