feat: add online definitions and FC/RP filters

Definitions for the built-in filters (RMT, FC, and RP) are now
downloaded from the git repo on plugin start and whenever the update
button is pressed. This will allow faster response to messages that
slip through, as updates can be pushed right away without needing to
release a new version.

After a successful download, the plugin writes the result to a cache
file in its config directory. In the event a download fails, the
plugin will fall back to that cached file. If that cached file does
not exist or fails to deserialise, the plugin will fall back to a file
included with each release after this commit called
default_definitions.yaml. If that file is missing (really only
possible because the user deleted it), an exception will be thrown.

Free Company recruitment messages and roleplaying advertisements are
now able to be filtered using built-in filters, hopefully making shout
chat and the Party Finder more bearable. As always, these are optional
filters (both default to disabled).
This commit is contained in:
Anna 2020-08-21 05:00:04 -04:00
parent 1e53af92f9
commit 37b9694144
10 changed files with 538 additions and 86 deletions

286
NoSoliciting/Definitions.cs Normal file
View File

@ -0,0 +1,286 @@
using Dalamud.Plugin;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using YamlDotNet.Core;
using YamlDotNet.Core.Events;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace NoSoliciting {
public class Definitions {
public static string LastError { get; private set; } = null;
public static DateTime? LastUpdate { get; set; } = null;
private const string URL = "https://git.sr.ht/~jkcclemens/NoSoliciting/blob/master/NoSoliciting/definitions.yaml";
public uint Version { get; private set; }
public ChatDefinitions Chat { get; private set; }
public PartyFinderDefinitions PartyFinder { get; private set; }
public GlobalDefinitions Global { get; private set; }
public static async Task<Definitions> UpdateAndCache(Plugin plugin) {
Definitions defs = null;
var download = await Download().ConfigureAwait(true);
if (download != null) {
defs = download.Item1;
try {
UpdateCache(plugin, download.Item2);
} catch (IOException e) {
PluginLog.Log($"Could not update cache.");
PluginLog.Log(e.ToString());
}
}
return defs ?? await CacheOrDefault(plugin).ConfigureAwait(true);
}
private static Definitions Load(string text) {
var de = new DeserializerBuilder()
.WithNamingConvention(UnderscoredNamingConvention.Instance)
.WithTypeConverter(new MatcherConverter())
.Build();
return de.Deserialize<Definitions>(text);
}
private static string PluginFolder(Plugin plugin) {
return Path.Combine(new string[] {
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
"XIVLauncher",
"pluginConfigs",
plugin.Name,
});
}
private static async Task<Definitions> CacheOrDefault(Plugin plugin) {
if (plugin == null) {
throw new ArgumentNullException(nameof(plugin), "Plugin cannot be null");
}
string pluginFolder = PluginFolder(plugin);
string cachedPath = Path.Combine(pluginFolder, "definitions.yaml");
if (!File.Exists(cachedPath)) {
goto LoadDefaults;
}
string text;
using (var file = File.OpenText(cachedPath)) {
text = await file.ReadToEndAsync().ConfigureAwait(true);
}
try {
return Load(text);
} catch (YamlException e) {
PluginLog.Log($"Could not load cached definitions: {e}. Loading defaults.");
}
LoadDefaults:
return await LoadDefaults().ConfigureAwait(true);
}
private static async Task<Definitions> LoadDefaults() {
string defaultPath = Path.Combine(
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location),
"default_definitions.yaml"
);
string text;
using (StreamReader file = File.OpenText(defaultPath)) {
text = await file.ReadToEndAsync().ConfigureAwait(true);
}
return Load(text);
}
private static async Task<Tuple<Definitions, string>> Download() {
try {
using (WebClient client = new WebClient()) {
string text = await client.DownloadStringTaskAsync(URL).ConfigureAwait(true);
LastError = null;
return Tuple.Create(Load(text), text);
}
} catch (WebException e) {
PluginLog.Log($"Could not download newest definitions.");
PluginLog.Log(e.ToString());
LastError = e.Message;
return null;
}
}
private static async void UpdateCache(Plugin plugin, string defs) {
string pluginFolder = PluginFolder(plugin);
Directory.CreateDirectory(pluginFolder);
string cachePath = Path.Combine(pluginFolder, "definitions.yaml");
byte[] b = Encoding.UTF8.GetBytes(defs);
using (var file = File.OpenWrite(cachePath)) {
await file.WriteAsync(b, 0, b.Length).ConfigureAwait(true);
}
}
internal void Initialise() {
Definition[] all = {
this.Chat.RMT,
this.Chat.FreeCompany,
this.PartyFinder.RMT,
this.Global.Roleplay,
};
foreach (Definition def in all) {
def.Initialise();
}
}
}
public class ChatDefinitions {
[YamlMember(Alias = "rmt")]
public Definition RMT { get; private set; }
public Definition FreeCompany { get; private set; }
}
public class PartyFinderDefinitions {
[YamlMember(Alias = "rmt")]
public Definition RMT { get; private set; }
}
public class GlobalDefinitions {
public Definition Roleplay { get; private set; }
}
public class Definition {
private bool initialised = false;
public List<List<Matcher>> RequiredMatchers { get; private set; } = new List<List<Matcher>>();
public List<List<Matcher>> LikelyMatchers { get; private set; } = new List<List<Matcher>>();
public int LikelihoodThreshold { get; private set; } = 0;
public bool IgnoreCase { get; private set; } = false;
internal void Initialise() {
if (this.initialised) {
return;
}
this.initialised = true;
if (!this.IgnoreCase) {
return;
}
IEnumerable<Matcher> allMatchers = this.LikelyMatchers
.Concat(this.RequiredMatchers)
.SelectMany(matchers => matchers);
foreach (Matcher matcher in allMatchers) {
matcher.MakeIgnoreCase();
}
}
public bool Matches(string text) {
if (text == null) {
throw new ArgumentNullException(nameof(text), "string cannot be null");
}
if (this.IgnoreCase) {
text = text.ToLowerInvariant();
}
// ensure all required matchers match
bool allRequired = this.RequiredMatchers.All(matchers => matchers.Any(matcher => matcher.Matches(text)));
if (!allRequired) {
return false;
}
// calculate likelihood
int likelihood = 0;
foreach (var matchers in this.LikelyMatchers) {
if (matchers.Any(matcher => matcher.Matches(text))) {
likelihood += 1;
}
}
// matches only if likelihood is greater than or equal the threshold
return likelihood >= this.LikelihoodThreshold;
}
}
public class Matcher {
private string substring;
private Regex regex;
public Matcher(string substring) {
this.substring = substring ?? throw new ArgumentNullException(nameof(substring), "string cannot be null");
}
public Matcher(Regex regex) {
this.regex = regex ?? throw new ArgumentNullException(nameof(regex), "Regex cannot be null");
}
internal void MakeIgnoreCase() {
if (this.substring != null) {
this.substring = this.substring.ToLowerInvariant();
}
if (this.regex != null) {
this.regex = new Regex(this.regex.ToString(), regex.Options | RegexOptions.IgnoreCase);
}
}
public bool Matches(string text) {
if (text == null) {
throw new ArgumentNullException(nameof(text), "string cannot be null");
}
if (this.substring != null) {
return text.Contains(substring);
}
if (this.regex != null) {
return this.regex.IsMatch(text);
}
throw new ApplicationException("Matcher created without substring or regex");
}
}
internal sealed class MatcherConverter : IYamlTypeConverter {
public bool Accepts(Type type) {
return type == typeof(Matcher);
}
public object ReadYaml(IParser parser, Type type) {
Matcher matcher;
if (parser.TryConsume(out Scalar scalar)) {
matcher = new Matcher(scalar.Value);
} else if (parser.TryConsume(out MappingStart _)) {
if (parser.Consume<Scalar>().Value != "regex") {
throw new ArgumentException("matcher was an object but did not specify regex key");
}
Regex regex = new Regex(parser.Consume<Scalar>().Value, RegexOptions.Compiled);
matcher = new Matcher(regex);
parser.Consume<MappingEnd>();
} else {
throw new ArgumentException("invalid matcher");
}
return matcher;
}
public void WriteYaml(IEmitter emitter, object value, Type type) {
throw new NotImplementedException();
}
}
}

View File

@ -62,8 +62,12 @@
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
<Reference Include="YamlDotNet, Version=8.0.0.0, Culture=neutral, PublicKeyToken=ec19458f3c15af5e, processorArchitecture=MSIL">
<HintPath>..\packages\YamlDotNet.8.1.2\lib\net45\YamlDotNet.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="Definitions.cs" />
<Compile Include="GlobalSuppressions.cs" />
<Compile Include="PFPacket.cs" />
<Compile Include="Plugin.cs" />
@ -76,6 +80,7 @@
<Compile Include="RMTUtil.cs" />
</ItemGroup>
<ItemGroup>
<None Include="definitions.yaml" />
<None Include="NoSoliciting.json" />
<None Include="packages.config" />
</ItemGroup>

View File

@ -1,6 +1,7 @@
using Dalamud.Game.Command;
using Dalamud.Plugin;
using System;
using System.Threading.Tasks;
namespace NoSoliciting {
public partial class Plugin : IDalamudPlugin {
@ -13,6 +14,7 @@ namespace NoSoliciting {
private RMTDetection rmt;
public PluginConfiguration Config { get; private set; }
public Definitions Definitions { get; private set; }
public void Initialize(DalamudPluginInterface pluginInterface) {
this.pi = pluginInterface ?? throw new ArgumentNullException(nameof(pluginInterface), "DalamudPluginInterface cannot be null");
@ -21,6 +23,8 @@ namespace NoSoliciting {
this.Config = this.pi.GetPluginConfig() as PluginConfiguration ?? new PluginConfiguration();
this.Config.Initialise(this.pi);
this.UpdateDefinitions();
this.rmt = new RMTDetection(this);
this.pi.Framework.Network.OnNetworkMessage += this.rmt.OnNetwork;
@ -32,6 +36,18 @@ namespace NoSoliciting {
});
}
internal void UpdateDefinitions() {
Task.Run(async () => {
Definitions defs = await Definitions.UpdateAndCache(this).ConfigureAwait(true);
// this shouldn't be possible, but what do I know
if (defs != null) {
defs.Initialise();
this.Definitions = defs;
Definitions.LastUpdate = DateTime.Now;
}
});
}
public void OnCommand(string command, string args) {
this.ui.OpenSettings(null, null);
}

View File

@ -12,7 +12,11 @@ namespace NoSoliciting {
public int Version { get; set; } = 1;
public bool FilterChat { get; set; } = true;
public bool FilterFCRecruitments { get; set; } = false;
public bool FilterChatRPAds { get; set; } = false;
public bool FilterPartyFinder { get; set; } = true;
public bool FilterPartyFinderRPAds { get; set; } = false;
public bool AdvancedMode { get; set; } = false;

View File

@ -46,7 +46,7 @@ namespace NoSoliciting {
}
private void DrawBasicSettings() {
ImGui.SetWindowSize(new Vector2(225, 125));
ImGui.SetWindowSize(new Vector2(250, 225));
bool filterChat = this.plugin.Config.FilterChat;
if (ImGui.Checkbox("Filter RMT from chat", ref filterChat)) {
@ -54,15 +54,35 @@ namespace NoSoliciting {
this.plugin.Config.Save();
}
bool filterFC = this.plugin.Config.FilterFCRecruitments;
if (ImGui.Checkbox("Filter FC recruitments from chat", ref filterFC)) {
this.plugin.Config.FilterFCRecruitments = filterFC;
this.plugin.Config.Save();
}
bool filterChatRP = this.plugin.Config.FilterChatRPAds;
if (ImGui.Checkbox("Filter RP ads from chat", ref filterChatRP)) {
this.plugin.Config.FilterChatRPAds = filterChatRP;
this.plugin.Config.Save();
}
ImGui.Separator();
bool filterPartyFinder = this.plugin.Config.FilterPartyFinder;
if (ImGui.Checkbox("Filter RMT from Party Finder", ref filterPartyFinder)) {
this.plugin.Config.FilterPartyFinder = filterPartyFinder;
this.plugin.Config.Save();
}
bool filterPFRP = this.plugin.Config.FilterPartyFinderRPAds;
if (ImGui.Checkbox("Filter RP ads from Party Finder", ref filterPFRP)) {
this.plugin.Config.FilterPartyFinderRPAds = filterPFRP;
this.plugin.Config.Save();
}
}
private void DrawAdvancedSettings() {
ImGui.SetWindowSize(new Vector2(600, 400));
ImGui.SetWindowSize(new Vector2(600, 450));
if (ImGui.BeginTabBar("##nosoliciting-tabs")) {
if (ImGui.BeginTabItem("Chat")) {
@ -72,6 +92,18 @@ namespace NoSoliciting {
this.plugin.Config.Save();
}
bool filterFC = this.plugin.Config.FilterFCRecruitments;
if (ImGui.Checkbox("Enable built-in FC recruitment filter", ref filterFC)) {
this.plugin.Config.FilterFCRecruitments = filterFC;
this.plugin.Config.Save();
}
bool filterChatRP = this.plugin.Config.FilterChatRPAds;
if (ImGui.Checkbox("Enable built-in RP ad filter", ref filterChatRP)) {
this.plugin.Config.FilterChatRPAds = filterChatRP;
this.plugin.Config.Save();
}
bool customChat = this.plugin.Config.CustomChatFilter;
if (ImGui.Checkbox("Enable custom chat filters", ref customChat)) {
this.plugin.Config.CustomChatFilter = customChat;
@ -94,6 +126,12 @@ namespace NoSoliciting {
this.plugin.Config.Save();
}
bool filterPFRP = this.plugin.Config.FilterPartyFinderRPAds;
if (ImGui.Checkbox("Enable built-in Party Finder RP filter", ref filterPFRP)) {
this.plugin.Config.FilterPartyFinderRPAds = filterPFRP;
this.plugin.Config.Save();
}
bool customPF = this.plugin.Config.CustomPFFilter;
if (ImGui.Checkbox("Enable custom Party Finder filters", ref customPF)) {
this.plugin.Config.CustomPFFilter = customPF;
@ -109,6 +147,25 @@ namespace NoSoliciting {
ImGui.EndTabItem();
}
if (ImGui.BeginTabItem("Definitions")) {
if (this.plugin.Definitions != null) {
ImGui.Text($"Version: {this.plugin.Definitions.Version}");
}
if (Definitions.LastUpdate != null) {
ImGui.Text($"Last update: {Definitions.LastUpdate}");
}
string error = Definitions.LastError;
if (error != null) {
ImGui.Text($"Last error: {error}");
}
if (ImGui.Button("Update definitions")) {
this.plugin.UpdateDefinitions();
}
}
ImGui.EndTabBar();
}
}

View File

@ -1,52 +1,11 @@
using System;
using Dalamud.Game.Chat;
using System;
using System.Linq;
using System.Text.RegularExpressions;
namespace NoSoliciting {
public partial class RMTDetection {
public static class Chat {
private static readonly string[] rmtSubstrings = {
"4KGOLD",
"We have sufficient stock",
"PVPBANK.COM",
"Gil for free",
"www.so9.com",
"Fast & Convenient",
"Cheap & Safety Guarantee",
"【Code|A O A U E",
"igfans",
"4KGOLD.COM",
"Cheapest Gil with",
"pvp and bank on google",
"Selling Cheap GIL",
"ff14mogstation.com",
"Cheap Gil 1000k",
"gilsforyou",
"server 1000K =",
"gils_selling",
"E A S Y.C O M",
"bonus code",
"mins delivery guarantee",
"Sell cheap",
"Salegm.com",
"cheap Mog",
"Off Code:",
"FF14Mog.com",
"使用する5オ",
"offers Fantasia",
"finalfantasyxiv.com-se.ru", // phishing
};
private static readonly Regex[] rmtRegexes = {
new Regex(@"Off Code( *)", RegexOptions.Compiled),
};
public static bool IsRMT(string msg) {
msg = RMTUtil.Normalise(msg);
return rmtSubstrings.Any(needle => msg.Contains(needle))
|| rmtRegexes.Any(needle => needle.IsMatch(msg));
}
public static bool MatchesCustomFilters(string msg, PluginConfiguration config) {
if (config == null) {
throw new ArgumentNullException(nameof(config), "PluginConfiguration cannot be null");

View File

@ -5,38 +5,6 @@ using System.Text.RegularExpressions;
namespace NoSoliciting {
public partial class RMTDetection {
public static class PartyFinder {
private static readonly Regex[] discord = {
new Regex(@".#\d{4}", RegexOptions.Compiled),
new Regex(@"discord\.(gg|io)/\w+", RegexOptions.Compiled),
};
private static readonly string[] content = {
"eden",
"savage",
"primal",
"ultimate",
};
private static readonly string[] selling = {
"sell",
"$ell",
"sale",
"price",
"cheap",
};
public static bool IsRMT(string desc) {
if (desc == null) {
throw new ArgumentNullException(nameof(desc), "description string cannot be null");
}
desc = RMTUtil.Normalise(desc).ToLowerInvariant();
bool containsSell = selling.Any(needle => desc.Contains(needle));
bool containsContent = content.Any(needle => desc.Contains(needle));
bool containsDiscord = discord.Any(needle => needle.IsMatch(desc));
return containsSell && containsDiscord && containsContent;
}
public static bool MatchesCustomFilters(string msg, PluginConfiguration config) {
if (config == null) {
throw new ArgumentNullException(nameof(config), "PluginConfiguration cannot be null");

View File

@ -3,6 +3,7 @@ using Dalamud.Game.Chat.SeStringHandling;
using Dalamud.Game.Internal.Network;
using Dalamud.Plugin;
using System;
using System.IO;
using System.Runtime.InteropServices;
namespace NoSoliciting {
@ -18,8 +19,12 @@ namespace NoSoliciting {
[System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "fulfilling a delegate")]
public void OnNetwork(IntPtr dataPtr, ushort opCode, uint sourceActorId, uint targetActorId, NetworkMessageDirection direction) {
// only filter when enabled
if (!this.plugin.Config.FilterPartyFinder) {
if (this.plugin.Definitions == null) {
return;
}
bool anyFilter = this.plugin.Config.FilterPartyFinder || this.plugin.Config.FilterPartyFinderRPAds || this.plugin.Config.CustomPFFilter;
if (!anyFilter) {
return;
}
@ -46,15 +51,23 @@ namespace NoSoliciting {
string desc = listing.Description();
// only look at listings that are RMT
if (!PartyFinder.IsRMT(desc) && !PartyFinder.MatchesCustomFilters(desc, this.plugin.Config)) {
bool filter = false;
// check for RMT if enabled
filter |= this.plugin.Config.FilterPartyFinder && this.plugin.Definitions.PartyFinder.RMT.Matches(desc);
// check for RP if enabled
filter |= this.plugin.Config.FilterPartyFinderRPAds && this.plugin.Definitions.Global.Roleplay.Matches(desc);
// check for custom filters if enabled
filter |= this.plugin.Config.CustomPFFilter && PartyFinder.MatchesCustomFilters(desc, this.plugin.Config);
if (!filter) {
continue;
}
// replace the listing with an empty one
packet.listings[i] = new PFListing();
PluginLog.Log($"Filtered out PF listing from {listing.Name()}: {listing.Description()}");
PluginLog.Log($"Filtered PF listing from {listing.Name()}: {listing.Description()}");
}
// get some memory for writing to
@ -78,18 +91,28 @@ namespace NoSoliciting {
throw new ArgumentNullException(nameof(message), "SeString cannot be null");
}
// only filter when enabled
if (!this.plugin.Config.FilterChat) {
if (this.plugin.Definitions == null) {
return;
}
string text = message.TextValue;
if (!Chat.IsRMT(text) && !Chat.MatchesCustomFilters(text, this.plugin.Config)) {
bool filter = false;
// check for RMT if enabled
filter |= this.plugin.Config.FilterChat && this.plugin.Definitions.Chat.RMT.Matches(text);
// check for RP ads if enabled
filter |= this.plugin.Config.FilterChatRPAds && this.plugin.Definitions.Global.Roleplay.Matches(text);
// check for FC recruitment if enabled
filter |= this.plugin.Config.FilterFCRecruitments && this.plugin.Definitions.Chat.FreeCompany.Matches(text);
// check for custom filters if enabled
filter |= this.plugin.Config.CustomChatFilter && Chat.MatchesCustomFilters(text, this.plugin.Config);
if (!filter) {
return;
}
PluginLog.Log($"Handled RMT message: {text}");
PluginLog.Log($"Filtered chat message: {text}");
isHandled = true;
}
}

View File

@ -0,0 +1,133 @@
version: 1
chat:
rmt:
required_matchers:
- - "4KGOLD"
- "We have sufficient stock"
- "PVPBANK.COM"
- "Gil for free"
- "www.so9.com"
- "Fast & Convenient"
- "Cheap & Safety Guarantee"
- "【Code|A O A U E"
- "igfans"
- "4KGOLD.COM"
- "Cheapest Gil with"
- "pvp and bank on google"
- "Selling Cheap GIL"
- "ff14mogstation.com"
- "Cheap Gil 1000k"
- "gilsforyou"
- "server 1000K ="
- "gils_selling"
- "E A S Y.C O M"
- "bonus code"
- "mins delivery guarantee"
- "Sell cheap"
- "Salegm.com"
- "cheap Mog"
- "Off Code"
- "FF14Mog.com"
- "使用する5オ"
- "offers Fantasia"
- "finalfantasyxiv.com-se.ru" # phishing
free_company:
ignore_case: true
likelihood_threshold: 4
likely_matchers:
# mentions free company
- - fc
- free company
# contains a call to action
- - join
- apply
- /tell
- /t
- dm
- whisper
# mentions benefits
- - discord
- map
- rank
- active
- weekly
- social
- friendly
- buff
- event
- house
# has common keywords
- - family
- community
- new
- veteran
- seasoned
- help
- recruit
party_finder:
rmt:
ignore_case: true
required_matchers:
# discord
- - regex: '.#\d{4}'
- regex: 'discord\.(gg|io)/\w+'
# content
- - eden
- savage
- primal
- ultimate
- ex
# selling
- - sell
- $ell
- sale
- price
- cheap
global:
roleplay:
ignore_case: true
required_matchers:
- - regex: 'w(ard)?\s*\d+[\s/,]*(p(lot)?|a(pt\.?|partment)?)\s*\d+'
likelihood_threshold: 1
likely_matchers:
# mentions roleplaying or fairly rp-only keywords
- - rp
- roleplay
- role play
- sfw # also catches nsfw
- '18+'
- '18 +'
- open
# has venue type
- - cafe
- bar
- lounge
- brothel
- casino
- venue
- restaurant
- library
- bookstore
- book store
# mentions services/activities
- - entertainment
- live
- raffle
- menu
- atmosphere
- drink
- food
- eat
- dance
- dancing
- music
- contest
# mentions a housing zone
- - gob # catches goblet, as well
- mist
- lb
- lav # catches lavender/lavender beds, as well
- shiro # catches shirogane, as well

View File

@ -5,4 +5,5 @@
<package id="Microsoft.CodeQuality.Analyzers" version="2.9.6" targetFramework="net48" developmentDependency="true" />
<package id="Microsoft.NetCore.Analyzers" version="2.9.6" targetFramework="net48" developmentDependency="true" />
<package id="Microsoft.NetFramework.Analyzers" version="2.9.6" targetFramework="net48" developmentDependency="true" />
<package id="YamlDotNet" version="8.1.2" targetFramework="net48" />
</packages>