feat: add normalisation to pipeline

Add a step to normalise messages to the ML pipeline. This ensures
computed properties run on the raw data (which is actually partially
normalised by the compute context). This prevents properties which
rely on symbols (e.g. "B>") from being unable to work properly when
normalisation happens before they have access to the input.

NoSoliciting.Interface/NoSolUtil.cs

@@ -51,6 +51,7 @@ namespace NoSoliciting.Interface {
             '/', '|',
             '(', ')',
             '[', ']',
+            '{', '}',
             '<', '>',
             '=', '+',
             '.', ',',

NoSoliciting.Internal.Interface/Data.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using System.Text.RegularExpressions;
 using Microsoft.ML.Data;
 using Microsoft.ML.Transforms;
+using NoSoliciting.Interface;
 
 namespace NoSoliciting.Internal.Interface {
     [SuppressMessage("ReSharper", "UnusedMember.Global")]
@@ -30,6 +31,26 @@ namespace NoSoliciting.Internal.Interface {
             this.Message = message;
         }
 
+        #region normalisation
+
+        [CustomMappingFactoryAttribute("Normalise")]
+        [SuppressMessage("ReSharper", "UnusedType.Global")]
+        public class Normalise : CustomMappingFactory<Data, Normalise.Normalised> {
+            public override Action<Data, Normalised> GetMapping() {
+                return Convert;
+            }
+
+            private static void Convert(Data data, Normalised normalised) {
+                normalised.NormalisedMessage = NoSolUtil.Normalise(data.Message, true);
+            }
+
+            public class Normalised {
+                public string? NormalisedMessage { get; set; }
+            }
+        }
+
+        #endregion
+
         #region computed
 
         [CustomMappingFactoryAttribute("Compute")]
@@ -103,13 +124,15 @@ namespace NoSoliciting.Internal.Interface {
                 output.Weight = weight;
             }
 
+            var normalised = NoSolUtil.Normalise(this.Message);
+
             output.PartyFinder = this.Channel == 0;
             output.Shout = this.Channel == 11 || this.Channel == 30;
-            output.ContainsWard = WardWords.Any(word => word.IsMatch(this.Message));
-            output.ContainsPlot = PlotWords.Any(word => word.IsMatch(this.Message));
-            output.ContainsHousingNumbers = NumbersRegex.IsMatch(this.Message);
-            output.ContainsTradeWords = TradeWords.Any(word => this.Message.ContainsIgnoreCase(word));
-            output.ContainsSketchUrl = SketchUrlRegex.IsMatch(this.Message);
+            output.ContainsWard = WardWords.Any(word => word.IsMatch(normalised));
+            output.ContainsPlot = PlotWords.Any(word => word.IsMatch(normalised));
+            output.ContainsHousingNumbers = NumbersRegex.IsMatch(normalised);
+            output.ContainsTradeWords = TradeWords.Any(word => normalised.ContainsIgnoreCase(word));
+            output.ContainsSketchUrl = SketchUrlRegex.IsMatch(normalised);
         }
 
         #endregion

NoSoliciting.Internal.Interface/NoSoliciting.Internal.Interface.csproj

@@ -10,4 +10,8 @@
       <PackageReference Include="Microsoft.ML" Version="1.5.4" />
     </ItemGroup>
 
+    <ItemGroup>
+      <ProjectReference Include="..\NoSoliciting.Interface\NoSoliciting.Interface.csproj" />
+    </ItemGroup>
+
 </Project>

NoSoliciting.Trainer/Program.cs

@@ -45,9 +45,6 @@ namespace NoSoliciting.Trainer {
             var classes = new Dictionary<string, uint>();
 
             foreach (var record in records) {
-                // normalise the message
-                record.Message = NoSolUtil.Normalise(record.Message, true);
-
                 // keep track of how many message of each category we have
                 if (!classes.ContainsKey(record.Category!)) {
                     classes[record.Category] = 0;
@@ -73,12 +70,14 @@ namespace NoSoliciting.Trainer {
             var ttd = ctx.Data.TrainTestSplit(df, 0.2, seed: 1);
 
             var compute = new Data.ComputeContext(weights);
+            var normalise = new Data.Normalise();
 
             ctx.ComponentCatalog.RegisterAssembly(typeof(Data).Assembly);
 
             var pipeline = ctx.Transforms.Conversion.MapValueToKey("Label", nameof(Data.Category))
                 .Append(ctx.Transforms.CustomMapping(compute.GetMapping(), "Compute"))
-                .Append(ctx.Transforms.Text.NormalizeText("MsgNormal", nameof(Data.Message), keepPunctuations: false))
+                .Append(ctx.Transforms.CustomMapping(normalise.GetMapping(), "Normalise"))
+                .Append(ctx.Transforms.Text.NormalizeText("MsgNormal", nameof(Data.Normalise.Normalised.NormalisedMessage), keepPunctuations: false))
                 .Append(ctx.Transforms.Text.TokenizeIntoWords("MsgTokens", "MsgNormal"))
                 .Append(ctx.Transforms.Text.RemoveDefaultStopWords("MsgNoDefStop", "MsgTokens"))
                 .Append(ctx.Transforms.Text.RemoveStopWords("MsgNoStop", "MsgNoDefStop",
@@ -161,12 +160,7 @@ namespace NoSoliciting.Trainer {
 
                 ushort.TryParse(parts[0], out var channel);
 
-                var input = new Data {
-                    Channel = channel,
-                    // PartyFinder = channel == 0,
-                    Message = NoSolUtil.Normalise(parts[1], true),
-                };
-
+                var input = new Data(channel, parts[1]);
                 var pred = predEngine.Predict(input);
 
                 Console.WriteLine(pred.Category);

NoSoliciting.Trainer/data.csv

@@ -36,6 +36,7 @@ FC,11,"""House Beaufort"" is now recruiting ! We're an international Guild looki
 FC,11,"""The Guild offers bonus XP an estate and has a Discord server available, if you would like to join us please feel free to send me a tell or just simply apply !"
 FC,11,<<Skyward Roses Blooms>> is looking for new and old  players alike! We're a casual FC looking to help run content and just chat! Send a tell or submit an application!!♥
 FC,11,"<Death> FC is throwing a server wide christmas party starting now, We will be doing a christmas video for the server, screenshots oppertunities, a christmas party and an item give away. If interested head to Lavender beds Ward 7, Plot 6 and we will see you inside the FC house. See you in a few minutes!"
+FC,11,"<Exiled Knights> FC is recruiting quality people to join us in making a fun community. We have a Medium house, all amenities you need."
 FC,11,"<Memoria> is a small, social FC looking for new or active players who regularly want to socialize and do content together. If this sounds like something you’re interested in, send me a /tell or apply via an application"
 FC,11,"☆★☆ We're completely LGBT friendly and have two events every month like costume contests, trivia nights and more! Send a /tell if you're interested! ☆★☆"
 FC,11,"☆★☆ We're completely LGBT friendly and have two events every month like costume contests, trivia nights and more! Send a /tell if you're interested! ☆★☆"
@@ -105,6 +106,7 @@ FC,11,"Tired of wandering through the game a lonely weeb? Want Senpai to notice
 FC,11,"treasure hunts, and leveling/dungeons are also common. If this sounds like the place for you, send a tell or apply today."
 FC,11,"Want to join a social FC?, Horizon Bonds likes to help novice players and we do a lot of events for all the members to enjoy the game =) Spanish speakers too. Pm me, we have a Mansion in Lavender Beds plot 33,10th Ward, come and visit!"
 FC,11,"We aim for a friendly and fun atmosphere. If you have questions and want to know more about us, don't hesitate to /tell me or Kenji Morty who is my co leader."
+FC,11,"We do not use discord, if you wish to chat we use playstation chat. If interested in joining PST."
 FC,11,"We do Raids, Mount Farms, Maps, PvP, Deep Dungeons, Eurekas and more."
 FC,11,We enjoy a vast variety of content together and we never shy away from tackling new challenges.
 FC,11,"We have a beautiful mansion in the Mist with all comforts, a fleet farming for us and +20% battle exp buff on Saturdays! Send me a /tell if you want more info or a invite!"
@@ -150,6 +152,7 @@ NORMAL,0,2222 practice group with experience > hopefully clears later on :D
 NORMAL,0,"2-2-2-2, phase 2 prog, got some vets, seen enrage"
 NORMAL,0,2s in p1. farming weapons/gwiber. If you're salty join Kyoko's party
 NORMAL,0,2x2x2x2 Farm no salt know your stuff
+NORMAL,0,2x2x2x2 FC fun 6/8 come get the mount with us
 NORMAL,0,"5/8 static running at 6-8 PST, wide/anti air prog. Happy strat. (9-11 EST)"
 NORMAL,0,5/8 Static. Cleaning up P1 hopefully going to P2. doing line strat.
 NORMAL,0,"5th week of this shit... P2 after basic onwards, Chinese Lions"
@@ -174,6 +177,7 @@ NORMAL,0,BLU Shock Strike #47 + Log
 NORMAL,0,"Blu Spell farming. Shiva, then Levi, then Alexander spells."
 NORMAL,0,"BLU spell, dont know the fight but willing to learn! Join as roles says"
 NORMAL,0,blue mage gear grinding!
+NORMAL,0,"Blue Mages - KNOW FIGHT, Have decent gear, LEVEL 70 BLUES please! Thank you! "
 NORMAL,0,Cinder prog for the fc all are welcome no salt! :D
 NORMAL,0,Cleaning up to heart phase. Tank swap at 2 stacks.
 NORMAL,0,clear party • braindead sundered • blue uptime • fox mist
@@ -182,6 +186,7 @@ NORMAL,0,come 2's and chill
 NORMAL,0,Come bonk the pope to get your bird
 NORMAL,0,come get your time worn artifacts.
 NORMAL,0,Come have some fun and watch out for those 8-way landslides <3
+NORMAL,0,"Come join the fun, get  a win.  Must hug Lily for drops."
 NORMAL,0,"Completed or Not, I'm recruiting folks to help complete the Containment Bay series."
 NORMAL,0,Deltascape 2-4 Savage for glams - Unsynced - Please have at least watched a vid on it
 NORMAL,0,Deltascape4.0 WT - - usnynced
@@ -202,6 +207,8 @@ NORMAL,0,"e9s learning party, low sodium pls, have discord if needed :D come enj
 NORMAL,0,farm
 NORMAL,0,"Farm. Reppin if necessary. T/H cards, DPS intercard. DPS in towers during adds."
 NORMAL,0,farming dragons / post macro if u want / DONT JOIN IF U WANT TO LEAVE CAUSE some stupid mistakes in first RUN!
+NORMAL,0,Farming Heavens' Ward helm Fragments
+NORMAL,0,Farming Heavens' Ward helm Fragments
 NORMAL,0,Fat fuk needs to drop a mount for us
 NORMAL,0,first go around! feel free to join! i learn better hands on! beginners are more than welcome!!!!! whoop whoop
 NORMAL,0,First time doing this.  Anyone is welcome to join.
@@ -256,6 +263,7 @@ NORMAL,0,Lf Clear have bonus as well
 NORMAL,0,LF for group to clear so that we can get to emerald weapon
 NORMAL,0,LF Home Designer Join or PM for details. High Budget
 NORMAL,0,Lf Housing Decorater please join group as dm might not be returned
+NORMAL,0,LF More Clears - 2's - Let's get some more weapons!
 NORMAL,0,"LF new LGBT friends to chill with in Eorzea! Join up, let's chat, and hang out. I have discord as well :)"
 NORMAL,0,Looking for a few veterans to teach from basic relativity onward. Obviously confident p1 clears only. Discord available.
 NORMAL,0,Looking for a SYNC community to run old extreme and savages :)
@@ -273,6 +281,7 @@ NORMAL,0,Looking to just complete these to see cutscenes and whatnot.
 NORMAL,0,"Looking to learn second half of the fight, and then if all goes well farm!!"
 NORMAL,0,Looking to Meld Materia icon status is still around. Also it seems to hide the party leader icon
 NORMAL,0,Looking to sell medium odder otter walls(2mil) join or tell.
+NORMAL,0,Magia 1 || No callouts wipe run || Eurekan Academy || I bet 12 gil on Channant dying.
 NORMAL,0,Mermaid Themed Character Art! Come join at the chocokeep in Gridania on Siren: https://www.twitch.tv/kikilove135
 NORMAL,0,mount farming party !!!! come join! down to help others!
 NORMAL,0,need of new friends c: discord: nine#0069
@@ -322,6 +331,7 @@ NORMAL,0,"Prog, new and experienced players present, discord w/ callouts availab
 NORMAL,0,"quick 2 chest, 4/8 static"
 NORMAL,0,"R > Crossworld Linkshell dedicated to doing Castrum!  Get your clears, come farm for the title, or because it's fun. Join for inv!"
 NORMAL,0,raid with static 7/8   weekly kill
+NORMAL,0,Running 10 times for scales. Come hang!
 NORMAL,0,Running Synced to second boss for Blue Spell .E<>G<EFBFBD> #61
 NORMAL,0,Shared fate farming if anyone wants to join.
 NORMAL,0,Shifting Sky and onwards. Let's make some progress!
@@ -337,6 +347,7 @@ NORMAL,0,"Team clear for new players need help pleaaase, it's our duty :D"
 NORMAL,0,There are savage undercutters on Twintania mb if you're looking for dirt cheap HQ Exarchic Ring of Casting/Slaying
 NORMAL,0,This is the first time for me so anyone is welcome to join as long as you are nice :)
 NORMAL,0,Tiles Practice! Discord Mandatory! Happy Brables  https://discord.gg/UEQNWw8
+NORMAL,0,Titan clean up/ statues prog. 6/8 Static. Discord available. DPS ccw Line Lions E safe side.  8:30 EST to 10:30 EST
 NORMAL,0,Titan prog| g1-NW g2-SE| TH-CCW
 NORMAL,0,Training from start. Only intuition and talent.
 NORMAL,0,Training p2 (1st holy link)
@@ -347,6 +358,7 @@ NORMAL,0,"Trying to learn the spell, come join!"
 NORMAL,0,"Unsync clear for my alt. Come for WT, bonus poetics, or out of the goodness of your heart."
 NORMAL,0,Unsync'd loot running; come get yer glams. Bring whatever.
 NORMAL,0,Unsynced! come for the fun ! ty
+NORMAL,0,"VG2 practice! JP VG2, G1-NW G2-SE. "
 NORMAL,0,VG2 prog JP strat. Group 1 N/W Group 2 S/E.
 NORMAL,0,Voidgate 1 proggers. Come have fun and pet a doggo owo
 NORMAL,0,Want some free ishgard material? Come on over to malboro head over to ceaderwood in Lower La Noscea
@@ -1899,6 +1911,7 @@ RP,0,[18+] CLUB AFTERLIFE is open! Come through our gates and drink the night aw
 RP,0,[18+] Please join us tonight at 9PM EST LB P33W22 Yorha theme For more info: https://milkym.carrd.co/
 RP,0,[18+] Scarlet Night is open! Live Music w/ Two imaginary Girls/Baths/Bar/Trivia! Adamantoise Shirogane W16 P7
 RP,0,[18+][LGBTQ+] CLUB AFTERLIFE is open! Come through our gates and fall into our embrace! (10PM-2AM EST)[Jeno/Shiro/W14/P16]
+RP,0,"[18+][LGBTQ+][Jenova]Club Spectrum will be opening 9pm cst on W13, P5 LB. Come for the drinks and dancing, or buy a courtesan! "
 RP,0,[18+]Need a way to get paid for your RP/ERP skills? The Tender Tonberry is hiring! For info go to @ discord.gg/TnvBgGQjuk
 RP,0,[18+]Need an FC or a venue to get paid for your rp/erp skills? The Tender Tonberry is hiring! Hit us up @ discord.gg/TnvBgGQjuk
 RP,0,[18+]Need to get your RP/ERP fix in? Wishing to become or buy a courtesan? C'mere to Touch Fluffy Tail @ discord.gg/fCS8Zng
@@ -1927,6 +1940,7 @@ RP,0,[RP] Ivory Tiger Tavern open for business. come relax and enjoy the meals w
 RP,0,[RP] Karaoke Night at NRAID HQ! Sign up for a chance to perform on our stage! Spectators welcome! https://discord.gg/ZhgqEqf
 RP,0,"[RP] Looking for adventure? Bounties, work, odd jobs? Join Bounty Call! More info in our discord: https://discord.gg/SnjZWRf"
 RP,0,"[RP] Rosa 'de Rescht' OPEN MIC NIGHT 11 PM-2 AM EST. Bards welcome, fine dining and wine served! Sarg Goblet W16 P55"
+RP,0,[RP] The Dead Dragoon is opening its doors for the first time! Come check out and have fun with us! opens in 1 hr!
 RP,0,"[RP] The House of illos - Open for all until 8am ST! Adult night life RP- LGTBQ+ Friendly, Lalas welcome - Faerie, LB W11 P28"
 RP,0,"[RP] The Ivory Tiger Tavern is open, come enjoy a meal and drink with a rogue. Gilga | Mists 2 Sub | APT 82"
 RP,0,"[RP] The Light House is open - Come Visit and dance, meet ppl and drink! Free drinks all night - W13 P4 Mist Gilg - Basement."
@@ -1943,6 +1957,7 @@ RP,0,[RP][18+] The dungeon Located at (Siren Mist w20 p1) is OPEN plz come enjoy
 RP,0,[RP][18+]Lunar Lounge is opening Tonight @11EST~! Bring all of your friends and lets reach for the Moon! Ada Mist W16 P60
 RP,0,"[RP][2 LIVES GONE CAFE] ""Take a break from housing. Inside a house""  LB W23 P17"
 RP,0,[RP][SFW] SAKE & UMESHI NIGHT!!! | Kusuo Izakaya & Bath (Kobai Goten Branch) | Gil/Shiro/W24/Apt32 (AFK'ers welcome)
+RP,0,"[RP][SFWish] The Paw and Tail bar is open, now with less Syrcus Tower! Gilgamesh, Lavender Beds, Ward 3, Plot 41"
 RP,0,"[RP]Calling all Pirates, Cutthroats, and sellswords tonight 5pm Central. check discord #events https://discord.gg/ChRB9b4ynz"
 RP,0,[RP|SFW] The Silver Garden is open for tea and conversation! Always looking to make new friends. | Cactuar LB w7 p6
 RP,0,"[RP18+] Oschon's Lounge is open! Party today, plan for tomorrow! 8pm-12a PST Gilgamesh Goblet W23 P23 "
@@ -2154,6 +2169,7 @@ STATIC,0,{4/8} {sHC} {5.5/opt/reclears} Wed/Fri 6pm-9pm PST | Tues/Thurs/Sat 6pm
 STATIC,0,"511 MNK LF MC/sHC static. Availability Sat-Monday 5pm-1am EST, Tues-Fri 6pm-10pm EST. Rowen#3731 msg for more info"
 STATIC,0,"6/8 MC static lf listed roles. Tu/Thu/Fri 9pm-12am est. message/join or if afk, https://discord.gg/kzf7RPE. We are on vg2"
 STATIC,0,6/8 MC static lf2 tanks 630 - 930pst must have cleared 9s discord Selayn#0391
+STATIC,0,"7/8 casual fun static LF meleeDPS(Non-DRG) Progging E10S but no savage experience required Thurs,Fri 7-10CT Disc:Mars#5334"
 STATIC,0,7/8 Casual static looking for 1 healer sundays and mondays 7-10 pm est clear the tier and future raids join pf
 STATIC,0,"7/8 LFM Melee. 7-10pm est Tu, We, Thu, Fri. E11S on Turn of the heavens prog. Contact me @ FirePyro#1689"
 STATIC,0,7/8 mC looking for Tank/Heal 4.4 Discord required//pm Hiro#0000 or here Tues/Fri 8:30-10:30 pm EST Sat 8:30-11:30 pm EST
@@ -2184,6 +2200,7 @@ STATIC,0,i527 GNB LF MC-sHC Static for Reclears. Available 6pm-11:30 EST  Discor
 STATIC,0,iLvl 506 AST LF static for eden; I can go as any healer. I'm not perfect but I learn fast.. :) can /tell or discord: Ché#0001
 STATIC,0,"ilvl 509 RDM lf end game raid group. Played since ARR, burnt out after SB but recently picked back up and really wanna clear E9-12S"
 STATIC,0,ILVL 527 WHM and ILVL 517 WAR LFG for E9-E12S. New to the data center and looking for Savage and/or Ultimate group. Rylai#5947
+STATIC,0,"ILVLV 530 WHM LFG Ultimates/current raid tier, on disc Quorra#5947"
 STATIC,0,LF 1 sub for ttonight for E10S clear @ 8 PM PST. We r on 2nd towers. Jobs below! Maybe AFK. Calli#0027 on disc :)
 STATIC,0,LF a melee 4 tuesday / thursday static 7-9pm 7/8 lgbtqia+
 STATIC,0,LF DPS Listed MC Static 7/8. Raid times are Tues-Thur 8p-10p est. Progging e12s p2. Looking to reclear today. Disc: @Ausoy#0118
@@ -2206,6 +2223,7 @@ STATIC,0,New Prog static 7/8 recruiting a Frontline Dps <510 Non Sam> for tuesda
 STATIC,0,relatively new 7/8 static LF 1 non-pld mt for the new eden tier! Message me at granger#0966 on discord if interested!
 STATIC,0,semi serious casual static looking for dps. on e11s enrage. Tues and thurs 8pm-1030 est. Discord me Ionsta#2751
 STATIC,0,Semi-Hardcore Static LF1M for 5.4 Raids. phys-rng/melee. M/T/F/Sat 8pm est.
+STATIC,0,SoftCore no Salt Static LF frontline dps replacement. Working on Tiles E9S. Monday/Thursday 9-11 EST. join/tell intrest
 STATIC,0,"static LF 1 dps rep for e9s tonight, 2 chest pref"
 STATIC,0,"Static LF 1 non-nin melee DPS, 7:30pm - 9:30pm EST Tues - Thurs, currently on e12 phase 1, discord Snow#6988 for info"
 STATIC,0,"Static lf 2 healers for 12S P1. Seen to Lions. Contact Cass#1413 on discord for more info. T, Thr, F 7PM-10PM EST"
@@ -2224,6 +2242,8 @@ TRADE,0,(Spriggan) WTB any rank 6 or higher FC (NO interest in Fc with plot)
 TRADE,0,"**Looking for a Large Plot, Coeurl Server** Ancient Order is looking to upgrade our house<3"
 TRADE,0,".:♥:. Crafter/Gatherer looking for work!  gear, raid pots/food, levekits and more! .:♥:."
 TRADE,0,"[Cactuar] [Sell-FC] [Goblet] [Small] [Ward 7, Plot 53] Beautiful view above, mb directly below. Only 50m instabuy!"
+TRADE,0,[Cerberus] Selling shirogane Plot 48 (small) - afk pls msg in discord Amina Shai#0001
+TRADE,0,[Louisoix][SellFC][Medium][Plot 51][Lavender Beds] 25m b/o
 TRADE,0,[OPEN] V's Armory Need gear? Crafting FULL Exarch sets an affordable price.
 TRADE,0,[Ragnarok] [Relo] [Medium] [Shirohane] [P13]
 TRADE,0,<Midgar> Looking for Free Company with plot any city and any size. Workshop is NOT required. Discord @Lunar#5999
@@ -2269,6 +2289,7 @@ TRADE,0,"Seliing i 490 Doh pieces,  1  mil per piece, fully melded full set 11 p
 TRADE,0,"selling /malevolent 1m, pleasant dot parasol 400k"
 TRADE,0,Selling 15-80 DoH Leveling Kits / Any Other Request / PST!  o/
 TRADE,0,Selling 1x Eldthurs Horn for 8mil. Skip MB taxes \o/ Join if interested
+TRADE,0,Selling 510 HQ gear sets [Low price] :3
 TRADE,0,Selling 510  sets 900k //  Individual prices available // Join Party //
 TRADE,0,Selling a Mist Medium relocation on bryn. Plot 37 join/tell offers.
 TRADE,0,"Selling Calfskin jacket/pants 1m ea, boots/hat/gloves 550k ea PM me"

NoSoliciting/Ml/MlFilter.cs

@@ -37,7 +37,6 @@ namespace NoSoliciting.Ml {
         }
 
         public MessageCategory ClassifyMessage(ushort channel, string message) {
-            message = NoSolUtil.Normalise(message, true);
             var prediction = this.Classifier.InvokeAsync(classifier => classifier.Classify(channel, message)).Result;
             var category = MessageCategoryExt.FromString(prediction);