fix(trainer): ignore numbers
This commit is contained in:
parent
1640310941
commit
238407c879
|
@ -124,7 +124,7 @@ namespace NoSoliciting.Trainer {
|
|||
var pipeline = ctx.Transforms.Conversion.MapValueToKey("Label", nameof(Data.Category))
|
||||
.Append(ctx.Transforms.CustomMapping(compute.GetMapping(), "Compute"))
|
||||
.Append(ctx.Transforms.CustomMapping(normalise.GetMapping(), "Normalise"))
|
||||
.Append(ctx.Transforms.Text.NormalizeText("MsgNormal", nameof(Data.Normalise.Normalised.NormalisedMessage), keepPunctuations: false))
|
||||
.Append(ctx.Transforms.Text.NormalizeText("MsgNormal", nameof(Data.Normalise.Normalised.NormalisedMessage), keepPunctuations: false, keepNumbers: false))
|
||||
.Append(ctx.Transforms.Text.TokenizeIntoWords("MsgTokens", "MsgNormal"))
|
||||
.Append(ctx.Transforms.Text.RemoveDefaultStopWords("MsgNoDefStop", "MsgTokens"))
|
||||
.Append(ctx.Transforms.Text.RemoveStopWords("MsgNoStop", "MsgNoDefStop", StopWords))
|
||||
|
|
Loading…
Reference in New Issue