using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Dalamud.Plugin; using ExpandedSearchInfo.Providers; using ExpandedSearchInfo.Sections; using Nager.PublicSuffix; namespace ExpandedSearchInfo { public class ExpandedSearchInfo { public string Info { get; } public List Sections { get; } public ExpandedSearchInfo(string info, List sections) { this.Info = info; this.Sections = sections; } } public class SearchInfoRepository : IDisposable { private Plugin Plugin { get; } private DomainParser Parser { get; } internal ConcurrentDictionary SearchInfos { get; } = new(); internal int LastActorId { get; private set; } private List Providers { get; } = new(); internal SearchInfoRepository(Plugin plugin) { this.Plugin = plugin; // create the public suffix list parser var provider = new WebTldRuleProvider(); if (!provider.CacheProvider.IsCacheValid()) { provider.BuildAsync().GetAwaiter().GetResult(); } this.Parser = new DomainParser(provider); // add providers this.AddProviders(); // listen for search info this.Plugin.Functions.ReceiveSearchInfo += this.ProcessSearchInfo; } public void Dispose() { this.Plugin.Functions.ReceiveSearchInfo -= this.ProcessSearchInfo; } private void AddProviders() { this.Providers.Add(new PastebinProvider()); this.Providers.Add(new CarrdProvider()); this.Providers.Add(new FListProvider(this.Plugin)); this.Providers.Add(new RefsheetProvider()); this.Providers.Add(new PlainTextProvider()); } private void ProcessSearchInfo(int actorId, string info) { this.LastActorId = actorId; // if empty search info, short circuit if (string.IsNullOrWhiteSpace(info)) { // remove any existing search info this.SearchInfos.TryRemove(actorId, out _); return; } // check to see if info has changed #if RELEASE if (this.SearchInfos.TryGetValue(actorId, out var existing)) { if (existing.Info == info) { return; } } #endif new Thread(async () => { try { await this.DoExtraction(actorId, info); } catch (Exception ex) { PluginLog.LogError($"Error in extraction thread:\n{ex}"); } }).Start(); } private async Task DoExtraction(int actorId, string info) { var downloadUris = new List(); // extract uris from the search info with providers var extractedUris = this.Providers .Where(provider => provider.ExtractsUris) .Select(provider => provider.ExtractUris(actorId, info)) .Where(uris => uris != null) .SelectMany(uris => uris); // add the extracted uris to the list downloadUris.AddRange(extractedUris!); // go word-by-word and try to parse a uri foreach (var word in info.Split(' ', '\n', '\r')) { Uri found; try { found = new UriBuilder(word.Trim()).Uri; } catch (UriFormatException) { continue; } // make sure the hostname is a valid domain try { if (!this.Parser.IsValidDomain(found.Host)) { continue; } } catch (ParseException) { continue; } downloadUris.Add(found); } // if there were no uris found or extracted, remove existing search info and stop if (downloadUris.Count == 0) { this.SearchInfos.TryRemove(actorId, out _); return; } // do the downloads await this.DownloadAndExtract(actorId, info, downloadUris); } private async Task DownloadAndExtract(int actorId, string info, IEnumerable uris) { var handler = new HttpClientHandler { UseCookies = true, AllowAutoRedirect = true, MaxAutomaticRedirections = 5, }; handler.CookieContainer.Add(new Cookie("warning", "1", "/", "www.f-list.net")); var client = new HttpClient(handler); var sections = new List(); // run through each extracted uri foreach (var uri in uris) { if (uri.Scheme != "http" && uri.Scheme != "https") { continue; } // find the providers that run on this uri var matching = this.Providers .Where(provider => provider.Matches(uri)) .ToList(); // skip the uri if no providers if (matching.Count == 0) { continue; } // get the http response from the uri and make sure it's ok var resp = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead); if (resp.StatusCode != HttpStatusCode.OK) { continue; } // go through each provider in order and take the first one that provides info foreach (var provider in matching) { var extracted = await provider.ExtractInfo(resp); if (extracted == null) { continue; } sections.Add(extracted); break; } } // remove expanded search info if no sections resulted if (sections.Count == 0) { this.SearchInfos.TryRemove(actorId, out _); return; } // otherwise set the expanded search info for this actor id this.SearchInfos[actorId] = new ExpandedSearchInfo(info, sections); } } }