ouczbs/FastGithub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

应用CancellationToken

Browse Source
This commit is contained in:
xljiulang 2021-06-18 22:37:53 +08:00
parent 01fac4fcb8
commit 9882e73678
16 changed files with 74 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
FastGithub.Core/PipelineBuilder.cs
View File

@ -11,7 +11,7 @@ namespace FastGithub
public class PipelineBuilder<TContext> : IPipelineBuilder<TContext>
{
private readonly InvokeDelegate<TContext> completedHandler;
private readonly List<Func<InvokeDelegate<TContext>, InvokeDelegate<TContext>>> middlewares = new List<Func<InvokeDelegate<TContext>, InvokeDelegate<TContext>>>();
private readonly List<Func<InvokeDelegate<TContext>, InvokeDelegate<TContext>>> middlewares = new();
/// <summary>
/// 获取服务提供者

5
FastGithub.Scanner/DomainAddressFacotry.cs
View File

@ -1,6 +1,7 @@
using Microsoft.Extensions.DependencyInjection;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner
@ -26,12 +27,12 @@ namespace FastGithub.Scanner
/// 创建域名与ip的关系
/// </summary>
/// <returns></returns>
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
{
var hashSet = new HashSet<DomainAddress>();
foreach (var provider in this.providers)
{
var domainAddresses = await provider.CreateDomainAddressesAsync();
var domainAddresses = await provider.CreateDomainAddressesAsync(cancellationToken);
foreach (var item in domainAddresses)
{
hashSet.Add(item);

13
FastGithub.Scanner/DomainAddressProviders/GithubMetaProvider.cs
View File

@ -8,6 +8,7 @@ using System.Net.Http;
using System.Net.Http.Json;
using System.Net.Sockets;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.DomainAddressProviders
@ -47,7 +48,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// 创建域名与ip的关系
/// </summary>
/// <returns></returns>
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
{
var setting = this.options.CurrentValue.DominAddressProviders.GithubMetaProvider;
if (setting.Enable == false)
@ -58,7 +59,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
try
{
var httpClient = this.httpClientFactory.CreateClient(nameof(FastGithub));
var meta = await this.GetMetaAsync(httpClient, setting.MetaUri);
var meta = await GetMetaAsync(httpClient, setting.MetaUri, cancellationToken);
if (meta != null)
{
return meta.ToDomainAddresses();
@ -66,6 +67,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
}
catch (Exception ex)
{
cancellationToken.ThrowIfCancellationRequested();
this.logger.LogWarning($"加载远程的ip列表异常：{ex.Message}");
}
@ -79,15 +81,16 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// <param name="httpClient"></param>
/// <param name="metaUri"></param>
/// <returns></returns>
private async Task<Meta?> GetMetaAsync(HttpClient httpClient, Uri metaUri)
private static async Task<Meta?> GetMetaAsync(HttpClient httpClient, Uri metaUri, CancellationToken cancellationToken)
{
try
{
return await httpClient.GetFromJsonAsync<Meta>(META_URI);
return await httpClient.GetFromJsonAsync<Meta>(META_URI, cancellationToken);
}
catch (Exception)
{
return await httpClient.GetFromJsonAsync<Meta>(metaUri);
cancellationToken.ThrowIfCancellationRequested();
return await httpClient.GetFromJsonAsync<Meta>(metaUri, cancellationToken);
}
}

12
FastGithub.Scanner/DomainAddressProviders/IPAddressComProvider.cs
View File

@ -7,6 +7,7 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.DomainAddressProviders
@ -46,7 +47,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// 创建域名与ip的关系
/// </summary>
/// <returns></returns>
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
{
var setting = this.options.CurrentValue.DominAddressProviders.IPAddressComProvider;
if (setting.Enable == false)
@ -60,7 +61,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
{
try
{
var addresses = await this.LookupAsync(httpClient, domain);
var addresses = await this.LookupAsync(httpClient, domain, cancellationToken);
foreach (var address in addresses)
{
result.Add(new DomainAddress(domain, address));
@ -68,6 +69,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
}
catch (Exception)
{
cancellationToken.ThrowIfCancellationRequested();
this.logger.LogWarning($"ipaddress.com无法解析{domain}");
}
}
@ -80,7 +82,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// <param name="httpClient"></param>
/// <param name="domain"></param>
/// <returns></returns>
private async Task<List<IPAddress>> LookupAsync(HttpClient httpClient, string domain)
private async Task<List<IPAddress>> LookupAsync(HttpClient httpClient, string domain, CancellationToken cancellationToken)
{
var keyValue = new KeyValuePair<string?, string?>("host", domain);
var content = new FormUrlEncodedContent(Enumerable.Repeat(keyValue, 1));
@ -91,8 +93,8 @@ namespace FastGithub.Scanner.DomainAddressProviders
Content = content
};
using var response = await httpClient.SendAsync(request);
var html = await response.Content.ReadAsStringAsync();
using var response = await httpClient.SendAsync(request, cancellationToken);
var html = await response.Content.ReadAsStringAsync(cancellationToken);
var match = Regex.Match(html, @"(?<=<h1>IP Lookup : )\d+\.\d+\.\d+\.\d+", RegexOptions.IgnoreCase);
if (match.Success && IPAddress.TryParse(match.Value, out var address))

10
FastGithub.Scanner/DomainAddressProviders/PublicDnsProvider.cs
View File

@ -6,6 +6,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.DomainAddressProviders
@ -41,7 +42,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// 创建域名与ip的关系
/// </summary>
/// <returns></returns>
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
{
var setting = this.options.CurrentValue.DominAddressProviders.PublicDnsProvider;
if (setting.Enable == false)
@ -52,7 +53,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
var result = new HashSet<DomainAddress>();
foreach (var dns in setting.Dnss)
{
var domainAddresses = await this.LookupAsync(dns, setting.Domains);
var domainAddresses = await this.LookupAsync(dns, setting.Domains, cancellationToken);
foreach (var item in domainAddresses)
{
result.Add(item);
@ -68,7 +69,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// <param name="dns">dns服务器</param>
/// <param name="domains">域名</param>
/// <returns></returns>
private async Task<List<DomainAddress>> LookupAsync(string dns, IEnumerable<string> domains)
private async Task<List<DomainAddress>> LookupAsync(string dns, IEnumerable<string> domains, CancellationToken cancellationToken)
{
var client = new DnsClient(dns);
var result = new List<DomainAddress>();
@ -77,7 +78,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
{
try
{
var addresses = await client.Lookup(domain);
var addresses = await client.Lookup(domain, cancellationToken: cancellationToken);
foreach (var address in addresses)
{
if (address.AddressFamily == AddressFamily.InterNetwork)
@ -88,6 +89,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
}
catch (Exception)
{
cancellationToken.ThrowIfCancellationRequested();
this.logger.LogWarning($"dns({dns})无法解析{domain}");
}
}

18
FastGithub.Scanner/GithubContext.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Net;
using System.Threading;
namespace FastGithub.Scanner
{
@ -13,6 +14,11 @@ namespace FastGithub.Scanner
/// </summary>
public bool Available { get; set; }
/// <summary>
/// 设置取消令牌
/// </summary>
public CancellationToken CancellationToken { get; }
/// <summary>
/// 获取扫描历史信息
/// </summary>
@ -25,8 +31,20 @@ namespace FastGithub.Scanner
/// <param name="domain"></param>
/// <param name="address"></param>
public GithubContext(string domain, IPAddress address)
: this(domain, address, CancellationToken.None)
{
}
/// <summary>
/// Github扫描上下文
/// </summary>
/// <param name="domain"></param>
/// <param name="address"></param>
/// <param name="cancellationToken"></param>
public GithubContext(string domain, IPAddress address, CancellationToken cancellationToken)
: base(domain, address)
{
this.CancellationToken = cancellationToken;
}
public bool Equals(GithubContext? other)

2
FastGithub.Scanner/GithubFullScanHostedService.cs
View File

@ -36,7 +36,7 @@ namespace FastGithub
{
while (stoppingToken.IsCancellationRequested == false)
{
await githubScanService.ScanAllAsync();
await githubScanService.ScanAllAsync(stoppingToken);
await Task.Delay(this.options.CurrentValue.Scan.FullScanInterval, stoppingToken);
}
}

2
FastGithub.Scanner/GithubResultScanHostedService.cs
View File

@ -39,6 +39,6 @@ namespace FastGithub
await Task.Delay(this.options.CurrentValue.Scan.ResultScanInterval, stoppingToken);
await githubScanService.ScanResultAsync();
}
}
}
}
}

7
FastGithub.Scanner/GithubScanService.cs
View File

@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner
@ -54,13 +55,13 @@ namespace FastGithub.Scanner
/// 扫描所有的ip
/// </summary>
/// <returns></returns>
public async Task ScanAllAsync()
public async Task ScanAllAsync(CancellationToken cancellationToken)
{
this.logger.LogInformation("完整扫描开始..");
var domainAddresses = await this.domainAddressFactory.CreateDomainAddressesAsync();
var domainAddresses = await this.domainAddressFactory.CreateDomainAddressesAsync(cancellationToken);
var scanTasks = domainAddresses
.Select(item => new GithubContext(item.Domain, item.Address))
.Select(item => new GithubContext(item.Domain, item.Address, cancellationToken))
.Select(ctx => ScanAsync(ctx));
var results = await Task.WhenAll(scanTasks);

3
FastGithub.Scanner/IDomainAddressProvider.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner
@ -17,6 +18,6 @@ namespace FastGithub.Scanner
/// 创建域名与ip的关系
/// </summary>
/// <returns></returns>
Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync();
Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken);
}
}

2
FastGithub.Scanner/IPAddressRange.cs
View File

@ -73,7 +73,7 @@ namespace FastGithub.Scanner
if (index >= 0)
{
var start = range.Slice(0, index);
var end = range.Slice(index + 1);
var end = range[(index + 1)..];
if (IPAddress.TryParse(start, out var startIp) &&
IPAddress.TryParse(end, out var endIp) &&

2
FastGithub.Scanner/ScanMiddlewares/ConcurrentMiddleware.cs
View File

@ -32,7 +32,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
{
try
{
await this.semaphoreSlim.WaitAsync();
await this.semaphoreSlim.WaitAsync(context.CancellationToken);
await next();
}
finally

16
FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs
View File

@ -53,26 +53,24 @@ namespace FastGithub.Scanner.ScanMiddlewares
Method = HttpMethod.Head,
RequestUri = new Uri($"https://{context.Address}"),
};
request.Headers.Host = context.Domain;
request.Headers.Host = context.Domain;
var timeout = this.options.CurrentValue.Scan.HttpsScanTimeout;
using var cancellationTokenSource = new CancellationTokenSource(timeout);
using var timeoutTokenSource = new CancellationTokenSource(timeout);
using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeoutTokenSource.Token, context.CancellationToken);
var httpClient = this.httpClientFactory.CreateClient(nameof(FastGithub));
using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationTokenSource.Token);
using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, linkedTokenSource.Token);
VerifyHttpsResponse(context.Domain, response);
context.Available = true;
await next();
}
catch (TaskCanceledException)
{
this.logger.LogTrace($"{context.Domain} {context.Address}连接超时");
}
catch (Exception ex)
{
var message = GetInnerMessage(ex);
this.logger.LogTrace($"{context.Domain} {context.Address} {message}");
context.CancellationToken.ThrowIfCancellationRequested();
this.logger.LogTrace($"{context.Domain} {context.Address} { GetInnerMessage(ex)}");
}
}

11
FastGithub.Scanner/ScanMiddlewares/StatisticsMiddleware.cs
View File

@ -32,20 +32,23 @@ namespace FastGithub.Scanner.ScanMiddlewares
public async Task InvokeAsync(GithubContext context, Func<Task> next)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
stopwatch.Start();
await next();
}
finally
{
stopwatch.Stop();
context.History.Add(context.Available, stopwatch.Elapsed);
if (context.History.AvailableRate > 0d)
if (context.CancellationToken.IsCancellationRequested == false)
{
this.logger.LogInformation(context.ToString());
context.History.Add(context.Available, stopwatch.Elapsed);
if (context.History.AvailableRate > 0d)
{
this.logger.LogInformation(context.ToString());
}
}
}
}

9
FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs
View File

@ -71,14 +71,17 @@ namespace FastGithub.Scanner.ScanMiddlewares
{
try
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
var timeout = this.options.CurrentValue.Scan.TcpScanTimeout;
using var cancellationTokenSource = new CancellationTokenSource(timeout);
await socket.ConnectAsync(context.Address, PORT, cancellationTokenSource.Token);
using var timeoutTokenSource = new CancellationTokenSource(timeout);
using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeoutTokenSource.Token, context.CancellationToken);
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await socket.ConnectAsync(context.Address, PORT, linkedTokenSource.Token);
return true;
}
catch (Exception)
{
context.CancellationToken.ThrowIfCancellationRequested();
return false;
}
}

2
FastGithub/HostBuilderExtensions.cs
View File

@ -31,7 +31,7 @@ namespace FastGithub
c.Service<IHost>(service => service
.ConstructUsing(() => hostBuilder.Build())
.WhenStarted(service => service.Start())
.WhenStopped(service => service.StopAsync())
.WhenStopped(service => service.StopAsync().Wait())
);
});
}