From 73b5914136405f0c66354e8da641c45ba4d77093 Mon Sep 17 00:00:00 2001 From: xljiulang <366193849@qq.com> Date: Thu, 17 Jun 2021 21:13:58 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../LocalDomainAddressProvider.cs | 20 +++++++--- .../RemoteDomainAddressProvider.cs | 19 +++++---- FastGithub.Scanner/FastGithub.Scanner.csproj | 5 +-- FastGithub.Scanner/GithubContext.cs | 6 +-- .../GithubFullScanHostedService.cs | 2 +- FastGithub.Scanner/GithubOptions.cs | 32 ++++++++++++--- .../GithubResultScanHostedService.cs | 2 +- FastGithub.Scanner/GithubScanService.cs | 2 +- .../{IPRange.cs => IPAddressRange.cs} | 10 ++--- FastGithub.Scanner/IPRange.json | 37 ++++++------------ .../ScanMiddlewares/ConcurrentMiddleware.cs | 2 +- .../ScanMiddlewares/HttpsScanMiddleware.cs | 39 ++++++++++++++----- ...ScanMiddleware.cs => TcpScanMiddleware.cs} | 11 +++--- .../ScannerServiceCollectionExtensions.cs | 3 +- FastGithub/appsettings.json | 19 ++++++--- 15 files changed, 128 insertions(+), 81 deletions(-) rename FastGithub.Scanner/{IPRange.cs => IPAddressRange.cs} (93%) rename FastGithub.Scanner/ScanMiddlewares/{PortScanMiddleware.cs => TcpScanMiddleware.cs} (79%) diff --git a/FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs b/FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs index eef9856..9647b14 100644 --- a/FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs +++ b/FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using System; using System.Collections.Generic; using System.IO; @@ -12,21 +13,30 @@ namespace FastGithub.Scanner.DomainMiddlewares [Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))] sealed class LocalDomainAddressProvider : IDomainAddressProvider { - private const string JsonFile = "IPRange.json"; + private readonly IOptionsMonitor options; private readonly ILogger logger; - public LocalDomainAddressProvider(ILogger logger) + public LocalDomainAddressProvider( + IOptionsMonitor options, + ILogger logger) { + this.options = options; this.logger = logger; } public async Task> CreateDomainAddressesAsync() { + var setting = this.options.CurrentValue.LocalAddressProvider; + if (setting.Enable == false) + { + return Enumerable.Empty(); + } + try { - if (File.Exists(JsonFile) == true) + if (File.Exists(setting.IPRangeFilePath) == true) { - using var fileStream = File.OpenRead(JsonFile); + using var fileStream = File.OpenRead(setting.IPRangeFilePath); var datas = await JsonSerializer.DeserializeAsync>(fileStream); if (datas != null) { @@ -49,7 +59,7 @@ namespace FastGithub.Scanner.DomainMiddlewares var domain = kv.Key; foreach (var item in kv.Value) { - if (IPRange.TryParse(item, out var range)) + if (IPAddressRange.TryParse(item, out var range)) { foreach (var address in range) { diff --git a/FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs b/FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs index 638dcdb..6c6d0f9 100644 --- a/FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs +++ b/FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs @@ -15,26 +15,29 @@ namespace FastGithub.Scanner.DomainMiddlewares [Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))] sealed class RemoteDomainAddressProvider : IDomainAddressProvider { - private readonly IHttpClientFactory httpClientFactory; private readonly IOptionsMonitor options; private readonly ILogger logger; public RemoteDomainAddressProvider( - IHttpClientFactory httpClientFactory, IOptionsMonitor options, ILogger logger) { - this.httpClientFactory = httpClientFactory; this.options = options; this.logger = logger; } public async Task> CreateDomainAddressesAsync() { + var setting = this.options.CurrentValue.RemoteAddressProvider; + if (setting.Enable == false) + { + return Enumerable.Empty(); + } + try { - var httpClient = this.httpClientFactory.CreateClient(); - var meta = await httpClient.GetFromJsonAsync(this.options.CurrentValue.MetaUri); + using var httpClient = new HttpClient(); + var meta = await httpClient.GetFromJsonAsync(setting.MetaUri); if (meta != null) { return meta.ToDomainAddresses(); @@ -50,7 +53,7 @@ namespace FastGithub.Scanner.DomainMiddlewares private class Meta { - [JsonPropertyName("web")] + [JsonPropertyName("web")] public string[] Web { get; set; } = Array.Empty(); [JsonPropertyName("api")] @@ -59,7 +62,7 @@ namespace FastGithub.Scanner.DomainMiddlewares public IEnumerable ToDomainAddresses() { - foreach (var range in IPRange.From(this.Web).OrderBy(item => item.Size)) + foreach (var range in IPAddressRange.From(this.Web).OrderBy(item => item.Size)) { if (range.AddressFamily == AddressFamily.InterNetwork) { @@ -70,7 +73,7 @@ namespace FastGithub.Scanner.DomainMiddlewares } } - foreach (var range in IPRange.From(this.Api).OrderBy(item => item.Size)) + foreach (var range in IPAddressRange.From(this.Api).OrderBy(item => item.Size)) { if (range.AddressFamily == AddressFamily.InterNetwork) { diff --git a/FastGithub.Scanner/FastGithub.Scanner.csproj b/FastGithub.Scanner/FastGithub.Scanner.csproj index bc449fe..fd9657f 100644 --- a/FastGithub.Scanner/FastGithub.Scanner.csproj +++ b/FastGithub.Scanner/FastGithub.Scanner.csproj @@ -1,4 +1,4 @@ - + net5.0 @@ -7,8 +7,7 @@ - - + diff --git a/FastGithub.Scanner/GithubContext.cs b/FastGithub.Scanner/GithubContext.cs index 43724b4..abeb8de 100644 --- a/FastGithub.Scanner/GithubContext.cs +++ b/FastGithub.Scanner/GithubContext.cs @@ -8,9 +8,8 @@ namespace FastGithub.Scanner private record Github( string Domain, IPAddress Address, - bool Available, double AvailableRate, - TimeSpan AvgElapsed); + double AvgElapsed); /// /// 获取或设置是否可用 @@ -38,9 +37,8 @@ namespace FastGithub.Scanner return new Github( this.Domain, this.Address, - this.Available, this.History.AvailableRate, - this.History.AvgElapsed + this.History.AvgElapsed.TotalSeconds ).ToString(); } } diff --git a/FastGithub.Scanner/GithubFullScanHostedService.cs b/FastGithub.Scanner/GithubFullScanHostedService.cs index 6216fcc..33eb41f 100644 --- a/FastGithub.Scanner/GithubFullScanHostedService.cs +++ b/FastGithub.Scanner/GithubFullScanHostedService.cs @@ -25,7 +25,7 @@ namespace FastGithub while (stoppingToken.IsCancellationRequested == false) { await githubScanService.ScanAllAsync(); - await Task.Delay(this.options.CurrentValue.ScanAllInterval, stoppingToken); + await Task.Delay(this.options.CurrentValue.Scan.FullScanInterval, stoppingToken); } } } diff --git a/FastGithub.Scanner/GithubOptions.cs b/FastGithub.Scanner/GithubOptions.cs index 0a4a88e..324584e 100644 --- a/FastGithub.Scanner/GithubOptions.cs +++ b/FastGithub.Scanner/GithubOptions.cs @@ -5,14 +5,36 @@ namespace FastGithub.Scanner [Options("Github")] sealed class GithubOptions { - public TimeSpan ScanAllInterval { get; set; } = TimeSpan.FromHours(2d); + public ScanSetting Scan { get; set; } = new ScanSetting(); - public TimeSpan ScanResultInterval { get; set; } = TimeSpan.FromMinutes(1d); + public RemoteAddressProviderSetting RemoteAddressProvider { get; set; } = new RemoteAddressProviderSetting(); - public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json"); + public LocalAddressProviderSetting LocalAddressProvider { get; set; } = new LocalAddressProviderSetting(); - public TimeSpan PortScanTimeout { get; set; } = TimeSpan.FromSeconds(1d); + public class ScanSetting + { + public TimeSpan FullScanInterval = TimeSpan.FromHours(2d); - public TimeSpan HttpsScanTimeout { get; set; } = TimeSpan.FromSeconds(5d); + public TimeSpan ResultScanInterval = TimeSpan.FromMinutes(1d); + + public TimeSpan TcpScanTimeout { get; set; } = TimeSpan.FromSeconds(1d); + + public TimeSpan HttpsScanTimeout { get; set; } = TimeSpan.FromSeconds(2d); + } + + + public class RemoteAddressProviderSetting + { + public bool Enable { get; set; } = true; + + public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json"); + } + + public class LocalAddressProviderSetting + { + public bool Enable { get; set; } = true; + + public string IPRangeFilePath { get; set; } = "./IPRange.json"; + } } } diff --git a/FastGithub.Scanner/GithubResultScanHostedService.cs b/FastGithub.Scanner/GithubResultScanHostedService.cs index 62c1e26..9ec46b1 100644 --- a/FastGithub.Scanner/GithubResultScanHostedService.cs +++ b/FastGithub.Scanner/GithubResultScanHostedService.cs @@ -23,7 +23,7 @@ namespace FastGithub { while (stoppingToken.IsCancellationRequested == false) { - await Task.Delay(this.options.CurrentValue.ScanResultInterval, stoppingToken); + await Task.Delay(this.options.CurrentValue.Scan.ResultScanInterval, stoppingToken); await githubScanService.ScanResultAsync(); } } diff --git a/FastGithub.Scanner/GithubScanService.cs b/FastGithub.Scanner/GithubScanService.cs index 96ba4e0..c57dad2 100644 --- a/FastGithub.Scanner/GithubScanService.cs +++ b/FastGithub.Scanner/GithubScanService.cs @@ -30,7 +30,7 @@ namespace FastGithub.Scanner this.fullScanDelegate = new PipelineBuilder(appService, ctx => Task.CompletedTask) .Use() .Use() - .Use() + .Use() .Use() .Build(); diff --git a/FastGithub.Scanner/IPRange.cs b/FastGithub.Scanner/IPAddressRange.cs similarity index 93% rename from FastGithub.Scanner/IPRange.cs rename to FastGithub.Scanner/IPAddressRange.cs index 78904fe..e47ecb6 100644 --- a/FastGithub.Scanner/IPRange.cs +++ b/FastGithub.Scanner/IPAddressRange.cs @@ -8,7 +8,7 @@ using System.Net.Sockets; namespace FastGithub.Scanner { - abstract class IPRange : IEnumerable + abstract class IPAddressRange : IEnumerable { public abstract int Size { get; } @@ -22,7 +22,7 @@ namespace FastGithub.Scanner return this.GetEnumerator(); } - public static IEnumerable From(IEnumerable ranges) + public static IEnumerable From(IEnumerable ranges) { foreach (var item in ranges) { @@ -34,7 +34,7 @@ namespace FastGithub.Scanner } - public static bool TryParse(ReadOnlySpan range, [MaybeNullWhen(false)] out IPRange value) + public static bool TryParse(ReadOnlySpan range, [MaybeNullWhen(false)] out IPAddressRange value) { if (range.IsEmpty == false && IPNetwork.TryParse(range.ToString(), out var ipNetwork)) { @@ -62,7 +62,7 @@ namespace FastGithub.Scanner } - private class NetworkIPAddressRange : IPRange + private class NetworkIPAddressRange : IPAddressRange { private readonly IPAddressCollection addressCollection; @@ -84,7 +84,7 @@ namespace FastGithub.Scanner } } - private class SplitIPAddressRange : IPRange + private class SplitIPAddressRange : IPAddressRange { private readonly IPAddress start; private readonly IPAddress end; diff --git a/FastGithub.Scanner/IPRange.json b/FastGithub.Scanner/IPRange.json index 86ab387..1f7ae34 100644 --- a/FastGithub.Scanner/IPRange.json +++ b/FastGithub.Scanner/IPRange.json @@ -1,10 +1,4 @@ { - "github.githubassets.com": [ - "185.199.108.154/32", - "185.199.109.154/32", - "185.199.110.154/32", - "185.199.111.154/32" - ], "github.com": [ "13.114.40.48/32", "52.192.72.89/32", @@ -27,25 +21,16 @@ "140.82.112.0/20", "143.55.64.0/20" ], - "api.github.com": [ - "13.230.158.120/32", - "18.179.245.253/32", - "52.69.239.207/32", - "13.209.163.61/32", - "54.180.75.25/32", - "13.233.76.15/32", - "13.234.168.60/32", - "13.250.168.23/32", - "13.250.94.254/32", - "54.169.195.247/32", - "13.236.14.80/32", - "13.238.54.232/32", - "52.63.231.178/32", - "18.229.199.252/32", - "54.207.47.76/32", - "192.30.252.0/22", - "185.199.108.0/22", - "140.82.112.0/20", - "143.55.64.0/20" + "github.githubassets.com": [ + "185.199.108.154/32", + "185.199.109.154/32", + "185.199.110.154/32", + "185.199.111.154/32" + ], + "avatars.githubusercontent.com": [ + "185.199.108.133/32", + "185.199.109.133/32", + "185.199.110.133/32", + "185.199.111.133/32" ] } diff --git a/FastGithub.Scanner/ScanMiddlewares/ConcurrentMiddleware.cs b/FastGithub.Scanner/ScanMiddlewares/ConcurrentMiddleware.cs index 31a503c..8a4af05 100644 --- a/FastGithub.Scanner/ScanMiddlewares/ConcurrentMiddleware.cs +++ b/FastGithub.Scanner/ScanMiddlewares/ConcurrentMiddleware.cs @@ -12,7 +12,7 @@ namespace FastGithub.Scanner.ScanMiddlewares public ConcurrentMiddleware() { - var currentCount = Environment.ProcessorCount * 4; + var currentCount = Environment.ProcessorCount * 2; this.semaphoreSlim = new SemaphoreSlim(currentCount, currentCount); } diff --git a/FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs b/FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs index c4e1eb0..4786dab 100644 --- a/FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs +++ b/FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System; +using System.ComponentModel.DataAnnotations; using System.Linq; using System.Net.Http; using System.Threading; @@ -35,23 +36,23 @@ namespace FastGithub.Scanner.ScanMiddlewares RequestUri = new Uri($"https://{context.Address}"), }; request.Headers.Host = context.Domain; + request.Headers.ConnectionClose = true; request.Headers.Accept.TryParseAdd("*/*"); - using var httpClient = new HttpClient(new HttpClientHandler + using var httpClient = new HttpMessageInvoker(new SocketsHttpHandler { Proxy = null, UseProxy = false, AllowAutoRedirect = false, }); - using var cancellationTokenSource = new CancellationTokenSource(this.options.CurrentValue.HttpsScanTimeout); - using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationTokenSource.Token); - var server = response.EnsureSuccessStatusCode().Headers.Server; - if (server.Any(s => string.Equals("GitHub.com", s.Product?.Name, StringComparison.OrdinalIgnoreCase))) - { - context.Available = true; - await next(); - } + var timeout = this.options.CurrentValue.Scan.HttpsScanTimeout; + using var cancellationTokenSource = new CancellationTokenSource(timeout); + using var response = await httpClient.SendAsync(request, cancellationTokenSource.Token); + this.VerifyHttpResponse(context.Domain, response); + context.Available = true; + + await next(); } catch (TaskCanceledException) { @@ -64,6 +65,26 @@ namespace FastGithub.Scanner.ScanMiddlewares } } + /// + /// 验证响应内容 + /// + /// + /// + /// + /// + private void VerifyHttpResponse(string domain, HttpResponseMessage response) + { + response.EnsureSuccessStatusCode(); + if (domain.EndsWith(".github.com")) + { + var server = response.Headers.Server; + if (server.Any(s => string.Equals("github.com", s.Product?.Name, StringComparison.OrdinalIgnoreCase)) == false) + { + throw new ValidationException("伪造的github服务"); + } + } + } + private string GetInnerMessage(Exception ex) { while (ex.InnerException != null) diff --git a/FastGithub.Scanner/ScanMiddlewares/PortScanMiddleware.cs b/FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs similarity index 79% rename from FastGithub.Scanner/ScanMiddlewares/PortScanMiddleware.cs rename to FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs index 8f61680..15d0bb1 100644 --- a/FastGithub.Scanner/ScanMiddlewares/PortScanMiddleware.cs +++ b/FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs @@ -9,15 +9,15 @@ using System.Threading.Tasks; namespace FastGithub.Scanner.ScanMiddlewares { [Service(ServiceLifetime.Singleton)] - sealed class PortScanMiddleware : IMiddleware + sealed class TcpScanMiddleware : IMiddleware { private const int PORT = 443; private readonly IOptionsMonitor options; - private readonly ILogger logger; + private readonly ILogger logger; - public PortScanMiddleware( + public TcpScanMiddleware( IOptionsMonitor options, - ILogger logger) + ILogger logger) { this.options = options; this.logger = logger; @@ -28,7 +28,8 @@ namespace FastGithub.Scanner.ScanMiddlewares try { using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - using var cancellationTokenSource = new CancellationTokenSource(this.options.CurrentValue.PortScanTimeout); + var timeout = this.options.CurrentValue.Scan.TcpScanTimeout; + using var cancellationTokenSource = new CancellationTokenSource(timeout); await socket.ConnectAsync(context.Address, PORT, cancellationTokenSource.Token); await next(); diff --git a/FastGithub.Scanner/ScannerServiceCollectionExtensions.cs b/FastGithub.Scanner/ScannerServiceCollectionExtensions.cs index 9fa7996..4c174e0 100644 --- a/FastGithub.Scanner/ScannerServiceCollectionExtensions.cs +++ b/FastGithub.Scanner/ScannerServiceCollectionExtensions.cs @@ -18,8 +18,7 @@ namespace FastGithub public static IServiceCollection AddGithubScanner(this IServiceCollection services, IConfiguration configuration) { var assembly = typeof(ScannerServiceCollectionExtensions).Assembly; - return services - .AddHttpClient() + return services .AddServiceAndOptions(assembly, configuration) .AddHostedService() .AddHostedService() diff --git a/FastGithub/appsettings.json b/FastGithub/appsettings.json index 36b51a5..b054d18 100644 --- a/FastGithub/appsettings.json +++ b/FastGithub/appsettings.json @@ -4,11 +4,20 @@ "GithubTTL": "00:10:00" }, "Github": { - "ScanAllInterval": "02:00:00", // ɨʱ - "ScanResultInterval": "00:01:00", // ɨʱ - "MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json", // ipԴļuri - "PortScanTimeout": "00:00:01", // ˿ɨ賬ʱʱ - "HttpsScanTimeout": "00:00:05" // httpsɨ賬ʱʱ + "Scan": { + "FullScanInterval": "02:00:00", // ɨʱ + "ResultScanInterval": "00:01:00", // ɨʱ + "TcpScanTimeout": "00:00:01", // tcpɨ賬ʱʱ + "HttpsScanTimeout": "00:00:02" // httpsɨ賬ʱʱ + }, + "RemoteAddressProvider": { + "Enable": true, + "MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json" // metaԴuri + }, + "LocalAddressProvider": { + "Enable": true, + "IPRangeFilePath": "./IPRange.json" + } }, "Logging": { "LogLevel": {