重构配置文件

FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs

@@ -1,5 +1,6 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -12,21 +13,30 @@ namespace FastGithub.Scanner.DomainMiddlewares
     [Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
     sealed class LocalDomainAddressProvider : IDomainAddressProvider
     {
-        private const string JsonFile = "IPRange.json";
+        private readonly IOptionsMonitor<GithubOptions> options;
         private readonly ILogger<LocalDomainAddressProvider> logger;
 
-        public LocalDomainAddressProvider(ILogger<LocalDomainAddressProvider> logger)
+        public LocalDomainAddressProvider(
+             IOptionsMonitor<GithubOptions> options,
+            ILogger<LocalDomainAddressProvider> logger)
         {
+            this.options = options;
             this.logger = logger;
         }
 
         public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
         {
+            var setting = this.options.CurrentValue.LocalAddressProvider;
+            if (setting.Enable == false)
+            {
+                return Enumerable.Empty<DomainAddress>();
+            }
+
             try
             {
-                if (File.Exists(JsonFile) == true)
+                if (File.Exists(setting.IPRangeFilePath) == true)
                 {
-                    using var fileStream = File.OpenRead(JsonFile);
+                    using var fileStream = File.OpenRead(setting.IPRangeFilePath);
                     var datas = await JsonSerializer.DeserializeAsync<Dictionary<string, string[]>>(fileStream);
                     if (datas != null)
                     {
@@ -49,7 +59,7 @@ namespace FastGithub.Scanner.DomainMiddlewares
                 var domain = kv.Key;
                 foreach (var item in kv.Value)
                 {
-                    if (IPRange.TryParse(item, out var range))
+                    if (IPAddressRange.TryParse(item, out var range))
                     {
                         foreach (var address in range)
                         {

FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs

@@ -15,26 +15,29 @@ namespace FastGithub.Scanner.DomainMiddlewares
     [Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
     sealed class RemoteDomainAddressProvider : IDomainAddressProvider
     {
-        private readonly IHttpClientFactory httpClientFactory;
         private readonly IOptionsMonitor<GithubOptions> options;
         private readonly ILogger<RemoteDomainAddressProvider> logger;
 
         public RemoteDomainAddressProvider(
-            IHttpClientFactory httpClientFactory,
             IOptionsMonitor<GithubOptions> options,
             ILogger<RemoteDomainAddressProvider> logger)
         {
-            this.httpClientFactory = httpClientFactory;
             this.options = options;
             this.logger = logger;
         }
 
         public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
         {
+            var setting = this.options.CurrentValue.RemoteAddressProvider;
+            if (setting.Enable == false)
+            {
+                return Enumerable.Empty<DomainAddress>();
+            }
+
             try
             {
-                var httpClient = this.httpClientFactory.CreateClient();
+                using var httpClient = new HttpClient();
-                var meta = await httpClient.GetFromJsonAsync<Meta>(this.options.CurrentValue.MetaUri);
+                var meta = await httpClient.GetFromJsonAsync<Meta>(setting.MetaUri);
                 if (meta != null)
                 {
                     return meta.ToDomainAddresses();
@@ -59,7 +62,7 @@ namespace FastGithub.Scanner.DomainMiddlewares
 
             public IEnumerable<DomainAddress> ToDomainAddresses()
             {
-                foreach (var range in IPRange.From(this.Web).OrderBy(item => item.Size))
+                foreach (var range in IPAddressRange.From(this.Web).OrderBy(item => item.Size))
                 {
                     if (range.AddressFamily == AddressFamily.InterNetwork)
                     {
@@ -70,7 +73,7 @@ namespace FastGithub.Scanner.DomainMiddlewares
                     }
                 }
 
-                foreach (var range in IPRange.From(this.Api).OrderBy(item => item.Size))
+                foreach (var range in IPAddressRange.From(this.Api).OrderBy(item => item.Size))
                 {
                     if (range.AddressFamily == AddressFamily.InterNetwork)
                     {

FastGithub.Scanner/FastGithub.Scanner.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 	<PropertyGroup>
 		<TargetFramework>net5.0</TargetFramework>
@@ -8,7 +8,6 @@
 	<ItemGroup>
 		<PackageReference Include="IPNetwork2" Version="2.5.320" />
 		<PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" /> 
-		<PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
 	</ItemGroup>
 
 	<ItemGroup>

FastGithub.Scanner/GithubContext.cs

@@ -8,9 +8,8 @@ namespace FastGithub.Scanner
         private record Github(
             string Domain,
             IPAddress Address,
-            bool Available,
             double AvailableRate,
-            TimeSpan AvgElapsed);
+            double AvgElapsed);
 
         /// <summary>
         /// 获取或设置是否可用
@@ -38,9 +37,8 @@ namespace FastGithub.Scanner
             return new Github(
                 this.Domain,
                 this.Address,
-                this.Available,
                 this.History.AvailableRate,
-                this.History.AvgElapsed
+                this.History.AvgElapsed.TotalSeconds
                 ).ToString();
         }
     }

FastGithub.Scanner/GithubFullScanHostedService.cs

@@ -25,7 +25,7 @@ namespace FastGithub
             while (stoppingToken.IsCancellationRequested == false)
             {
                 await githubScanService.ScanAllAsync();
-                await Task.Delay(this.options.CurrentValue.ScanAllInterval, stoppingToken);
+                await Task.Delay(this.options.CurrentValue.Scan.FullScanInterval, stoppingToken);
             }
         }
     }

FastGithub.Scanner/GithubOptions.cs

@@ -5,14 +5,36 @@ namespace FastGithub.Scanner
     [Options("Github")]
     sealed class GithubOptions
     {
-        public TimeSpan ScanAllInterval { get; set; } = TimeSpan.FromHours(2d);
+        public ScanSetting Scan { get; set; } = new ScanSetting();
 
-        public TimeSpan ScanResultInterval { get; set; } = TimeSpan.FromMinutes(1d);
+        public RemoteAddressProviderSetting RemoteAddressProvider { get; set; } = new RemoteAddressProviderSetting();
+
+        public LocalAddressProviderSetting LocalAddressProvider { get; set; } = new LocalAddressProviderSetting();
+
+        public class ScanSetting
+        {
+            public TimeSpan FullScanInterval = TimeSpan.FromHours(2d);
+
+            public TimeSpan ResultScanInterval = TimeSpan.FromMinutes(1d);
+
+            public TimeSpan TcpScanTimeout { get; set; } = TimeSpan.FromSeconds(1d);
+
+            public TimeSpan HttpsScanTimeout { get; set; } = TimeSpan.FromSeconds(2d);
+        }
+
+
+        public class RemoteAddressProviderSetting
+        {
+            public bool Enable { get; set; } = true;
 
             public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json");
+        }
 
-        public TimeSpan PortScanTimeout { get; set; } = TimeSpan.FromSeconds(1d);
+        public class LocalAddressProviderSetting
+        {
+            public bool Enable { get; set; } = true;
 
-        public TimeSpan HttpsScanTimeout { get; set; } = TimeSpan.FromSeconds(5d);
+            public string IPRangeFilePath { get; set; } = "./IPRange.json";
+        }
     }
 }

FastGithub.Scanner/GithubResultScanHostedService.cs

@@ -23,7 +23,7 @@ namespace FastGithub
         {
             while (stoppingToken.IsCancellationRequested == false)
             {
-                await Task.Delay(this.options.CurrentValue.ScanResultInterval, stoppingToken);
+                await Task.Delay(this.options.CurrentValue.Scan.ResultScanInterval, stoppingToken);
                 await githubScanService.ScanResultAsync();
             }
         }

FastGithub.Scanner/GithubScanService.cs

@@ -30,7 +30,7 @@ namespace FastGithub.Scanner
             this.fullScanDelegate = new PipelineBuilder<GithubContext>(appService, ctx => Task.CompletedTask)
                 .Use<ConcurrentMiddleware>()
                 .Use<StatisticsMiddleware>()
-                .Use<PortScanMiddleware>()
+                .Use<TcpScanMiddleware>()
                 .Use<HttpsScanMiddleware>()
                 .Build();
 

FastGithub.Scanner/IPAddressRange.cs

@@ -8,7 +8,7 @@ using System.Net.Sockets;
 
 namespace FastGithub.Scanner
 {
-    abstract class IPRange : IEnumerable<IPAddress>
+    abstract class IPAddressRange : IEnumerable<IPAddress>
     {
         public abstract int Size { get; }
 
@@ -22,7 +22,7 @@ namespace FastGithub.Scanner
             return this.GetEnumerator();
         }
 
-        public static IEnumerable<IPRange> From(IEnumerable<string> ranges)
+        public static IEnumerable<IPAddressRange> From(IEnumerable<string> ranges)
         {
             foreach (var item in ranges)
             {
@@ -34,7 +34,7 @@ namespace FastGithub.Scanner
         }
 
 
-        public static bool TryParse(ReadOnlySpan<char> range, [MaybeNullWhen(false)] out IPRange value)
+        public static bool TryParse(ReadOnlySpan<char> range, [MaybeNullWhen(false)] out IPAddressRange value)
         {
             if (range.IsEmpty == false && IPNetwork.TryParse(range.ToString(), out var ipNetwork))
             {
@@ -62,7 +62,7 @@ namespace FastGithub.Scanner
         }
 
 
-        private class NetworkIPAddressRange : IPRange
+        private class NetworkIPAddressRange : IPAddressRange
         {
             private readonly IPAddressCollection addressCollection;
 
@@ -84,7 +84,7 @@ namespace FastGithub.Scanner
             }
         }
 
-        private class SplitIPAddressRange : IPRange
+        private class SplitIPAddressRange : IPAddressRange
         {
             private readonly IPAddress start;
             private readonly IPAddress end;

FastGithub.Scanner/IPRange.json

@@ -1,10 +1,4 @@
 {
-  "github.githubassets.com": [
-    "185.199.108.154/32",
-    "185.199.109.154/32",
-    "185.199.110.154/32",
-    "185.199.111.154/32"
-  ],
   "github.com": [
     "13.114.40.48/32",
     "52.192.72.89/32",
@@ -27,25 +21,16 @@
     "140.82.112.0/20",
     "143.55.64.0/20"
   ],
-  "api.github.com": [
+  "github.githubassets.com": [
-    "13.230.158.120/32",
+    "185.199.108.154/32",
-    "18.179.245.253/32",
+    "185.199.109.154/32",
-    "52.69.239.207/32",
+    "185.199.110.154/32",
-    "13.209.163.61/32",
+    "185.199.111.154/32"
-    "54.180.75.25/32",
+  ],
-    "13.233.76.15/32",
+  "avatars.githubusercontent.com": [
-    "13.234.168.60/32",
+    "185.199.108.133/32",
-    "13.250.168.23/32",
+    "185.199.109.133/32",
-    "13.250.94.254/32",
+    "185.199.110.133/32",
-    "54.169.195.247/32",
+    "185.199.111.133/32"
-    "13.236.14.80/32",
-    "13.238.54.232/32",
-    "52.63.231.178/32",
-    "18.229.199.252/32",
-    "54.207.47.76/32",
-    "192.30.252.0/22",
-    "185.199.108.0/22",
-    "140.82.112.0/20",
-    "143.55.64.0/20"
   ]
 }

FastGithub.Scanner/ScanMiddlewares/ConcurrentMiddleware.cs

@@ -12,7 +12,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
 
         public ConcurrentMiddleware()
         {
-            var currentCount = Environment.ProcessorCount * 4;
+            var currentCount = Environment.ProcessorCount * 2;
             this.semaphoreSlim = new SemaphoreSlim(currentCount, currentCount);
         }
 

FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs

@@ -2,6 +2,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using System;
+using System.ComponentModel.DataAnnotations;
 using System.Linq;
 using System.Net.Http;
 using System.Threading;
@@ -35,24 +36,24 @@ namespace FastGithub.Scanner.ScanMiddlewares
                     RequestUri = new Uri($"https://{context.Address}"),
                 };
                 request.Headers.Host = context.Domain;
+                request.Headers.ConnectionClose = true;
                 request.Headers.Accept.TryParseAdd("*/*");
 
-                using var httpClient = new HttpClient(new HttpClientHandler
+                using var httpClient = new HttpMessageInvoker(new SocketsHttpHandler
                 {
                     Proxy = null,
                     UseProxy = false,
                     AllowAutoRedirect = false,
                 });
 
-                using var cancellationTokenSource = new CancellationTokenSource(this.options.CurrentValue.HttpsScanTimeout);
+                var timeout = this.options.CurrentValue.Scan.HttpsScanTimeout;
-                using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationTokenSource.Token);
+                using var cancellationTokenSource = new CancellationTokenSource(timeout);
-                var server = response.EnsureSuccessStatusCode().Headers.Server;
+                using var response = await httpClient.SendAsync(request, cancellationTokenSource.Token);
-                if (server.Any(s => string.Equals("GitHub.com", s.Product?.Name, StringComparison.OrdinalIgnoreCase)))
+                this.VerifyHttpResponse(context.Domain, response);
-                {
                 context.Available = true;
+
                 await next();
             }
-            }
             catch (TaskCanceledException)
             {
                 this.logger.LogTrace($"{context.Domain} {context.Address}连接超时");
@@ -64,6 +65,26 @@ namespace FastGithub.Scanner.ScanMiddlewares
             }
         }
 
+        /// <summary>
+        /// 验证响应内容
+        /// </summary>
+        /// <param name="domain"></param>
+        /// <param name="response"></param>
+        /// <exception cref="HttpRequestException"></exception>
+        /// <exception cref="ValidationException"></exception>
+        private void VerifyHttpResponse(string domain, HttpResponseMessage response)
+        {
+            response.EnsureSuccessStatusCode();
+            if (domain.EndsWith(".github.com"))
+            {
+                var server = response.Headers.Server;
+                if (server.Any(s => string.Equals("github.com", s.Product?.Name, StringComparison.OrdinalIgnoreCase)) == false)
+                {
+                    throw new ValidationException("伪造的github服务");
+                }
+            }
+        }
+
         private string GetInnerMessage(Exception ex)
         {
             while (ex.InnerException != null)

FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs

@@ -9,15 +9,15 @@ using System.Threading.Tasks;
 namespace FastGithub.Scanner.ScanMiddlewares
 {
     [Service(ServiceLifetime.Singleton)]
-    sealed class PortScanMiddleware : IMiddleware<GithubContext>
+    sealed class TcpScanMiddleware : IMiddleware<GithubContext>
     {
         private const int PORT = 443;
         private readonly IOptionsMonitor<GithubOptions> options;
-        private readonly ILogger<PortScanMiddleware> logger;
+        private readonly ILogger<TcpScanMiddleware> logger;
 
-        public PortScanMiddleware(
+        public TcpScanMiddleware(
             IOptionsMonitor<GithubOptions> options,
-            ILogger<PortScanMiddleware> logger)
+            ILogger<TcpScanMiddleware> logger)
         {
             this.options = options;
             this.logger = logger;
@@ -28,7 +28,8 @@ namespace FastGithub.Scanner.ScanMiddlewares
             try
             {
                 using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-                using var cancellationTokenSource = new CancellationTokenSource(this.options.CurrentValue.PortScanTimeout);
+                var timeout = this.options.CurrentValue.Scan.TcpScanTimeout;
+                using var cancellationTokenSource = new CancellationTokenSource(timeout);
                 await socket.ConnectAsync(context.Address, PORT, cancellationTokenSource.Token);
 
                 await next();

FastGithub.Scanner/ScannerServiceCollectionExtensions.cs

@@ -19,7 +19,6 @@ namespace FastGithub
         {
             var assembly = typeof(ScannerServiceCollectionExtensions).Assembly;
             return services 
-                .AddHttpClient()
                 .AddServiceAndOptions(assembly, configuration)
                 .AddHostedService<GithubFullScanHostedService>()
                 .AddHostedService<GithubResultScanHostedService>()

FastGithub/appsettings.json

@@ -4,11 +4,20 @@
     "GithubTTL": "00:10:00"
   },
   "Github": {
-    "ScanAllInterval": "02:00:00", // 完整扫描时间间隔
+    "Scan": {
-    "ScanResultInterval": "00:01:00", // 结果扫描时间间隔
+      "FullScanInterval": "02:00:00",   // 完整扫描时间间隔
-    "MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json", // ip资源文件uri    
+      "ResultScanInterval": "00:01:00", // 结果扫描时间间隔
-    "PortScanTimeout": "00:00:01", // 端口扫描超时时间
+      "TcpScanTimeout": "00:00:01",     // tcp扫描超时时间
-    "HttpsScanTimeout": "00:00:05" // https扫描超时时间
+      "HttpsScanTimeout": "00:00:02"    // https扫描超时时间
+    },
+    "RemoteAddressProvider": {
+      "Enable": true,
+      "MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json"  // meta资源uri  
+    },
+    "LocalAddressProvider": {
+      "Enable": true,
+      "IPRangeFilePath": "./IPRange.json"
+    }
   },
   "Logging": {
     "LogLevel": {