增加https扫描规则

FastGithub.Scanner/GithubLookupFacotry.cs

@@ -30,10 +30,10 @@ namespace FastGithub.Scanner
         }
 
         /// <summary>
-        /// 创建域名与ip的关系
+        /// 查找域名与ip关系
         /// </summary>
         /// <returns></returns>
-        public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
+        public async Task<IEnumerable<DomainAddress>> LookupAsync(CancellationToken cancellationToken)
         {
             var hashSet = new HashSet<DomainAddress>();
             var domains = this.options.CurrentValue.Domains;

FastGithub.Scanner/GithubScanService.cs

@@ -14,7 +14,7 @@ namespace FastGithub.Scanner
     [Service(ServiceLifetime.Singleton)]
     sealed class GithubScanService
     {
-        private readonly GithubLookupFacotry domainAddressFactory;
+        private readonly GithubLookupFacotry lookupFactory;
         private readonly GithubContextCollection scanResults;
         private readonly ILogger<GithubScanService> logger;
 
@@ -24,17 +24,17 @@ namespace FastGithub.Scanner
         /// <summary>
         /// github扫描服务
         /// </summary>
-        /// <param name="domainAddressFactory"></param>
+        /// <param name="lookupFactory"></param>
         /// <param name="scanResults"></param>
         /// <param name="appService"></param>
         /// <param name="logger"></param>
         public GithubScanService(
-            GithubLookupFacotry domainAddressFactory,
+            GithubLookupFacotry lookupFactory,
             GithubContextCollection scanResults,
             IServiceProvider appService,
             ILogger<GithubScanService> logger)
         {
-            this.domainAddressFactory = domainAddressFactory;
+            this.lookupFactory = lookupFactory;
             this.scanResults = scanResults;
             this.logger = logger;
 
@@ -58,7 +58,7 @@ namespace FastGithub.Scanner
         public async Task ScanAllAsync(CancellationToken cancellationToken)
         {
             this.logger.LogInformation("完整扫描开始..");
-            var domainAddresses = await this.domainAddressFactory.CreateDomainAddressesAsync(cancellationToken);
+            var domainAddresses = await this.lookupFactory.LookupAsync(cancellationToken);
 
             var scanTasks = domainAddresses
                 .Select(item => new GithubContext(item.Domain, item.Address, cancellationToken))

FastGithub.Scanner/LookupProviders/GithubMetaProviderOptions.cs

@@ -2,6 +2,9 @@
 
 namespace FastGithub.Scanner.LookupProviders
 {
+    /// <summary>
+    /// Github公开的域名与ip关系提供者选项
+    /// </summary>
     [Options("Github:Lookup:GithubMetaProvider")]
     sealed class GithubMetaProviderOptions
     {

FastGithub.Scanner/LookupProviders/IPAddressComProviderOptions.cs

@@ -1,5 +1,8 @@
 namespace FastGithub.Scanner.LookupProviders
 {
+    /// <summary>
+    /// ipaddress.com的域名与ip关系提供者选项
+    /// </summary>
     [Options("Github:Lookup:IPAddressComProvider")]
     sealed class IPAddressComProviderOptions
     {

FastGithub.Scanner/LookupProviders/PublicDnsProviderOptions.cs

@@ -2,11 +2,20 @@
 
 namespace FastGithub.Scanner.LookupProviders
 {
+    /// <summary>
+    /// 公共dns的域名与ip关系提供者选项
+    /// </summary>
     [Options("Github:Lookup:PublicDnsProvider")]
     sealed class PublicDnsProviderOptions
     {
+        /// <summary>
+        /// 是否启用
+        /// </summary>
         public bool Enable { get; set; } = true;
 
+        /// <summary>
+        /// dns列表
+        /// </summary>
         public string[] Dnss { get; set; } = Array.Empty<string>();
     }
 }

FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs

@@ -48,11 +48,15 @@ namespace FastGithub.Scanner.ScanMiddlewares
             {
                 context.Available = false;
 
-                var request = new HttpRequestMessage
+                var setting = this.options.CurrentValue;
+                if (setting.Rules.TryGetValue(context.Domain, out var rule) == false)
                 {
-                    Method = HttpMethod.Head,
-                    RequestUri = new Uri($"https://{context.Address}"),
-                };
+                    rule = new HttpsScanOptions.ScanRule();
+                }
+
+                using var request = new HttpRequestMessage();
+                request.Method = new HttpMethod(rule.Method);
+                request.RequestUri = new Uri(new Uri($"https://{context.Address}"), rule.Path);
                 request.Headers.Host = context.Domain;
 
                 var timeout = this.options.CurrentValue.Timeout;

FastGithub.Scanner/ScanMiddlewares/HttpsScanOptions.cs

@@ -1,10 +1,38 @@
 using System;
+using System.Collections.Generic;
 
 namespace FastGithub.Scanner.ScanMiddlewares
 {
+    /// <summary>
+    /// https扫描选项
+    /// </summary>
     [Options("Github:Scan:HttpsScan")]
     sealed class HttpsScanOptions
     {
+        /// <summary>
+        /// 扫描超时时长
+        /// </summary>
         public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(5d);
+
+        /// <summary>
+        /// 各域名扫描规则
+        /// </summary>
+        public Dictionary<string, ScanRule> Rules { get; set; } = new Dictionary<string, ScanRule>();
+
+        /// <summary>
+        /// 扫描规则
+        /// </summary>
+        public class ScanRule
+        {
+            /// <summary>
+            /// 请求方式
+            /// </summary>
+            public string Method { get; set; } = "HEAD";
+
+            /// <summary>
+            /// 请求路径
+            /// </summary>
+            public string Path { get; set; } = "/";
+        }
     }
 }

FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs

@@ -74,7 +74,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
                 using var timeoutTokenSource = new CancellationTokenSource(timeout);
                 using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeoutTokenSource.Token, context.CancellationToken);
 
-                using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+                using var socket = new Socket(context.Address.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
                 await socket.ConnectAsync(context.Address, PORT, linkedTokenSource.Token);
                 return true;
             }

FastGithub.Scanner/ScanMiddlewares/TcpScanOptions.cs

@@ -2,6 +2,9 @@
 
 namespace FastGithub.Scanner.ScanMiddlewares
 {
+    /// <summary>
+    /// tcp扫描选项
+    /// </summary>
     [Options("Github:Scan:TcpScan")]
     sealed class TcpScanOptions
     {

FastGithub/appsettings.json

@@ -12,14 +12,27 @@
         "CacheExpiration": "00:20:00" // 扫描结果缓存时长
       },
       "HttpsScan": {
-        "Timeout": "00:00:05" // https扫描超时时间
+        "Timeout": "00:00:05", // https扫描超时时间
+        "Rules": {
+          "github.githubassets.com": {
+            "Path": "/favicons/favicon.png"
+          },
+          "avatars.githubusercontent.com": {
+            "Path": "/u/8308014?s=40&v=4"
+          },
+          "raw.githubusercontent.com": {
+            "Path": "/xljiulang/FastGithub/master/README.md"
+          }
+        }
       }
     },
     "Lookup": {
       "Domains": [
         "github.com",
         "api.github.com",
-        "github.githubassets.com"
+        "github.githubassets.com",
+        "raw.githubusercontent.com",
+        "avatars.githubusercontent.com"
       ],
       "IPAddressComProvider": {
         "Enable": true