diff --git a/FastGithub.Scanner/GithubLookupFacotry.cs b/FastGithub.Scanner/GithubLookupFacotry.cs index 398410f..b48692f 100644 --- a/FastGithub.Scanner/GithubLookupFacotry.cs +++ b/FastGithub.Scanner/GithubLookupFacotry.cs @@ -30,10 +30,10 @@ namespace FastGithub.Scanner } /// - /// 创建域名与ip的关系 + /// 查找域名与ip关系 /// /// - public async Task> CreateDomainAddressesAsync(CancellationToken cancellationToken) + public async Task> LookupAsync(CancellationToken cancellationToken) { var hashSet = new HashSet(); var domains = this.options.CurrentValue.Domains; diff --git a/FastGithub.Scanner/GithubScanService.cs b/FastGithub.Scanner/GithubScanService.cs index 97502f3..85a3891 100644 --- a/FastGithub.Scanner/GithubScanService.cs +++ b/FastGithub.Scanner/GithubScanService.cs @@ -14,7 +14,7 @@ namespace FastGithub.Scanner [Service(ServiceLifetime.Singleton)] sealed class GithubScanService { - private readonly GithubLookupFacotry domainAddressFactory; + private readonly GithubLookupFacotry lookupFactory; private readonly GithubContextCollection scanResults; private readonly ILogger logger; @@ -24,17 +24,17 @@ namespace FastGithub.Scanner /// /// github扫描服务 /// - /// + /// /// /// /// public GithubScanService( - GithubLookupFacotry domainAddressFactory, + GithubLookupFacotry lookupFactory, GithubContextCollection scanResults, IServiceProvider appService, ILogger logger) { - this.domainAddressFactory = domainAddressFactory; + this.lookupFactory = lookupFactory; this.scanResults = scanResults; this.logger = logger; @@ -58,7 +58,7 @@ namespace FastGithub.Scanner public async Task ScanAllAsync(CancellationToken cancellationToken) { this.logger.LogInformation("完整扫描开始.."); - var domainAddresses = await this.domainAddressFactory.CreateDomainAddressesAsync(cancellationToken); + var domainAddresses = await this.lookupFactory.LookupAsync(cancellationToken); var scanTasks = domainAddresses .Select(item => new GithubContext(item.Domain, item.Address, cancellationToken)) diff --git a/FastGithub.Scanner/LookupProviders/GithubMetaProviderOptions.cs b/FastGithub.Scanner/LookupProviders/GithubMetaProviderOptions.cs index 489b2db..9539134 100644 --- a/FastGithub.Scanner/LookupProviders/GithubMetaProviderOptions.cs +++ b/FastGithub.Scanner/LookupProviders/GithubMetaProviderOptions.cs @@ -2,6 +2,9 @@ namespace FastGithub.Scanner.LookupProviders { + /// + /// Github公开的域名与ip关系提供者选项 + /// [Options("Github:Lookup:GithubMetaProvider")] sealed class GithubMetaProviderOptions { diff --git a/FastGithub.Scanner/LookupProviders/IPAddressComProviderOptions.cs b/FastGithub.Scanner/LookupProviders/IPAddressComProviderOptions.cs index acda328..1985cc6 100644 --- a/FastGithub.Scanner/LookupProviders/IPAddressComProviderOptions.cs +++ b/FastGithub.Scanner/LookupProviders/IPAddressComProviderOptions.cs @@ -1,5 +1,8 @@ namespace FastGithub.Scanner.LookupProviders { + /// + /// ipaddress.com的域名与ip关系提供者选项 + /// [Options("Github:Lookup:IPAddressComProvider")] sealed class IPAddressComProviderOptions { diff --git a/FastGithub.Scanner/LookupProviders/PublicDnsProviderOptions.cs b/FastGithub.Scanner/LookupProviders/PublicDnsProviderOptions.cs index bbe2892..b6d0972 100644 --- a/FastGithub.Scanner/LookupProviders/PublicDnsProviderOptions.cs +++ b/FastGithub.Scanner/LookupProviders/PublicDnsProviderOptions.cs @@ -2,11 +2,20 @@ namespace FastGithub.Scanner.LookupProviders { + /// + /// 公共dns的域名与ip关系提供者选项 + /// [Options("Github:Lookup:PublicDnsProvider")] sealed class PublicDnsProviderOptions { + /// + /// 是否启用 + /// public bool Enable { get; set; } = true; + /// + /// dns列表 + /// public string[] Dnss { get; set; } = Array.Empty(); } } diff --git a/FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs b/FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs index f6d1e1c..534f216 100644 --- a/FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs +++ b/FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs @@ -48,11 +48,15 @@ namespace FastGithub.Scanner.ScanMiddlewares { context.Available = false; - var request = new HttpRequestMessage + var setting = this.options.CurrentValue; + if (setting.Rules.TryGetValue(context.Domain, out var rule) == false) { - Method = HttpMethod.Head, - RequestUri = new Uri($"https://{context.Address}"), - }; + rule = new HttpsScanOptions.ScanRule(); + } + + using var request = new HttpRequestMessage(); + request.Method = new HttpMethod(rule.Method); + request.RequestUri = new Uri(new Uri($"https://{context.Address}"), rule.Path); request.Headers.Host = context.Domain; var timeout = this.options.CurrentValue.Timeout; diff --git a/FastGithub.Scanner/ScanMiddlewares/HttpsScanOptions.cs b/FastGithub.Scanner/ScanMiddlewares/HttpsScanOptions.cs index 945d06b..6acadc4 100644 --- a/FastGithub.Scanner/ScanMiddlewares/HttpsScanOptions.cs +++ b/FastGithub.Scanner/ScanMiddlewares/HttpsScanOptions.cs @@ -1,10 +1,38 @@ using System; +using System.Collections.Generic; namespace FastGithub.Scanner.ScanMiddlewares { + /// + /// https扫描选项 + /// [Options("Github:Scan:HttpsScan")] sealed class HttpsScanOptions { + /// + /// 扫描超时时长 + /// public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(5d); + + /// + /// 各域名扫描规则 + /// + public Dictionary Rules { get; set; } = new Dictionary(); + + /// + /// 扫描规则 + /// + public class ScanRule + { + /// + /// 请求方式 + /// + public string Method { get; set; } = "HEAD"; + + /// + /// 请求路径 + /// + public string Path { get; set; } = "/"; + } } } diff --git a/FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs b/FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs index e132c5e..9268a05 100644 --- a/FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs +++ b/FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs @@ -74,7 +74,7 @@ namespace FastGithub.Scanner.ScanMiddlewares using var timeoutTokenSource = new CancellationTokenSource(timeout); using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeoutTokenSource.Token, context.CancellationToken); - using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + using var socket = new Socket(context.Address.AddressFamily, SocketType.Stream, ProtocolType.Tcp); await socket.ConnectAsync(context.Address, PORT, linkedTokenSource.Token); return true; } diff --git a/FastGithub.Scanner/ScanMiddlewares/TcpScanOptions.cs b/FastGithub.Scanner/ScanMiddlewares/TcpScanOptions.cs index 10d0290..646eb06 100644 --- a/FastGithub.Scanner/ScanMiddlewares/TcpScanOptions.cs +++ b/FastGithub.Scanner/ScanMiddlewares/TcpScanOptions.cs @@ -2,6 +2,9 @@ namespace FastGithub.Scanner.ScanMiddlewares { + /// + /// tcp扫描选项 + /// [Options("Github:Scan:TcpScan")] sealed class TcpScanOptions { diff --git a/FastGithub/appsettings.json b/FastGithub/appsettings.json index a1afa55..09b598c 100644 --- a/FastGithub/appsettings.json +++ b/FastGithub/appsettings.json @@ -12,14 +12,27 @@ "CacheExpiration": "00:20:00" // ɨʱ }, "HttpsScan": { - "Timeout": "00:00:05" // httpsɨ賬ʱʱ + "Timeout": "00:00:05", // httpsɨ賬ʱʱ + "Rules": { + "github.githubassets.com": { + "Path": "/favicons/favicon.png" + }, + "avatars.githubusercontent.com": { + "Path": "/u/8308014?s=40&v=4" + }, + "raw.githubusercontent.com": { + "Path": "/xljiulang/FastGithub/master/README.md" + } + } } }, "Lookup": { "Domains": [ "github.com", "api.github.com", - "github.githubassets.com" + "github.githubassets.com", + "raw.githubusercontent.com", + "avatars.githubusercontent.com" ], "IPAddressComProvider": { "Enable": true