拆分Options

This commit is contained in:
xljiulang 2021-06-19 13:27:34 +08:00
parent 3004842a18
commit 960f4efe18
19 changed files with 208 additions and 144 deletions

View File

@ -12,7 +12,7 @@ namespace FastGithub
sealed class GithubFullScanHostedService : BackgroundService
{
private readonly GithubScanService githubScanService;
private readonly IOptionsMonitor<GithubOptions> options;
private readonly IOptionsMonitor<GithubScanOptions> options;
/// <summary>
/// 完整扫描后台服务
@ -21,7 +21,7 @@ namespace FastGithub
/// <param name="options"></param>
public GithubFullScanHostedService(
GithubScanService githubScanService,
IOptionsMonitor<GithubOptions> options)
IOptionsMonitor<GithubScanOptions> options)
{
this.githubScanService = githubScanService;
this.options = options;
@ -37,7 +37,7 @@ namespace FastGithub
while (stoppingToken.IsCancellationRequested == false)
{
await githubScanService.ScanAllAsync(stoppingToken);
await Task.Delay(this.options.CurrentValue.Scan.FullScanInterval, stoppingToken);
await Task.Delay(this.options.CurrentValue.FullScanInterval, stoppingToken);
}
}
}

View File

@ -1,4 +1,5 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
@ -10,17 +11,22 @@ namespace FastGithub.Scanner
/// 域名与ip关系工厂
/// </summary>
[Service(ServiceLifetime.Singleton)]
sealed class DomainAddressFacotry
sealed class GithubLookupFacotry
{
private readonly IEnumerable<IDomainAddressProvider> providers;
private readonly IEnumerable<IGithubLookupProvider> providers;
private readonly IOptionsMonitor<GithubLookupFactoryOptions> options;
/// <summary>
/// 域名与ip关系工厂
/// </summary>
/// <param name="providers"></param>
public DomainAddressFacotry(IEnumerable<IDomainAddressProvider> providers)
/// <param name="options"></param>
public GithubLookupFacotry(
IEnumerable<IGithubLookupProvider> providers,
IOptionsMonitor<GithubLookupFactoryOptions> options)
{
this.providers = providers.OrderBy(item => item.Order);
this.options = options;
}
/// <summary>
@ -30,9 +36,11 @@ namespace FastGithub.Scanner
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
{
var hashSet = new HashSet<DomainAddress>();
var domains = this.options.CurrentValue.Domains;
foreach (var provider in this.providers)
{
var domainAddresses = await provider.CreateDomainAddressesAsync(cancellationToken);
var domainAddresses = await provider.LookupAsync(domains, cancellationToken);
foreach (var item in domainAddresses)
{
hashSet.Add(item);

View File

@ -0,0 +1,16 @@
using System;
namespace FastGithub.Scanner
{
/// <summary>
/// 域名
/// </summary>
[Options("Github:Lookup")]
sealed class GithubLookupFactoryOptions
{
/// <summary>
/// 反查的域名
/// </summary>
public string[] Domains { get; set; } = Array.Empty<string>();
}
}

View File

@ -1,67 +0,0 @@
using System;
namespace FastGithub.Scanner
{
/// <summary>
/// github选项
/// </summary>
[Options("Github")]
sealed class GithubOptions
{
/// <summary>
/// 扫描配置
/// </summary>
public ScanSetting Scan { get; set; } = new ScanSetting();
/// <summary>
/// 域名与ip关系提供者配置
/// </summary>
public DomainAddressProvidersSetting DominAddressProviders { get; set; } = new DomainAddressProvidersSetting();
/// <summary>
/// 扫描配置
/// </summary>
public class ScanSetting
{
public TimeSpan FullScanInterval = TimeSpan.FromHours(2d);
public TimeSpan ResultScanInterval = TimeSpan.FromMinutes(1d);
public TimeSpan TcpScanTimeout { get; set; } = TimeSpan.FromSeconds(1d);
public TimeSpan HttpsScanTimeout { get; set; } = TimeSpan.FromSeconds(5d);
}
/// <summary>
/// 域名与ip关系提供者配置
/// </summary>
public class DomainAddressProvidersSetting
{
public GithubMetaProviderSetting GithubMetaProvider { get; set; } = new GithubMetaProviderSetting();
public IPAddressComProviderSetting IPAddressComProvider { get; set; } = new IPAddressComProviderSetting();
public PublicDnsProviderSetting PublicDnsProvider { get; set; } = new PublicDnsProviderSetting();
public class GithubMetaProviderSetting
{
public bool Enable { get; set; } = true;
public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json");
}
public class IPAddressComProviderSetting
{
public bool Enable { get; set; } = true;
public string[] Domains { get; set; } = Array.Empty<string>();
}
public class PublicDnsProviderSetting
{
public bool Enable { get; set; } = true;
public string[] Dnss { get; set; } = Array.Empty<string>();
public string[] Domains { get; set; } = Array.Empty<string>();
}
}
}
}

View File

@ -12,7 +12,7 @@ namespace FastGithub
sealed class GithubResultScanHostedService : BackgroundService
{
private readonly GithubScanService githubScanService;
private readonly IOptionsMonitor<GithubOptions> options;
private readonly IOptionsMonitor<GithubScanOptions> options;
/// <summary>
/// 扫描结果轮询扫描后台服务
@ -21,7 +21,7 @@ namespace FastGithub
/// <param name="options"></param>
public GithubResultScanHostedService(
GithubScanService githubScanService,
IOptionsMonitor<GithubOptions> options)
IOptionsMonitor<GithubScanOptions> options)
{
this.githubScanService = githubScanService;
this.options = options;
@ -36,7 +36,7 @@ namespace FastGithub
{
while (stoppingToken.IsCancellationRequested == false)
{
await Task.Delay(this.options.CurrentValue.Scan.ResultScanInterval, stoppingToken);
await Task.Delay(this.options.CurrentValue.ResultScanInterval, stoppingToken);
await githubScanService.ScanResultAsync();
}
}

View File

@ -0,0 +1,22 @@
using System;
namespace FastGithub.Scanner
{
/// <summary>
/// 扫描选项
/// </summary>
[Options("Github:Scan")]
sealed class GithubScanOptions
{
/// <summary>
/// 完整扫描轮询时间间隔
/// </summary>
public TimeSpan FullScanInterval = TimeSpan.FromHours(2d);
/// <summary>
/// 结果扫描轮询时间间隔
/// </summary>
public TimeSpan ResultScanInterval = TimeSpan.FromMinutes(1d);
}
}

View File

@ -14,7 +14,7 @@ namespace FastGithub.Scanner
[Service(ServiceLifetime.Singleton)]
sealed class GithubScanService
{
private readonly DomainAddressFacotry domainAddressFactory;
private readonly GithubLookupFacotry domainAddressFactory;
private readonly GithubContextCollection scanResults;
private readonly ILogger<GithubScanService> logger;
@ -29,7 +29,7 @@ namespace FastGithub.Scanner
/// <param name="appService"></param>
/// <param name="logger"></param>
public GithubScanService(
DomainAddressFacotry domainAddressFactory,
GithubLookupFacotry domainAddressFactory,
GithubContextCollection scanResults,
IServiceProvider appService,
ILogger<GithubScanService> logger)

View File

@ -7,7 +7,7 @@ namespace FastGithub.Scanner
/// <summary>
/// 定义域名的ip提值者
/// </summary>
interface IDomainAddressProvider
interface IGithubLookupProvider
{
/// <summary>
/// 获取排序
@ -15,9 +15,11 @@ namespace FastGithub.Scanner
int Order { get; }
/// <summary>
/// 创建域名与ip的关系
/// 查找域名与ip关系
/// </summary>
/// <param name="domains"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken);
Task<IEnumerable<DomainAddress>> LookupAsync(IEnumerable<string> domains, CancellationToken cancellationToken);
}
}

View File

@ -11,15 +11,15 @@ using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.DomainAddressProviders
namespace FastGithub.Scanner.LookupProviders
{
/// <summary>
/// Github公开的域名与ip关系提供者
/// </summary>
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
sealed class GithubMetaProvider : IDomainAddressProvider
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IGithubLookupProvider))]
sealed class GithubMetaProvider : IGithubLookupProvider
{
private readonly IOptionsMonitor<GithubOptions> options;
private readonly IOptionsMonitor<GithubMetaProviderOptions> options;
private readonly IHttpClientFactory httpClientFactory;
private readonly ILogger<GithubMetaProvider> logger;
private const string META_URI = "https://api.github.com/meta";
@ -35,7 +35,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// <param name="options"></param>
/// <param name="logger"></param>
public GithubMetaProvider(
IOptionsMonitor<GithubOptions> options,
IOptionsMonitor<GithubMetaProviderOptions> options,
IHttpClientFactory httpClientFactory,
ILogger<GithubMetaProvider> logger)
{
@ -45,12 +45,14 @@ namespace FastGithub.Scanner.DomainAddressProviders
}
/// <summary>
/// 创建域名与ip的关系
/// 查找域名与ip关系
/// </summary>
/// <param name="domains"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
public async Task<IEnumerable<DomainAddress>> LookupAsync(IEnumerable<string> domains, CancellationToken cancellationToken)
{
var setting = this.options.CurrentValue.DominAddressProviders.GithubMetaProvider;
var setting = this.options.CurrentValue;
if (setting.Enable == false)
{
return Enumerable.Empty<DomainAddress>();
@ -62,7 +64,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
var meta = await GetMetaAsync(httpClient, setting.MetaUri, cancellationToken);
if (meta != null)
{
return meta.ToDomainAddresses();
return meta.ToDomainAddresses(domains);
}
}
catch (Exception ex)
@ -109,7 +111,12 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// 转换为域名与ip关系
/// </summary>
/// <returns></returns>
public IEnumerable<DomainAddress> ToDomainAddresses()
public IEnumerable<DomainAddress> ToDomainAddresses(IEnumerable<string> domains)
{
const string github = "github.com";
const string apiGithub = "api.github.com";
if (domains.Contains(github) == true)
{
foreach (var range in IPAddressRange.From(this.Web).OrderBy(item => item.Size))
{
@ -117,18 +124,22 @@ namespace FastGithub.Scanner.DomainAddressProviders
{
foreach (var address in range)
{
yield return new DomainAddress("github.com", address);
yield return new DomainAddress(github, address);
}
}
}
}
if (domains.Contains(apiGithub) == true)
{
foreach (var range in IPAddressRange.From(this.Api).OrderBy(item => item.Size))
{
if (range.AddressFamily == AddressFamily.InterNetwork)
{
foreach (var address in range)
{
yield return new DomainAddress("api.github.com", address);
yield return new DomainAddress(apiGithub, address);
}
}
}
}

View File

@ -0,0 +1,18 @@
using System;
namespace FastGithub.Scanner.LookupProviders
{
[Options("Github:Lookup:GithubMetaProvider")]
sealed class GithubMetaProviderOptions
{
/// <summary>
/// 是否启用
/// </summary>
public bool Enable { get; set; }
/// <summary>
/// meta请求uri
/// </summary>
public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json");
}
}

View File

@ -10,15 +10,15 @@ using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.DomainAddressProviders
namespace FastGithub.Scanner.LookupProviders
{
/// <summary>
/// ipaddress.com的域名与ip关系提供者
/// </summary>
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
sealed class IPAddressComProvider : IDomainAddressProvider
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IGithubLookupProvider))]
sealed class IPAddressComProvider : IGithubLookupProvider
{
private readonly IOptionsMonitor<GithubOptions> options;
private readonly IOptionsMonitor<IPAddressComProviderOptions> options;
private readonly IHttpClientFactory httpClientFactory;
private readonly ILogger<IPAddressComProvider> logger;
private readonly Uri lookupUri = new("https://www.ipaddress.com/ip-lookup");
@ -34,7 +34,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// <param name="options"></param>
/// <param name="logger"></param>
public IPAddressComProvider(
IOptionsMonitor<GithubOptions> options,
IOptionsMonitor<IPAddressComProviderOptions> options,
IHttpClientFactory httpClientFactory,
ILogger<IPAddressComProvider> logger)
{
@ -44,12 +44,14 @@ namespace FastGithub.Scanner.DomainAddressProviders
}
/// <summary>
/// 创建域名与ip的关系
/// 查找域名与ip关系
/// </summary>
/// <param name="domains"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
public async Task<IEnumerable<DomainAddress>> LookupAsync(IEnumerable<string> domains, CancellationToken cancellationToken)
{
var setting = this.options.CurrentValue.DominAddressProviders.IPAddressComProvider;
var setting = this.options.CurrentValue;
if (setting.Enable == false)
{
return Enumerable.Empty<DomainAddress>();
@ -57,7 +59,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
var httpClient = this.httpClientFactory.CreateClient(nameof(FastGithub));
var result = new HashSet<DomainAddress>();
foreach (var domain in setting.Domains)
foreach (var domain in domains)
{
try
{

View File

@ -0,0 +1,11 @@
namespace FastGithub.Scanner.LookupProviders
{
[Options("Github:Lookup:IPAddressComProvider")]
sealed class IPAddressComProviderOptions
{
/// <summary>
/// 是否启用
/// </summary>
public bool Enable { get; set; }
}
}

View File

@ -9,15 +9,15 @@ using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.DomainAddressProviders
namespace FastGithub.Scanner.LookupProviders
{
/// <summary>
/// 公共dns的域名与ip关系提供者
/// </summary>
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
sealed class PublicDnsProvider : IDomainAddressProvider
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IGithubLookupProvider))]
sealed class PublicDnsProvider : IGithubLookupProvider
{
private readonly IOptionsMonitor<GithubOptions> options;
private readonly IOptionsMonitor<PublicDnsProviderOptions> options;
private readonly ILogger<PublicDnsProvider> logger;
/// <summary>
@ -31,7 +31,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
/// <param name="options"></param>
/// <param name="logger"></param>
public PublicDnsProvider(
IOptionsMonitor<GithubOptions> options,
IOptionsMonitor<PublicDnsProviderOptions> options,
ILogger<PublicDnsProvider> logger)
{
this.options = options;
@ -39,12 +39,14 @@ namespace FastGithub.Scanner.DomainAddressProviders
}
/// <summary>
/// 创建域名与ip的关系
/// 查找域名与ip关系
/// </summary>
/// <param name="domains"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync(CancellationToken cancellationToken)
public async Task<IEnumerable<DomainAddress>> LookupAsync(IEnumerable<string> domains, CancellationToken cancellationToken)
{
var setting = this.options.CurrentValue.DominAddressProviders.PublicDnsProvider;
var setting = this.options.CurrentValue;
if (setting.Enable == false)
{
return Enumerable.Empty<DomainAddress>();
@ -53,7 +55,7 @@ namespace FastGithub.Scanner.DomainAddressProviders
var result = new HashSet<DomainAddress>();
foreach (var dns in setting.Dnss)
{
var domainAddresses = await this.LookupAsync(dns, setting.Domains, cancellationToken);
var domainAddresses = await this.LookupAsync(dns, domains, cancellationToken);
foreach (var item in domainAddresses)
{
result.Add(item);

View File

@ -0,0 +1,12 @@
using System;
namespace FastGithub.Scanner.LookupProviders
{
[Options("Github:Lookup:PublicDnsProvider")]
sealed class PublicDnsProviderOptions
{
public bool Enable { get; set; } = true;
public string[] Dnss { get; set; } = Array.Empty<string>();
}
}

View File

@ -17,7 +17,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
[Service(ServiceLifetime.Singleton)]
sealed class HttpsScanMiddleware : IMiddleware<GithubContext>
{
private readonly IOptionsMonitor<GithubOptions> options;
private readonly IOptionsMonitor<HttpsScanOptions> options;
private readonly IHttpClientFactory httpClientFactory;
private readonly ILogger<HttpsScanMiddleware> logger;
@ -27,7 +27,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
/// <param name="options"></param>
/// <param name="logger"></param>
public HttpsScanMiddleware(
IOptionsMonitor<GithubOptions> options,
IOptionsMonitor<HttpsScanOptions> options,
IHttpClientFactory httpClientFactory,
ILogger<HttpsScanMiddleware> logger)
{
@ -55,7 +55,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
};
request.Headers.Host = context.Domain;
var timeout = this.options.CurrentValue.Scan.HttpsScanTimeout;
var timeout = this.options.CurrentValue.Timeout;
using var timeoutTokenSource = new CancellationTokenSource(timeout);
using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeoutTokenSource.Token, context.CancellationToken);

View File

@ -0,0 +1,10 @@
using System;
namespace FastGithub.Scanner.ScanMiddlewares
{
[Options("Github:Scan:HttpsScan")]
sealed class HttpsScanOptions
{
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(5d);
}
}

View File

@ -16,8 +16,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
sealed class TcpScanMiddleware : IMiddleware<GithubContext>
{
private const int PORT = 443;
private readonly TimeSpan cacheTimeSpan = TimeSpan.FromMinutes(20d);
private readonly IOptionsMonitor<GithubOptions> options;
private readonly IOptionsMonitor<TcpScanOptions> options;
private readonly IMemoryCache memoryCache;
private readonly ILogger<TcpScanMiddleware> logger;
@ -27,7 +26,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
/// <param name="options"></param>
/// <param name="logger"></param>
public TcpScanMiddleware(
IOptionsMonitor<GithubOptions> options,
IOptionsMonitor<TcpScanOptions> options,
IMemoryCache memoryCache,
ILogger<TcpScanMiddleware> logger)
{
@ -48,7 +47,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
if (this.memoryCache.TryGetValue<bool>(key, out var available) == false)
{
available = await this.TcpScanAsync(context);
this.memoryCache.Set(key, available, cacheTimeSpan);
this.memoryCache.Set(key, available, this.options.CurrentValue.CacheExpiration);
}
if (available == true)
@ -71,7 +70,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
{
try
{
var timeout = this.options.CurrentValue.Scan.TcpScanTimeout;
var timeout = this.options.CurrentValue.Timeout;
using var timeoutTokenSource = new CancellationTokenSource(timeout);
using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeoutTokenSource.Token, context.CancellationToken);

View File

@ -0,0 +1,18 @@
using System;
namespace FastGithub.Scanner.ScanMiddlewares
{
[Options("Github:Scan:TcpScan")]
sealed class TcpScanOptions
{
/// <summary>
/// 扫描超时时长
/// </summary>
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(1d);
/// <summary>
/// 扫描结果缓存时长
/// </summary>
public TimeSpan CacheExpiration { get; set; } = TimeSpan.FromMinutes(20d);
}
}

View File

@ -7,21 +7,26 @@
"Scan": {
"FullScanInterval": "02:00:00", //
"ResultScanInterval": "00:01:00", //
"TcpScanTimeout": "00:00:01", // tcp
"HttpsScanTimeout": "00:00:05" // https
"TcpScan": {
"Timeout": "00:00:01", // tcp
"CacheExpiration": "00:20:00" //
},
"DominAddressProviders": {
"GithubMetaProvider": {
"Enable": true,
"MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json"
"HttpsScan": {
"Timeout": "00:00:05" // https
}
},
"IPAddressComProvider": {
"Enable": true,
"Lookup": {
"Domains": [
"github.com",
"api.github.com",
"github.githubassets.com"
]
],
"IPAddressComProvider": {
"Enable": true
},
"GithubMetaProvider": {
"Enable": true,
"MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json"
},
"PublicDnsProvider": {
"Enable": true,
@ -34,11 +39,6 @@
"1.2.4.8",
"208.67.220.220",
"123.125.81.6"
],
"Domains": [
"github.com",
"api.github.com",
"github.githubassets.com"
]
}
}