ouczbs/FastGithub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

重构配置文件

Browse Source
This commit is contained in:
xljiulang 2021-06-17 21:13:58 +08:00
parent 733220adcf
commit 73b5914136
15 changed files with 128 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs
View File

@ -1,5 +1,6 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.IO;
@ -12,21 +13,30 @@ namespace FastGithub.Scanner.DomainMiddlewares
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
sealed class LocalDomainAddressProvider : IDomainAddressProvider
{
private const string JsonFile = "IPRange.json";
private readonly IOptionsMonitor<GithubOptions> options;
private readonly ILogger<LocalDomainAddressProvider> logger;
public LocalDomainAddressProvider(ILogger<LocalDomainAddressProvider> logger)
public LocalDomainAddressProvider(
IOptionsMonitor<GithubOptions> options,
ILogger<LocalDomainAddressProvider> logger)
{
this.options = options;
this.logger = logger;
}
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
{
var setting = this.options.CurrentValue.LocalAddressProvider;
if (setting.Enable == false)
{
return Enumerable.Empty<DomainAddress>();
}
try
{
if (File.Exists(JsonFile) == true)
if (File.Exists(setting.IPRangeFilePath) == true)
{
using var fileStream = File.OpenRead(JsonFile);
using var fileStream = File.OpenRead(setting.IPRangeFilePath);
var datas = await JsonSerializer.DeserializeAsync<Dictionary<string, string[]>>(fileStream);
if (datas != null)
{
@ -49,7 +59,7 @@ namespace FastGithub.Scanner.DomainMiddlewares
var domain = kv.Key;
foreach (var item in kv.Value)
{
if (IPRange.TryParse(item, out var range))
if (IPAddressRange.TryParse(item, out var range))
{
foreach (var address in range)
{

19
FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs
View File

@ -15,26 +15,29 @@ namespace FastGithub.Scanner.DomainMiddlewares
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
sealed class RemoteDomainAddressProvider : IDomainAddressProvider
{
private readonly IHttpClientFactory httpClientFactory;
private readonly IOptionsMonitor<GithubOptions> options;
private readonly ILogger<RemoteDomainAddressProvider> logger;
public RemoteDomainAddressProvider(
IHttpClientFactory httpClientFactory,
IOptionsMonitor<GithubOptions> options,
ILogger<RemoteDomainAddressProvider> logger)
{
this.httpClientFactory = httpClientFactory;
this.options = options;
this.logger = logger;
}
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
{
var setting = this.options.CurrentValue.RemoteAddressProvider;
if (setting.Enable == false)
{
return Enumerable.Empty<DomainAddress>();
}
try
{
var httpClient = this.httpClientFactory.CreateClient();
var meta = await httpClient.GetFromJsonAsync<Meta>(this.options.CurrentValue.MetaUri);
using var httpClient = new HttpClient();
var meta = await httpClient.GetFromJsonAsync<Meta>(setting.MetaUri);
if (meta != null)
{
return meta.ToDomainAddresses();
@ -50,7 +53,7 @@ namespace FastGithub.Scanner.DomainMiddlewares
private class Meta
{
[JsonPropertyName("web")]
[JsonPropertyName("web")]
public string[] Web { get; set; } = Array.Empty<string>();
[JsonPropertyName("api")]
@ -59,7 +62,7 @@ namespace FastGithub.Scanner.DomainMiddlewares
public IEnumerable<DomainAddress> ToDomainAddresses()
{
foreach (var range in IPRange.From(this.Web).OrderBy(item => item.Size))
foreach (var range in IPAddressRange.From(this.Web).OrderBy(item => item.Size))
{
if (range.AddressFamily == AddressFamily.InterNetwork)
{
@ -70,7 +73,7 @@ namespace FastGithub.Scanner.DomainMiddlewares
}
}
foreach (var range in IPRange.From(this.Api).OrderBy(item => item.Size))
foreach (var range in IPAddressRange.From(this.Api).OrderBy(item => item.Size))
{
if (range.AddressFamily == AddressFamily.InterNetwork)
{

5
FastGithub.Scanner/FastGithub.Scanner.csproj
View File

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
@ -7,8 +7,7 @@
<ItemGroup>
<PackageReference Include="IPNetwork2" Version="2.5.320" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" />
</ItemGroup>
<ItemGroup>

6
FastGithub.Scanner/GithubContext.cs
View File

@ -8,9 +8,8 @@ namespace FastGithub.Scanner
private record Github(
string Domain,
IPAddress Address,
bool Available,
double AvailableRate,
TimeSpan AvgElapsed);
double AvgElapsed);
/// <summary>
/// 获取或设置是否可用
@ -38,9 +37,8 @@ namespace FastGithub.Scanner
return new Github(
this.Domain,
this.Address,
this.Available,
this.History.AvailableRate,
this.History.AvgElapsed
this.History.AvgElapsed.TotalSeconds
).ToString();
}
}

2
FastGithub.Scanner/GithubFullScanHostedService.cs
View File

@ -25,7 +25,7 @@ namespace FastGithub
while (stoppingToken.IsCancellationRequested == false)
{
await githubScanService.ScanAllAsync();
await Task.Delay(this.options.CurrentValue.ScanAllInterval, stoppingToken);
await Task.Delay(this.options.CurrentValue.Scan.FullScanInterval, stoppingToken);
}
}
}

32
FastGithub.Scanner/GithubOptions.cs
View File

@ -5,14 +5,36 @@ namespace FastGithub.Scanner
[Options("Github")]
sealed class GithubOptions
{
public TimeSpan ScanAllInterval { get; set; } = TimeSpan.FromHours(2d);
public ScanSetting Scan { get; set; } = new ScanSetting();
public TimeSpan ScanResultInterval { get; set; } = TimeSpan.FromMinutes(1d);
public RemoteAddressProviderSetting RemoteAddressProvider { get; set; } = new RemoteAddressProviderSetting();
public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json");
public LocalAddressProviderSetting LocalAddressProvider { get; set; } = new LocalAddressProviderSetting();
public TimeSpan PortScanTimeout { get; set; } = TimeSpan.FromSeconds(1d);
public class ScanSetting
{
public TimeSpan FullScanInterval = TimeSpan.FromHours(2d);
public TimeSpan HttpsScanTimeout { get; set; } = TimeSpan.FromSeconds(5d);
public TimeSpan ResultScanInterval = TimeSpan.FromMinutes(1d);
public TimeSpan TcpScanTimeout { get; set; } = TimeSpan.FromSeconds(1d);
public TimeSpan HttpsScanTimeout { get; set; } = TimeSpan.FromSeconds(2d);
}
public class RemoteAddressProviderSetting
{
public bool Enable { get; set; } = true;
public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json");
}
public class LocalAddressProviderSetting
{
public bool Enable { get; set; } = true;
public string IPRangeFilePath { get; set; } = "./IPRange.json";
}
}
}

2
FastGithub.Scanner/GithubResultScanHostedService.cs
View File

@ -23,7 +23,7 @@ namespace FastGithub
{
while (stoppingToken.IsCancellationRequested == false)
{
await Task.Delay(this.options.CurrentValue.ScanResultInterval, stoppingToken);
await Task.Delay(this.options.CurrentValue.Scan.ResultScanInterval, stoppingToken);
await githubScanService.ScanResultAsync();
}
}

2
FastGithub.Scanner/GithubScanService.cs
View File

@ -30,7 +30,7 @@ namespace FastGithub.Scanner
this.fullScanDelegate = new PipelineBuilder<GithubContext>(appService, ctx => Task.CompletedTask)
.Use<ConcurrentMiddleware>()
.Use<StatisticsMiddleware>()
.Use<PortScanMiddleware>()
.Use<TcpScanMiddleware>()
.Use<HttpsScanMiddleware>()
.Build();

10
FastGithub.Scanner/IPRange.cs → FastGithub.Scanner/IPAddressRange.cs
View File

@ -8,7 +8,7 @@ using System.Net.Sockets;
namespace FastGithub.Scanner
{
abstract class IPRange : IEnumerable<IPAddress>
abstract class IPAddressRange : IEnumerable<IPAddress>
{
public abstract int Size { get; }
@ -22,7 +22,7 @@ namespace FastGithub.Scanner
return this.GetEnumerator();
}
public static IEnumerable<IPRange> From(IEnumerable<string> ranges)
public static IEnumerable<IPAddressRange> From(IEnumerable<string> ranges)
{
foreach (var item in ranges)
{
@ -34,7 +34,7 @@ namespace FastGithub.Scanner
}
public static bool TryParse(ReadOnlySpan<char> range, [MaybeNullWhen(false)] out IPRange value)
public static bool TryParse(ReadOnlySpan<char> range, [MaybeNullWhen(false)] out IPAddressRange value)
{
if (range.IsEmpty == false && IPNetwork.TryParse(range.ToString(), out var ipNetwork))
{
@ -62,7 +62,7 @@ namespace FastGithub.Scanner
}
private class NetworkIPAddressRange : IPRange
private class NetworkIPAddressRange : IPAddressRange
{
private readonly IPAddressCollection addressCollection;
@ -84,7 +84,7 @@ namespace FastGithub.Scanner
}
}
private class SplitIPAddressRange : IPRange
private class SplitIPAddressRange : IPAddressRange
{
private readonly IPAddress start;
private readonly IPAddress end;

37
FastGithub.Scanner/IPRange.json
View File

@ -1,10 +1,4 @@
{
"github.githubassets.com": [
"185.199.108.154/32",
"185.199.109.154/32",
"185.199.110.154/32",
"185.199.111.154/32"
],
"github.com": [
"13.114.40.48/32",
"52.192.72.89/32",
@ -27,25 +21,16 @@
"140.82.112.0/20",
"143.55.64.0/20"
],
"api.github.com": [
"13.230.158.120/32",
"18.179.245.253/32",
"52.69.239.207/32",
"13.209.163.61/32",
"54.180.75.25/32",
"13.233.76.15/32",
"13.234.168.60/32",
"13.250.168.23/32",
"13.250.94.254/32",
"54.169.195.247/32",
"13.236.14.80/32",
"13.238.54.232/32",
"52.63.231.178/32",
"18.229.199.252/32",
"54.207.47.76/32",
"192.30.252.0/22",
"185.199.108.0/22",
"140.82.112.0/20",
"143.55.64.0/20"
"github.githubassets.com": [
"185.199.108.154/32",
"185.199.109.154/32",
"185.199.110.154/32",
"185.199.111.154/32"
],
"avatars.githubusercontent.com": [
"185.199.108.133/32",
"185.199.109.133/32",
"185.199.110.133/32",
"185.199.111.133/32"
]
}

2
FastGithub.Scanner/ScanMiddlewares/ConcurrentMiddleware.cs
View File

@ -12,7 +12,7 @@ namespace FastGithub.Scanner.ScanMiddlewares
public ConcurrentMiddleware()
{
var currentCount = Environment.ProcessorCount * 4;
var currentCount = Environment.ProcessorCount * 2;
this.semaphoreSlim = new SemaphoreSlim(currentCount, currentCount);
}

39
FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs
View File

@ -2,6 +2,7 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Net.Http;
using System.Threading;
@ -35,23 +36,23 @@ namespace FastGithub.Scanner.ScanMiddlewares
RequestUri = new Uri($"https://{context.Address}"),
};
request.Headers.Host = context.Domain;
request.Headers.ConnectionClose = true;
request.Headers.Accept.TryParseAdd("*/*");
using var httpClient = new HttpClient(new HttpClientHandler
using var httpClient = new HttpMessageInvoker(new SocketsHttpHandler
{
Proxy = null,
UseProxy = false,
AllowAutoRedirect = false,
});
using var cancellationTokenSource = new CancellationTokenSource(this.options.CurrentValue.HttpsScanTimeout);
using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationTokenSource.Token);
var server = response.EnsureSuccessStatusCode().Headers.Server;
if (server.Any(s => string.Equals("GitHub.com", s.Product?.Name, StringComparison.OrdinalIgnoreCase)))
{
context.Available = true;
await next();
}
var timeout = this.options.CurrentValue.Scan.HttpsScanTimeout;
using var cancellationTokenSource = new CancellationTokenSource(timeout);
using var response = await httpClient.SendAsync(request, cancellationTokenSource.Token);
this.VerifyHttpResponse(context.Domain, response);
context.Available = true;
await next();
}
catch (TaskCanceledException)
{
@ -64,6 +65,26 @@ namespace FastGithub.Scanner.ScanMiddlewares
}
}
/// <summary>
/// 验证响应内容
/// </summary>
/// <param name="domain"></param>
/// <param name="response"></param>
/// <exception cref="HttpRequestException"></exception>
/// <exception cref="ValidationException"></exception>
private void VerifyHttpResponse(string domain, HttpResponseMessage response)
{
response.EnsureSuccessStatusCode();
if (domain.EndsWith(".github.com"))
{
var server = response.Headers.Server;
if (server.Any(s => string.Equals("github.com", s.Product?.Name, StringComparison.OrdinalIgnoreCase)) == false)
{
throw new ValidationException("伪造的github服务");
}
}
}
private string GetInnerMessage(Exception ex)
{
while (ex.InnerException != null)

11
FastGithub.Scanner/ScanMiddlewares/PortScanMiddleware.cs → FastGithub.Scanner/ScanMiddlewares/TcpScanMiddleware.cs
View File

@ -9,15 +9,15 @@ using System.Threading.Tasks;
namespace FastGithub.Scanner.ScanMiddlewares
{
[Service(ServiceLifetime.Singleton)]
sealed class PortScanMiddleware : IMiddleware<GithubContext>
sealed class TcpScanMiddleware : IMiddleware<GithubContext>
{
private const int PORT = 443;
private readonly IOptionsMonitor<GithubOptions> options;
private readonly ILogger<PortScanMiddleware> logger;
private readonly ILogger<TcpScanMiddleware> logger;
public PortScanMiddleware(
public TcpScanMiddleware(
IOptionsMonitor<GithubOptions> options,
ILogger<PortScanMiddleware> logger)
ILogger<TcpScanMiddleware> logger)
{
this.options = options;
this.logger = logger;
@ -28,7 +28,8 @@ namespace FastGithub.Scanner.ScanMiddlewares
try
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
using var cancellationTokenSource = new CancellationTokenSource(this.options.CurrentValue.PortScanTimeout);
var timeout = this.options.CurrentValue.Scan.TcpScanTimeout;
using var cancellationTokenSource = new CancellationTokenSource(timeout);
await socket.ConnectAsync(context.Address, PORT, cancellationTokenSource.Token);
await next();

3
FastGithub.Scanner/ScannerServiceCollectionExtensions.cs
View File

@ -18,8 +18,7 @@ namespace FastGithub
public static IServiceCollection AddGithubScanner(this IServiceCollection services, IConfiguration configuration)
{
var assembly = typeof(ScannerServiceCollectionExtensions).Assembly;
return services
.AddHttpClient()
return services
.AddServiceAndOptions(assembly, configuration)
.AddHostedService<GithubFullScanHostedService>()
.AddHostedService<GithubResultScanHostedService>()

19
FastGithub/appsettings.json
View File

@ -4,11 +4,20 @@
"GithubTTL": "00:10:00"
},
"Github": {
"ScanAllInterval": "02:00:00", // 完整扫描时间间隔
"ScanResultInterval": "00:01:00", // 结果扫描时间间隔
"MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json", // ip资源文件uri
"PortScanTimeout": "00:00:01", // 端口扫描超时时间
"HttpsScanTimeout": "00:00:05" // https扫描超时时间
"Scan": {
"FullScanInterval": "02:00:00", // 完整扫描时间间隔
"ResultScanInterval": "00:01:00", // 结果扫描时间间隔
"TcpScanTimeout": "00:00:01", // tcp扫描超时时间
"HttpsScanTimeout": "00:00:02" // https扫描超时时间
},
"RemoteAddressProvider": {
"Enable": true,
"MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json" // meta资源uri
},
"LocalAddressProvider": {
"Enable": true,
"IPRangeFilePath": "./IPRange.json"
}
},
"Logging": {
"LogLevel": {