ouczbs/FastGithub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加ip提供者功能

Browse Source
This commit is contained in:
陈国伟 2021-06-17 17:50:44 +08:00
parent 67f839e7df
commit 733220adcf
17 changed files with 428 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
FastGithub.Scanner/DomainAddress.cs Normal file
View File

@ -0,0 +1,45 @@
using System;
using System.Net;
namespace FastGithub.Scanner
{
class DomainAddress : IEquatable<DomainAddress>
{
/// <summary>
/// 获取域名
/// </summary>
public string Domain { get; }
/// <summary>
/// 获取ip
/// </summary>
public IPAddress Address { get; }
public DomainAddress(string domain, IPAddress address)
{
this.Domain = domain;
this.Address = address;
}
public override bool Equals(object? obj)
{
return obj is DomainAddress other && this.Equals(other);
}
public bool Equals(DomainAddress? other)
{
return other != null && other.Address.Equals(this.Address) && other.Domain == this.Domain;
}
public override int GetHashCode()
{
return HashCode.Combine(this.Domain, this.Address);
}
public override string ToString()
{
return $"{this.Domain} {this.Address}";
}
}
}

31
FastGithub.Scanner/DomainAddressFacotry.cs Normal file
View File

@ -0,0 +1,31 @@
using Microsoft.Extensions.DependencyInjection;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace FastGithub.Scanner
{
[Service(ServiceLifetime.Singleton)]
sealed class DomainAddressFacotry
{
private readonly IEnumerable<IDomainAddressProvider> providers;
public DomainAddressFacotry(IEnumerable<IDomainAddressProvider> providers)
{
this.providers = providers;
}
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
{
var hashSet = new HashSet<DomainAddress>();
foreach (var provider in this.providers)
{
var domainAddresses = await provider.CreateDomainAddressesAsync();
foreach (var item in domainAddresses)
{
hashSet.Add(item);
}
}
return hashSet;
}
}
}

63
FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs Normal file
View File

@ -0,0 +1,63 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
namespace FastGithub.Scanner.DomainMiddlewares
{
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
sealed class LocalDomainAddressProvider : IDomainAddressProvider
{
private const string JsonFile = "IPRange.json";
private readonly ILogger<LocalDomainAddressProvider> logger;
public LocalDomainAddressProvider(ILogger<LocalDomainAddressProvider> logger)
{
this.logger = logger;
}
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
{
try
{
if (File.Exists(JsonFile) == true)
{
using var fileStream = File.OpenRead(JsonFile);
var datas = await JsonSerializer.DeserializeAsync<Dictionary<string, string[]>>(fileStream);
if (datas != null)
{
return this.GetDomainAddresses(datas);
}
}
}
catch (Exception ex)
{
this.logger.LogWarning($"加载本机的ip列表异常：{ex.Message}");
}
return Enumerable.Empty<DomainAddress>();
}
private IEnumerable<DomainAddress> GetDomainAddresses(Dictionary<string, string[]> datas)
{
foreach (var kv in datas)
{
var domain = kv.Key;
foreach (var item in kv.Value)
{
if (IPRange.TryParse(item, out var range))
{
foreach (var address in range)
{
yield return new DomainAddress(domain, address);
}
}
}
}
}
}
}

86
FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs Normal file
View File

@ -0,0 +1,86 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Json;
using System.Net.Sockets;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace FastGithub.Scanner.DomainMiddlewares
{
[Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))]
sealed class RemoteDomainAddressProvider : IDomainAddressProvider
{
private readonly IHttpClientFactory httpClientFactory;
private readonly IOptionsMonitor<GithubOptions> options;
private readonly ILogger<RemoteDomainAddressProvider> logger;
public RemoteDomainAddressProvider(
IHttpClientFactory httpClientFactory,
IOptionsMonitor<GithubOptions> options,
ILogger<RemoteDomainAddressProvider> logger)
{
this.httpClientFactory = httpClientFactory;
this.options = options;
this.logger = logger;
}
public async Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync()
{
try
{
var httpClient = this.httpClientFactory.CreateClient();
var meta = await httpClient.GetFromJsonAsync<Meta>(this.options.CurrentValue.MetaUri);
if (meta != null)
{
return meta.ToDomainAddresses();
}
}
catch (Exception ex)
{
this.logger.LogWarning($"加载远程的ip列表异常：{ex.Message}");
}
return Enumerable.Empty<DomainAddress>();
}
private class Meta
{
[JsonPropertyName("web")]
public string[] Web { get; set; } = Array.Empty<string>();
[JsonPropertyName("api")]
public string[] Api { get; set; } = Array.Empty<string>();
public IEnumerable<DomainAddress> ToDomainAddresses()
{
foreach (var range in IPRange.From(this.Web).OrderBy(item => item.Size))
{
if (range.AddressFamily == AddressFamily.InterNetwork)
{
foreach (var address in range)
{
yield return new DomainAddress("github.com", address);
}
}
}
foreach (var range in IPRange.From(this.Api).OrderBy(item => item.Size))
{
if (range.AddressFamily == AddressFamily.InterNetwork)
{
foreach (var address in range)
{
yield return new DomainAddress("api.github.com", address);
}
}
}
}
}
}
}

6
FastGithub.Scanner/FastGithub.Scanner.csproj
View File

@ -15,4 +15,10 @@
<ProjectReference Include="..\FastGithub.Core\FastGithub.Core.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="IPRange.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

27
FastGithub.Scanner/GithubContext.cs
View File

@ -3,7 +3,7 @@ using System.Net;
namespace FastGithub.Scanner
{
sealed class GithubContext : IEquatable<GithubContext>
sealed class GithubContext : DomainAddress, IEquatable<GithubContext>
{
private record Github(
string Domain,
@ -12,16 +12,6 @@ namespace FastGithub.Scanner
double AvailableRate,
TimeSpan AvgElapsed);
/// <summary>
/// 获取域名
/// </summary>
public string Domain { get; }
/// <summary>
/// 获取ip
/// </summary>
public IPAddress Address { get; }
/// <summary>
/// 获取或设置是否可用
/// </summary>
@ -34,24 +24,13 @@ namespace FastGithub.Scanner
public GithubContext(string domain, IPAddress address)
: base(domain, address)
{
this.Domain = domain;
this.Address = address;
}
public override bool Equals(object? obj)
{
return obj is GithubContext other && this.Equals(other);
}
public bool Equals(GithubContext? other)
{
return other != null && other.Address.Equals(this.Address) && other.Domain == this.Domain;
}
public override int GetHashCode()
{
return HashCode.Combine(this.Domain, this.Address);
return base.Equals(other);
}
public override string ToString()

2
FastGithub.Scanner/GithubFullScanHostedService.cs
View File

@ -24,7 +24,7 @@ namespace FastGithub
{
while (stoppingToken.IsCancellationRequested == false)
{
await githubScanService.ScanAllAsync(stoppingToken);
await githubScanService.ScanAllAsync();
await Task.Delay(this.options.CurrentValue.ScanAllInterval, stoppingToken);
}
}

43
FastGithub.Scanner/GithubMetaService.cs
View File

@ -1,43 +0,0 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner
{
[Service(ServiceLifetime.Singleton)]
sealed class GithubMetaService
{
private readonly IHttpClientFactory httpClientFactory;
private readonly IOptionsMonitor<GithubOptions> options;
private readonly ILogger<GithubMetaService> logger;
public GithubMetaService(
IHttpClientFactory httpClientFactory,
IOptionsMonitor<GithubOptions> options,
ILogger<GithubMetaService> logger)
{
this.httpClientFactory = httpClientFactory;
this.options = options;
this.logger = logger;
}
public async Task<Meta?> GetMetaAsync(CancellationToken cancellationToken = default)
{
try
{
var httpClient = this.httpClientFactory.CreateClient();
return await httpClient.GetFromJsonAsync<Meta>(this.options.CurrentValue.MetaUri, cancellationToken);
}
catch (Exception ex)
{
this.logger.LogError(ex, "获取meta.json文件失败");
return default;
}
}
}
}

29
FastGithub.Scanner/GithubScanService.cs
View File

@ -1,9 +1,8 @@
using FastGithub.Scanner.Middlewares;
using FastGithub.Scanner.ScanMiddlewares;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner
@ -11,7 +10,7 @@ namespace FastGithub.Scanner
[Service(ServiceLifetime.Singleton)]
sealed class GithubScanService
{
private readonly GithubMetaService metaService;
private readonly DomainAddressFacotry domainAddressFactory;
private readonly ILogger<GithubScanService> logger;
private readonly GithubContextCollection contextCollection;
@ -19,12 +18,12 @@ namespace FastGithub.Scanner
private readonly InvokeDelegate<GithubContext> resultScanDelegate;
public GithubScanService(
GithubMetaService metaService,
DomainAddressFacotry domainAddressFactory,
GithubContextCollection contextCollection,
IServiceProvider appService,
ILogger<GithubScanService> logger)
{
this.metaService = metaService;
this.domainAddressFactory = domainAddressFactory;
this.contextCollection = contextCollection;
this.logger = logger;
@ -41,17 +40,19 @@ namespace FastGithub.Scanner
.Build();
}
public async Task ScanAllAsync(CancellationToken cancellationToken = default)
public async Task ScanAllAsync()
{
this.logger.LogInformation("完整扫描开始..");
var meta = await this.metaService.GetMetaAsync(cancellationToken);
if (meta != null)
{
var scanTasks = meta.ToGithubContexts().Select(ctx => ScanAsync(ctx));
var results = await Task.WhenAll(scanTasks);
var successCount = results.Count(item => item);
this.logger.LogInformation($"完整扫描结束，成功{successCount}条共{results.Length}条");
}
var domainAddresses = await this.domainAddressFactory.CreateDomainAddressesAsync();
var scanTasks = domainAddresses
.Select(item => new GithubContext(item.Domain, item.Address))
.Select(ctx => ScanAsync(ctx));
var results = await Task.WhenAll(scanTasks);
var successCount = results.Count(item => item);
this.logger.LogInformation($"完整扫描结束，成功{successCount}条共{results.Length}条");
async Task<bool> ScanAsync(GithubContext context)
{

10
FastGithub.Scanner/IDomainAddressProvider.cs Normal file
View File

@ -0,0 +1,10 @@
using System.Collections.Generic;
using System.Threading.Tasks;
namespace FastGithub.Scanner
{
interface IDomainAddressProvider
{
Task<IEnumerable<DomainAddress>> CreateDomainAddressesAsync();
}
}

187
FastGithub.Scanner/IPRange.cs
View File

@ -8,111 +8,144 @@ using System.Net.Sockets;
namespace FastGithub.Scanner
{
sealed class IPRange : IEnumerable<IPAddress>
abstract class IPRange : IEnumerable<IPAddress>
{
private readonly IPNetwork network;
public abstract int Size { get; }
public AddressFamily AddressFamily => this.network.AddressFamily;
public abstract AddressFamily AddressFamily { get; }
public int Size => (int)this.network.Total;
public abstract IEnumerator<IPAddress> GetEnumerator();
private IPRange(IPNetwork network)
{
this.network = network;
}
public IEnumerator<IPAddress> GetEnumerator()
{
return new Enumerator(this.network);
}
IEnumerator IEnumerable.GetEnumerator()
{
return this.GetEnumerator();
}
private class Enumerator : IEnumerator<IPAddress>
public static IEnumerable<IPRange> From(IEnumerable<string> ranges)
{
private IPAddress? currrent;
private readonly IPNetwork network;
private readonly IPAddress maxAddress;
public Enumerator(IPNetwork network)
foreach (var item in ranges)
{
this.network = network;
this.maxAddress = Add(network.LastUsable, 1);
}
public IPAddress Current => this.currrent ?? throw new NotImplementedException();
object IEnumerator.Current => this.Current;
public void Dispose()
{
}
public bool MoveNext()
{
var value = this.currrent == null
? this.network.FirstUsable
: Add(this.currrent, 1);
if (value.Equals(maxAddress))
if (TryParse(item, out var range))
{
return false;
}
this.currrent = value;
return true;
}
public void Reset()
{
this.currrent = null;
}
}
/// <summary>
/// 添加值
/// </summary>
/// <param name="ip"></param>
/// <param name="value"></param>
/// <returns></returns>
private static IPAddress Add(IPAddress ip, int value)
{
var span = ip.GetAddressBytes().AsSpan();
var hostValue = BinaryPrimitives.ReadInt32BigEndian(span);
BinaryPrimitives.WriteInt32BigEndian(span, hostValue + value);
return new IPAddress(span);
}
public static IEnumerable<IPRange> From(IEnumerable<string> networks)
{
foreach (var item in networks)
{
if (TryParse(item, out var value))
{
yield return value;
yield return range;
}
}
}
public static bool TryParse(ReadOnlySpan<char> network, [MaybeNullWhen(false)] out IPRange value)
public static bool TryParse(ReadOnlySpan<char> range, [MaybeNullWhen(false)] out IPRange value)
{
if (network.IsEmpty == false && IPNetwork.TryParse(network.ToString(), out var ipNetwork))
if (range.IsEmpty == false && IPNetwork.TryParse(range.ToString(), out var ipNetwork))
{
value = new IPRange(ipNetwork);
value = new NetworkIPAddressRange(ipNetwork);
return true;
}
var index = range.IndexOf('-');
if (index >= 0)
{
var start = range.Slice(0, index);
var end = range.Slice(index + 1);
if (IPAddress.TryParse(start, out var startIp) &&
IPAddress.TryParse(end, out var endIp) &&
startIp.AddressFamily == endIp.AddressFamily)
{
value = new SplitIPAddressRange(startIp, endIp);
return true;
}
}
value = null;
return false;
}
public override string ToString()
private class NetworkIPAddressRange : IPRange
{
return this.network.ToString();
private readonly IPAddressCollection addressCollection;
private readonly AddressFamily addressFamily;
public override int Size => (int)this.addressCollection.Count;
public override AddressFamily AddressFamily => this.addressFamily;
public NetworkIPAddressRange(IPNetwork network)
{
this.addressCollection = network.ListIPAddress(FilterEnum.All);
this.addressFamily = network.AddressFamily;
}
public override IEnumerator<IPAddress> GetEnumerator()
{
return ((IEnumerable<IPAddress>)this.addressCollection).GetEnumerator();
}
}
private class SplitIPAddressRange : IPRange
{
private readonly IPAddress start;
private readonly IPAddress end;
private readonly AddressFamily addressFamily;
public override AddressFamily AddressFamily => this.addressFamily;
public SplitIPAddressRange(IPAddress start, IPAddress end)
{
this.start = start;
this.end = end;
this.addressFamily = start.AddressFamily;
}
public override int Size
{
get
{
if (this.start.AddressFamily == AddressFamily.InterNetworkV6)
{
var startValue = BinaryPrimitives.ReadInt64BigEndian(this.start.GetAddressBytes());
var endValue = BinaryPrimitives.ReadInt64BigEndian(this.end.GetAddressBytes());
return (int)(endValue - startValue) + 1;
}
else
{
var startValue = BinaryPrimitives.ReadInt32BigEndian(this.start.GetAddressBytes());
var endValue = BinaryPrimitives.ReadInt32BigEndian(this.end.GetAddressBytes());
return endValue - startValue + 1;
}
}
}
public override IEnumerator<IPAddress> GetEnumerator()
{
return this.GetIPAddresses().GetEnumerator();
}
private IEnumerable<IPAddress> GetIPAddresses()
{
for (var i = 0; i < this.Size; i++)
{
var value = i;
yield return Add(this.start, value);
}
}
/// <summary>
/// 添加值
/// </summary>
/// <param name="address"></param>
/// <param name="value"></param>
/// <returns></returns>
private static IPAddress Add(IPAddress address, int value)
{
var span = address.GetAddressBytes().AsSpan();
var hostValue = BinaryPrimitives.ReadInt32BigEndian(span);
BinaryPrimitives.WriteInt32BigEndian(span, hostValue + value);
return new IPAddress(span);
}
}
}
}

51
FastGithub.Scanner/IPRange.json Normal file
View File

@ -0,0 +1,51 @@
{
"github.githubassets.com": [
"185.199.108.154/32",
"185.199.109.154/32",
"185.199.110.154/32",
"185.199.111.154/32"
],
"github.com": [
"13.114.40.48/32",
"52.192.72.89/32",
"52.69.186.44/32",
"15.164.81.167/32",
"52.78.231.108/32",
"13.234.176.102/32",
"13.234.210.38/32",
"13.229.188.59/32",
"13.250.177.223/32",
"52.74.223.119/32",
"13.236.229.21/32",
"13.237.44.5/32",
"52.64.108.95/32",
"18.228.52.138/32",
"18.228.67.229/32",
"18.231.5.6/32",
"192.30.252.0/22",
"185.199.108.0/22",
"140.82.112.0/20",
"143.55.64.0/20"
],
"api.github.com": [
"13.230.158.120/32",
"18.179.245.253/32",
"52.69.239.207/32",
"13.209.163.61/32",
"54.180.75.25/32",
"13.233.76.15/32",
"13.234.168.60/32",
"13.250.168.23/32",
"13.250.94.254/32",
"54.169.195.247/32",
"13.236.14.80/32",
"13.238.54.232/32",
"52.63.231.178/32",
"18.229.199.252/32",
"54.207.47.76/32",
"192.30.252.0/22",
"185.199.108.0/22",
"140.82.112.0/20",
"143.55.64.0/20"
]
}

53
FastGithub.Scanner/Meta.cs
View File

@ -1,53 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Sockets;
using System.Text.Json.Serialization;
namespace FastGithub.Scanner
{
sealed class Meta
{
[JsonPropertyName("hooks")]
public string[] Hooks { get; set; } = Array.Empty<string>();
[JsonPropertyName("web")]
public string[] Web { get; set; } = Array.Empty<string>();
[JsonPropertyName("api")]
public string[] Api { get; set; } = Array.Empty<string>();
[JsonPropertyName("git")]
public string[] Git { get; set; } = Array.Empty<string>();
[JsonPropertyName("packages")]
public string[] Packages { get; set; } = Array.Empty<string>();
[JsonPropertyName("pages")]
public string[] Pages { get; set; } = Array.Empty<string>();
[JsonPropertyName("importer")]
public string[] Importer { get; set; } = Array.Empty<string>();
[JsonPropertyName("actions")]
public string[] Actions { get; set; } = Array.Empty<string>();
[JsonPropertyName("dependabot")]
public string[] Dependabot { get; set; } = Array.Empty<string>();
public IEnumerable<GithubContext> ToGithubContexts()
{
foreach (var range in IPRange.From(this.Web).OrderBy(item => item.Size))
{
if (range.AddressFamily == AddressFamily.InterNetwork)
{
foreach (var address in range)
{
yield return new GithubContext("github.com", address);
}
}
}
}
}
}

2
FastGithub.Scanner/Middlewares/ConcurrentMiddleware.cs → FastGithub.Scanner/ScanMiddlewares/ConcurrentMiddleware.cs
View File

@ -3,7 +3,7 @@ using System;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.Middlewares
namespace FastGithub.Scanner.ScanMiddlewares
{
[Service(ServiceLifetime.Singleton)]
sealed class ConcurrentMiddleware : IMiddleware<GithubContext>

6
FastGithub.Scanner/Middlewares/HttpsScanMiddleware.cs → FastGithub.Scanner/ScanMiddlewares/HttpsScanMiddleware.cs
View File

@ -7,7 +7,7 @@ using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.Middlewares
namespace FastGithub.Scanner.ScanMiddlewares
{
[Service(ServiceLifetime.Singleton)]
sealed class HttpsScanMiddleware : IMiddleware<GithubContext>
@ -35,15 +35,17 @@ namespace FastGithub.Scanner.Middlewares
RequestUri = new Uri($"https://{context.Address}"),
};
request.Headers.Host = context.Domain;
request.Headers.Accept.TryParseAdd("*/*");
using var httpClient = new HttpClient(new HttpClientHandler
{
Proxy = null,
UseProxy = false,
AllowAutoRedirect = false,
});
using var cancellationTokenSource = new CancellationTokenSource(this.options.CurrentValue.HttpsScanTimeout);
var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationTokenSource.Token);
using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationTokenSource.Token);
var server = response.EnsureSuccessStatusCode().Headers.Server;
if (server.Any(s => string.Equals("GitHub.com", s.Product?.Name, StringComparison.OrdinalIgnoreCase)))
{

2
FastGithub.Scanner/Middlewares/PortScanMiddleware.cs → FastGithub.Scanner/ScanMiddlewares/PortScanMiddleware.cs
View File

@ -6,7 +6,7 @@ using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
namespace FastGithub.Scanner.Middlewares
namespace FastGithub.Scanner.ScanMiddlewares
{
[Service(ServiceLifetime.Singleton)]
sealed class PortScanMiddleware : IMiddleware<GithubContext>

2
FastGithub.Scanner/Middlewares/StatisticsMiddleware.cs → FastGithub.Scanner/ScanMiddlewares/StatisticsMiddleware.cs
View File

@ -4,7 +4,7 @@ using System;
using System.Diagnostics;
using System.Threading.Tasks;
namespace FastGithub.Scanner.Middlewares
namespace FastGithub.Scanner.ScanMiddlewares
{
[Service(ServiceLifetime.Singleton)]
sealed class StatisticsMiddleware : IMiddleware<GithubContext>