From 52b5272c180e7292ea2e99579b1cf819f40fddcb Mon Sep 17 00:00:00 2001 From: xljiulang <366193849@qq.com> Date: Thu, 17 Jun 2021 22:09:05 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0DnsDomainAddressProvider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- FastGithub.Dns/FastGithub.Dns.csproj | 6 +- .../DnsDomainAddressProvider.cs | 72 ++++++++++++++++++ .../LocalDomainAddressProvider.cs | 73 ------------------- ...ovider.cs => MetaDomainAddressProvider.cs} | 27 ++----- FastGithub.Scanner/FastGithub.Scanner.csproj | 7 +- FastGithub.Scanner/GithubOptions.cs | 14 ++-- FastGithub.Scanner/IPRange.json | 36 --------- FastGithub/appsettings.json | 32 ++++++-- 8 files changed, 114 insertions(+), 153 deletions(-) create mode 100644 FastGithub.Scanner/DomainAddressProviders/DnsDomainAddressProvider.cs delete mode 100644 FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs rename FastGithub.Scanner/DomainAddressProviders/{RemoteDomainAddressProvider.cs => MetaDomainAddressProvider.cs} (69%) delete mode 100644 FastGithub.Scanner/IPRange.json diff --git a/FastGithub.Dns/FastGithub.Dns.csproj b/FastGithub.Dns/FastGithub.Dns.csproj index a399977..fa0e740 100644 --- a/FastGithub.Dns/FastGithub.Dns.csproj +++ b/FastGithub.Dns/FastGithub.Dns.csproj @@ -6,10 +6,6 @@ - - - - - + diff --git a/FastGithub.Scanner/DomainAddressProviders/DnsDomainAddressProvider.cs b/FastGithub.Scanner/DomainAddressProviders/DnsDomainAddressProvider.cs new file mode 100644 index 0000000..e81787a --- /dev/null +++ b/FastGithub.Scanner/DomainAddressProviders/DnsDomainAddressProvider.cs @@ -0,0 +1,72 @@ +using DNS.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Sockets; +using System.Threading.Tasks; + +namespace FastGithub.Scanner.DomainMiddlewares +{ + [Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))] + sealed class DnsDomainAddressProvider : IDomainAddressProvider + { + private readonly IOptionsMonitor options; + private readonly ILogger logger; + + public DnsDomainAddressProvider( + IOptionsMonitor options, + ILogger logger) + { + this.options = options; + this.logger = logger; + } + + public async Task> CreateDomainAddressesAsync() + { + var setting = this.options.CurrentValue.DnsDomainAddress; + if (setting.Enable == false) + { + return Enumerable.Empty(); + } + + var result = new List(); + foreach (var dns in setting.Dnss) + { + var domainAddresses = await this.LookupAsync(dns, setting.Domains); + result.AddRange(domainAddresses); + } + + return result; + } + + private async Task> LookupAsync(string dns, IEnumerable domains) + { + var client = new DnsClient(dns); + var result = new List(); + + foreach (var domain in domains) + { + try + { + var addresses = await client.Lookup(domain); + foreach (var address in addresses) + { + if (address.AddressFamily == AddressFamily.InterNetwork) + { + result.Add(new DomainAddress(domain, address)); + } + } + } + catch (Exception) + { + this.logger.LogWarning($"dns({dns})无法解析{domain}"); + } + } + + return result; + } + } +} diff --git a/FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs b/FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs deleted file mode 100644 index 9647b14..0000000 --- a/FastGithub.Scanner/DomainAddressProviders/LocalDomainAddressProvider.cs +++ /dev/null @@ -1,73 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; - -namespace FastGithub.Scanner.DomainMiddlewares -{ - [Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))] - sealed class LocalDomainAddressProvider : IDomainAddressProvider - { - private readonly IOptionsMonitor options; - private readonly ILogger logger; - - public LocalDomainAddressProvider( - IOptionsMonitor options, - ILogger logger) - { - this.options = options; - this.logger = logger; - } - - public async Task> CreateDomainAddressesAsync() - { - var setting = this.options.CurrentValue.LocalAddressProvider; - if (setting.Enable == false) - { - return Enumerable.Empty(); - } - - try - { - if (File.Exists(setting.IPRangeFilePath) == true) - { - using var fileStream = File.OpenRead(setting.IPRangeFilePath); - var datas = await JsonSerializer.DeserializeAsync>(fileStream); - if (datas != null) - { - return this.GetDomainAddresses(datas); - } - } - } - catch (Exception ex) - { - this.logger.LogWarning($"加载本机的ip列表异常：{ex.Message}"); - } - - return Enumerable.Empty(); - } - - private IEnumerable GetDomainAddresses(Dictionary datas) - { - foreach (var kv in datas) - { - var domain = kv.Key; - foreach (var item in kv.Value) - { - if (IPAddressRange.TryParse(item, out var range)) - { - foreach (var address in range) - { - yield return new DomainAddress(domain, address); - } - } - } - } - } - } -} diff --git a/FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs b/FastGithub.Scanner/DomainAddressProviders/MetaDomainAddressProvider.cs similarity index 69% rename from FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs rename to FastGithub.Scanner/DomainAddressProviders/MetaDomainAddressProvider.cs index 6c6d0f9..3507ce3 100644 --- a/FastGithub.Scanner/DomainAddressProviders/RemoteDomainAddressProvider.cs +++ b/FastGithub.Scanner/DomainAddressProviders/MetaDomainAddressProvider.cs @@ -13,14 +13,14 @@ using System.Threading.Tasks; namespace FastGithub.Scanner.DomainMiddlewares { [Service(ServiceLifetime.Singleton, ServiceType = typeof(IDomainAddressProvider))] - sealed class RemoteDomainAddressProvider : IDomainAddressProvider + sealed class MetaDomainAddressProvider : IDomainAddressProvider { private readonly IOptionsMonitor options; - private readonly ILogger logger; + private readonly ILogger logger; - public RemoteDomainAddressProvider( + public MetaDomainAddressProvider( IOptionsMonitor options, - ILogger logger) + ILogger logger) { this.options = options; this.logger = logger; @@ -28,7 +28,7 @@ namespace FastGithub.Scanner.DomainMiddlewares public async Task> CreateDomainAddressesAsync() { - var setting = this.options.CurrentValue.RemoteAddressProvider; + var setting = this.options.CurrentValue.MetaDomainAddress; if (setting.Enable == false) { return Enumerable.Empty(); @@ -53,13 +53,9 @@ namespace FastGithub.Scanner.DomainMiddlewares private class Meta { - [JsonPropertyName("web")] + [JsonPropertyName("web")] public string[] Web { get; set; } = Array.Empty(); - [JsonPropertyName("api")] - public string[] Api { get; set; } = Array.Empty(); - - public IEnumerable ToDomainAddresses() { foreach (var range in IPAddressRange.From(this.Web).OrderBy(item => item.Size)) @@ -72,17 +68,6 @@ namespace FastGithub.Scanner.DomainMiddlewares } } } - - foreach (var range in IPAddressRange.From(this.Api).OrderBy(item => item.Size)) - { - if (range.AddressFamily == AddressFamily.InterNetwork) - { - foreach (var address in range) - { - yield return new DomainAddress("api.github.com", address); - } - } - } } } } diff --git a/FastGithub.Scanner/FastGithub.Scanner.csproj b/FastGithub.Scanner/FastGithub.Scanner.csproj index fd9657f..48f7e49 100644 --- a/FastGithub.Scanner/FastGithub.Scanner.csproj +++ b/FastGithub.Scanner/FastGithub.Scanner.csproj @@ -6,6 +6,7 @@ + @@ -14,10 +15,4 @@ - - - Always - - - diff --git a/FastGithub.Scanner/GithubOptions.cs b/FastGithub.Scanner/GithubOptions.cs index 324584e..76bbdb8 100644 --- a/FastGithub.Scanner/GithubOptions.cs +++ b/FastGithub.Scanner/GithubOptions.cs @@ -1,5 +1,7 @@ using System; + + namespace FastGithub.Scanner { [Options("Github")] @@ -7,9 +9,9 @@ namespace FastGithub.Scanner { public ScanSetting Scan { get; set; } = new ScanSetting(); - public RemoteAddressProviderSetting RemoteAddressProvider { get; set; } = new RemoteAddressProviderSetting(); + public MetaDoaminAddressSetting MetaDomainAddress { get; set; } = new MetaDoaminAddressSetting(); - public LocalAddressProviderSetting LocalAddressProvider { get; set; } = new LocalAddressProviderSetting(); + public DnsDomainAddressSetting DnsDomainAddress { get; set; } = new DnsDomainAddressSetting(); public class ScanSetting { @@ -23,18 +25,20 @@ namespace FastGithub.Scanner } - public class RemoteAddressProviderSetting + public class MetaDoaminAddressSetting { public bool Enable { get; set; } = true; public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json"); } - public class LocalAddressProviderSetting + public class DnsDomainAddressSetting { public bool Enable { get; set; } = true; - public string IPRangeFilePath { get; set; } = "./IPRange.json"; + public string[] Dnss { get; set; } = Array.Empty(); + + public string[] Domains { get; set; } = Array.Empty(); } } } diff --git a/FastGithub.Scanner/IPRange.json b/FastGithub.Scanner/IPRange.json deleted file mode 100644 index 1f7ae34..0000000 --- a/FastGithub.Scanner/IPRange.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "github.com": [ - "13.114.40.48/32", - "52.192.72.89/32", - "52.69.186.44/32", - "15.164.81.167/32", - "52.78.231.108/32", - "13.234.176.102/32", - "13.234.210.38/32", - "13.229.188.59/32", - "13.250.177.223/32", - "52.74.223.119/32", - "13.236.229.21/32", - "13.237.44.5/32", - "52.64.108.95/32", - "18.228.52.138/32", - "18.228.67.229/32", - "18.231.5.6/32", - "192.30.252.0/22", - "185.199.108.0/22", - "140.82.112.0/20", - "143.55.64.0/20" - ], - "github.githubassets.com": [ - "185.199.108.154/32", - "185.199.109.154/32", - "185.199.110.154/32", - "185.199.111.154/32" - ], - "avatars.githubusercontent.com": [ - "185.199.108.133/32", - "185.199.109.133/32", - "185.199.110.133/32", - "185.199.111.133/32" - ] -} diff --git a/FastGithub/appsettings.json b/FastGithub/appsettings.json index b054d18..78457a3 100644 --- a/FastGithub/appsettings.json +++ b/FastGithub/appsettings.json @@ -5,18 +5,36 @@ }, "Github": { "Scan": { - "FullScanInterval": "02:00:00", // ɨʱ + "FullScanInterval": "02:00:00", // ɨʱ "ResultScanInterval": "00:01:00", // ɨʱ - "TcpScanTimeout": "00:00:01", // tcpɨ賬ʱʱ - "HttpsScanTimeout": "00:00:02" // httpsɨ賬ʱʱ + "TcpScanTimeout": "00:00:01", // tcpɨ賬ʱʱ + "HttpsScanTimeout": "00:00:02" // httpsɨ賬ʱʱ }, - "RemoteAddressProvider": { + "MetaDomainAddress": { "Enable": true, - "MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json" // metaԴuri + "MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json" // metaԴuri }, - "LocalAddressProvider": { + "DnsDomainAddress": { "Enable": true, - "IPRangeFilePath": "./IPRange.json" + "Dnss": [ + "119.29.29.29", + "180.76.76.76", + "8.8.8.8", + "8.8.4.4", + "114.114.114.114", + "114.114.115.115", + "223.5.5.5", + "223.6.6.6", + "1.2.4.8", + "210.2.4.8", + "208.67.220.220", + "208.67.222.222" + ], + "Domains": [ + "github.com", + "api.github.com", + "github.githubassets.com" + ] } }, "Logging": {