From bca67653a3adf6e5d103269081a3cca0d8cc9178 Mon Sep 17 00:00:00 2001 From: xljiulang <366193849@qq.com> Date: Fri, 11 Jun 2021 23:00:24 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=86=E5=BC=80=E6=A3=80=E6=B5=8B=E4=B8=8D?= =?UTF-8?q?=E5=90=8C=E5=9F=9F=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- FastGithub/GithubContext.cs | 8 ++++++ FastGithub/GithubOptions.cs | 4 +-- FastGithub/GithubService.cs | 11 ++++---- FastGithub/Meta.cs | 25 +++++++++++++------ .../Middlewares/ConcurrentMiddleware.cs | 10 ++------ FastGithub/Middlewares/HttpTestMiddleware.cs | 16 ++++++------ FastGithub/Middlewares/PortScanMiddleware.cs | 2 +- FastGithub/appsettings.json | 5 ++-- 8 files changed, 47 insertions(+), 34 deletions(-) diff --git a/FastGithub/GithubContext.cs b/FastGithub/GithubContext.cs index aa6824b..b414e80 100644 --- a/FastGithub/GithubContext.cs +++ b/FastGithub/GithubContext.cs @@ -6,9 +6,17 @@ namespace FastGithub { class GithubContext { + [AllowNull] + public string Domain { get; set; } + [AllowNull] public IPAddress Address { get; set; } public TimeSpan? HttpElapsed { get; set; } + + public override string ToString() + { + return $"{Address}\t{Domain}\t#{HttpElapsed}"; + } } } diff --git a/FastGithub/GithubOptions.cs b/FastGithub/GithubOptions.cs index a748152..79c0793 100644 --- a/FastGithub/GithubOptions.cs +++ b/FastGithub/GithubOptions.cs @@ -5,9 +5,7 @@ namespace FastGithub class GithubOptions { public Uri MetaUri { get; set; } = new Uri("https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json"); - - public int Concurrent { get; set; } = 50; - + public TimeSpan PortScanTimeout { get; set; } = TimeSpan.FromSeconds(1d); public TimeSpan HttpTestTimeout { get; set; } = TimeSpan.FromSeconds(5d); diff --git a/FastGithub/GithubService.cs b/FastGithub/GithubService.cs index c196e05..9b89e33 100644 --- a/FastGithub/GithubService.cs +++ b/FastGithub/GithubService.cs @@ -41,9 +41,9 @@ namespace FastGithub foreach (var context in sortedContexts) { - var content = $"{context.Address}\t{context.HttpElapsed}"; - this.logger.LogInformation(content); - await fileWriter.WriteLineAsync(content); + var message = context.ToString(); + this.logger.LogInformation(message); + await fileWriter.WriteLineAsync(message); } } @@ -53,11 +53,12 @@ namespace FastGithub private IEnumerable GetMetaScanTasks(Meta meta, IList contexts) { - foreach (var address in meta.ToIPv4Address()) + foreach (var item in meta.ToDomainAddress()) { var context = new GithubContext { - Address = address, + Domain = item.Domain, + Address = item.Address, }; contexts.Add(context); yield return this.githubDelegate(context); diff --git a/FastGithub/Meta.cs b/FastGithub/Meta.cs index 906afc1..8a8c5ba 100644 --- a/FastGithub/Meta.cs +++ b/FastGithub/Meta.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Net; using System.Text.Json.Serialization; @@ -36,19 +35,31 @@ namespace FastGithub public string[] Dependabot { get; set; } = Array.Empty(); - public IEnumerable ToIPv4Address() + public IEnumerable ToDomainAddress() { - var cidrs = this.Web.Concat(this.Api); - foreach (var cidr in cidrs) + foreach (var item in this.Web) { - if (IPv4CIDR.TryParse(cidr, out var value)) + if (IPv4CIDR.TryParse(item, out var cidr)) { - foreach (var ip in value.GetAllIPAddress()) + foreach (var address in cidr.GetAllIPAddress()) { - yield return ip; + yield return new DomainAddress("github.com", address); + } + } + } + + foreach (var item in this.Api) + { + if (IPv4CIDR.TryParse(item, out var cidr)) + { + foreach (var address in cidr.GetAllIPAddress()) + { + yield return new DomainAddress("api.github.com", address); } } } } + + public record DomainAddress(string Domain, IPAddress Address); } } diff --git a/FastGithub/Middlewares/ConcurrentMiddleware.cs b/FastGithub/Middlewares/ConcurrentMiddleware.cs index 5941216..dbf1012 100644 --- a/FastGithub/Middlewares/ConcurrentMiddleware.cs +++ b/FastGithub/Middlewares/ConcurrentMiddleware.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.Options; -using System; +using System; using System.Threading; using System.Threading.Tasks; @@ -7,12 +6,7 @@ namespace FastGithub.Middlewares { sealed class ConcurrentMiddleware : IGithubMiddleware { - private readonly SemaphoreSlim semaphoreSlim; - - public ConcurrentMiddleware(IOptions options) - { - this.semaphoreSlim = new SemaphoreSlim(options.Value.Concurrent); - } + private readonly SemaphoreSlim semaphoreSlim = new(Environment.ProcessorCount * 4); public async Task InvokeAsync(GithubContext context, Func next) { diff --git a/FastGithub/Middlewares/HttpTestMiddleware.cs b/FastGithub/Middlewares/HttpTestMiddleware.cs index c175b6d..2e61688 100644 --- a/FastGithub/Middlewares/HttpTestMiddleware.cs +++ b/FastGithub/Middlewares/HttpTestMiddleware.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System; +using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -27,9 +28,9 @@ namespace FastGithub.Middlewares var request = new HttpRequestMessage { Method = HttpMethod.Get, - RequestUri = new Uri($"https://{context.Address}/manifest.json"), + RequestUri = new Uri($"https://{context.Address}/"), }; - request.Headers.Host = "github.com"; + request.Headers.Host = context.Domain; using var httpClient = new HttpClient(new HttpClientHandler { @@ -38,18 +39,19 @@ namespace FastGithub.Middlewares var startTime = DateTime.Now; using var cancellationTokenSource = new CancellationTokenSource(this.options.CurrentValue.HttpTestTimeout); - var response = await httpClient.SendAsync(request, cancellationTokenSource.Token); - var media = response.EnsureSuccessStatusCode().Content.Headers.ContentType?.MediaType; - - if (string.Equals(media, "application/manifest+json", StringComparison.OrdinalIgnoreCase)) + var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationTokenSource.Token); + var server = response.EnsureSuccessStatusCode().Headers.Server; + if (server.Any(s => string.Equals("GitHub.com", s.Product?.Name, StringComparison.OrdinalIgnoreCase))) { context.HttpElapsed = DateTime.Now.Subtract(startTime); + this.logger.LogWarning(context.ToString()); + await next(); } } catch (Exception ex) { - this.logger.LogInformation($"{context.Address} {ex.Message}"); + this.logger.LogInformation($"{context.Domain} {context.Address} {ex.Message}"); } } } diff --git a/FastGithub/Middlewares/PortScanMiddleware.cs b/FastGithub/Middlewares/PortScanMiddleware.cs index f59a69f..fdc128f 100644 --- a/FastGithub/Middlewares/PortScanMiddleware.cs +++ b/FastGithub/Middlewares/PortScanMiddleware.cs @@ -33,7 +33,7 @@ namespace FastGithub.Middlewares } catch (Exception) { - this.logger.LogInformation($"{context.Address}的{PORT}端口未开放"); + this.logger.LogInformation($"{context.Domain} {context.Address}的{PORT}端口未开放"); } } } diff --git a/FastGithub/appsettings.json b/FastGithub/appsettings.json index 47b6e37..bfb22b7 100644 --- a/FastGithub/appsettings.json +++ b/FastGithub/appsettings.json @@ -1,8 +1,7 @@ { "Github": { - "MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json", // ipԴļuri - "Concurrent": 50, // ҵҪЧ - "PortScanTimeout": "00:00:00.500", // ˿ɨ賬ʱʱ + "MetaUri": "https://gitee.com/jiulang/fast-github/raw/master/FastGithub/meta.json", // ipԴļuri + "PortScanTimeout": "00:00:01", // ˿ɨ賬ʱʱ "HttpTestTimeout": "00:00:05" // httpԳʱʱ }, "Logging": {