Skip to content

Commit 6120a03

Browse files
vbilopavvbilopav
committed
Implement static files parser
1 parent c339415 commit 6120a03

File tree

11 files changed

+565
-105
lines changed

11 files changed

+565
-105
lines changed

‎BenchmarkTests/FormatStringTests.cs

-7
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,6 @@ public ReadOnlySpan<char> FormatStringMethod()
2222
return DefaultResponseParser.FormatString(input, replacements);
2323
}
2424

25-
//[Benchmark]
26-
//public ReadOnlySpan<char> FormatString_OLD_Method()
27-
//{
28-
// ReadOnlySpan<char> input = "Hello, {name}! Welcome to {place}.";
29-
// return DefaultResponseParser.FormatString_OLD(input, replacements);
30-
//}
31-
3225
[Benchmark]
3326
public string RegexMethod()
3427
{

‎BenchmarkTests/ParseStringsTests.cs

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
using System;
2+
using System.Net;
3+
using System.Text;
4+
using System.Text.RegularExpressions;
5+
using BenchmarkDotNet.Attributes;
6+
using NpgsqlRestClient;
7+
8+
namespace BenchmarkTests;
9+
10+
public class ParseStringsTests
11+
{
12+
string[] names = { "test.txt", "x", "verylongfilename.txt", "abcde" };
13+
string[] patterns = { "*.txt", "*?*", "*long*.txt", "a?c*" };
14+
15+
16+
[Benchmark]
17+
public void IsPatternMatchMethod()
18+
{
19+
foreach (var name in names)
20+
foreach (var pattern in patterns)
21+
{
22+
DefaultResponseParser.IsPatternMatch(name, pattern);
23+
}
24+
}
25+
26+
//[Benchmark]
27+
//public void LikePatternIsMethod()
28+
//{
29+
// foreach (var name in names)
30+
// foreach (var pattern in patterns)
31+
// {
32+
// DefaultResponseParser.LikePatternIsMatch(name, pattern);
33+
// }
34+
//}
35+
36+
//[Benchmark]
37+
//public void LikePatternIsMatchFastMethod()
38+
//{
39+
// foreach (var name in names)
40+
// foreach (var pattern in patterns)
41+
// {
42+
// DefaultResponseParser.LikePatternIsMatchFast(name, pattern);
43+
// }
44+
//}
45+
}

‎BenchmarkTests/Program.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
using BenchmarkTests;
77
using Perfolizer.Horology;
88

9-
BenchmarkRunner.Run<FormatStringTests>();
9+
BenchmarkRunner.Run<ParseStringsTests>();
1010

1111
//BenchmarkRunner
1212
// .Run<HttpClientTests>(

‎NpgsqlRestClient/App.cs

+36-29
Original file line numberDiff line numberDiff line change
@@ -61,40 +61,41 @@ public static void ConfigureStaticFiles(WebApplication app)
6161
return;
6262
}
6363

64-
var redirect = GetConfigStr("LoginRedirectPath", staticFilesCfg) ?? "/login/";
65-
var anonPaths = GetConfigEnumerable("AnonymousPaths", staticFilesCfg) ?? ["*"];
66-
HashSet<string>? anonPathsHash = anonPaths is null ?
67-
null :
68-
new(Config.GetConfigEnumerable("AnonymousPaths", staticFilesCfg) ?? ["*"]);
69-
7064
app.UseDefaultFiles();
71-
if (anonPathsHash?.Contains("*") is true)
65+
var parseCfg = staticFilesCfg.GetSection("ParseContentOptions");
66+
67+
if (parseCfg.Exists() is false || GetConfigBool("Enabled", parseCfg) is false)
7268
{
73-
app.UseStaticFiles();
69+
app.UseMiddleware<AppStaticFileMiddleware>();
70+
Logger?.Information("Serving static files from {0}", app.Environment.WebRootPath);
71+
return;
7472
}
75-
else
73+
74+
var filePaths = GetConfigEnumerable("FilePaths", parseCfg)?.ToArray();
75+
var userIdTag = GetConfigStr("UserIdTag", parseCfg);
76+
var userNameTag = GetConfigStr("UserNameTag", parseCfg);
77+
var userRolesTag = GetConfigStr("UserRolesTag", parseCfg);
78+
Dictionary<string, StringValues>? customTags = null;
79+
foreach (var section in parseCfg.GetSection("CustomTagToClaimMappings").GetChildren())
7680
{
77-
app.UseStaticFiles(new StaticFileOptions
81+
customTags ??= [];
82+
if (section?.Value is null)
7883
{
79-
OnPrepareResponse = ctx =>
80-
{
81-
if (anonPathsHash is not null && ctx?.Context?.User?.Identity?.IsAuthenticated is false)
82-
{
83-
var path = ctx.Context.Request.Path.Value?[..^ctx.File.Name.Length] ?? "/";
84-
if (anonPathsHash.Contains(path) is false)
85-
{
86-
Logger?.Information("Unauthorized access to {0}", path);
87-
ctx.Context.Response.StatusCode = (int)HttpStatusCode.Unauthorized;
88-
if (redirect is not null)
89-
{
90-
ctx.Context.Response.Redirect(redirect);
91-
}
92-
}
93-
}
94-
}
95-
});
84+
continue;
85+
}
86+
customTags.Add(section.Key, section.Value!);
9687
}
97-
Logger?.Information("Serving static files from {0}", app.Environment.WebRootPath);
88+
AppStaticFileMiddleware.ConfigureStaticFileMiddleware(
89+
true,
90+
filePaths,
91+
userIdTag,
92+
userNameTag,
93+
userRolesTag,
94+
customTags,
95+
Logger?.ForContext<AppStaticFileMiddleware>());
96+
97+
app.UseMiddleware<AppStaticFileMiddleware>();
98+
Logger?.Information("Serving static files from {0}. Parsing following file path patterns: {1}", app.Environment.WebRootPath, filePaths);
9899
}
99100

100101
public static string CreateUrl(Routine routine, NpgsqlRestOptions options) =>
@@ -409,8 +410,14 @@ public static List<IRoutineSource> CreateRoutineSources()
409410
ExcludeNames = GetConfigEnumerable("ExcludeNames", crudSourceCfg)?.ToArray(),
410411
CommentsMode = GetConfigEnum<CommentsMode?>("CommentsMode", crudSourceCfg),
411412
CrudTypes = GetConfigFlag<CrudCommandType>("CrudTypes", crudSourceCfg),
413+
414+
ReturningUrlPattern = GetConfigStr("ReturningUrlPattern", crudSourceCfg) ?? "{0}/returning",
415+
OnConflictDoNothingUrlPattern = GetConfigStr("OnConflictDoNothingUrlPattern", crudSourceCfg) ?? "{0}/on-conflict-do-nothing",
416+
OnConflictDoNothingReturningUrlPattern = GetConfigStr("OnConflictDoNothingReturningUrlPattern", crudSourceCfg) ?? "{0}/on-conflict-do-nothing/returning",
417+
OnConflictDoUpdateUrlPattern = GetConfigStr("OnConflictDoUpdateUrlPattern", crudSourceCfg) ?? "{0}/on-conflict-do-update",
418+
OnConflictDoUpdateReturningUrlPattern = GetConfigStr("OnConflictDoUpdateReturningUrlPattern", crudSourceCfg) ?? "{0}/on-conflict-do-update/returning",
412419
});
413420

414-
return [source];
421+
return sources;
415422
}
416423
}

‎NpgsqlRestClient/AppStaticFileMiddleware.cs

+217
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
using System.Buffers;
2+
using System.Collections.Concurrent;
3+
using System.IO.Pipelines;
4+
using System.Text;
5+
using Microsoft.AspNetCore.StaticFiles;
6+
using Microsoft.Extensions.FileProviders;
7+
using Microsoft.Extensions.Primitives;
8+
using Microsoft.Net.Http.Headers;
9+
10+
namespace NpgsqlRestClient;
11+
12+
public class AppStaticFileMiddleware
13+
{
14+
private readonly RequestDelegate _next;
15+
private readonly IWebHostEnvironment _hostingEnv;
16+
private static readonly FileExtensionContentTypeProvider _fileTypeProvider = new();
17+
18+
private static bool _parse;
19+
private static string[]? _parsePatterns = default!;
20+
private static string? _userIdTag = default!;
21+
private static string? _userNameTag = default!;
22+
private static string? _userRolesTag = default!;
23+
private static Dictionary<string, StringValues>? _customClaimTags = default!;
24+
private static Serilog.ILogger? _logger = default!;
25+
26+
private static readonly ConcurrentDictionary<string, bool> _pathInParsePattern = new();
27+
28+
const long StreamingThreshold = 1024 * 1024 * 10; // 10MB
29+
const int MaxBufferSize = 8192; // 8KB
30+
31+
public static void ConfigureStaticFileMiddleware(
32+
bool parse,
33+
string[]? parsePatterns,
34+
string? userIdTag,
35+
string? userNameTag,
36+
string? userRolesTag,
37+
Dictionary<string, StringValues>? customClaimTags,
38+
Serilog.ILogger? logger)
39+
{
40+
_parse = parse;
41+
_parsePatterns = parsePatterns == null || parsePatterns.Length == 0 ? null : parsePatterns?.Where(p => !string.IsNullOrEmpty(p)).ToArray();
42+
_userIdTag = string.IsNullOrEmpty(userIdTag) ? null : userIdTag;
43+
_userNameTag = string.IsNullOrEmpty(userNameTag) ? null : userNameTag;
44+
_userRolesTag = string.IsNullOrEmpty(userRolesTag) ? null : userRolesTag;
45+
_customClaimTags = customClaimTags == null || customClaimTags.Count == 0 ? null : customClaimTags;
46+
_logger = logger;
47+
}
48+
49+
public AppStaticFileMiddleware(RequestDelegate next, IWebHostEnvironment hostingEnv)
50+
{
51+
_next = next ?? throw new ArgumentNullException(nameof(next));
52+
_hostingEnv = hostingEnv ?? throw new ArgumentNullException(nameof(hostingEnv));
53+
}
54+
55+
public async Task InvokeAsync(HttpContext context)
56+
{
57+
string method = context.Request.Method;
58+
bool isGet = HttpMethods.IsGet(method);
59+
if (!isGet && !HttpMethods.IsHead(method))
60+
{
61+
await _next(context);
62+
return;
63+
}
64+
PathString path = context.Request.Path; // Cache PathString
65+
IFileInfo fileInfo = _hostingEnv.WebRootFileProvider.GetFileInfo(path);
66+
if (!fileInfo.Exists || fileInfo.IsDirectory)
67+
{
68+
await _next(context);
69+
return;
70+
}
71+
72+
string contentType = fileInfo.PhysicalPath != null && _fileTypeProvider.TryGetContentType(fileInfo.PhysicalPath, out var ct)
73+
? ct
74+
: "application/octet-stream";
75+
76+
DateTimeOffset lastModified = fileInfo.LastModified.ToUniversalTime();
77+
long length = fileInfo.Length;
78+
long etagHash = lastModified.ToFileTime() ^ length;
79+
string etagString = string.Concat("\"", Convert.ToString(etagHash, 16), "\"");
80+
81+
var ifNoneMatch = context.Request.Headers[HeaderNames.IfNoneMatch];
82+
if (!string.IsNullOrEmpty(ifNoneMatch) &&
83+
System.Net.Http.Headers.EntityTagHeaderValue.TryParse(ifNoneMatch, out var clientEtag) &&
84+
string.Equals(etagString, clientEtag.ToString(), StringComparison.Ordinal))
85+
{
86+
context.Response.StatusCode = StatusCodes.Status304NotModified;
87+
return;
88+
}
89+
90+
var ifModifiedSince = context.Request.Headers[HeaderNames.IfModifiedSince];
91+
if (!string.IsNullOrEmpty(ifModifiedSince) && DateTimeOffset.TryParse(ifModifiedSince, out var since) && since >= lastModified)
92+
{
93+
context.Response.StatusCode = StatusCodes.Status304NotModified;
94+
return;
95+
}
96+
context.Response.StatusCode = StatusCodes.Status200OK;
97+
context.Response.ContentType = contentType;
98+
context.Response.Headers[HeaderNames.LastModified] = lastModified.ToString("R");
99+
context.Response.Headers[HeaderNames.ETag] = etagString;
100+
context.Response.Headers[HeaderNames.AcceptRanges] = "bytes";
101+
102+
if (isGet)
103+
{
104+
try
105+
{
106+
using var fileStream = new FileStream(fileInfo.PhysicalPath!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 8192, useAsync: true);
107+
108+
if (_parse is false || _parsePatterns is null ||
109+
(_userIdTag is null && _userNameTag is null && _userRolesTag is null && _customClaimTags is null)
110+
)
111+
{
112+
context.Response.ContentLength = length;
113+
await fileStream.CopyToAsync(context.Response.Body, context.RequestAborted);
114+
return;
115+
}
116+
var pathString = path.ToString();
117+
if (_pathInParsePattern.TryGetValue(pathString, out bool isInParsePattern) is false)
118+
{
119+
isInParsePattern = false;
120+
for (int i = 0; i < _parsePatterns.Length; i++)
121+
{
122+
if (DefaultResponseParser.IsPatternMatch(pathString, _parsePatterns[i]))
123+
{
124+
isInParsePattern = true;
125+
break;
126+
}
127+
}
128+
_pathInParsePattern.TryAdd(pathString, isInParsePattern);
129+
}
130+
131+
if (isInParsePattern is false)
132+
{
133+
context.Response.ContentLength = length;
134+
await fileStream.CopyToAsync(context.Response.Body, context.RequestAborted);
135+
return;
136+
}
137+
138+
var parser = new DefaultResponseParser(
139+
userIdParameterName: _userIdTag,
140+
userNameParameterName: _userNameTag,
141+
userRolesParameterName: _userRolesTag,
142+
ipAddressParameterName: null,
143+
customClaims: _customClaimTags,
144+
customParameters: null);
145+
146+
147+
if (fileInfo.Length < StreamingThreshold)
148+
{
149+
byte[] buffer = new byte[(int)fileInfo.Length];
150+
await fileStream.ReadExactlyAsync(buffer, context.RequestAborted);
151+
int charCount = Encoding.UTF8.GetCharCount(buffer);
152+
char[] chars = ArrayPool<char>.Shared.Rent(charCount);
153+
try
154+
{
155+
Encoding.UTF8.GetChars(buffer, 0, buffer.Length, chars, 0);
156+
ReadOnlySpan<char> result = parser.Parse(new ReadOnlySpan<char>(chars, 0, charCount), context);
157+
var writer = PipeWriter.Create(context.Response.Body);
158+
try
159+
{
160+
int maxBytesNeeded = Encoding.UTF8.GetMaxByteCount(result.Length);
161+
Memory<byte> memory = writer.GetMemory(maxBytesNeeded);
162+
int actualBytesWritten = Encoding.UTF8.GetBytes(result, memory.Span);
163+
writer.Advance(actualBytesWritten);
164+
context.Response.ContentLength = actualBytesWritten;
165+
await writer.FlushAsync(context.RequestAborted);
166+
}
167+
finally
168+
{
169+
await writer.CompleteAsync();
170+
}
171+
}
172+
finally
173+
{
174+
ArrayPool<char>.Shared.Return(chars);
175+
}
176+
}
177+
else
178+
{
179+
var writer = PipeWriter.Create(context.Response.Body);
180+
try
181+
{
182+
using var reader = new StreamReader(fileStream, Encoding.UTF8, leaveOpen: true);
183+
char[] chars = ArrayPool<char>.Shared.Rent(MaxBufferSize);
184+
try
185+
{
186+
int charsRead;
187+
while ((charsRead = await reader.ReadAsync(chars, context.RequestAborted)) > 0)
188+
{
189+
var result = parser.Parse(chars.AsSpan(0, charsRead), context);
190+
int bytesWritten = Encoding.UTF8.GetBytes(result, writer.GetSpan(result.Length));
191+
writer.Advance(bytesWritten);
192+
await writer.FlushAsync(context.RequestAborted);
193+
}
194+
}
195+
finally
196+
{
197+
ArrayPool<char>.Shared.Return(chars);
198+
}
199+
}
200+
finally
201+
{
202+
await writer.CompleteAsync();
203+
}
204+
}
205+
}
206+
catch (IOException ex)
207+
{
208+
_logger?.Error(ex, "Failed to serve static file {Path}", path.ToString());
209+
context.Response.Clear();
210+
await _next(context);
211+
return;
212+
}
213+
}
214+
215+
// HEAD request completes here
216+
}
217+
}

0 commit comments

Comments
 (0)