Skip to content

Commit cddbec5

Browse files
committed
* partial revert f210ca5 to fix 8477e81 @ c#/crawler
1 parent efb62cb commit cddbec5

File tree

8 files changed

+15
-28
lines changed

8 files changed

+15
-28
lines changed

c#/crawler/src/Db/CrawlerDbContext.cs

+2-4
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,10 @@ protected override void OnModelCreating(ModelBuilder b)
6969
b.Entity<User>().ToTable("tbmc_user");
7070
b.Entity<ThreadPost>().ToTable($"tbmc_f{Fid}_thread");
7171
b.Entity<ThreadMissingFirstReply>().ToTable("tbmc_thread_missingFirstReply");
72-
b.Entity<ReplyPost>().ToTable($"tbmc_f{Fid}_reply")
73-
.HasOne(e => e.Content).WithOne().HasForeignKey<ReplyContent>(e => e.Pid);
72+
b.Entity<ReplyPost>().ToTable($"tbmc_f{Fid}_reply");
7473
b.Entity<ReplyContent>().ToTable($"tbmc_f{Fid}_reply_content");
7574
b.Entity<ReplySignature>().ToTable("tbmc_reply_signature").HasKey(e => new {e.SignatureId, e.XxHash3});
76-
b.Entity<SubReplyPost>().ToTable($"tbmc_f{Fid}_subReply")
77-
.HasOne(e => e.Content).WithOne().HasForeignKey<SubReplyContent>(e => e.Spid);
75+
b.Entity<SubReplyPost>().ToTable($"tbmc_f{Fid}_subReply");
7876
b.Entity<SubReplyContent>().ToTable($"tbmc_f{Fid}_subReply_content");
7977

8078
_ = new RevisionWithSplitting<BaseThreadRevision>

c#/crawler/src/Db/Post/PostWithContentAndAuthorExpGrade.cs

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
// ReSharper disable PropertyCanBeMadeInitOnly.Global
22
namespace tbm.Crawler.Db.Post;
33

4-
public abstract class PostWithContentAndAuthorExpGrade<TPostContent> : PostWithAuthorExpGrade
5-
where TPostContent : BasePostContent
4+
public abstract class PostWithContentAndAuthorExpGrade : PostWithAuthorExpGrade
65
{
7-
public required TPostContent Content { get; set; }
6+
[NotMapped] public byte[]? Content { get; set; }
87

98
[JsonConverter(typeof(ProtoBufRepeatedFieldJsonConverter<Content>))]
109
[NotMapped]

c#/crawler/src/Db/Post/ReplyPost.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// ReSharper disable PropertyCanBeMadeInitOnly.Global
22
namespace tbm.Crawler.Db.Post;
33

4-
public class ReplyPost : PostWithContentAndAuthorExpGrade<ReplyContent>
4+
public class ReplyPost : PostWithContentAndAuthorExpGrade
55
{
66
[Key] [Column(TypeName = "bigint")]
77
public ulong Pid { get; set; }

c#/crawler/src/Db/Post/SubReplyPost.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// ReSharper disable PropertyCanBeMadeInitOnly.Global
22
namespace tbm.Crawler.Db.Post;
33

4-
public class SubReplyPost : PostWithContentAndAuthorExpGrade<SubReplyContent>
4+
public class SubReplyPost : PostWithContentAndAuthorExpGrade
55
{
66
[Column(TypeName = "bigint")]
77
public ulong Pid { get; set; }

c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs

+2-10
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,13 @@ protected override IEnumerable<ReplyPost> ParseInternal
1414

1515
protected override ReplyPost Convert(Reply inPost)
1616
{
17-
var o = new ReplyPost
18-
{
19-
Content = null!, // will get mutated by SimplifyImagesInReplyContent()
20-
ContentsProtoBuf = inPost.Content
21-
};
17+
var o = new ReplyPost {ContentsProtoBuf = inPost.Content};
2218
try
2319
{
2420
o.Pid = inPost.Pid;
2521
o.Floor = inPost.Floor;
2622
SimplifyImagesInReplyContent(logger, ref inPost);
27-
o.Content = new()
28-
{
29-
Pid = inPost.Pid,
30-
ProtoBufBytes = Helper.SerializedProtoBufWrapperOrNullIfEmpty(inPost.Content, Helper.WrapPostContent)
31-
};
23+
o.Content = Helper.SerializedProtoBufWrapperOrNullIfEmpty(inPost.Content, Helper.WrapPostContent);
3224

3325
// AuthorId rarely respond with 0, Author should always be null with no guarantee
3426
o.AuthorUid = inPost.AuthorId.NullIfZero() ?? inPost.Author?.Uid ?? 0;

c#/crawler/src/Tieba/Crawl/Parser/Post/SubReplyParser.cs

+2-9
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,12 @@ protected override IEnumerable<SubReplyPost> ParseInternal
1313

1414
protected override SubReplyPost Convert(SubReply inPost)
1515
{
16-
var o = new SubReplyPost
17-
{
18-
Content = new()
19-
{
20-
Spid = inPost.Spid,
21-
ProtoBufBytes = Helper.SerializedProtoBufWrapperOrNullIfEmpty(inPost.Content, Helper.WrapPostContent)
22-
},
23-
ContentsProtoBuf = inPost.Content
24-
};
16+
var o = new SubReplyPost {ContentsProtoBuf = inPost.Content};
2517
try
2618
{
2719
var author = inPost.Author;
2820
o.Spid = inPost.Spid;
21+
o.Content = Helper.SerializedProtoBufWrapperOrNullIfEmpty(inPost.Content, Helper.WrapPostContent);
2922
o.AuthorUid = author.Uid;
3023
o.AuthorExpGrade = (byte)author.LevelId;
3124
o.PostedAt = inPost.Time;

c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs

+2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ public override SaverChangeSet<ReplyPost> Save(CrawlerDbContext db)
1717
r => new ReplyRevision {TakenAt = r.UpdatedAt ?? r.CreatedAt, Pid = r.Pid},
1818
LinqKit.PredicateBuilder.New<ReplyPost>(r => Posts.Keys.Contains(r.Pid)));
1919

20+
db.ReplyContents.AddRange(changeSet.NewlyAdded
21+
.Select(r => new ReplyContent {Pid = r.Pid, ProtoBufBytes = r.Content}));
2022
PostSaveHandlers += replyContentImageSaver.Save(db, changeSet.NewlyAdded).Invoke;
2123
PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
2224
PostSaveHandlers += replySignatureSaver.Save(db, changeSet.AllAfter).Invoke;

c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ public override SaverChangeSet<SubReplyPost> Save(CrawlerDbContext db)
1414
var changeSet = Save(db, sr => sr.Spid,
1515
sr => new SubReplyRevision {TakenAt = sr.UpdatedAt ?? sr.CreatedAt, Spid = sr.Spid},
1616
LinqKit.PredicateBuilder.New<SubReplyPost>(sr => Posts.Keys.Contains(sr.Spid)));
17+
18+
db.SubReplyContents.AddRange(changeSet.NewlyAdded.Select(sr =>
19+
new SubReplyContent {Spid = sr.Spid, ProtoBufBytes = sr.Content}));
1720
PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
1821

1922
return changeSet;

0 commit comments

Comments
 (0)