Feature/add reels models final

Andromeda.Commands/MigrateCommand.cs

@@ -37,6 +37,9 @@ public static void Migrate(Databases db, bool force_update = false) {
                 case Databases.LakeTikTok:
                     DatabaseOperations.Migrate<DataLakeTikTokContext>();
                     break;
+                case Databases.LakeReels:
+                    DatabaseOperations.Migrate<DataLakeReelsContext>();
+                    break;
                 default:
                     throw new Exception("Invalid database");
             }
@@ -51,6 +54,7 @@ public static void MigrateDataLake() {
             DatabaseOperations.Migrate<DataLakeTwitterAdsContext>();
             DatabaseOperations.Migrate<DataLakeLoggingContext>();
             DatabaseOperations.Migrate<DataLakeTikTokContext>();
+            DatabaseOperations.Migrate<DataLakeReelsContext>();
             Console.WriteLine("Successfully migrated data-lake.");
         }
 

Andromeda.Commands/RunJobsCommand.cs

@@ -9,6 +9,7 @@
 using Jobs.Fetcher.Facebook;
 using Jobs.Fetcher.Twitter;
 using Jobs.Fetcher.TikTok;
+using Jobs.Fetcher.Reels;
 
 namespace Andromeda.Commands {
 
@@ -21,6 +22,7 @@ public static class RunJobsCommand {
             new TwitterFetchers(),
             new YouTubeFetchers(),
             new YouTubeStudioFetchers(),
+            new ReelsFetchers()
         };
 
         public static Dictionary<string, AbstractJob> BuildJobsDict(IEnumerable<AbstractJob> jobList) {

Andromeda.Common/DatabaseHelper.cs

@@ -19,7 +19,8 @@ public enum Databases {
         LakeFacebook,
         LakeTwitterData,
         LakeTwitterAds,
-        LakeTikTok
+        LakeTikTok,
+        LakeReels,
     }
 
     public enum YearDatabase {
@@ -37,6 +38,8 @@ public static void Drop(Databases db) {
                 case Databases.LakeAdWords:
                 case Databases.LakeLogging:
                 case Databases.LakeTwitterData:
+                case Databases.LakeTikTok:
+                case Databases.LakeReels:
                     settingsField = "DataLakeDatabase";
                     break;
                 case Databases.LakeFacebook:

Andromeda.Common/Jobs/Enumerations.cs

@@ -13,6 +13,7 @@ public enum JobScope {
         Facebook,
         Instagram,
         Twitter,
-        TikTok
+        TikTok,
+        Reels
     }
 }

Andromeda.ConsoleApp/Andromeda.ConsoleApp.csproj

@@ -16,6 +16,7 @@
     <ProjectReference Include="..\Jobs.Fetcher.Facebook\Jobs.Fetcher.Facebook.csproj" />
     <ProjectReference Include="..\Jobs.Fetcher.Twitter\Jobs.Fetcher.Twitter.csproj" />
     <ProjectReference Include="..\Jobs.Fetcher.TikTok\Jobs.Fetcher.TikTok.csproj" />
+    <ProjectReference Include="..\Jobs.Fetcher.Reels\Jobs.Fetcher.Reels.csproj" />
   </ItemGroup>
 
   <ItemGroup>

DataLakeModels/DataLakeReelsContext.cs

@@ -0,0 +1,156 @@
+using System;
+using Microsoft.EntityFrameworkCore;
+using DataLakeModels.Models.Reels;
+
+namespace DataLakeModels {
+
+    public partial class DataLakeReelsContext : AbstractDataLakeContext {
+
+        public static string ReelsScraperVersion = "1";
+
+        protected override void OnModelCreating(ModelBuilder modelBuilder) {
+            base.OnModelCreating(modelBuilder);
+
+            modelBuilder.HasDefaultSchema($"reels_v{ReelsScraperVersion}");
+
+            modelBuilder.Entity<AnimatedThumbnail>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<Caption>()
+                .HasKey(table => new { table.Pk });
+
+            modelBuilder.Entity<ClipsMeta>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<CommentInfo>()
+                .HasKey(table => new { table.Pk });
+
+            modelBuilder.Entity<CommentInfo>()
+                .HasOne(table => table.Reel)
+                .WithMany(reel => reel.Comments)
+                .HasForeignKey(table => table.ReelId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<ConsumptionInfo>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<Friction>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<ImageVersion>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<Image>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<Image>()
+                .HasOne(table => table.ImageVersion)
+                .WithMany(imageVersion => imageVersion.Candidates)
+                .HasForeignKey(table => table.ImageVersionId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<AnimatedThumbnail>()
+                .HasOne(table => table.ImageVersion)
+                .WithOne(imageVersion => imageVersion.AnimatedThumbnailSpritesheetInfo)
+                .HasForeignKey<AnimatedThumbnail>(table => table.ImageVersionId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<MashupInfo>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<MashupInfo>()
+                .HasOne(table => table.Clip)
+                .WithOne(clip => clip.MashupInfo)
+                .HasForeignKey<MashupInfo>(table => table.ClipsId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<OriginalSound>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<OriginalSound>()
+                .HasOne(table => table.User)
+                .WithMany(reel => reel.Sounds)
+                .HasForeignKey(table => table.UserId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<Reel>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<Reel>()
+                .HasOne(table => table.User)
+                .WithMany(reel => reel.Reels)
+                .HasForeignKey(table => table.UserId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<Reel>()
+                .HasOne(table => table.Caption)
+                .WithOne(caption => caption.Reel)
+                .HasForeignKey<Caption>(table => table.ReelId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<Reel>()
+                .HasOne(table => table.ClipsMetaData)
+                .WithOne(clipsMeta => clipsMeta.Reel)
+                .HasForeignKey<ClipsMeta>(table => table.ReelId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<Reel>()
+                .HasOne(table => table.ImageVersions)
+                .WithOne(imageVersion => imageVersion.Reel)
+                .HasForeignKey<ImageVersion>(table => table.ReelId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<Reel>()
+                .HasOne(table => table.MediaCroppingInfo)
+                .WithOne(squareCrop => squareCrop.Reel)
+                .HasForeignKey<SquareCrop>(table => table.ReelId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<Reel>()
+                .HasOne(table => table.SharingFrictionInfo)
+                .WithOne(friction => friction.Reel)
+                .HasForeignKey<Friction>(table => table.ReelId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<ReelStats>()
+                .HasKey(table => new { table.ReelId, table.ValidityStart });
+
+            modelBuilder.Entity<ReelStats>()
+                .HasOne(table => table.Reel)
+                .WithMany(reel => reel.Stats)
+                .HasForeignKey(table => table.ReelId)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            modelBuilder.Entity<User>()
+                .HasKey(table => new { table.Pk });
+
+            modelBuilder.Entity<SquareCrop>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<VideoVersion>()
+                .HasKey(table => new { table.Id });
+
+            modelBuilder.Entity<VideoVersion>()
+                .HasOne(table => table.Reel)
+                .WithMany(reel => reel.VideoVersions)
+                .HasForeignKey(table => table.ReelId)
+                .OnDelete(DeleteBehavior.Cascade);
+        }
+
+        public virtual DbSet<User> Users { get; set; }
+        public virtual DbSet<Reel> Reels { get; set; }
+        public virtual DbSet<ImageVersion> ImageVersions { get; set; }
+        public virtual DbSet<Image> Images { get; set; }
+        public virtual DbSet<AnimatedThumbnail> AnimatedThumbnails { get; set; }
+        public virtual DbSet<Caption> Captions { get; set; }
+        public virtual DbSet<ClipsMeta> ClipsMetas { get; set; }
+        public virtual DbSet<CommentInfo> Comments { get; set; }
+        public virtual DbSet<ConsumptionInfo> ConsumptionInfos { get; set; }
+        public virtual DbSet<Friction> Frictions { get; set; }
+        public virtual DbSet<MashupInfo> MashupInfos { get; set; }
+        public virtual DbSet<OriginalSound> OriginalSounds { get; set; }
+        public virtual DbSet<ReelStats> ReelStats { get; set; }
+        public virtual DbSet<SquareCrop> SquareCrops { get; set; }
+        public virtual DbSet<VideoVersion> VideoVersions { get; set; }
+    }
+}

DataLakeModels/TikTokScraperDatabaseManager.cs → DataLakeModels/GeneralScraperDatabaseManager.cs

@@ -5,7 +5,7 @@
 
 namespace DataLakeModels {
 
-    public class TikTokScraperDatabaseManager {
+    public class GeneralScraperDatabaseManager {
 
         public const string ApiVersion = "1";
 
@@ -17,6 +17,6 @@ public static string ConnectionString() {
             return String.Format("{0};SearchPath={1};", ConnectionStringHelper.GetConnStr("DataLakeDatabase"), SchemaName());
         }
 
-        protected static ILogger Logger { get => Log.ForContext<TikTokScraperDatabaseManager>(); }
+        protected static ILogger Logger { get => Log.ForContext<GeneralScraperDatabaseManager>(); }
     }
 }