From 933fb9d4cd908be611587fed9f51fbe7fb7d1599 Mon Sep 17 00:00:00 2001 From: Uncled1023 Date: Wed, 19 Dec 2018 21:52:46 -0800 Subject: [PATCH] Added service worker for handling routine tasks outside the scope of the main site. --- .gitignore | 3 +- ServiceWorker/ArgumentOptions.cs | 23 +++ ServiceWorker/Program.cs | 213 ++++++++++++++++++++++++ ServiceWorker/ServiceWorker.csproj | 18 ++ Teknik.sln | 8 + Teknik/Controllers/DefaultController.cs | 5 - Teknik/Data/DbInitializer.cs | 11 -- Teknik/Startup.cs | 3 + Teknik/TeknikMigration.cs | 19 +++ 9 files changed, 286 insertions(+), 17 deletions(-) create mode 100644 ServiceWorker/ArgumentOptions.cs create mode 100644 ServiceWorker/Program.cs create mode 100644 ServiceWorker/ServiceWorker.csproj delete mode 100644 Teknik/Data/DbInitializer.cs create mode 100644 Teknik/TeknikMigration.cs diff --git a/.gitignore b/.gitignore index 893511b..7c6fdc3 100644 --- a/.gitignore +++ b/.gitignore @@ -265,4 +265,5 @@ __pycache__/ /Teknik/App_Data/version.json **/appsettings.*.json -**/tempkey.rsa \ No newline at end of file +**/tempkey.rsa +/ServiceWorker/Properties/launchSettings.json diff --git a/ServiceWorker/ArgumentOptions.cs b/ServiceWorker/ArgumentOptions.cs new file mode 100644 index 0000000..ca24f00 --- /dev/null +++ b/ServiceWorker/ArgumentOptions.cs @@ -0,0 +1,23 @@ +using CommandLine; +using System; +using System.Collections.Generic; +using System.Text; + +namespace ServiceWorker +{ + public class ArgumentOptions + { + [Option('a', "all", Default = false, Required = false, HelpText = "Run All Processes")] + public bool RunAll { get; set; } + + [Option('c', "config", Required = false, HelpText = "The path to the teknik config file")] + public string Config { get; set; } + + [Option('s', "scan", Default = false, Required = false, HelpText = "Scan all uploads for viruses")] + public bool ScanUploads { get; set; } + + // Omitting long name, default --verbose + [Option(HelpText = "Prints all messages to standard output.")] + public bool Verbose { get; set; } + } +} diff --git a/ServiceWorker/Program.cs b/ServiceWorker/Program.cs new file mode 100644 index 0000000..8c3011b --- /dev/null +++ b/ServiceWorker/Program.cs @@ -0,0 +1,213 @@ +using CommandLine; +using Microsoft.EntityFrameworkCore; +using nClam; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Teknik.Areas.Stats.Models; +using Teknik.Areas.Upload.Models; +using Teknik.Areas.Users.Models; +using Teknik.Areas.Users.Utility; +using Teknik.Configuration; +using Teknik.Data; +using Teknik.Utilities; +using Teknik.Utilities.Cryptography; + +namespace ServiceWorker +{ + public class Program + { + private static string currentPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); + private static string virusFile = Path.Combine(currentPath, "virusLogs.txt"); + private static string errorFile = Path.Combine(currentPath, "errorLogs.txt"); + private static string configPath = currentPath; + + private const string TAKEDOWN_REPORTER = "Teknik Automated System"; + + private static readonly object dbLock = new object(); + private static readonly object scanStatsLock = new object(); + + public static event Action OutputEvent; + + public static int Main(string[] args) + { + try + { + Parser.Default.ParseArguments(args).WithParsed(options => + { + if (!string.IsNullOrEmpty(options.Config)) + configPath = options.Config; + + if (Directory.Exists(configPath)) + { + Config config = Config.Load(configPath); + Output(string.Format("[{0}] Started Server Maintenance Process.", DateTime.Now)); + + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseSqlServer("Data Source=blog.db"); + + using (TeknikEntities db = new TeknikEntities(optionsBuilder.Options)) + { + // Scan all the uploads for viruses, and remove the bad ones + if (options.ScanUploads && config.UploadConfig.VirusScanEnable) + { + ScanUploads(config, db); + } + } + + Output(string.Format("[{0}] Finished Server Maintenance Process.", DateTime.Now)); + } + else + { + string msg = string.Format("[{0}] Config File does not exist.", DateTime.Now); + File.AppendAllLines(errorFile, new List { msg }); + Output(msg); + } + }); + } + catch (Exception ex) + { + string msg = string.Format("[{0}] Exception: {1}", DateTime.Now, ex.GetFullMessage(true)); + File.AppendAllLines(errorFile, new List { msg }); + Output(msg); + } + return -1; + } + + public static void ScanUploads(Config config, TeknikEntities db) + { + Output(string.Format("[{0}] Started Virus Scan.", DateTime.Now)); + List uploads = db.Uploads.ToList(); + + int totalCount = uploads.Count(); + int totalScans = 0; + int totalViruses = 0; + List runningTasks = new List(); + foreach (Upload upload in uploads) + { + int currentScan = totalScans++; + Task scanTask = Task.Factory.StartNew(async () => + { + var virusDetected = await ScanUpload(config, db, upload, totalCount, currentScan); + if (virusDetected) + totalViruses++; + }); + if (scanTask != null) + { + runningTasks.Add(scanTask); + } + } + bool running = true; + + while (running) + { + running = runningTasks.Exists(s => s != null && !s.IsCompleted && !s.IsCanceled && !s.IsFaulted); + } + + Output(string.Format("Scanning Complete. {0} Scanned | {1} Viruses Found | {2} Total Files", totalScans, totalViruses, totalCount)); + } + + private static async Task ScanUpload(Config config, TeknikEntities db, Upload upload, int totalCount, int currentCount) + { + bool virusDetected = false; + string subDir = upload.FileName[0].ToString(); + string filePath = Path.Combine(config.UploadConfig.UploadDirectory, subDir, upload.FileName); + if (File.Exists(filePath)) + { + // If the IV is set, and Key is set, then scan it + if (!string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV)) + { + byte[] keyBytes = Encoding.UTF8.GetBytes(upload.Key); + byte[] ivBytes = Encoding.UTF8.GetBytes(upload.IV); + + + long maxUploadSize = config.UploadConfig.MaxUploadSize; + if (upload.User != null) + { + maxUploadSize = config.UploadConfig.MaxUploadSizeBasic; + IdentityUserInfo userInfo = await IdentityHelper.GetIdentityUserInfo(config, upload.User.Username); + if (userInfo.AccountType == AccountType.Premium) + { + maxUploadSize = config.UploadConfig.MaxUploadSizePremium; + } + } + + using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read)) + using (AesCounterStream aesStream = new AesCounterStream(fs, false, keyBytes, ivBytes)) + { + ClamClient clam = new ClamClient(config.UploadConfig.ClamServer, config.UploadConfig.ClamPort); + clam.MaxStreamSize = maxUploadSize; + ClamScanResult scanResult = await clam.SendAndScanFileAsync(fs); + + switch (scanResult.Result) + { + case ClamScanResults.Clean: + string cleanMsg = string.Format("[{0}] Clean Scan: {1}/{2} Scanned | {3} - {4}", DateTime.Now, currentCount, totalCount, upload.Url, upload.FileName); + Output(cleanMsg); + break; + case ClamScanResults.VirusDetected: + string msg = string.Format("[{0}] Virus Detected: {1} - {2} - {3}", DateTime.Now, upload.Url, upload.FileName, scanResult.InfectedFiles.First().VirusName); + Output(msg); + lock (scanStatsLock) + { + virusDetected = true; + File.AppendAllLines(virusFile, new List { msg }); + } + + lock (dbLock) + { + string urlName = upload.Url; + // Delete from the DB + db.Uploads.Remove(upload); + + // Delete the File + if (File.Exists(filePath)) + { + File.Delete(filePath); + } + + // Add to transparency report if any were found + Takedown report = new Takedown(); + report.Requester = TAKEDOWN_REPORTER; + report.RequesterContact = config.SupportEmail; + report.DateRequested = DateTime.Now; + report.Reason = "Malware Found"; + report.ActionTaken = string.Format("Upload removed: {0}", urlName); + report.DateActionTaken = DateTime.Now; + db.Takedowns.Add(report); + + // Save Changes + db.SaveChanges(); + } + break; + case ClamScanResults.Error: + string errorMsg = string.Format("[{0}] Scan Error: {1}", DateTime.Now, scanResult.RawResult); + File.AppendAllLines(errorFile, new List { errorMsg }); + Output(errorMsg); + break; + case ClamScanResults.Unknown: + string unkMsg = string.Format("[{0}] Unknown Scan Result: {1}", DateTime.Now, scanResult.RawResult); + File.AppendAllLines(errorFile, new List { unkMsg }); + Output(unkMsg); + break; + } + } + } + } + return virusDetected; + } + + public static void Output(string message) + { + Console.WriteLine(message); + if (OutputEvent != null) + { + OutputEvent(message); + } + } + } +} diff --git a/ServiceWorker/ServiceWorker.csproj b/ServiceWorker/ServiceWorker.csproj new file mode 100644 index 0000000..d11974a --- /dev/null +++ b/ServiceWorker/ServiceWorker.csproj @@ -0,0 +1,18 @@ + + + + Exe + netcoreapp2.1 + + + + + + + + + + + + + diff --git a/Teknik.sln b/Teknik.sln index 5f57d9a..81c874c 100644 --- a/Teknik.sln +++ b/Teknik.sln @@ -26,6 +26,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MailService", "MailService\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GitService", "GitService\GitService.csproj", "{014879B1-DDD5-4F8C-9597-6D7960912CF0}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceWorker", "ServiceWorker\ServiceWorker.csproj", "{0B712243-994C-4AC3-893C-B86B59F63F53}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -75,6 +77,12 @@ Global {014879B1-DDD5-4F8C-9597-6D7960912CF0}.Release|Any CPU.Build.0 = Release|Any CPU {014879B1-DDD5-4F8C-9597-6D7960912CF0}.Test|Any CPU.ActiveCfg = Release|Any CPU {014879B1-DDD5-4F8C-9597-6D7960912CF0}.Test|Any CPU.Build.0 = Release|Any CPU + {0B712243-994C-4AC3-893C-B86B59F63F53}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0B712243-994C-4AC3-893C-B86B59F63F53}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0B712243-994C-4AC3-893C-B86B59F63F53}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0B712243-994C-4AC3-893C-B86B59F63F53}.Release|Any CPU.Build.0 = Release|Any CPU + {0B712243-994C-4AC3-893C-B86B59F63F53}.Test|Any CPU.ActiveCfg = Debug|Any CPU + {0B712243-994C-4AC3-893C-B86B59F63F53}.Test|Any CPU.Build.0 = Debug|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Teknik/Controllers/DefaultController.cs b/Teknik/Controllers/DefaultController.cs index 2e160ba..b6e4677 100644 --- a/Teknik/Controllers/DefaultController.cs +++ b/Teknik/Controllers/DefaultController.cs @@ -37,11 +37,6 @@ namespace Teknik.Controllers protected readonly Config _config; protected readonly TeknikEntities _dbContext; - //protected virtual new TeknikPrincipal User - //{ - // get { return HttpContext.User as TeknikPrincipal; } - //} - public DefaultController(ILogger logger, Config config, TeknikEntities dbContext) { _logger = logger; diff --git a/Teknik/Data/DbInitializer.cs b/Teknik/Data/DbInitializer.cs deleted file mode 100644 index f754d9b..0000000 --- a/Teknik/Data/DbInitializer.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace Teknik.Data -{ - public static class DbInitializer - { - } -} diff --git a/Teknik/Startup.cs b/Teknik/Startup.cs index fbeeb1c..5854c29 100644 --- a/Teknik/Startup.cs +++ b/Teknik/Startup.cs @@ -237,6 +237,9 @@ namespace Teknik // Create and Migrate the database dbContext.Database.Migrate(); + // Run the overall migration calls + TeknikMigration.RunMigration(); + // Initiate Logging loggerFactory.AddLogger(config); diff --git a/Teknik/TeknikMigration.cs b/Teknik/TeknikMigration.cs new file mode 100644 index 0000000..f6a62cf --- /dev/null +++ b/Teknik/TeknikMigration.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Teknik +{ + public static class TeknikMigration + { + public static bool RunMigration() + { + bool success = false; + + // Transfer User security info to + + return success; + } + } +}