using CommandLine; using Microsoft.EntityFrameworkCore; using nClam; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Text; using System.Threading; using System.Threading.Tasks; using Teknik.Areas.Paste.Models; using Teknik.Areas.Stats.Models; using Teknik.Areas.Upload.Models; using Teknik.Areas.Users.Models; using Teknik.Areas.Users.Utility; using Teknik.Configuration; using Teknik.Data; using Teknik.StorageService; using Teknik.Utilities; using Teknik.Utilities.Cryptography; namespace Teknik.ServiceWorker { public class Program { private static string currentPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); private static string virusFile = Path.Combine(currentPath, "virusLogs.txt"); private static string errorFile = Path.Combine(currentPath, "errorLogs.txt"); private static string orphansFile = Path.Combine(currentPath, "orphanedFiles.txt"); private static string configPath = currentPath; private const string TAKEDOWN_REPORTER = "Teknik Automated System"; private static readonly object dbLock = new object(); private static readonly object scanStatsLock = new object(); public static event Action OutputEvent; public static int Main(string[] args) { try { Parser.Default.ParseArguments(args).WithParsed(options => { if (!string.IsNullOrEmpty(options.Config)) configPath = options.Config; if (Directory.Exists(configPath)) { Config config = Config.Load(configPath); Output(string.Format("[{0}] Started Server Maintenance Process.", DateTime.Now)); var optionsBuilder = new DbContextOptionsBuilder(); optionsBuilder.UseSqlServer(config.DbConnection); using (TeknikEntities db = new TeknikEntities(optionsBuilder.Options)) { // Scan all the uploads for viruses, and remove the bad ones if (options.ScanUploads && config.UploadConfig.ClamConfig.Enabled) { ScanUploads(config, db); } // Runs the migration if (options.Migrate) { // Run the overall migration calls TeknikMigration.RunMigration(db, config); } if (options.Expire) { ProcessExpirations(config, db); } if (options.Clean) { CleanStorage(config, db); } } Output(string.Format("[{0}] Finished Server Maintenance Process.", DateTime.Now)); } else { string msg = string.Format("[{0}] Config File does not exist.", DateTime.Now); File.AppendAllLines(errorFile, new List { msg }); Output(msg); } }); } catch (Exception ex) { string msg = string.Format("[{0}] Exception: {1}", DateTime.Now, ex.GetFullMessage(true)); File.AppendAllLines(errorFile, new List { msg }); Output(msg); } return -1; } public static void ScanUploads(Config config, TeknikEntities db) { Output(string.Format("[{0}] Started Virus Scan.", DateTime.Now)); List uploads = db.Uploads.ToList(); int maxConcurrency = 100; int totalCount = uploads.Count(); int totalScans = 0; int currentScan = 0; int totalViruses = 0; using (SemaphoreSlim concurrencySemaphore = new SemaphoreSlim(maxConcurrency)) { List runningTasks = new List(); foreach (Upload upload in uploads) { concurrencySemaphore.Wait(); currentScan++; Task scanTask = Task.Factory.StartNew(async () => { try { var virusDetected = await ScanUpload(config, db, upload, totalCount, currentScan); if (virusDetected) totalViruses++; totalScans++; } catch (Exception ex) { string errorMsg = string.Format("[{0}] Scan Error: {1}", DateTime.Now, ex.GetFullMessage(true, true)); File.AppendAllLines(errorFile, new List { errorMsg }); } finally { concurrencySemaphore.Release(); } }); if (scanTask != null) { runningTasks.Add(scanTask); } } Task.WaitAll(runningTasks.ToArray()); } Output(string.Format("Scanning Complete. {0} Scanned | {1} Viruses Found | {2} Total Files", totalScans, totalViruses, totalCount)); } private static async Task ScanUpload(Config config, TeknikEntities db, Upload upload, int totalCount, int currentCount) { bool virusDetected = false; var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig); var fileStream = storageService.GetFile(upload.FileName); if (fileStream != null) { // If the IV is set, and Key is set, then scan it if (!string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV)) { byte[] keyBytes = Encoding.UTF8.GetBytes(upload.Key); byte[] ivBytes = Encoding.UTF8.GetBytes(upload.IV); long maxUploadSize = config.UploadConfig.MaxUploadFileSize; if (upload.User != null) { // Check account total limits if (upload.User.UploadSettings.MaxUploadFileSize != null) maxUploadSize = upload.User.UploadSettings.MaxUploadFileSize.Value; } using (AesCounterStream aesStream = new AesCounterStream(fileStream, false, keyBytes, ivBytes)) { ClamClient clam = new ClamClient(config.UploadConfig.ClamConfig.Server, config.UploadConfig.ClamConfig.Port); clam.MaxStreamSize = maxUploadSize; ClamScanResult scanResult = await clam.SendAndScanFileAsync(aesStream); switch (scanResult.Result) { case ClamScanResults.Clean: string cleanMsg = string.Format("[{0}] Clean Scan: {1}/{2} Scanned | {3} - {4}", DateTime.Now, currentCount, totalCount, upload.Url, upload.FileName); Output(cleanMsg); break; case ClamScanResults.VirusDetected: string msg = string.Format("[{0}] Virus Detected: {1} - {2} - {3}", DateTime.Now, upload.Url, upload.FileName, scanResult.InfectedFiles.First().VirusName); Output(msg); lock (scanStatsLock) { virusDetected = true; File.AppendAllLines(virusFile, new List { msg }); } lock (dbLock) { string urlName = upload.Url; // Delete the File storageService.DeleteFile(upload.FileName); // Delete from the DB db.Uploads.Remove(upload); // Add to transparency report if any were found Takedown report = new Takedown(); report.Requester = TAKEDOWN_REPORTER; report.RequesterContact = config.SupportEmail; report.DateRequested = DateTime.Now; report.Reason = "Malware Found"; report.ActionTaken = string.Format("Upload removed: {0}", urlName); report.DateActionTaken = DateTime.Now; db.Takedowns.Add(report); // Save Changes db.SaveChanges(); } break; case ClamScanResults.Error: string errorMsg = string.Format("[{0}] Scan Error: {1}", DateTime.Now, scanResult.RawResult); File.AppendAllLines(errorFile, new List { errorMsg }); Output(errorMsg); break; case ClamScanResults.Unknown: string unkMsg = string.Format("[{0}] Unknown Scan Result: {1}", DateTime.Now, scanResult.RawResult); File.AppendAllLines(errorFile, new List { unkMsg }); Output(unkMsg); break; } } } } return virusDetected; } public static void ProcessExpirations(Config config, TeknikEntities db) { Output(string.Format("[{0}] Starting processing expirations.", DateTime.Now)); var curDate = DateTime.Now; // Process uploads List uploads = db.Uploads.Where(u => u.ExpireDate != null && u.ExpireDate < curDate).ToList(); var uploadStorageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig); foreach (Upload upload in uploads) { DeleteFile(uploadStorageService, upload.FileName); } db.RemoveRange(uploads); db.SaveChanges(); // Process Pastes List pastes = db.Pastes.Where(p => p.ExpireDate != null && p.ExpireDate < curDate).ToList(); var pasteStorageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig); foreach (Paste paste in pastes) { // Delete the File DeleteFile(pasteStorageService, paste.FileName); } db.RemoveRange(pastes); db.SaveChanges(); } public static void CleanStorage(Config config, TeknikEntities db) { // Process upload data Output(string.Format("[{0}] Starting processing upload storage cleaning.", DateTime.Now)); CleanUploadFiles(config, db); // Process paste data Output(string.Format("[{0}] Starting processing upload storage cleaning.", DateTime.Now)); CleanPasteFiles(config, db); } public static void CleanUploadFiles(Config config, TeknikEntities db) { var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig); List files = storageService.GetFileNames(); List uploads = db.Uploads.Where(u => !string.IsNullOrEmpty(u.FileName)).Select(u => u.FileName.ToLower()).ToList(); var orphans = files.Except(uploads); File.AppendAllLines(orphansFile, orphans); foreach (var orphan in orphans) { DeleteFile(storageService, orphan); } } public static void CleanPasteFiles(Config config, TeknikEntities db) { var storageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig); List files = storageService.GetFileNames(); List pastes = db.Pastes.Where(p => !string.IsNullOrEmpty(p.FileName)).Select(p => p.FileName.ToLower()).ToList(); var orphans = files.Except(pastes); File.AppendAllLines(orphansFile, orphans); foreach (var orphan in orphans) { DeleteFile(storageService, orphan); } } public static void DeleteFile(IStorageService storageService, string fileName) { try { storageService.DeleteFile(fileName); } catch (Exception ex) { Output(string.Format("[{0}] Unable to delete file: {1} | {2}", DateTime.Now, fileName, ex.ToString())); } } public static void Output(string message) { Console.WriteLine(message); if (OutputEvent != null) { OutputEvent(message); } } } }