using nClam; using System; using System.Collections.Generic; using System.Data.Entity; using System.IO; using System.Linq; using System.Net; using System.Net.Mail; using System.Reflection; using System.Text; using Teknik.Areas.Stats.Models; using Teknik.Areas.Upload.Models; using Teknik.Areas.Users.Models; using Teknik.Areas.Users.Utility; using Teknik.Configuration; using Teknik.Utilities; using Teknik.Models; using System.Threading.Tasks; using Teknik.Utilities.Cryptography; namespace ServerMaint { public class Program { private static string currentPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); private static string virusFile = Path.Combine(currentPath, "virusLogs.txt"); private static string errorFile = Path.Combine(currentPath, "errorLogs.txt"); private static string configPath = currentPath; private const string TAKEDOWN_REPORTER = "Teknik Automated System"; private static readonly object dbLock = new object(); private static readonly object scanStatsLock = new object(); public static event Action OutputEvent; public static int Main(string[] args) { try { ArgumentOptions options = new ArgumentOptions(); var parser = new CommandLine.Parser(config => config.HelpWriter = Console.Out); if (parser.ParseArguments(args, options)) { if (!string.IsNullOrEmpty(options.Config)) configPath = options.Config; if (Directory.Exists(configPath)) { Config config = Config.Load(configPath); Output(string.Format("[{0}] Started Server Maintenance Process.", DateTime.Now)); using (TeknikEntities db = new TeknikEntities()) { // Scan all the uploads for viruses, and remove the bad ones if (options.ScanUploads && config.UploadConfig.VirusScanEnable) { ScanUploads(config, db); } // Warns all the invalid accounts via email if (options.WarnAccounts) { WarnInvalidAccounts(config, db); } // Cleans all inactive users if (options.CleanUsers) { CleanAccounts(config, db, options.DaysBeforeDeletion); } // Cleans the email for unused accounts if (options.CleanEmails) { CleanEmail(config, db); } // Cleans all the git accounts that are unused if (options.CleanGit) { CleanGit(config, db); } // Generates a file for all of the user's last seen dates if (options.GenerateLastSeen) { GenerateLastSeen(config, db, options.LastSeenFile); } // Generates a file for all of the invalid accounts if (options.GenerateInvalid) { GenerateInvalidAccounts(config, db, options.InvalidFile); } // Generates a file for all of the accounts to be cleaned if (options.GenerateCleaning) { GenerateCleaningList(config, db, options.CleaningFile, options.DaysBeforeDeletion); } } Output(string.Format("[{0}] Finished Server Maintenance Process.", DateTime.Now)); return 0; } else { string msg = string.Format("[{0}] Config File does not exist.", DateTime.Now); File.AppendAllLines(errorFile, new List { msg }); Output(msg); } } else { Output(options.GetUsage()); } } catch (Exception ex) { string msg = string.Format("[{0}] Exception: {1}", DateTime.Now, ex.GetFullMessage(true)); File.AppendAllLines(errorFile, new List { msg }); Output(msg); } return -1; } public static void ScanUploads(Config config, TeknikEntities db) { Output(string.Format("[{0}] Started Virus Scan.", DateTime.Now)); List uploads = db.Uploads.ToList(); int totalCount = uploads.Count(); int totalScans = 0; int totalViruses = 0; List runningTasks = new List(); foreach (Upload upload in uploads) { int currentScan = totalScans++; Task scanTask = Task.Factory.StartNew(() => ScanUpload(config, db, upload, totalCount, currentScan, ref totalViruses)); if (scanTask != null) { runningTasks.Add(scanTask); } } bool running = true; while (running) { running = runningTasks.Exists(s => s != null && !s.IsCompleted && !s.IsCanceled && !s.IsFaulted); } Output(string.Format("Scanning Complete. {0} Scanned | {1} Viruses Found | {2} Total Files", totalScans, totalViruses, totalCount)); } private static void ScanUpload(Config config, TeknikEntities db, Upload upload, int totalCount, int currentCount, ref int totalViruses) { // Initialize ClamAV ClamClient clam = new ClamClient(config.UploadConfig.ClamServer, config.UploadConfig.ClamPort); clam.MaxStreamSize = config.UploadConfig.MaxUploadSize; string subDir = upload.FileName[0].ToString(); string filePath = Path.Combine(config.UploadConfig.UploadDirectory, subDir, upload.FileName); if (File.Exists(filePath)) { // If the IV is set, and Key is set, then scan it if (!string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV)) { byte[] keyBytes = Encoding.UTF8.GetBytes(upload.Key); byte[] ivBytes = Encoding.UTF8.GetBytes(upload.IV); FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read); AesCounterStream aesStream = new AesCounterStream(fs, false, keyBytes, ivBytes); // We have the data, let's scan it ClamScanResult scanResult = clam.SendAndScanFile(aesStream); // Close file stream fs.Close(); switch (scanResult.Result) { case ClamScanResults.Clean: string cleanMsg = string.Format("[{0}] Clean Scan: {1}/{2} Scanned | {3} - {4}", DateTime.Now, currentCount, totalCount, upload.Url, upload.FileName); Output(cleanMsg); break; case ClamScanResults.VirusDetected: string msg = string.Format("[{0}] Virus Detected: {1} - {2} - {3}", DateTime.Now, upload.Url, upload.FileName, scanResult.InfectedFiles.First().VirusName); Output(msg); lock (scanStatsLock) { totalViruses++; File.AppendAllLines(virusFile, new List { msg }); } lock (dbLock) { string urlName = upload.Url; // Delete from the DB db.Uploads.Remove(upload); // Delete the File if (File.Exists(filePath)) { File.Delete(filePath); } // Add to transparency report if any were found Takedown report = db.Takedowns.Create(); report.Requester = TAKEDOWN_REPORTER; report.RequesterContact = config.SupportEmail; report.DateRequested = DateTime.Now; report.Reason = "Malware Found"; report.ActionTaken = string.Format("Upload removed: {0}", urlName); report.DateActionTaken = DateTime.Now; db.Takedowns.Add(report); // Save Changes db.SaveChanges(); } break; case ClamScanResults.Error: string errorMsg = string.Format("[{0}] Scan Error: {1}", DateTime.Now, scanResult.RawResult); File.AppendAllLines(errorFile, new List { errorMsg }); Output(errorMsg); break; case ClamScanResults.Unknown: string unkMsg = string.Format("[{0}] Unknown Scan Result: {1}", DateTime.Now, scanResult.RawResult); File.AppendAllLines(errorFile, new List { unkMsg }); Output(unkMsg); break; } } } } public static void WarnInvalidAccounts(Config config, TeknikEntities db) { Output(string.Format("[{0}] Started Warning of Invalid Accounts.", DateTime.Now)); List invalidAccounts = GetInvalidAccounts(config, db); foreach (string account in invalidAccounts) { // Let's send them an email :D string email = UserHelper.GetUserEmailAddress(config, account); SmtpClient client = new SmtpClient(); client.Host = config.ContactConfig.EmailAccount.Host; client.Port = config.ContactConfig.EmailAccount.Port; client.EnableSsl = config.ContactConfig.EmailAccount.SSL; client.DeliveryMethod = SmtpDeliveryMethod.Network; client.UseDefaultCredentials = true; client.Credentials = new NetworkCredential(config.ContactConfig.EmailAccount.Username, config.ContactConfig.EmailAccount.Password); client.Timeout = 5000; try { MailMessage mail = new MailMessage(config.ContactConfig.EmailAccount.EmailAddress, email); mail.Subject = "Invalid Account Notice"; mail.Body = string.Format(@" The account {0} does not meet the requirements for a valid username. The username must meet the following requirements: {1} It must also be greater than or equal to {2} characters in length, and less than or equal to {3} characters in length. This email is to let you know that this account will be deleted in {4} days ({5}) in order to comply with the username restrictions. If you would like to keep your data, you should create a new account and transfer the data over to the new account. In order to make the process as easy as possible, you can reply to this email to ask for your current account to be renamed to another available account. This would keep all your data intact, and just require you to change all references to your email/git/user to the new username. If you wish to do this, please respond within {6} days ({7}) with the new username you would like to use. Thank you for your continued use of Teknik! - Teknik Administration", account, config.UserConfig.UsernameFilterLabel, config.UserConfig.MinUsernameLength, config.UserConfig.MaxUsernameLength, 30, DateTime.Now.AddDays(30).ToShortDateString(), 15, DateTime.Now.AddDays(15).ToShortDateString()); mail.BodyEncoding = UTF8Encoding.UTF8; mail.DeliveryNotificationOptions = DeliveryNotificationOptions.Never; client.Send(mail); } catch (Exception ex) { Output(string.Format("[{0}] Unable to send email to {1}. Exception: {2}", DateTime.Now, email, ex.Message)); } } Output(string.Format("[{0}] Finished Warning of Invalid Accounts. {1} Accounts Warned.", DateTime.Now, invalidAccounts.Count)); } public static void CleanAccounts(Config config, TeknikEntities db, int maxDays) { Output(string.Format("[{0}] Started Cleaning of Inactive/Invalid Users.", DateTime.Now)); List invalidAccounts = GetInvalidAccounts(config, db); List inactiveAccounts = GetInactiveAccounts(config, db, maxDays); // Delete invalid accounts foreach (string account in invalidAccounts) { UserHelper.DeleteAccount(db, config, UserHelper.GetUser(db, account)); } if (invalidAccounts.Count > 0) { // Add to transparency report if any users were removed Takedown report = db.Takedowns.Create(); report.Requester = TAKEDOWN_REPORTER; report.RequesterContact = config.SupportEmail; report.DateRequested = DateTime.Now; report.Reason = "Username Invalid"; report.ActionTaken = string.Format("{0} Accounts Removed", invalidAccounts.Count); report.DateActionTaken = DateTime.Now; db.Takedowns.Add(report); db.SaveChanges(); } // Delete inactive accounts foreach (string account in inactiveAccounts) { UserHelper.DeleteAccount(db, config, UserHelper.GetUser(db, account)); } if (invalidAccounts.Count > 0) { // Add to transparency report if any users were removed Takedown report = db.Takedowns.Create(); report.Requester = TAKEDOWN_REPORTER; report.RequesterContact = config.SupportEmail; report.DateRequested = DateTime.Now; report.Reason = "Account Inactive"; report.ActionTaken = string.Format("{0} Accounts Removed", inactiveAccounts.Count); report.DateActionTaken = DateTime.Now; db.Takedowns.Add(report); db.SaveChanges(); } Output(string.Format("[{0}] Finished Cleaning of Inactive/Invalid Users. {1} Accounts Removed.", DateTime.Now, invalidAccounts.Count + inactiveAccounts.Count)); } public static void CleanEmail(Config config, TeknikEntities db) { Output(string.Format("[{0}] Started Cleaning of Orphaned Email Accounts.", DateTime.Now)); List emails = GetOrphanedEmail(config, db); foreach (string email in emails) { // User doesn't exist, and it isn't reserved. Let's nuke it. UserHelper.DeleteUserEmail(config, email); } if (emails.Count > 0) { // Add to transparency report if any users were removed Takedown report = db.Takedowns.Create(); report.Requester = TAKEDOWN_REPORTER; report.RequesterContact = config.SupportEmail; report.DateRequested = DateTime.Now; report.Reason = "Orphaned Email Account"; report.ActionTaken = string.Format("{0} Accounts Removed", emails.Count); report.DateActionTaken = DateTime.Now; db.Takedowns.Add(report); db.SaveChanges(); } Output(string.Format("[{0}] Finished Cleaning of Orphaned Email Accounts. {1} Accounts Removed.", DateTime.Now, emails.Count)); } public static void CleanGit(Config config, TeknikEntities db) { Output(string.Format("[{0}] Started Cleaning of Orphaned Git Accounts.", DateTime.Now)); List gitAccounts = GetOrphanedGit(config, db); foreach (string account in gitAccounts) { // User doesn't exist, and it isn't reserved. Let's nuke it. UserHelper.DeleteUserGit(config, account); } if (gitAccounts.Count > 0) { // Add to transparency report if any users were removed Takedown report = db.Takedowns.Create(); report.Requester = TAKEDOWN_REPORTER; report.RequesterContact = config.SupportEmail; report.DateRequested = DateTime.Now; report.Reason = "Orphaned Git Account"; report.ActionTaken = string.Format("{0} Accounts Removed", gitAccounts.Count); report.DateActionTaken = DateTime.Now; db.Takedowns.Add(report); db.SaveChanges(); } Output(string.Format("[{0}] Finished Cleaning of Orphaned Git Accounts. {1} Accounts Removed.", DateTime.Now, gitAccounts.Count)); } public static void GenerateLastSeen(Config config, TeknikEntities db, string fileName) { Output(string.Format("[{0}] Started Generation of Last Activity List.", DateTime.Now)); List curUsers = db.Users.ToList(); StringBuilder sb = new StringBuilder(); sb.AppendLine("Username,Last Activity,Creation Date,Last Website Activity,Last Email Activity,Last Git Activity"); foreach (User user in curUsers) { sb.AppendLine(string.Format("{0},{1},{2},{3},{4},{5}", user.Username, UserHelper.GetLastAccountActivity(db, config, user).ToString("g"), user.JoinDate.ToString("g"), user.LastSeen.ToString("g"), UserHelper.UserEmailLastActive(config, UserHelper.GetUserEmailAddress(config, user.Username)).ToString("g"), UserHelper.UserGitLastActive(config, user.Username).ToString("g"))); } string dir = Path.GetDirectoryName(fileName); if (!Directory.Exists(dir)) Directory.CreateDirectory(dir); File.WriteAllText(fileName, sb.ToString()); Output(string.Format("[{0}] Finished Generating Last Activity List.", DateTime.Now)); } public static void GenerateInvalidAccounts(Config config, TeknikEntities db, string fileName) { Output(string.Format("[{0}] Started Generation of Invalid Account List.", DateTime.Now)); List invalidAccounts = GetInvalidAccounts(config, db); StringBuilder sb = new StringBuilder(); sb.AppendLine("Username,Last Activity,Creation Date,Last Website Activity,Last Email Activity,Last Git Activity"); foreach (string account in invalidAccounts) { User user = UserHelper.GetUser(db, account); sb.AppendLine(string.Format("{0},{1},{2},{3},{4},{5}", user.Username, UserHelper.GetLastAccountActivity(db, config, user).ToString("g"), user.JoinDate.ToString("g"), user.LastSeen.ToString("g"), UserHelper.UserEmailLastActive(config, UserHelper.GetUserEmailAddress(config, user.Username)).ToString("g"), UserHelper.UserGitLastActive(config, user.Username).ToString("g"))); } string dir = Path.GetDirectoryName(fileName); if (!Directory.Exists(dir)) Directory.CreateDirectory(dir); File.WriteAllText(fileName, sb.ToString()); Output(string.Format("[{0}] Finished Generating Invalid Account List.", DateTime.Now)); } public static void GenerateCleaningList(Config config, TeknikEntities db, string fileName, int maxDays) { Output(string.Format("[{0}] Started Generation of Accounts to Clean List.", DateTime.Now)); List invalidAccounts = GetInvalidAccounts(config, db); List inactiveAccounts = GetInactiveAccounts(config, db, maxDays); List emailAccounts = GetOrphanedEmail(config, db); List gitAccounts = GetOrphanedGit(config, db); StringBuilder sb = new StringBuilder(); sb.AppendLine("Invalid Account Cleaning"); sb.AppendLine("Username,Last Activity,Creation Date,Last Website Activity,Last Email Activity,Last Git Activity"); foreach (string account in invalidAccounts) { User user = UserHelper.GetUser(db, account); sb.AppendLine(string.Format("{0},{1},{2},{3},{4},{5}", user.Username, UserHelper.GetLastAccountActivity(db, config, user).ToString("g"), user.JoinDate.ToString("g"), user.LastSeen.ToString("g"), UserHelper.UserEmailExists(config, UserHelper.GetUserEmailAddress(config, user.Username)) ? UserHelper.UserEmailLastActive(config, UserHelper.GetUserEmailAddress(config, user.Username)).ToString("g") : null, UserHelper.UserGitExists(config, user.Username) ? UserHelper.UserGitLastActive(config, user.Username).ToString("g") : null)); } sb.AppendLine(); sb.AppendLine("Inactive Account Cleaning"); sb.AppendLine("Username,Last Activity,Creation Date,Last Website Activity,Last Email Activity,Last Git Activity"); foreach (string account in inactiveAccounts) { User user = UserHelper.GetUser(db, account); sb.AppendLine(string.Format("{0},{1},{2},{3},{4},{5}", user.Username, UserHelper.GetLastAccountActivity(db, config, user).ToString("g"), user.JoinDate.ToString("g"), user.LastSeen.ToString("g"), UserHelper.UserEmailExists(config, UserHelper.GetUserEmailAddress(config, user.Username)) ? UserHelper.UserEmailLastActive(config, UserHelper.GetUserEmailAddress(config, user.Username)).ToString("g") : null, UserHelper.UserGitExists(config, user.Username) ? UserHelper.UserGitLastActive(config, user.Username).ToString("g") : null)); } sb.AppendLine(); sb.AppendLine("Orphaned Email Cleaning"); sb.AppendLine("Email,Last Activity"); foreach (string account in emailAccounts) { sb.AppendLine(string.Format("{0},{1}", account, UserHelper.UserEmailExists(config, account) ? UserHelper.UserEmailLastActive(config, account).ToString("g") : null)); } sb.AppendLine(); sb.AppendLine("Orphaned Git Cleaning"); sb.AppendLine("Username,Last Activity"); foreach (string account in gitAccounts) { sb.AppendLine(string.Format("{0},{1}", account, UserHelper.UserGitExists(config, account) ? UserHelper.UserGitLastActive(config, account).ToString("g") : null)); } string dir = Path.GetDirectoryName(fileName); if (!Directory.Exists(dir)) Directory.CreateDirectory(dir); File.WriteAllText(fileName, sb.ToString()); Output(string.Format("[{0}] Finished Generating Accounts to Clean List.", DateTime.Now)); } public static List GetInvalidAccounts(Config config, TeknikEntities db) { List foundUsers = new List(); List curUsers = db.Users.ToList(); foreach (User user in curUsers) { // If the username is reserved, let's add it to the list if (UserHelper.UsernameReserved(config, user.Username) && user.Username != Constants.SERVERUSER) { foundUsers.Add(user.Username); continue; } // If the username is invalid, let's add it to the list if (!UserHelper.ValidUsername(config, user.Username) && user.Username != Constants.SERVERUSER) { foundUsers.Add(user.Username); continue; } } return foundUsers; } public static List GetInactiveAccounts(Config config, TeknikEntities db, int maxDays) { List foundUsers = new List(); List curUsers = db.Users.ToList(); foreach (User user in curUsers) { // If the username is reserved, don't worry about it if (UserHelper.UsernameReserved(config, user.Username) || user.Username == Constants.SERVERUSER) { continue; } // If they are Premium, don't worry about it either if (user.AccountType == AccountType.Premium && user.AccountStatus != AccountStatus.Banned) { continue; } #region Inactivity Finding DateTime lastActivity = DateTime.Now; try { lastActivity = UserHelper.GetLastAccountActivity(db, config, user); } catch { continue; } TimeSpan inactiveTime = DateTime.Now.Subtract(lastActivity); // If older than max days, check their current usage if (inactiveTime >= new TimeSpan(maxDays, 0, 0, 0, 0)) { // Check the user's usage of the service. bool noData = true; // Any blog comments? var blogCom = db.BlogComments.Where(c => c.UserId == user.UserId); noData &= !(blogCom != null && blogCom.Any()); // Any blog posts? var blogPosts = db.BlogPosts.Where(p => p.Blog.UserId == user.UserId); noData &= !(blogPosts != null && blogPosts.Any()); // Any podcast comments? var podCom = db.PodcastComments.Where(p => p.UserId == user.UserId); noData &= !(podCom != null && podCom.Any()); // Any email? if (config.EmailConfig.Enabled && UserHelper.UserEmailExists(config, UserHelper.GetUserEmailAddress(config, user.Username))) { var app = new hMailServer.Application(); app.Connect(); app.Authenticate(config.EmailConfig.Username, config.EmailConfig.Password); try { var domain = app.Domains.ItemByName[config.EmailConfig.Domain]; var account = domain.Accounts.ItemByAddress[UserHelper.GetUserEmailAddress(config, user.Username)]; noData &= ((account.Messages.Count == 0) && ((int)account.Size == 0)); } catch { } } // Any git repos? if (config.GitConfig.Enabled && UserHelper.UserGitExists(config, user.Username)) { string email = UserHelper.GetUserEmailAddress(config, user.Username); // We need to check the actual git database MysqlDatabase mySQL = new MysqlDatabase(config.GitConfig.Database.Server, config.GitConfig.Database.Database, config.GitConfig.Database.Username, config.GitConfig.Database.Password, config.GitConfig.Database.Port); string sql = @"SELECT * FROM gogs.repository LEFT JOIN gogs.action ON gogs.user.id = gogs.action.act_user_id WHERE gogs.user.login_name = {0}"; var results = mySQL.Query(sql, new object[] { email }); noData &= !(results != null && results.Any()); } if (noData) { // They have no data, so safe to delete them. foundUsers.Add(user.Username); } continue; } #endregion } return foundUsers; } public static List GetOrphanedEmail(Config config, TeknikEntities db) { List foundEmail = new List(); if (config.EmailConfig.Enabled) { List curUsers = db.Users.ToList(); // Connect to hmailserver COM var app = new hMailServer.Application(); app.Connect(); app.Authenticate(config.EmailConfig.Username, config.EmailConfig.Password); var domain = app.Domains.ItemByName[config.EmailConfig.Domain]; var accounts = domain.Accounts; for (int i = 0; i < accounts.Count; i++) { var account = accounts[i]; bool userExists = curUsers.Exists(u => UserHelper.GetUserEmailAddress(config, u.Username) == account.Address); bool isReserved = UserHelper.GetReservedUsernames(config).Exists(r => UserHelper.GetUserEmailAddress(config, r).ToLower() == account.Address.ToLower()); if (!userExists && !isReserved) { foundEmail.Add(account.Address); } } } return foundEmail; } public static List GetOrphanedGit(Config config, TeknikEntities db) { List foundGit = new List(); if (config.GitConfig.Enabled) { List curUsers = db.Users.ToList(); // We need to check the actual git database MysqlDatabase mySQL = new MysqlDatabase(config.GitConfig.Database.Server, config.GitConfig.Database.Database, config.GitConfig.Database.Username, config.GitConfig.Database.Password, config.GitConfig.Database.Port); string sql = @"SELECT gogs.user.login_name AS login_name, gogs.user.lower_name AS username FROM gogs.user"; var results = mySQL.Query(sql); if (results != null && results.Any()) { foreach (var account in results) { bool userExists = curUsers.Exists(u => UserHelper.GetUserEmailAddress(config, u.Username).ToLower() == account["login_name"].ToString().ToLower()); bool isReserved = UserHelper.GetReservedUsernames(config).Exists(r => UserHelper.GetUserEmailAddress(config, r) == account["login_name"].ToString().ToLower()); if (!userExists && !isReserved) { foundGit.Add(account["username"].ToString()); } } } } return foundGit; } public static void Output(string message) { Console.WriteLine(message); if (OutputEvent != null) { OutputEvent(message); } } } }