Implement separate Storage Service to decouple file storage for uploads and pastes

Configuration/PasteConfig.cs

@@ -8,14 +8,12 @@ namespace Teknik.Configuration
         public int UrlLength { get; set; }
         public int DeleteKeyLength { get; set; }
         public string SyntaxVisualStyle { get; set; }
-        // Location of the upload directory
-        public string PasteDirectory { get; set; }
-        // File Extension for saved files
-        public string FileExtension { get; set; }
         public int KeySize { get; set; }
         public int BlockSize { get; set; }
         // The size of the chunk that the file will be encrypted/decrypted in (bytes)
         public int ChunkSize { get; set; }
+        // Storage settings
+        public StorageConfig StorageConfig { get; set; }
 
         public PasteConfig()
         {
@@ -25,9 +23,8 @@ namespace Teknik.Configuration
             KeySize = 256;
             BlockSize = 128;
             ChunkSize = 1040;
-            PasteDirectory = Directory.GetCurrentDirectory();
-            FileExtension = "enc";
             SyntaxVisualStyle = "vs";
+            StorageConfig = new StorageConfig("pastes");
         }
     }
 }

Configuration/StorageConfig.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Teknik.Configuration
+{
+    public class StorageConfig
+    {
+        public StorageType Type { get; set; }
+        // File Extension for saved files
+        public string FileExtension { get; set; }
+        // Length of filename
+        public int FileNameLength { get; set; }
+
+        // Local Storage Options
+        public string LocalDirectory { get; set; }
+
+        // S3 Options
+        public string Container { get; set; }
+
+        public StorageConfig(string container)
+        {
+            Type = StorageType.Local;
+            Container = container;
+            LocalDirectory = Directory.GetCurrentDirectory();
+            FileExtension = "enc";
+            FileNameLength = 10;
+        }
+    }
+}

Configuration/StorageType.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Teknik.Configuration
+{
+    public enum StorageType
+    {
+        Local,
+        S3
+    }
+}

Configuration/UploadConfig.cs

@@ -19,10 +19,6 @@ namespace Teknik.Configuration
         public long MaxTotalSizeBasic { get; set; }
         // Maximum total size for basic users
         public long MaxTotalSizePremium { get; set; }
-        // Location of the upload directory
-        public string UploadDirectory { get; set; }
-        // File Extension for saved files
-        public string FileExtension { get; set; }
         public int UrlLength { get; set; }
         public int DeleteKeyLength { get; set; }
         public int KeySize { get; set; }
@@ -34,6 +30,8 @@ namespace Teknik.Configuration
         public ClamConfig ClamConfig { get; set; }
         // Hash Scanning Settings
         public HashScanConfig HashScanConfig { get; set; }
+        // Storage settings
+        public StorageConfig StorageConfig { get; set; }
         // Content Type Restrictions
         public List<string> RestrictedContentTypes { get; set; }
         public List<string> RestrictedExtensions { get; set; }
@@ -53,8 +51,6 @@ namespace Teknik.Configuration
             MaxDownloadSize = 100000000;
             MaxTotalSizeBasic = 1000000000;
             MaxTotalSizePremium = 5000000000;
-            UploadDirectory = Directory.GetCurrentDirectory();
-            FileExtension = "enc";
             UrlLength = 5;
             DeleteKeyLength = 24;
             KeySize = 256;
@@ -63,6 +59,7 @@ namespace Teknik.Configuration
             ChunkSize = 1024;
             ClamConfig = new ClamConfig();
             HashScanConfig = new HashScanConfig();
+            StorageConfig = new StorageConfig("uploads");
             RestrictedContentTypes = new List<string>();
             RestrictedExtensions = new List<string>();
         }

ServiceWorker/Program.cs

@@ -1,6 +1,7 @@
 using CommandLine;
 using Microsoft.EntityFrameworkCore;
 using nClam;
+using StorageService;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -151,9 +152,9 @@ namespace Teknik.ServiceWorker
         private static async Task<bool> ScanUpload(Config config, TeknikEntities db, Upload upload, int totalCount, int currentCount)
         {
             bool virusDetected = false;
-            string subDir = upload.FileName[0].ToString();
+            var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
-            string filePath = Path.Combine(config.UploadConfig.UploadDirectory, subDir, upload.FileName);
+            var fileStream = storageService.GetFile(upload.FileName);
-            if (File.Exists(filePath))
+            if (fileStream != null)
             {
                 // If the IV is set, and Key is set, then scan it
                 if (!string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV))
@@ -173,12 +174,11 @@ namespace Teknik.ServiceWorker
                         }
                     }
 
-                    using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read))
+                    using (AesCounterStream aesStream = new AesCounterStream(fileStream, false, keyBytes, ivBytes))
-                    using (AesCounterStream aesStream = new AesCounterStream(fs, false, keyBytes, ivBytes))
                     {
                         ClamClient clam = new ClamClient(config.UploadConfig.ClamConfig.Server, config.UploadConfig.ClamConfig.Port);
                         clam.MaxStreamSize = maxUploadSize;
-                        ClamScanResult scanResult = await clam.SendAndScanFileAsync(fs);
+                        ClamScanResult scanResult = await clam.SendAndScanFileAsync(fileStream);
 
                         switch (scanResult.Result)
                         {
@@ -198,11 +198,12 @@ namespace Teknik.ServiceWorker
                                 lock (dbLock)
                                 {
                                     string urlName = upload.Url;
-                                    // Delete from the DB
-                                    db.Uploads.Remove(upload);
 
                                     // Delete the File
-                                    DeleteFile(filePath);
+                                    storageService.DeleteFile(upload.FileName);
+
+                                    // Delete from the DB
+                                    db.Uploads.Remove(upload);
 
                                     // Add to transparency report if any were found
                                     Takedown report = new Takedown();
@@ -244,13 +245,10 @@ namespace Teknik.ServiceWorker
             // Process uploads
             List<Upload> uploads = db.Uploads.Where(u => u.ExpireDate != null && u.ExpireDate < curDate).ToList();
 
+            var uploadStorageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
             foreach (Upload upload in uploads)
             {
-                string subDir = upload.FileName[0].ToString();
+                DeleteFile(uploadStorageService, upload.FileName);
-                string filePath = Path.Combine(config.UploadConfig.UploadDirectory, subDir, upload.FileName);
-
-                // Delete the File
-                DeleteFile(filePath);
             }
             db.RemoveRange(uploads);
             db.SaveChanges();
@@ -258,13 +256,11 @@ namespace Teknik.ServiceWorker
             // Process Pastes
             List<Paste> pastes = db.Pastes.Where(p => p.ExpireDate != null && p.ExpireDate < curDate).ToList();
 
+            var pasteStorageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig);
             foreach (Paste paste in pastes)
             {
-                string subDir = paste.FileName[0].ToString();
-                string filePath = Path.Combine(config.PasteConfig.PasteDirectory, subDir, paste.FileName);
-
                 // Delete the File
-                DeleteFile(filePath);
+                DeleteFile(pasteStorageService, paste.FileName);
             }
             db.RemoveRange(pastes);
             db.SaveChanges();
@@ -283,37 +279,39 @@ namespace Teknik.ServiceWorker
 
         public static void CleanUploadFiles(Config config, TeknikEntities db)
         {
-            List<string> uploads = db.Uploads.Where(u => !string.IsNullOrEmpty(u.FileName)).Select(u => Path.Combine(config.UploadConfig.UploadDirectory, u.FileName[0].ToString(), u.FileName)).Select(u => u.ToLower()).ToList();
+            var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
-            List<string> files = Directory.GetFiles(config.UploadConfig.UploadDirectory, "*.*", SearchOption.AllDirectories).Select(f => f.ToLower()).ToList();
+            List<string> files = storageService.GetFileNames();
+            List<string> uploads = db.Uploads.Where(u => !string.IsNullOrEmpty(u.FileName)).Select(u => u.FileName.ToLower()).ToList();
             var orphans = files.Except(uploads);
             File.AppendAllLines(orphansFile, orphans);
             foreach (var orphan in orphans)
             {
-                DeleteFile(orphan);
+                DeleteFile(storageService, orphan);
             }
         }
 
         public static void CleanPasteFiles(Config config, TeknikEntities db)
         {
-            List<string> pastes = db.Pastes.Where(p => !string.IsNullOrEmpty(p.FileName)).Select(p => Path.Combine(config.PasteConfig.PasteDirectory, p.FileName[0].ToString(), p.FileName)).Select(p => p.ToLower()).ToList();
+            var storageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig);
-            List<string> files = Directory.GetFiles(config.PasteConfig.PasteDirectory, "*.*", SearchOption.AllDirectories).Select(f => f.ToLower()).ToList();
+            List<string> files = storageService.GetFileNames();
+            List<string> pastes = db.Pastes.Where(p => !string.IsNullOrEmpty(p.FileName)).Select(p => p.FileName.ToLower()).ToList();
             var orphans = files.Except(pastes);
             File.AppendAllLines(orphansFile, orphans);
             foreach (var orphan in orphans)
             {
-                DeleteFile(orphan);
+                DeleteFile(storageService, orphan);
             }
         }
 
-        public static void DeleteFile(string filePath)
+        public static void DeleteFile(IStorageService storageService, string fileName)
         {
             try
             {
-                File.Delete(filePath);
+                storageService.DeleteFile(fileName);
             }
             catch (Exception ex)
             {
-                Output(string.Format("[{0}] Unable to delete file: {1} | {2}", DateTime.Now, filePath, ex.ToString()));
+                Output(string.Format("[{0}] Unable to delete file: {1} | {2}", DateTime.Now, fileName, ex.ToString()));
             }
         }
 

StorageService/IStorageService.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Teknik.Configuration;
+
+namespace StorageService
+{
+    public interface IStorageService
+    {
+        public string GetUniqueFileName();
+        public Stream GetFile(string fileName);
+        public List<string> GetFileNames();
+        public void SaveFile(string fileName, Stream file);
+        public void SaveEncryptedFile(string fileName, Stream file, int chunkSize, byte[] key, byte[] iv);
+        public void DeleteFile(string fileName);
+    }
+}

StorageService/LocalStorageService.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Teknik.Configuration;
+using Teknik.Utilities;
+using Teknik.Utilities.Cryptography;
+
+namespace StorageService
+{
+    public class LocalStorageService : StorageService
+    {
+        public LocalStorageService(StorageConfig config) : base(config)
+        {
+        }
+
+        public override string GetUniqueFileName()
+        {
+            string filePath = FileHelper.GenerateRandomFileName(_config.LocalDirectory, _config.FileExtension, _config.FileNameLength);
+            return Path.GetFileName(filePath);
+        }
+
+        public override List<string> GetFileNames()
+        {
+            return Directory.GetFiles(_config.LocalDirectory, "*.*", SearchOption.AllDirectories).Select(f => Path.GetFileName(f).ToLower()).ToList();
+        }
+
+        public override Stream GetFile(string fileName)
+        {
+            if (string.IsNullOrEmpty(fileName))
+                return null;
+
+            string filePath = GetFilePath(fileName);
+            if (File.Exists(filePath))
+                return new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+            return null;
+        }
+
+        public override void SaveEncryptedFile(string fileName, Stream file, int chunkSize, byte[] key, byte[] iv)
+        {
+            if (!Directory.Exists(_config.LocalDirectory))
+                Directory.CreateDirectory(_config.LocalDirectory);
+
+            string filePath = GetFilePath(fileName);
+            AesCounterManaged.EncryptToFile(filePath, file, chunkSize, key, iv);
+        }
+
+        public override void SaveFile(string fileName, Stream file)
+        {
+            if (!Directory.Exists(_config.LocalDirectory))
+                Directory.CreateDirectory(_config.LocalDirectory);
+
+            string filePath = GetFilePath(fileName);
+            // Just write the stream to the file
+            using (FileStream fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
+            {
+                file.Seek(0, SeekOrigin.Begin);
+                file.CopyTo(fileStream);
+            }
+        }
+
+        public override void DeleteFile(string fileName)
+        {
+            string filePath = GetFilePath(fileName);
+
+            // Delete the File
+            if (File.Exists(filePath))
+            {
+                File.Delete(filePath);
+            }
+        }
+
+        private string GetFilePath(string fileName)
+        {
+            if (string.IsNullOrEmpty(fileName))
+                return null;
+
+            string subDir = fileName[0].ToString().ToLower();
+            return Path.Combine(_config.LocalDirectory, subDir, fileName);
+        }
+    }
+}

StorageService/StorageService.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Teknik.Configuration;
+
+namespace StorageService
+{
+    public abstract class StorageService : IStorageService
+    {
+        protected readonly StorageConfig _config;
+
+        public StorageService(StorageConfig config)
+        {
+            _config = config;
+        }
+
+        public abstract string GetUniqueFileName();
+        public abstract Stream GetFile(string fileName);
+        public abstract List<string> GetFileNames();
+        public abstract void SaveFile(string fileName, Stream file);
+        public abstract void SaveEncryptedFile(string fileName, Stream file, int chunkSize, byte[] key, byte[] iv);
+        public abstract void DeleteFile(string fileName);
+    }
+}

StorageService/StorageService.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Configuration\Configuration.csproj" />
+  </ItemGroup>
+
+</Project>

StorageService/StorageServiceFactory.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Teknik.Configuration;
+
+namespace StorageService
+{
+    public static class StorageServiceFactory
+    {
+        public static StorageService GetStorageService(StorageConfig config)
+        {
+            switch (config.Type)
+            {
+                case StorageType.Local:
+                    return new LocalStorageService(config);
+                case StorageType.S3:
+                default:
+                    return null;
+            }
+        }
+    }
+}

Teknik.sln

@@ -32,7 +32,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IdentityServer", "IdentityS
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContentScanningService", "ContentScanningService\ContentScanningService.csproj", "{491FE626-ABC8-4D00-8C7F-0849C357201A}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebCommon", "WebCommon\WebCommon.csproj", "{32E85A7F-871A-437C-9BA3-00499AAB442C}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCommon", "WebCommon\WebCommon.csproj", "{32E85A7F-871A-437C-9BA3-00499AAB442C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StorageService", "StorageService\StorageService.csproj", "{4A600C17-C772-462F-A37F-307E7893B2DB}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -107,6 +109,12 @@ Global
 		{32E85A7F-871A-437C-9BA3-00499AAB442C}.Release|Any CPU.Build.0 = Release|Any CPU
 		{32E85A7F-871A-437C-9BA3-00499AAB442C}.Test|Any CPU.ActiveCfg = Debug|Any CPU
 		{32E85A7F-871A-437C-9BA3-00499AAB442C}.Test|Any CPU.Build.0 = Debug|Any CPU
+		{4A600C17-C772-462F-A37F-307E7893B2DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4A600C17-C772-462F-A37F-307E7893B2DB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4A600C17-C772-462F-A37F-307E7893B2DB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4A600C17-C772-462F-A37F-307E7893B2DB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4A600C17-C772-462F-A37F-307E7893B2DB}.Test|Any CPU.ActiveCfg = Debug|Any CPU
+		{4A600C17-C772-462F-A37F-307E7893B2DB}.Test|Any CPU.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Teknik/Areas/Paste/Controllers/PasteController.cs

@@ -22,6 +22,7 @@ using System.IO;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Diagnostics;
 using Teknik.Utilities.Routing;
+using StorageService;
 
 namespace Teknik.Areas.Paste.Controllers
 {
@@ -58,7 +59,7 @@ namespace Teknik.Areas.Paste.Controllers
                 // Check Expiration
                 if (PasteHelper.CheckExpiration(paste))
                 {
-                    DeleteFile(paste);
+                    PasteHelper.DeleteFile(_dbContext, _config, _logger, paste);
                     return new StatusCodeResult(StatusCodes.Status404NotFound);
                 }
 
@@ -98,7 +99,7 @@ namespace Teknik.Areas.Paste.Controllers
                     if (string.IsNullOrEmpty(password) || hash != paste.HashedPassword)
                     {
                         PasswordViewModel passModel = new PasswordViewModel();
-                        passModel.ActionUrl = Url.SubRouteUrl("p", "Paste.View");
+                        passModel.ActionUrl = Url.SubRouteUrl("p", "Paste.View", new { type = type, url = url });
                         passModel.Url = url;
                         passModel.Type = type;
 
@@ -119,15 +120,12 @@ namespace Teknik.Areas.Paste.Controllers
                 // Read in the file
                 if (string.IsNullOrEmpty(paste.FileName))
                     return new StatusCodeResult(StatusCodes.Status404NotFound);
-                string subDir = paste.FileName[0].ToString();
+                var storageService = StorageServiceFactory.GetStorageService(_config.PasteConfig.StorageConfig);
-                string filePath = Path.Combine(_config.PasteConfig.PasteDirectory, subDir, paste.FileName);
+                var fileStream = storageService.GetFile(paste.FileName);
-                if (!System.IO.File.Exists(filePath))
+                if (fileStream == null)
-                {
                     return new StatusCodeResult(StatusCodes.Status404NotFound);
-                }
 
-                using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
+                using (AesCounterStream cs = new AesCounterStream(fileStream, false, keyBytes, ivBytes))
-                using (AesCounterStream cs = new AesCounterStream(fs, false, keyBytes, ivBytes))
                 using (StreamReader sr = new StreamReader(cs, Encoding.Unicode))
                 {
                     model.Content = await sr.ReadToEndAsync();
@@ -151,8 +149,7 @@ namespace Teknik.Areas.Paste.Controllers
 
                         Response.Headers.Add("Content-Disposition", cd.ToString());
 
-                        FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+                        return new BufferedFileStreamResult("application/octet-stream", async (response) => await ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fileStream, false, keyBytes, ivBytes), (int)fileStream.Length, _config.PasteConfig.ChunkSize), false);
-                        return new BufferedFileStreamResult("application/octet-stream", async (response) => await ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fs, false, keyBytes, ivBytes), (int)fs.Length, _config.PasteConfig.ChunkSize), false);
                     default:
                         return View("~/Areas/Paste/Views/Paste/Full.cshtml", model);
                 }
@@ -220,7 +217,7 @@ namespace Teknik.Areas.Paste.Controllers
                 // Check Expiration
                 if (PasteHelper.CheckExpiration(paste))
                 {
-                    DeleteFile(paste);
+                    PasteHelper.DeleteFile(_dbContext, _config, _logger, paste);
                     return new StatusCodeResult(StatusCodes.Status404NotFound);
                 }
 
@@ -271,15 +268,12 @@ namespace Teknik.Areas.Paste.Controllers
                 // Read in the file
                 if (string.IsNullOrEmpty(paste.FileName))
                     return new StatusCodeResult(StatusCodes.Status404NotFound);
-                string subDir = paste.FileName[0].ToString();
+                var storageService = StorageServiceFactory.GetStorageService(_config.PasteConfig.StorageConfig);
-                string filePath = Path.Combine(_config.PasteConfig.PasteDirectory, subDir, paste.FileName);
+                var fileStream = storageService.GetFile(paste.FileName);
-                if (!System.IO.File.Exists(filePath))
+                if (fileStream == null)
-                {
                     return new StatusCodeResult(StatusCodes.Status404NotFound);
-                }
 
-                using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
+                using (AesCounterStream cs = new AesCounterStream(fileStream, false, keyBytes, ivBytes))
-                using (AesCounterStream cs = new AesCounterStream(fs, false, keyBytes, ivBytes))
                 using (StreamReader sr = new StreamReader(cs, Encoding.Unicode))
                 {
                     model.Content = await sr.ReadToEndAsync();
@@ -333,17 +327,16 @@ namespace Teknik.Areas.Paste.Controllers
                         }
 
                         // get the old file
-                        string subDir = paste.FileName[0].ToString();
+                        var storageService = StorageServiceFactory.GetStorageService(_config.PasteConfig.StorageConfig);
-                        string oldFile = Path.Combine(_config.PasteConfig.PasteDirectory, subDir, paste.FileName);
+                        var oldFile = paste.FileName;
 
                         // Generate a unique file name that does not currently exist
-                        string newFilePath = FileHelper.GenerateRandomFileName(_config.PasteConfig.PasteDirectory, _config.PasteConfig.FileExtension, 10);
+                        string fileName = storageService.GetUniqueFileName();
-                        string fileName = Path.GetFileName(newFilePath);
 
                         string key = PasteHelper.GenerateKey(_config.PasteConfig.KeySize);
                         string iv = PasteHelper.GenerateIV(_config.PasteConfig.BlockSize);
 
-                        PasteHelper.EncryptContents(model.Content, newFilePath, password, key, iv, _config.PasteConfig.KeySize, _config.PasteConfig.ChunkSize);
+                        PasteHelper.EncryptContents(storageService, model.Content, fileName, password, key, iv, _config.PasteConfig.KeySize, _config.PasteConfig.ChunkSize);
 
                         paste.Key = key;
                         paste.KeySize = _config.PasteConfig.KeySize;
@@ -361,8 +354,7 @@ namespace Teknik.Areas.Paste.Controllers
                         _dbContext.SaveChanges();
 
                         // Delete the old file
-                        if (System.IO.File.Exists(oldFile))
+                        storageService.DeleteFile(oldFile);
-                            System.IO.File.Delete(oldFile);
 
                         return Redirect(Url.SubRouteUrl("p", "Paste.View", new { type = "Full", url = paste.Url }));
                     }
@@ -385,7 +377,7 @@ namespace Teknik.Areas.Paste.Controllers
                 if (foundPaste.User.Username == User.Identity.Name ||
                     User.IsInRole("Admin"))
                 {
-                    DeleteFile(foundPaste);
+                    PasteHelper.DeleteFile(_dbContext, _config, _logger, foundPaste);
 
                     return Json(new { result = true, redirect = Url.SubRouteUrl("p", "Paste.Index") });
                 }
@@ -418,31 +410,5 @@ namespace Teknik.Areas.Paste.Controllers
                 HttpContext.Session.Remove("PastePassword_" + url);
             }
         }
-
-        private void DeleteFile(Models.Paste paste)
-        {
-            if (!string.IsNullOrEmpty(paste.FileName))
-            {
-                string delSub = paste.FileName[0].ToString();
-                string delPath = Path.Combine(_config.PasteConfig.PasteDirectory, delSub, paste.FileName);
-
-                // Delete the File
-                if (System.IO.File.Exists(delPath))
-                {
-                    try
-                    {
-                        System.IO.File.Delete(delPath);
-                    }
-                    catch (Exception ex)
-                    {
-                        _logger.LogError(ex, "Unable to delete file: {0}", paste.FileName);
-                    }
-                }
-            }
-
-            // Delete from the DB
-            _dbContext.Pastes.Remove(paste);
-            _dbContext.SaveChanges();
-        }
     }
 }

Teknik/Areas/Paste/PasteHelper.cs

@@ -9,6 +9,9 @@ using Teknik.Models;
 using Teknik.Utilities.Cryptography;
 using Teknik.Data;
 using System.IO;
+using StorageService;
+using Teknik.Logging;
+using Microsoft.Extensions.Logging;
 
 namespace Teknik.Areas.Paste
 {
@@ -56,15 +59,6 @@ namespace Teknik.Areas.Paste
                     break;
             }
 
-            if (!Directory.Exists(config.PasteConfig.PasteDirectory))
-            {
-                Directory.CreateDirectory(config.PasteConfig.PasteDirectory);
-            }
-
-            // Generate a unique file name that does not currently exist
-            string filePath = FileHelper.GenerateRandomFileName(config.PasteConfig.PasteDirectory, config.PasteConfig.FileExtension, 10);
-            string fileName = Path.GetFileName(filePath);
-
             string key = GenerateKey(config.PasteConfig.KeySize);
             string iv = GenerateIV(config.PasteConfig.BlockSize);
 
@@ -73,8 +67,13 @@ namespace Teknik.Areas.Paste
                 paste.HashedPassword = HashPassword(key, password);
             }
 
+
+            // Generate a unique file name that does not currently exist
+            var storageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig);
+            var fileName = storageService.GetUniqueFileName();
+
             // Encrypt the contents to the file
-            EncryptContents(content, filePath, password, key, iv, config.PasteConfig.KeySize, config.PasteConfig.ChunkSize);
+            EncryptContents(storageService, content, fileName, password, key, iv, config.PasteConfig.KeySize, config.PasteConfig.ChunkSize);
 
             // Generate a deletion key
             string delKey = StringHelper.RandomString(config.PasteConfig.DeleteKeyLength);
@@ -118,7 +117,7 @@ namespace Teknik.Areas.Paste
             return SHA384.Hash(key, password).ToHex();
         }
 
-        public static void EncryptContents(string content, string filePath, string password, string key, string iv, int keySize, int chunkSize)
+        public static void EncryptContents(IStorageService storageService, string content, string fileName, string password, string key, string iv, int keySize, int chunkSize)
         {
             byte[] ivBytes = Encoding.Unicode.GetBytes(iv);
             byte[] keyBytes = AesCounterManaged.CreateKey(key, ivBytes, keySize);
@@ -133,8 +132,25 @@ namespace Teknik.Areas.Paste
             byte[] data = Encoding.Unicode.GetBytes(content);
             using (MemoryStream ms = new MemoryStream(data))
             {
-                AesCounterManaged.EncryptToFile(filePath, ms, chunkSize, keyBytes, ivBytes);
+                storageService.SaveEncryptedFile(fileName, ms, chunkSize, keyBytes, ivBytes);
             }
         }
+
+        public static void DeleteFile(TeknikEntities db, Config config, ILogger<Logger> logger, Models.Paste paste)
+        {
+            try
+            {
+                var storageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig);
+                storageService.DeleteFile(paste.FileName);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Unable to delete file: {0}", paste.FileName);
+            }
+
+            // Delete from the DB
+            db.Pastes.Remove(paste);
+            db.SaveChanges();
+        }
     }
 }

Teknik/Areas/Upload/Controllers/UploadController.cs

@@ -24,6 +24,7 @@ using Teknik.Logging;
 using Teknik.Areas.Users.Models;
 using Teknik.ContentScanningService;
 using Teknik.Utilities.Routing;
+using StorageService;
 
 namespace Teknik.Areas.Upload.Controllers
 {
@@ -240,7 +241,7 @@ namespace Teknik.Areas.Upload.Controllers
                     // Check Expiration
                     if (UploadHelper.CheckExpiration(upload))
                     {
-                        DeleteFile(upload);
+                        UploadHelper.DeleteFile(_dbContext, _config, _logger, upload);
                         return new StatusCodeResult(StatusCodes.Status404NotFound);
                     }
 
@@ -320,12 +321,12 @@ namespace Teknik.Areas.Upload.Controllers
                     }
                     else
                     {
-                        string subDir = fileName[0].ToString();
+                        var storageService = StorageServiceFactory.GetStorageService(_config.UploadConfig.StorageConfig);
-                        string filePath = Path.Combine(_config.UploadConfig.UploadDirectory, subDir, fileName);
+                        var fileStream = storageService.GetFile(fileName);
                         long startByte = 0;
                         long endByte = contentLength - 1;
                         long length = contentLength;
-                        if (System.IO.File.Exists(filePath))
+                        if (fileStream != null)
                         {
                             #region Range Calculation
                             // Are they downloading it by range?
@@ -413,11 +414,8 @@ namespace Teknik.Areas.Upload.Controllers
 
                             Response.Headers.Add("Content-Disposition", cd.ToString());
 
-                            // Read in the file
-                            FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
-
                             // Reset file stream to starting position (or start of range)
-                            fs.Seek(startByte, SeekOrigin.Begin);
+                            fileStream.Seek(startByte, SeekOrigin.Begin);
 
                             try
                             {
@@ -427,12 +425,12 @@ namespace Teknik.Areas.Upload.Controllers
                                     byte[] keyBytes = Encoding.UTF8.GetBytes(key);
                                     byte[] ivBytes = Encoding.UTF8.GetBytes(iv);
 
-                                    return new BufferedFileStreamResult(contentType, async (response) => await ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fs, false, keyBytes, ivBytes), (int)length, _config.UploadConfig.ChunkSize), false);
+                                    return new BufferedFileStreamResult(contentType, async (response) => await ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fileStream, false, keyBytes, ivBytes), (int)length, _config.UploadConfig.ChunkSize), false);
                                 }
                                 else // Otherwise just send it
                                 {
                                     // Send the file
-                                    return new BufferedFileStreamResult(contentType, async (response) => await ResponseHelper.StreamToOutput(response, true, fs, (int)length, _config.UploadConfig.ChunkSize), false);
+                                    return new BufferedFileStreamResult(contentType, async (response) => await ResponseHelper.StreamToOutput(response, true, fileStream, (int)length, _config.UploadConfig.ChunkSize), false);
                                 }
                             }
                             catch (Exception ex)
@@ -459,13 +457,13 @@ namespace Teknik.Areas.Upload.Controllers
                     // Check Expiration
                     if (UploadHelper.CheckExpiration(upload))
                     {
-                        DeleteFile(upload);
+                        UploadHelper.DeleteFile(_dbContext, _config, _logger, upload);
                         return Json(new { error = new { message = "File Does Not Exist" } });
                     }
 
-                    string subDir = upload.FileName[0].ToString();
+                    var storageService = StorageServiceFactory.GetStorageService(_config.UploadConfig.StorageConfig);
-                    string filePath = Path.Combine(_config.UploadConfig.UploadDirectory, subDir, upload.FileName);
+                    var fileStream = storageService.GetFile(upload.FileName);
-                    if (System.IO.File.Exists(filePath))
+                    if (fileStream != null)
                     {
                         // Notify the client the content length we'll be outputting 
                         Response.Headers.Add("Content-Length", upload.ContentLength.ToString());
@@ -482,21 +480,18 @@ namespace Teknik.Areas.Upload.Controllers
 
                         Response.Headers.Add("Content-Disposition", cd.ToString());
 
-                        // Read in the file
-                        FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
-
                         // If the IV is set, and Key is set, then decrypt it while sending
                         if (decrypt && !string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV))
                         {
                             byte[] keyBytes = Encoding.UTF8.GetBytes(upload.Key);
                             byte[] ivBytes = Encoding.UTF8.GetBytes(upload.IV);
 
-                            return new BufferedFileStreamResult(upload.ContentType, (response) => ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fs, false, keyBytes, ivBytes), (int)upload.ContentLength, _config.UploadConfig.ChunkSize), false);
+                            return new BufferedFileStreamResult(upload.ContentType, (response) => ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fileStream, false, keyBytes, ivBytes), (int)upload.ContentLength, _config.UploadConfig.ChunkSize), false);
                         }
                         else // Otherwise just send it
                         {
                             // Send the file
-                            return new BufferedFileStreamResult(upload.ContentType, (response) => ResponseHelper.StreamToOutput(response, true, fs, (int)upload.ContentLength, _config.UploadConfig.ChunkSize), false);
+                            return new BufferedFileStreamResult(upload.ContentType, (response) => ResponseHelper.StreamToOutput(response, true, fileStream, (int)upload.ContentLength, _config.UploadConfig.ChunkSize), false);
                         }
                     }
                 }
@@ -517,7 +512,7 @@ namespace Teknik.Areas.Upload.Controllers
                 model.File = file;
                 if (!string.IsNullOrEmpty(upload.DeleteKey) && upload.DeleteKey == key)
                 {
-                    DeleteFile(upload);
+                    UploadHelper.DeleteFile(_dbContext, _config, _logger, upload);
                     model.Deleted = true;
                 }
                 else
@@ -557,35 +552,12 @@ namespace Teknik.Areas.Upload.Controllers
             {
                 if (foundUpload.User.Username == User.Identity.Name)
                 {
-                    DeleteFile(foundUpload);
+                    UploadHelper.DeleteFile(_dbContext, _config, _logger, foundUpload);
                     return Json(new { result = true });
                 }
                 return Json(new { error = new { message = "You do not have permission to edit this Paste" } });
             }
             return Json(new { error = new { message = "This Upload does not exist" } });
         }
-
-        private void DeleteFile(Models.Upload upload)
-        {
-            string subDir = upload.FileName[0].ToString();
-            string filePath = Path.Combine(_config.UploadConfig.UploadDirectory, subDir, upload.FileName);
-
-            // Delete the File
-            if (System.IO.File.Exists(filePath))
-            {
-                try
-                {
-                    System.IO.File.Delete(filePath);
-                }
-                catch (Exception ex)
-                {
-                    _logger.LogError(ex, "Unable to delete file: {0}", upload.FileName);
-                }
-            }
-
-            // Delete from the DB
-            _dbContext.Uploads.Remove(upload);
-            _dbContext.SaveChanges();
-        }
     }
 }

Teknik/Areas/Upload/UploadHelper.cs

@@ -9,6 +9,9 @@ using Teknik.Utilities;
 using System.Text;
 using Teknik.Utilities.Cryptography;
 using Teknik.Data;
+using StorageService;
+using Teknik.Logging;
+using Microsoft.Extensions.Logging;
 
 namespace Teknik.Areas.Upload
 {
@@ -36,14 +39,10 @@ namespace Teknik.Areas.Upload
 
         public static Models.Upload SaveFile(TeknikEntities db, Config config, Stream file, string contentType, long contentLength, bool encrypt, ExpirationUnit expirationUnit, int expirationLength, string fileExt, string iv, string key, int keySize, int blockSize)
         {
-            if (!Directory.Exists(config.UploadConfig.UploadDirectory))
+            var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
-            {
-                Directory.CreateDirectory(config.UploadConfig.UploadDirectory);
-            }
 
             // Generate a unique file name that does not currently exist
-            string filePath = FileHelper.GenerateRandomFileName(config.UploadConfig.UploadDirectory, config.UploadConfig.FileExtension, 10);
+            var fileName = storageService.GetUniqueFileName();
-            string fileName = Path.GetFileName(filePath);
 
             // once we have the filename, lets save the file
             if (encrypt)
@@ -57,17 +56,11 @@ namespace Teknik.Areas.Upload
                 byte[] keyBytes = Encoding.UTF8.GetBytes(key);
                 byte[] ivBytes = Encoding.UTF8.GetBytes(iv);
 
-                // Encrypt the file to disk
+                storageService.SaveEncryptedFile(fileName, file, config.UploadConfig.ChunkSize, keyBytes, ivBytes);
-                AesCounterManaged.EncryptToFile(filePath, file, config.UploadConfig.ChunkSize, keyBytes, ivBytes);
             }
             else
             {
-                // Just write the stream to the file
+                storageService.SaveFile(fileName, file);
-                using (FileStream fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
-                {
-                    file.Seek(0, SeekOrigin.Begin);
-                    file.CopyTo(fileStream);
-                }
             }
 
             // Generate a unique url
@@ -142,5 +135,22 @@ namespace Teknik.Areas.Upload
 
             return upload;
         }
+
+        public static void DeleteFile(TeknikEntities db, Config config, ILogger<Logger> logger, Models.Upload upload)
+        {
+            try
+            {
+                var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
+                storageService.DeleteFile(upload.FileName);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Unable to delete file: {0}", upload.FileName);
+            }
+
+            // Delete from the DB
+            db.Uploads.Remove(upload);
+            db.SaveChanges();
+        }
     }
 }

Teknik/Areas/Vault/Controllers/VaultController.cs

@@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
+using StorageService;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -110,15 +111,14 @@ namespace Teknik.Areas.Vault.Controllers
                             if (!pasteModel.HasPassword)
                             {
                                 // Read in the file
-                                string subDir = paste.Paste.FileName[0].ToString();
+                                var storageService = StorageServiceFactory.GetStorageService(_config.PasteConfig.StorageConfig);
-                                string filePath = Path.Combine(_config.PasteConfig.PasteDirectory, subDir, paste.Paste.FileName);
+                                var fileStream = storageService.GetFile(paste.Paste.FileName);
-                                if (!System.IO.File.Exists(filePath))
+                                if (fileStream == null)
                                     continue;
 
                                 byte[] ivBytes = Encoding.Unicode.GetBytes(paste.Paste.IV);
                                 byte[] keyBytes = AesCounterManaged.CreateKey(paste.Paste.Key, ivBytes, paste.Paste.KeySize);
-                                using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
+                                using (AesCounterStream cs = new AesCounterStream(fileStream, false, keyBytes, ivBytes))
-                                using (AesCounterStream cs = new AesCounterStream(fs, false, keyBytes, ivBytes))
                                 using (StreamReader sr = new StreamReader(cs, Encoding.Unicode))
                                 {
                                     pasteModel.Content = await sr.ReadToEndAsync();

Teknik/Teknik.csproj

@@ -96,6 +96,7 @@
     <ProjectReference Include="..\GitService\GitService.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\MailService\MailService.csproj" />
+    <ProjectReference Include="..\StorageService\StorageService.csproj" />
     <ProjectReference Include="..\Tracking\Tracking.csproj" />
     <ProjectReference Include="..\Utilities\Utilities.csproj" />
     <ProjectReference Include="..\WebCommon\WebCommon.csproj" />

Utilities/Cryptography/AesCounterManaged.cs

@@ -77,39 +77,45 @@ namespace Teknik.Utilities.Cryptography
         }
 
         public static void EncryptToFile(string filePath, Stream input, int chunkSize, byte[] key, byte[] iv)
+        {
+
+            using (FileStream fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
+            {
+                EncryptToStream(input, fileStream, chunkSize, key, iv);
+            }
+        }
+
+        public static void EncryptToStream(Stream input, Stream output, int chunkSize, byte[] key, byte[] iv)
         {
             // Make sure the input stream is at the beginning
             input.Seek(0, SeekOrigin.Begin);
 
             AesCounterStream cryptoStream = new AesCounterStream(input, true, key, iv);
 
-            using (FileStream fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
+            int curByte = 0;
+            int processedBytes = 0;
+            byte[] buffer = new byte[chunkSize];
+            int bytesRemaining = (int)input.Length;
+            int bytesToRead = chunkSize;
+            do
             {
-                int curByte = 0;
+                if (chunkSize > bytesRemaining)
-                int processedBytes = 0;
-                byte[] buffer = new byte[chunkSize];
-                int bytesRemaining = (int)input.Length;
-                int bytesToRead = chunkSize;
-                do
                 {
-                    if (chunkSize > bytesRemaining)
+                    bytesToRead = bytesRemaining;
-                    {
-                        bytesToRead = bytesRemaining;
-                    }
-
-                    processedBytes = cryptoStream.Read(buffer, 0, bytesToRead);
-                    if (processedBytes > 0)
-                    {
-                        fileStream.Write(buffer, 0, processedBytes);
-
-                        // Clear the buffer
-                        Array.Clear(buffer, 0, chunkSize);
-                    }
-                    curByte += processedBytes;
-                    bytesRemaining -= processedBytes;
                 }
-                while (processedBytes > 0 && bytesRemaining > 0);
+
+                processedBytes = cryptoStream.Read(buffer, 0, bytesToRead);
+                if (processedBytes > 0)
+                {
+                    output.Write(buffer, 0, processedBytes);
+
+                    // Clear the buffer
+                    Array.Clear(buffer, 0, chunkSize);
+                }
+                curByte += processedBytes;
+                bytesRemaining -= processedBytes;
             }
+            while (processedBytes > 0 && bytesRemaining > 0);
         }
 
         public static byte[] CreateKey(string password, string iv, int keySize = 256)