Mirrors/Teknik
1
0
Fork 0
mirror of https://git.teknik.io/Teknikode/Teknik.git synced 2023-08-02 14:16:22 +02:00
Code Releases Activity

Implement separate Storage Service to decouple file storage for uploads and pastes

Browse Source
This commit is contained in:
Uncled1023 2021-08-06 22:26:09 -07:00
parent 3ffac5959f
commit 2331d1dd9f
18 changed files with 369 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Configuration/PasteConfig.cs
View File

@ -8,14 +8,12 @@ namespace Teknik.Configuration
public int UrlLength { get; set; }
public int DeleteKeyLength { get; set; }
public string SyntaxVisualStyle { get; set; }
// Location of the upload directory
public string PasteDirectory { get; set; }
// File Extension for saved files
public string FileExtension { get; set; }
public int KeySize { get; set; }
public int BlockSize { get; set; }
// The size of the chunk that the file will be encrypted/decrypted in (bytes)
public int ChunkSize { get; set; }
// Storage settings
public StorageConfig StorageConfig { get; set; }
public PasteConfig()
{
@ -25,9 +23,8 @@ namespace Teknik.Configuration
KeySize = 256;
BlockSize = 128;
ChunkSize = 1040;
PasteDirectory = Directory.GetCurrentDirectory();
FileExtension = "enc";
SyntaxVisualStyle = "vs";
StorageConfig = new StorageConfig("pastes");
}
}
}

33
Configuration/StorageConfig.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Teknik.Configuration
{
public class StorageConfig
{
public StorageType Type { get; set; }
// File Extension for saved files
public string FileExtension { get; set; }
// Length of filename
public int FileNameLength { get; set; }
// Local Storage Options
public string LocalDirectory { get; set; }
// S3 Options
public string Container { get; set; }
public StorageConfig(string container)
{
Type = StorageType.Local;
Container = container;
LocalDirectory = Directory.GetCurrentDirectory();
FileExtension = "enc";
FileNameLength = 10;
}
}
}

14
Configuration/StorageType.cs Normal file
View File

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Teknik.Configuration
{
public enum StorageType
{
Local,
S3
}
}

9
Configuration/UploadConfig.cs
View File

@ -19,10 +19,6 @@ namespace Teknik.Configuration
public long MaxTotalSizeBasic { get; set; }
// Maximum total size for basic users
public long MaxTotalSizePremium { get; set; }
// Location of the upload directory
public string UploadDirectory { get; set; }
// File Extension for saved files
public string FileExtension { get; set; }
public int UrlLength { get; set; }
public int DeleteKeyLength { get; set; }
public int KeySize { get; set; }
@ -34,6 +30,8 @@ namespace Teknik.Configuration
public ClamConfig ClamConfig { get; set; }
// Hash Scanning Settings
public HashScanConfig HashScanConfig { get; set; }
// Storage settings
public StorageConfig StorageConfig { get; set; }
// Content Type Restrictions
public List<string> RestrictedContentTypes { get; set; }
public List<string> RestrictedExtensions { get; set; }
@ -53,8 +51,6 @@ namespace Teknik.Configuration
MaxDownloadSize = 100000000;
MaxTotalSizeBasic = 1000000000;
MaxTotalSizePremium = 5000000000;
UploadDirectory = Directory.GetCurrentDirectory();
FileExtension = "enc";
UrlLength = 5;
DeleteKeyLength = 24;
KeySize = 256;
@ -63,6 +59,7 @@ namespace Teknik.Configuration
ChunkSize = 1024;
ClamConfig = new ClamConfig();
HashScanConfig = new HashScanConfig();
StorageConfig = new StorageConfig("uploads");
RestrictedContentTypes = new List<string>();
RestrictedExtensions = new List<string>();
}

52
ServiceWorker/Program.cs
View File

@ -1,6 +1,7 @@
using CommandLine;
using Microsoft.EntityFrameworkCore;
using nClam;
using StorageService;
using System;
using System.Collections.Generic;
using System.IO;
@ -151,9 +152,9 @@ namespace Teknik.ServiceWorker
private static async Task<bool> ScanUpload(Config config, TeknikEntities db, Upload upload, int totalCount, int currentCount)
{
bool virusDetected = false;
string subDir = upload.FileName[0].ToString();
string filePath = Path.Combine(config.UploadConfig.UploadDirectory, subDir, upload.FileName);
if (File.Exists(filePath))
var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
var fileStream = storageService.GetFile(upload.FileName);
if (fileStream != null)
{
// If the IV is set, and Key is set, then scan it
if (!string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV))
@ -173,12 +174,11 @@ namespace Teknik.ServiceWorker
}
}
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read))
using (AesCounterStream aesStream = new AesCounterStream(fs, false, keyBytes, ivBytes))
using (AesCounterStream aesStream = new AesCounterStream(fileStream, false, keyBytes, ivBytes))
{
ClamClient clam = new ClamClient(config.UploadConfig.ClamConfig.Server, config.UploadConfig.ClamConfig.Port);
clam.MaxStreamSize = maxUploadSize;
ClamScanResult scanResult = await clam.SendAndScanFileAsync(fs);
ClamScanResult scanResult = await clam.SendAndScanFileAsync(fileStream);
switch (scanResult.Result)
{
@ -198,11 +198,12 @@ namespace Teknik.ServiceWorker
lock (dbLock)
{
string urlName = upload.Url;
// Delete from the DB
db.Uploads.Remove(upload);
// Delete the File
DeleteFile(filePath);
storageService.DeleteFile(upload.FileName);
// Delete from the DB
db.Uploads.Remove(upload);
// Add to transparency report if any were found
Takedown report = new Takedown();
@ -244,13 +245,10 @@ namespace Teknik.ServiceWorker
// Process uploads
List<Upload> uploads = db.Uploads.Where(u => u.ExpireDate != null && u.ExpireDate < curDate).ToList();
var uploadStorageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
foreach (Upload upload in uploads)
{
string subDir = upload.FileName[0].ToString();
string filePath = Path.Combine(config.UploadConfig.UploadDirectory, subDir, upload.FileName);
// Delete the File
DeleteFile(filePath);
DeleteFile(uploadStorageService, upload.FileName);
}
db.RemoveRange(uploads);
db.SaveChanges();
@ -258,13 +256,11 @@ namespace Teknik.ServiceWorker
// Process Pastes
List<Paste> pastes = db.Pastes.Where(p => p.ExpireDate != null && p.ExpireDate < curDate).ToList();
var pasteStorageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig);
foreach (Paste paste in pastes)
{
string subDir = paste.FileName[0].ToString();
string filePath = Path.Combine(config.PasteConfig.PasteDirectory, subDir, paste.FileName);
// Delete the File
DeleteFile(filePath);
DeleteFile(pasteStorageService, paste.FileName);
}
db.RemoveRange(pastes);
db.SaveChanges();
@ -283,37 +279,39 @@ namespace Teknik.ServiceWorker
public static void CleanUploadFiles(Config config, TeknikEntities db)
{
List<string> uploads = db.Uploads.Where(u => !string.IsNullOrEmpty(u.FileName)).Select(u => Path.Combine(config.UploadConfig.UploadDirectory, u.FileName[0].ToString(), u.FileName)).Select(u => u.ToLower()).ToList();
List<string> files = Directory.GetFiles(config.UploadConfig.UploadDirectory, "*.*", SearchOption.AllDirectories).Select(f => f.ToLower()).ToList();
var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
List<string> files = storageService.GetFileNames();
List<string> uploads = db.Uploads.Where(u => !string.IsNullOrEmpty(u.FileName)).Select(u => u.FileName.ToLower()).ToList();
var orphans = files.Except(uploads);
File.AppendAllLines(orphansFile, orphans);
foreach (var orphan in orphans)
{
DeleteFile(orphan);
DeleteFile(storageService, orphan);
}
}
public static void CleanPasteFiles(Config config, TeknikEntities db)
{
List<string> pastes = db.Pastes.Where(p => !string.IsNullOrEmpty(p.FileName)).Select(p => Path.Combine(config.PasteConfig.PasteDirectory, p.FileName[0].ToString(), p.FileName)).Select(p => p.ToLower()).ToList();
List<string> files = Directory.GetFiles(config.PasteConfig.PasteDirectory, "*.*", SearchOption.AllDirectories).Select(f => f.ToLower()).ToList();
var storageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig);
List<string> files = storageService.GetFileNames();
List<string> pastes = db.Pastes.Where(p => !string.IsNullOrEmpty(p.FileName)).Select(p => p.FileName.ToLower()).ToList();
var orphans = files.Except(pastes);
File.AppendAllLines(orphansFile, orphans);
foreach (var orphan in orphans)
{
DeleteFile(orphan);
DeleteFile(storageService, orphan);
}
}
public static void DeleteFile(string filePath)
public static void DeleteFile(IStorageService storageService, string fileName)
{
try
{
File.Delete(filePath);
storageService.DeleteFile(fileName);
}
catch (Exception ex)
{
Output(string.Format("[{0}] Unable to delete file: {1} | {2}", DateTime.Now, filePath, ex.ToString()));
Output(string.Format("[{0}] Unable to delete file: {1} | {2}", DateTime.Now, fileName, ex.ToString()));
}
}

17
StorageService/IStorageService.cs Normal file
View File

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.IO;
using Teknik.Configuration;
namespace StorageService
{
public interface IStorageService
{
public string GetUniqueFileName();
public Stream GetFile(string fileName);
public List<string> GetFileNames();
public void SaveFile(string fileName, Stream file);
public void SaveEncryptedFile(string fileName, Stream file, int chunkSize, byte[] key, byte[] iv);
public void DeleteFile(string fileName);
}
}

84
StorageService/LocalStorageService.cs Normal file
View File

@ -0,0 +1,84 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Teknik.Configuration;
using Teknik.Utilities;
using Teknik.Utilities.Cryptography;
namespace StorageService
{
public class LocalStorageService : StorageService
{
public LocalStorageService(StorageConfig config) : base(config)
{
}
public override string GetUniqueFileName()
{
string filePath = FileHelper.GenerateRandomFileName(_config.LocalDirectory, _config.FileExtension, _config.FileNameLength);
return Path.GetFileName(filePath);
}
public override List<string> GetFileNames()
{
return Directory.GetFiles(_config.LocalDirectory, "*.*", SearchOption.AllDirectories).Select(f => Path.GetFileName(f).ToLower()).ToList();
}
public override Stream GetFile(string fileName)
{
if (string.IsNullOrEmpty(fileName))
return null;
string filePath = GetFilePath(fileName);
if (File.Exists(filePath))
return new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
return null;
}
public override void SaveEncryptedFile(string fileName, Stream file, int chunkSize, byte[] key, byte[] iv)
{
if (!Directory.Exists(_config.LocalDirectory))
Directory.CreateDirectory(_config.LocalDirectory);
string filePath = GetFilePath(fileName);
AesCounterManaged.EncryptToFile(filePath, file, chunkSize, key, iv);
}
public override void SaveFile(string fileName, Stream file)
{
if (!Directory.Exists(_config.LocalDirectory))
Directory.CreateDirectory(_config.LocalDirectory);
string filePath = GetFilePath(fileName);
// Just write the stream to the file
using (FileStream fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
{
file.Seek(0, SeekOrigin.Begin);
file.CopyTo(fileStream);
}
}
public override void DeleteFile(string fileName)
{
string filePath = GetFilePath(fileName);
// Delete the File
if (File.Exists(filePath))
{
File.Delete(filePath);
}
}
private string GetFilePath(string fileName)
{
if (string.IsNullOrEmpty(fileName))
return null;
string subDir = fileName[0].ToString().ToLower();
return Path.Combine(_config.LocalDirectory, subDir, fileName);
}
}
}

24
StorageService/StorageService.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.IO;
using Teknik.Configuration;
namespace StorageService
{
public abstract class StorageService : IStorageService
{
protected readonly StorageConfig _config;
public StorageService(StorageConfig config)
{
_config = config;
}
public abstract string GetUniqueFileName();
public abstract Stream GetFile(string fileName);
public abstract List<string> GetFileNames();
public abstract void SaveFile(string fileName, Stream file);
public abstract void SaveEncryptedFile(string fileName, Stream file, int chunkSize, byte[] key, byte[] iv);
public abstract void DeleteFile(string fileName);
}
}

11
StorageService/StorageService.csproj Normal file
View File

@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Configuration\Configuration.csproj" />
</ItemGroup>
</Project>

24
StorageService/StorageServiceFactory.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Teknik.Configuration;
namespace StorageService
{
public static class StorageServiceFactory
{
public static StorageService GetStorageService(StorageConfig config)
{
switch (config.Type)
{
case StorageType.Local:
return new LocalStorageService(config);
case StorageType.S3:
default:
return null;
}
}
}
}

10
Teknik.sln
View File

@ -32,7 +32,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IdentityServer", "IdentityS
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContentScanningService", "ContentScanningService\ContentScanningService.csproj", "{491FE626-ABC8-4D00-8C7F-0849C357201A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebCommon", "WebCommon\WebCommon.csproj", "{32E85A7F-871A-437C-9BA3-00499AAB442C}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCommon", "WebCommon\WebCommon.csproj", "{32E85A7F-871A-437C-9BA3-00499AAB442C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StorageService", "StorageService\StorageService.csproj", "{4A600C17-C772-462F-A37F-307E7893B2DB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -107,6 +109,12 @@ Global
{32E85A7F-871A-437C-9BA3-00499AAB442C}.Release|Any CPU.Build.0 = Release|Any CPU
{32E85A7F-871A-437C-9BA3-00499AAB442C}.Test|Any CPU.ActiveCfg = Debug|Any CPU
{32E85A7F-871A-437C-9BA3-00499AAB442C}.Test|Any CPU.Build.0 = Debug|Any CPU
{4A600C17-C772-462F-A37F-307E7893B2DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4A600C17-C772-462F-A37F-307E7893B2DB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4A600C17-C772-462F-A37F-307E7893B2DB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4A600C17-C772-462F-A37F-307E7893B2DB}.Release|Any CPU.Build.0 = Release|Any CPU
{4A600C17-C772-462F-A37F-307E7893B2DB}.Test|Any CPU.ActiveCfg = Debug|Any CPU
{4A600C17-C772-462F-A37F-307E7893B2DB}.Test|Any CPU.Build.0 = Debug|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

72
Teknik/Areas/Paste/Controllers/PasteController.cs
View File

@ -22,6 +22,7 @@ using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Diagnostics;
using Teknik.Utilities.Routing;
using StorageService;
namespace Teknik.Areas.Paste.Controllers
{
@ -58,7 +59,7 @@ namespace Teknik.Areas.Paste.Controllers
// Check Expiration
if (PasteHelper.CheckExpiration(paste))
{
DeleteFile(paste);
PasteHelper.DeleteFile(_dbContext, _config, _logger, paste);
return new StatusCodeResult(StatusCodes.Status404NotFound);
}
@ -98,7 +99,7 @@ namespace Teknik.Areas.Paste.Controllers
if (string.IsNullOrEmpty(password) || hash != paste.HashedPassword)
{
PasswordViewModel passModel = new PasswordViewModel();
passModel.ActionUrl = Url.SubRouteUrl("p", "Paste.View");
passModel.ActionUrl = Url.SubRouteUrl("p", "Paste.View", new { type = type, url = url });
passModel.Url = url;
passModel.Type = type;
@ -119,15 +120,12 @@ namespace Teknik.Areas.Paste.Controllers
// Read in the file
if (string.IsNullOrEmpty(paste.FileName))
return new StatusCodeResult(StatusCodes.Status404NotFound);
string subDir = paste.FileName[0].ToString();
string filePath = Path.Combine(_config.PasteConfig.PasteDirectory, subDir, paste.FileName);
if (!System.IO.File.Exists(filePath))
{
var storageService = StorageServiceFactory.GetStorageService(_config.PasteConfig.StorageConfig);
var fileStream = storageService.GetFile(paste.FileName);
if (fileStream == null)
return new StatusCodeResult(StatusCodes.Status404NotFound);
}
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (AesCounterStream cs = new AesCounterStream(fs, false, keyBytes, ivBytes))
using (AesCounterStream cs = new AesCounterStream(fileStream, false, keyBytes, ivBytes))
using (StreamReader sr = new StreamReader(cs, Encoding.Unicode))
{
model.Content = await sr.ReadToEndAsync();
@ -151,8 +149,7 @@ namespace Teknik.Areas.Paste.Controllers
Response.Headers.Add("Content-Disposition", cd.ToString());
FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
return new BufferedFileStreamResult("application/octet-stream", async (response) => await ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fs, false, keyBytes, ivBytes), (int)fs.Length, _config.PasteConfig.ChunkSize), false);
return new BufferedFileStreamResult("application/octet-stream", async (response) => await ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fileStream, false, keyBytes, ivBytes), (int)fileStream.Length, _config.PasteConfig.ChunkSize), false);
default:
return View("~/Areas/Paste/Views/Paste/Full.cshtml", model);
}
@ -220,7 +217,7 @@ namespace Teknik.Areas.Paste.Controllers
// Check Expiration
if (PasteHelper.CheckExpiration(paste))
{
DeleteFile(paste);
PasteHelper.DeleteFile(_dbContext, _config, _logger, paste);
return new StatusCodeResult(StatusCodes.Status404NotFound);
}
@ -271,15 +268,12 @@ namespace Teknik.Areas.Paste.Controllers
// Read in the file
if (string.IsNullOrEmpty(paste.FileName))
return new StatusCodeResult(StatusCodes.Status404NotFound);
string subDir = paste.FileName[0].ToString();
string filePath = Path.Combine(_config.PasteConfig.PasteDirectory, subDir, paste.FileName);
if (!System.IO.File.Exists(filePath))
{
var storageService = StorageServiceFactory.GetStorageService(_config.PasteConfig.StorageConfig);
var fileStream = storageService.GetFile(paste.FileName);
if (fileStream == null)
return new StatusCodeResult(StatusCodes.Status404NotFound);
}
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (AesCounterStream cs = new AesCounterStream(fs, false, keyBytes, ivBytes))
using (AesCounterStream cs = new AesCounterStream(fileStream, false, keyBytes, ivBytes))
using (StreamReader sr = new StreamReader(cs, Encoding.Unicode))
{
model.Content = await sr.ReadToEndAsync();
@ -333,17 +327,16 @@ namespace Teknik.Areas.Paste.Controllers
}
// get the old file
string subDir = paste.FileName[0].ToString();
string oldFile = Path.Combine(_config.PasteConfig.PasteDirectory, subDir, paste.FileName);
var storageService = StorageServiceFactory.GetStorageService(_config.PasteConfig.StorageConfig);
var oldFile = paste.FileName;
// Generate a unique file name that does not currently exist
string newFilePath = FileHelper.GenerateRandomFileName(_config.PasteConfig.PasteDirectory, _config.PasteConfig.FileExtension, 10);
string fileName = Path.GetFileName(newFilePath);
string fileName = storageService.GetUniqueFileName();
string key = PasteHelper.GenerateKey(_config.PasteConfig.KeySize);
string iv = PasteHelper.GenerateIV(_config.PasteConfig.BlockSize);
PasteHelper.EncryptContents(model.Content, newFilePath, password, key, iv, _config.PasteConfig.KeySize, _config.PasteConfig.ChunkSize);
PasteHelper.EncryptContents(storageService, model.Content, fileName, password, key, iv, _config.PasteConfig.KeySize, _config.PasteConfig.ChunkSize);
paste.Key = key;
paste.KeySize = _config.PasteConfig.KeySize;
@ -361,8 +354,7 @@ namespace Teknik.Areas.Paste.Controllers
_dbContext.SaveChanges();
// Delete the old file
if (System.IO.File.Exists(oldFile))
System.IO.File.Delete(oldFile);
storageService.DeleteFile(oldFile);
return Redirect(Url.SubRouteUrl("p", "Paste.View", new { type = "Full", url = paste.Url }));
}
@ -385,7 +377,7 @@ namespace Teknik.Areas.Paste.Controllers
if (foundPaste.User.Username == User.Identity.Name ||
User.IsInRole("Admin"))
{
DeleteFile(foundPaste);
PasteHelper.DeleteFile(_dbContext, _config, _logger, foundPaste);
return Json(new { result = true, redirect = Url.SubRouteUrl("p", "Paste.Index") });
}
@ -418,31 +410,5 @@ namespace Teknik.Areas.Paste.Controllers
HttpContext.Session.Remove("PastePassword_" + url);
}
}
private void DeleteFile(Models.Paste paste)
{
if (!string.IsNullOrEmpty(paste.FileName))
{
string delSub = paste.FileName[0].ToString();
string delPath = Path.Combine(_config.PasteConfig.PasteDirectory, delSub, paste.FileName);
// Delete the File
if (System.IO.File.Exists(delPath))
{
try
{
System.IO.File.Delete(delPath);
}
catch (Exception ex)
{
_logger.LogError(ex, "Unable to delete file: {0}", paste.FileName);
}
}
}
// Delete from the DB
_dbContext.Pastes.Remove(paste);
_dbContext.SaveChanges();
}
}
}

40
Teknik/Areas/Paste/PasteHelper.cs
View File

@ -9,6 +9,9 @@ using Teknik.Models;
using Teknik.Utilities.Cryptography;
using Teknik.Data;
using System.IO;
using StorageService;
using Teknik.Logging;
using Microsoft.Extensions.Logging;
namespace Teknik.Areas.Paste
{
@ -56,15 +59,6 @@ namespace Teknik.Areas.Paste
break;
}
if (!Directory.Exists(config.PasteConfig.PasteDirectory))
{
Directory.CreateDirectory(config.PasteConfig.PasteDirectory);
}
// Generate a unique file name that does not currently exist
string filePath = FileHelper.GenerateRandomFileName(config.PasteConfig.PasteDirectory, config.PasteConfig.FileExtension, 10);
string fileName = Path.GetFileName(filePath);
string key = GenerateKey(config.PasteConfig.KeySize);
string iv = GenerateIV(config.PasteConfig.BlockSize);
@ -73,8 +67,13 @@ namespace Teknik.Areas.Paste
paste.HashedPassword = HashPassword(key, password);
}
// Generate a unique file name that does not currently exist
var storageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig);
var fileName = storageService.GetUniqueFileName();
// Encrypt the contents to the file
EncryptContents(content, filePath, password, key, iv, config.PasteConfig.KeySize, config.PasteConfig.ChunkSize);
EncryptContents(storageService, content, fileName, password, key, iv, config.PasteConfig.KeySize, config.PasteConfig.ChunkSize);
// Generate a deletion key
string delKey = StringHelper.RandomString(config.PasteConfig.DeleteKeyLength);
@ -118,7 +117,7 @@ namespace Teknik.Areas.Paste
return SHA384.Hash(key, password).ToHex();
}
public static void EncryptContents(string content, string filePath, string password, string key, string iv, int keySize, int chunkSize)
public static void EncryptContents(IStorageService storageService, string content, string fileName, string password, string key, string iv, int keySize, int chunkSize)
{
byte[] ivBytes = Encoding.Unicode.GetBytes(iv);
byte[] keyBytes = AesCounterManaged.CreateKey(key, ivBytes, keySize);
@ -133,8 +132,25 @@ namespace Teknik.Areas.Paste
byte[] data = Encoding.Unicode.GetBytes(content);
using (MemoryStream ms = new MemoryStream(data))
{
AesCounterManaged.EncryptToFile(filePath, ms, chunkSize, keyBytes, ivBytes);
storageService.SaveEncryptedFile(fileName, ms, chunkSize, keyBytes, ivBytes);
}
}
public static void DeleteFile(TeknikEntities db, Config config, ILogger<Logger> logger, Models.Paste paste)
{
try
{
var storageService = StorageServiceFactory.GetStorageService(config.PasteConfig.StorageConfig);
storageService.DeleteFile(paste.FileName);
}
catch (Exception ex)
{
logger.LogError(ex, "Unable to delete file: {0}", paste.FileName);
}
// Delete from the DB
db.Pastes.Remove(paste);
db.SaveChanges();
}
}
}

60
Teknik/Areas/Upload/Controllers/UploadController.cs
View File

@ -24,6 +24,7 @@ using Teknik.Logging;
using Teknik.Areas.Users.Models;
using Teknik.ContentScanningService;
using Teknik.Utilities.Routing;
using StorageService;
namespace Teknik.Areas.Upload.Controllers
{
@ -240,7 +241,7 @@ namespace Teknik.Areas.Upload.Controllers
// Check Expiration
if (UploadHelper.CheckExpiration(upload))
{
DeleteFile(upload);
UploadHelper.DeleteFile(_dbContext, _config, _logger, upload);
return new StatusCodeResult(StatusCodes.Status404NotFound);
}
@ -320,12 +321,12 @@ namespace Teknik.Areas.Upload.Controllers
}
else
{
string subDir = fileName[0].ToString();
string filePath = Path.Combine(_config.UploadConfig.UploadDirectory, subDir, fileName);
var storageService = StorageServiceFactory.GetStorageService(_config.UploadConfig.StorageConfig);
var fileStream = storageService.GetFile(fileName);
long startByte = 0;
long endByte = contentLength - 1;
long length = contentLength;
if (System.IO.File.Exists(filePath))
if (fileStream != null)
{
#region Range Calculation
// Are they downloading it by range?
@ -413,11 +414,8 @@ namespace Teknik.Areas.Upload.Controllers
Response.Headers.Add("Content-Disposition", cd.ToString());
// Read in the file
FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
// Reset file stream to starting position (or start of range)
fs.Seek(startByte, SeekOrigin.Begin);
fileStream.Seek(startByte, SeekOrigin.Begin);
try
{
@ -427,12 +425,12 @@ namespace Teknik.Areas.Upload.Controllers
byte[] keyBytes = Encoding.UTF8.GetBytes(key);
byte[] ivBytes = Encoding.UTF8.GetBytes(iv);
return new BufferedFileStreamResult(contentType, async (response) => await ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fs, false, keyBytes, ivBytes), (int)length, _config.UploadConfig.ChunkSize), false);
return new BufferedFileStreamResult(contentType, async (response) => await ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fileStream, false, keyBytes, ivBytes), (int)length, _config.UploadConfig.ChunkSize), false);
}
else // Otherwise just send it
{
// Send the file
return new BufferedFileStreamResult(contentType, async (response) => await ResponseHelper.StreamToOutput(response, true, fs, (int)length, _config.UploadConfig.ChunkSize), false);
return new BufferedFileStreamResult(contentType, async (response) => await ResponseHelper.StreamToOutput(response, true, fileStream, (int)length, _config.UploadConfig.ChunkSize), false);
}
}
catch (Exception ex)
@ -459,13 +457,13 @@ namespace Teknik.Areas.Upload.Controllers
// Check Expiration
if (UploadHelper.CheckExpiration(upload))
{
DeleteFile(upload);
UploadHelper.DeleteFile(_dbContext, _config, _logger, upload);
return Json(new { error = new { message = "File Does Not Exist" } });
}
string subDir = upload.FileName[0].ToString();
string filePath = Path.Combine(_config.UploadConfig.UploadDirectory, subDir, upload.FileName);
if (System.IO.File.Exists(filePath))
var storageService = StorageServiceFactory.GetStorageService(_config.UploadConfig.StorageConfig);
var fileStream = storageService.GetFile(upload.FileName);
if (fileStream != null)
{
// Notify the client the content length we'll be outputting
Response.Headers.Add("Content-Length", upload.ContentLength.ToString());
@ -482,21 +480,18 @@ namespace Teknik.Areas.Upload.Controllers
Response.Headers.Add("Content-Disposition", cd.ToString());
// Read in the file
FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
// If the IV is set, and Key is set, then decrypt it while sending
if (decrypt && !string.IsNullOrEmpty(upload.Key) && !string.IsNullOrEmpty(upload.IV))
{
byte[] keyBytes = Encoding.UTF8.GetBytes(upload.Key);
byte[] ivBytes = Encoding.UTF8.GetBytes(upload.IV);
return new BufferedFileStreamResult(upload.ContentType, (response) => ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fs, false, keyBytes, ivBytes), (int)upload.ContentLength, _config.UploadConfig.ChunkSize), false);
return new BufferedFileStreamResult(upload.ContentType, (response) => ResponseHelper.StreamToOutput(response, true, new AesCounterStream(fileStream, false, keyBytes, ivBytes), (int)upload.ContentLength, _config.UploadConfig.ChunkSize), false);
}
else // Otherwise just send it
{
// Send the file
return new BufferedFileStreamResult(upload.ContentType, (response) => ResponseHelper.StreamToOutput(response, true, fs, (int)upload.ContentLength, _config.UploadConfig.ChunkSize), false);
return new BufferedFileStreamResult(upload.ContentType, (response) => ResponseHelper.StreamToOutput(response, true, fileStream, (int)upload.ContentLength, _config.UploadConfig.ChunkSize), false);
}
}
}
@ -517,7 +512,7 @@ namespace Teknik.Areas.Upload.Controllers
model.File = file;
if (!string.IsNullOrEmpty(upload.DeleteKey) && upload.DeleteKey == key)
{
DeleteFile(upload);
UploadHelper.DeleteFile(_dbContext, _config, _logger, upload);
model.Deleted = true;
}
else
@ -557,35 +552,12 @@ namespace Teknik.Areas.Upload.Controllers
{
if (foundUpload.User.Username == User.Identity.Name)
{
DeleteFile(foundUpload);
UploadHelper.DeleteFile(_dbContext, _config, _logger, foundUpload);
return Json(new { result = true });
}
return Json(new { error = new { message = "You do not have permission to edit this Paste" } });
}
return Json(new { error = new { message = "This Upload does not exist" } });
}
private void DeleteFile(Models.Upload upload)
{
string subDir = upload.FileName[0].ToString();
string filePath = Path.Combine(_config.UploadConfig.UploadDirectory, subDir, upload.FileName);
// Delete the File
if (System.IO.File.Exists(filePath))
{
try
{
System.IO.File.Delete(filePath);
}
catch (Exception ex)
{
_logger.LogError(ex, "Unable to delete file: {0}", upload.FileName);
}
}
// Delete from the DB
_dbContext.Uploads.Remove(upload);
_dbContext.SaveChanges();
}
}
}

38
Teknik/Areas/Upload/UploadHelper.cs
View File

@ -9,6 +9,9 @@ using Teknik.Utilities;
using System.Text;
using Teknik.Utilities.Cryptography;
using Teknik.Data;
using StorageService;
using Teknik.Logging;
using Microsoft.Extensions.Logging;
namespace Teknik.Areas.Upload
{
@ -36,14 +39,10 @@ namespace Teknik.Areas.Upload
public static Models.Upload SaveFile(TeknikEntities db, Config config, Stream file, string contentType, long contentLength, bool encrypt, ExpirationUnit expirationUnit, int expirationLength, string fileExt, string iv, string key, int keySize, int blockSize)
{
if (!Directory.Exists(config.UploadConfig.UploadDirectory))
{
Directory.CreateDirectory(config.UploadConfig.UploadDirectory);
}
var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
// Generate a unique file name that does not currently exist
string filePath = FileHelper.GenerateRandomFileName(config.UploadConfig.UploadDirectory, config.UploadConfig.FileExtension, 10);
string fileName = Path.GetFileName(filePath);
var fileName = storageService.GetUniqueFileName();
// once we have the filename, lets save the file
if (encrypt)
@ -57,17 +56,11 @@ namespace Teknik.Areas.Upload
byte[] keyBytes = Encoding.UTF8.GetBytes(key);
byte[] ivBytes = Encoding.UTF8.GetBytes(iv);
// Encrypt the file to disk
AesCounterManaged.EncryptToFile(filePath, file, config.UploadConfig.ChunkSize, keyBytes, ivBytes);
storageService.SaveEncryptedFile(fileName, file, config.UploadConfig.ChunkSize, keyBytes, ivBytes);
}
else
{
// Just write the stream to the file
using (FileStream fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
{
file.Seek(0, SeekOrigin.Begin);
file.CopyTo(fileStream);
}
storageService.SaveFile(fileName, file);
}
// Generate a unique url
@ -142,5 +135,22 @@ namespace Teknik.Areas.Upload
return upload;
}
public static void DeleteFile(TeknikEntities db, Config config, ILogger<Logger> logger, Models.Upload upload)
{
try
{
var storageService = StorageServiceFactory.GetStorageService(config.UploadConfig.StorageConfig);
storageService.DeleteFile(upload.FileName);
}
catch (Exception ex)
{
logger.LogError(ex, "Unable to delete file: {0}", upload.FileName);
}
// Delete from the DB
db.Uploads.Remove(upload);
db.SaveChanges();
}
}
}

10
Teknik/Areas/Vault/Controllers/VaultController.cs
View File

@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using StorageService;
using System;
using System.Collections.Generic;
using System.IO;
@ -110,15 +111,14 @@ namespace Teknik.Areas.Vault.Controllers
if (!pasteModel.HasPassword)
{
// Read in the file
string subDir = paste.Paste.FileName[0].ToString();
string filePath = Path.Combine(_config.PasteConfig.PasteDirectory, subDir, paste.Paste.FileName);
if (!System.IO.File.Exists(filePath))
var storageService = StorageServiceFactory.GetStorageService(_config.PasteConfig.StorageConfig);
var fileStream = storageService.GetFile(paste.Paste.FileName);
if (fileStream == null)
continue;
byte[] ivBytes = Encoding.Unicode.GetBytes(paste.Paste.IV);
byte[] keyBytes = AesCounterManaged.CreateKey(paste.Paste.Key, ivBytes, paste.Paste.KeySize);
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (AesCounterStream cs = new AesCounterStream(fs, false, keyBytes, ivBytes))
using (AesCounterStream cs = new AesCounterStream(fileStream, false, keyBytes, ivBytes))
using (StreamReader sr = new StreamReader(cs, Encoding.Unicode))
{
pasteModel.Content = await sr.ReadToEndAsync();

1
Teknik/Teknik.csproj
View File

@ -96,6 +96,7 @@
<ProjectReference Include="..\GitService\GitService.csproj" />
<ProjectReference Include="..\Logging\Logging.csproj" />
<ProjectReference Include="..\MailService\MailService.csproj" />
<ProjectReference Include="..\StorageService\StorageService.csproj" />
<ProjectReference Include="..\Tracking\Tracking.csproj" />
<ProjectReference Include="..\Utilities\Utilities.csproj" />
<ProjectReference Include="..\WebCommon\WebCommon.csproj" />

52
Utilities/Cryptography/AesCounterManaged.cs
View File

@ -77,39 +77,45 @@ namespace Teknik.Utilities.Cryptography
}
public static void EncryptToFile(string filePath, Stream input, int chunkSize, byte[] key, byte[] iv)
{
using (FileStream fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
{
EncryptToStream(input, fileStream, chunkSize, key, iv);
}
}
public static void EncryptToStream(Stream input, Stream output, int chunkSize, byte[] key, byte[] iv)
{
// Make sure the input stream is at the beginning
input.Seek(0, SeekOrigin.Begin);
AesCounterStream cryptoStream = new AesCounterStream(input, true, key, iv);
using (FileStream fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
int curByte = 0;
int processedBytes = 0;
byte[] buffer = new byte[chunkSize];
int bytesRemaining = (int)input.Length;
int bytesToRead = chunkSize;
do
{
int curByte = 0;
int processedBytes = 0;
byte[] buffer = new byte[chunkSize];
int bytesRemaining = (int)input.Length;
int bytesToRead = chunkSize;
do
if (chunkSize > bytesRemaining)
{
if (chunkSize > bytesRemaining)
{
bytesToRead = bytesRemaining;
}
processedBytes = cryptoStream.Read(buffer, 0, bytesToRead);
if (processedBytes > 0)
{
fileStream.Write(buffer, 0, processedBytes);
// Clear the buffer
Array.Clear(buffer, 0, chunkSize);
}
curByte += processedBytes;
bytesRemaining -= processedBytes;
bytesToRead = bytesRemaining;
}
while (processedBytes > 0 && bytesRemaining > 0);
processedBytes = cryptoStream.Read(buffer, 0, bytesToRead);
if (processedBytes > 0)
{
output.Write(buffer, 0, processedBytes);
// Clear the buffer
Array.Clear(buffer, 0, chunkSize);
}
curByte += processedBytes;
bytesRemaining -= processedBytes;
}
while (processedBytes > 0 && bytesRemaining > 0);
}
public static byte[] CreateKey(string password, string iv, int keySize = 256)