Mirrors/Teknik
1
0
Fork 0
mirror of https://git.teknik.io/Teknikode/Teknik.git synced 2023-08-02 14:16:22 +02:00
Code Releases Activity

Added scanning based on sha1 hash of the file to an endpoint

Browse Source
This commit is contained in:
Uncled1023 2020-07-17 00:48:24 -07:00
parent 2b14547c25
commit c701493859
15 changed files with 431 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Configuration/ClamConfig.cs Normal file
View File

@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Teknik.Configuration
{
public class ClamConfig
{
// Virus Scanning Settings
public bool Enabled { get; set; }
public string Server { get; set; }
public int Port { get; set; }
public ClamConfig()
{
Enabled = false;
Server = "localhost";
Port = 3310;
}
}
}

24
Configuration/HashScanConfig.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Teknik.Configuration
{
public class HashScanConfig
{
public bool Enabled { get; set; }
public string Endpoint { get; set; }
public bool Authenticate { get; set; }
public string Username { get; set; }
public string Password { get; set; }
public HashScanConfig()
{
Enabled = false;
Endpoint = string.Empty;
Authenticate = false;
Username = string.Empty;
Password = string.Empty;
}
}
}

11
Configuration/UploadConfig.cs
View File

@ -31,9 +31,9 @@ namespace Teknik.Configuration
// The size of the chunk that the file will be encrypted/decrypted in (bytes)
public int ChunkSize { get; set; }
// Virus Scanning Settings
public bool VirusScanEnable { get; set; }
public string ClamServer { get; set; }
public int ClamPort { get; set; }
public ClamConfig ClamConfig { get; set; }
// Hash Scanning Settings
public HashScanConfig HashScanConfig { get; set; }
// Content Type Restrictions
public List<string> RestrictedContentTypes { get; set; }
public List<string> RestrictedExtensions { get; set; }
@ -61,9 +61,8 @@ namespace Teknik.Configuration
BlockSize = 128;
IncludeExtension = true;
ChunkSize = 1024;
VirusScanEnable = false;
ClamServer = "localhost";
ClamPort = 3310;
ClamConfig = new ClamConfig();
HashScanConfig = new HashScanConfig();
RestrictedContentTypes = new List<string>();
RestrictedExtensions = new List<string>();
}

50
ContentScanningService/ClamScanner.cs Normal file
View File

@ -0,0 +1,50 @@
using nClam;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Teknik.Configuration;
namespace Teknik.ContentScanningService
{
public class ClamScanner : ContentScanner
{
public ClamScanner(Config config) : base(config)
{ }
public async override Task<ScanResult> ScanFile(Stream stream)
{
var result = new ScanResult();
if (stream != null)
{
// Set the start of the stream
stream.Seek(0, SeekOrigin.Begin);
ClamClient clam = new ClamClient(_config.UploadConfig.ClamConfig.Server, _config.UploadConfig.ClamConfig.Port);
clam.MaxStreamSize = stream.Length;
ClamScanResult scanResult = await clam.SendAndScanFileAsync(stream);
result.RawResult = scanResult.RawResult;
switch (scanResult.Result)
{
case ClamScanResults.Clean:
result.ResultType = ScanResultType.Clean;
break;
case ClamScanResults.VirusDetected:
result.ResultType = ScanResultType.VirusDetected;
result.RawResult = scanResult.InfectedFiles.First().VirusName;
break;
case ClamScanResults.Error:
result.ResultType = ScanResultType.Error;
break;
case ClamScanResults.Unknown:
result.ResultType = ScanResultType.Unknown;
break;
}
}
return result;
}
}
}

21
ContentScanningService/ContentScanner.cs Normal file
View File

@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Teknik.Configuration;
namespace Teknik.ContentScanningService
{
public abstract class ContentScanner
{
protected readonly Config _config;
public ContentScanner(Config config)
{
_config = config;
}
public abstract Task<ScanResult> ScanFile(Stream stream);
}
}

19
ContentScanningService/ContentScanningService.csproj Normal file
View File

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp2.2</TargetFramework>
<AssemblyName>Teknik.ContentScanningService</AssemblyName>
<RootNamespace>Teknik.ContentScanningService</RootNamespace>
<RuntimeIdentifiers>win-x86;win-x64;linux-x64;linux-arm;osx-x64</RuntimeIdentifiers>
<Configurations>Debug;Release;Test</Configurations>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="nClam" Version="4.0.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Configuration\Configuration.csproj" />
</ItemGroup>
</Project>

55
ContentScanningService/HashScanner.cs Normal file
View File

@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Teknik.Configuration;
using Teknik.Utilities.Cryptography;
namespace Teknik.ContentScanningService
{
public class HashScanner : ContentScanner
{
private static readonly HttpClient _client = new HttpClient();
public HashScanner(Config config) : base(config)
{ }
public async override Task<ScanResult> ScanFile(Stream stream)
{
var result = new ScanResult();
if (stream != null)
{
// Set the start of the stream
stream.Seek(0, SeekOrigin.Begin);
if (_config.UploadConfig.HashScanConfig.Authenticate)
{
var byteArray = Encoding.UTF8.GetBytes($"{_config.UploadConfig.HashScanConfig.Username}:{_config.UploadConfig.HashScanConfig.Password}");
_client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
}
// compute the hash of the stream
var hash = Utilities.Cryptography.SHA1.Hash(stream);
HttpResponseMessage response = await _client.PostAsync(_config.UploadConfig.HashScanConfig.Endpoint, new StringContent(hash));
HttpContent content = response.Content;
string resultStr = await content.ReadAsStringAsync();
if (resultStr == "true")
{
// The hash matched a CP entry, let's return the result as such
result.ResultType = ScanResultType.ChildPornography;
}
else
{
result.RawResult = resultStr + " | " + hash + _client.DefaultRequestHeaders.Authorization.ToString();
}
}
return result;
}
}
}

19
ContentScanningService/ScanResult.cs Normal file
View File

@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Text;
namespace Teknik.ContentScanningService
{
public class ScanResult
{
public string RawResult { get; set;}
public ScanResultType ResultType { get; set; }
public ScanResult()
{
RawResult = string.Empty;
ResultType = ScanResultType.Clean;
}
}
}

15
ContentScanningService/ScanResultType.cs Normal file
View File

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Teknik.ContentScanningService
{
public enum ScanResultType
{
Unknown = 0,
Clean = 1,
VirusDetected = 2,
ChildPornography = 3,
Error = 4
}
}

8
Teknik.sln
View File

@ -31,6 +31,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceWorker", "ServiceWor
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IdentityServer", "IdentityServer\IdentityServer.csproj", "{3434645B-B8B4-457A-8C85-342E6727CCEE}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ContentScanningService", "ContentScanningService\ContentScanningService.csproj", "{491FE626-ABC8-4D00-8C7F-0849C357201A}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -92,6 +94,12 @@ Global
{3434645B-B8B4-457A-8C85-342E6727CCEE}.Release|Any CPU.Build.0 = Release|Any CPU
{3434645B-B8B4-457A-8C85-342E6727CCEE}.Test|Any CPU.ActiveCfg = Test|Any CPU
{3434645B-B8B4-457A-8C85-342E6727CCEE}.Test|Any CPU.Build.0 = Test|Any CPU
{491FE626-ABC8-4D00-8C7F-0849C357201A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{491FE626-ABC8-4D00-8C7F-0849C357201A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{491FE626-ABC8-4D00-8C7F-0849C357201A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{491FE626-ABC8-4D00-8C7F-0849C357201A}.Release|Any CPU.Build.0 = Release|Any CPU
{491FE626-ABC8-4D00-8C7F-0849C357201A}.Test|Any CPU.ActiveCfg = Debug|Any CPU
{491FE626-ABC8-4D00-8C7F-0849C357201A}.Test|Any CPU.Build.0 = Debug|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

52
Teknik/Areas/API/V1/Controllers/UploadAPIv1Controller.cs
View File

@ -16,6 +16,7 @@ using Teknik.Areas.Upload;
using Teknik.Areas.Users.Models;
using Teknik.Areas.Users.Utility;
using Teknik.Configuration;
using Teknik.ContentScanningService;
using Teknik.Data;
using Teknik.Filters;
using Teknik.Logging;
@ -29,7 +30,6 @@ namespace Teknik.Areas.API.V1.Controllers
public UploadAPIv1Controller(ILogger<Logger> logger, Config config, TeknikEntities dbContext) : base(logger, config, dbContext) { }
[HttpPost]
[AllowAnonymous]
[TrackPageView]
public async Task<IActionResult> Upload(UploadAPIv1Model model)
{
@ -73,24 +73,36 @@ namespace Teknik.Areas.API.V1.Controllers
string fileExt = FileHelper.GetFileExtension(model.file.FileName);
long contentLength = model.file.Length;
// Scan the file to detect a virus
if (_config.UploadConfig.VirusScanEnable)
using (var fs = model.file.OpenReadStream())
{
ClamClient clam = new ClamClient(_config.UploadConfig.ClamServer, _config.UploadConfig.ClamPort);
clam.MaxStreamSize = maxUploadSize;
ClamScanResult scanResult = await clam.SendAndScanFileAsync(model.file.OpenReadStream());
ScanResult scanResult = null;
switch (scanResult.Result)
// Scan the file to detect a virus
if (_config.UploadConfig.ClamConfig.Enabled)
{
case ClamScanResults.Clean:
break;
case ClamScanResults.VirusDetected:
return Json(new { error = new { message = string.Format("Virus Detected: {0}. As per our <a href=\"{1}\">Terms of Service</a>, Viruses are not permited.", scanResult.InfectedFiles.First().VirusName, Url.SubRouteUrl("tos", "TOS.Index")) } });
case ClamScanResults.Error:
break;
case ClamScanResults.Unknown:
break;
var clamScanner = new ClamScanner(_config);
scanResult = await clamScanner.ScanFile(fs);
}
// Scan the files against an endpoint based on hash
if (_config.UploadConfig.HashScanConfig.Enabled && (scanResult == null || scanResult.ResultType == ScanResultType.Clean))
{
var hashScanner = new HashScanner(_config);
scanResult = await hashScanner.ScanFile(fs);
}
switch (scanResult?.ResultType)
{
case ScanResultType.Clean:
break;
case ScanResultType.VirusDetected:
return Json(new { error = new { message = string.Format("Virus Detected: {0}. As per our <a href=\"{1}\">Terms of Service</a>, Viruses are not permited.", scanResult.RawResult, Url.SubRouteUrl("tos", "TOS.Index")) } });
case ScanResultType.ChildPornography:
return Json(new { error = new { message = string.Format("Child Pornography Detected: As per our <a href=\"{0}\">Terms of Service</a>, Child Pornography is strictly not permited.", Url.SubRouteUrl("tos", "TOS.Index")) } });
case ScanResultType.Error:
return Json(new { error = new { message = string.Format("Error scanning the file upload. {0}", scanResult.RawResult) } });
case ScanResultType.Unknown:
return Json(new { error = new { message = string.Format("Unknown result while scanning the file upload. {0}", scanResult.RawResult) } });
}
// Need to grab the contentType if it's empty
@ -100,10 +112,8 @@ namespace Teknik.Areas.API.V1.Controllers
if (string.IsNullOrEmpty(model.contentType))
{
using (Stream fileStream = model.file.OpenReadStream())
{
fileStream.Seek(0, SeekOrigin.Begin);
FileType fileType = fileStream.GetFileType();
fs.Seek(0, SeekOrigin.Begin);
FileType fileType = fs.GetFileType();
if (fileType != null)
model.contentType = fileType.Mime;
if (string.IsNullOrEmpty(model.contentType))
@ -112,7 +122,6 @@ namespace Teknik.Areas.API.V1.Controllers
}
}
}
}
// Check content type restrictions (Only for encrypting server side
if (model.encrypt || !string.IsNullOrEmpty(model.key))
@ -130,7 +139,7 @@ namespace Teknik.Areas.API.V1.Controllers
model.blockSize = _config.UploadConfig.BlockSize;
// Save the file data
Upload.Models.Upload upload = UploadHelper.SaveFile(_dbContext, _config, model.file.OpenReadStream(), model.contentType, contentLength, model.encrypt, model.expirationUnit, model.expirationLength, fileExt, model.iv, model.key, model.keySize, model.blockSize);
Upload.Models.Upload upload = UploadHelper.SaveFile(_dbContext, _config, fs, model.contentType, contentLength, model.encrypt, model.expirationUnit, model.expirationLength, fileExt, model.iv, model.key, model.keySize, model.blockSize);
if (upload != null)
{
@ -183,6 +192,7 @@ namespace Teknik.Areas.API.V1.Controllers
};
return Json(new { result = returnData });
}
}
return Json(new { error = new { message = "Unable to save file" } });
}

53
Teknik/Areas/Upload/Controllers/UploadController.cs
View File

@ -22,6 +22,7 @@ using Microsoft.EntityFrameworkCore;
using Microsoft.AspNetCore.Http;
using Teknik.Logging;
using Teknik.Areas.Users.Models;
using Teknik.ContentScanningService;
namespace Teknik.Areas.Upload.Controllers
{
@ -32,7 +33,6 @@ namespace Teknik.Areas.Upload.Controllers
public UploadController(ILogger<Logger> logger, Config config, TeknikEntities dbContext) : base(logger, config, dbContext) { }
[HttpGet]
[AllowAnonymous]
[TrackPageView]
public async Task<IActionResult> Index()
{
@ -77,7 +77,6 @@ namespace Teknik.Areas.Upload.Controllers
}
[HttpPost]
[AllowAnonymous]
[DisableRequestSizeLimit]
public async Task<IActionResult> Upload([FromForm] UploadFileViewModel uploadFile)
{
@ -118,27 +117,36 @@ namespace Teknik.Areas.Upload.Controllers
// convert file to bytes
long contentLength = uploadFile.file.Length;
// Scan the file to detect a virus
if (_config.UploadConfig.VirusScanEnable)
{
using (Stream fs = uploadFile.file.OpenReadStream())
{
ClamClient clam = new ClamClient(_config.UploadConfig.ClamServer, _config.UploadConfig.ClamPort);
clam.MaxStreamSize = maxUploadSize;
ClamScanResult scanResult = await clam.SendAndScanFileAsync(fs);
ScanResult scanResult = null;
switch (scanResult.Result)
// Scan the file to detect a virus
if (_config.UploadConfig.ClamConfig.Enabled)
{
case ClamScanResults.Clean:
var clamScanner = new ClamScanner(_config);
scanResult = await clamScanner.ScanFile(fs);
}
// Scan the files against an endpoint based on hash
if (_config.UploadConfig.HashScanConfig.Enabled && (scanResult == null || scanResult.ResultType == ScanResultType.Clean))
{
var hashScanner = new HashScanner(_config);
scanResult = await hashScanner.ScanFile(fs);
}
switch (scanResult?.ResultType)
{
case ScanResultType.Clean:
break;
case ClamScanResults.VirusDetected:
return Json(new { error = new { message = string.Format("Virus Detected: {0}. As per our <a href=\"{1}\">Terms of Service</a>, Viruses are not permited.", scanResult.InfectedFiles.First().VirusName, Url.SubRouteUrl("tos", "TOS.Index")) } });
case ClamScanResults.Error:
return Json(new { error = new { message = string.Format("Error scanning the file upload for viruses. {0}", scanResult.RawResult) } });
case ClamScanResults.Unknown:
return Json(new { error = new { message = string.Format("Unknown result while scanning the file upload for viruses. {0}", scanResult.RawResult) } });
}
}
case ScanResultType.VirusDetected:
return Json(new { error = new { message = string.Format("Virus Detected: {0}. As per our <a href=\"{1}\">Terms of Service</a>, Viruses are not permited.", scanResult.RawResult, Url.SubRouteUrl("tos", "TOS.Index")) } });
case ScanResultType.ChildPornography:
return Json(new { error = new { message = string.Format("Child Pornography Detected: As per our <a href=\"{0}\">Terms of Service</a>, Child Pornography is not permited.", Url.SubRouteUrl("tos", "TOS.Index")) } });
case ScanResultType.Error:
return Json(new { error = new { message = string.Format("Error scanning the file upload. {0}", scanResult.RawResult) } });
case ScanResultType.Unknown:
return Json(new { error = new { message = string.Format("Unknown result while scanning the file upload. {0}", scanResult.RawResult) } });
}
// Check content type restrictions (Only for encrypting server side
@ -150,8 +158,6 @@ namespace Teknik.Areas.Upload.Controllers
}
}
using (Stream fs = uploadFile.file.OpenReadStream())
{
Models.Upload upload = UploadHelper.SaveFile(_dbContext,
_config,
fs,
@ -176,7 +182,9 @@ namespace Teknik.Areas.Upload.Controllers
_dbContext.SaveChanges();
}
}
return Json(new { result = new
return Json(new
{
result = new
{
name = upload.Url,
url = Url.SubRouteUrl("u", "Upload.Download", new { file = upload.Url }),
@ -186,7 +194,8 @@ namespace Teknik.Areas.Upload.Controllers
deleteUrl = Url.SubRouteUrl("u", "Upload.DeleteByKey", new { file = upload.Url, key = upload.DeleteKey }),
expirationUnit = uploadFile.options.ExpirationUnit.ToString(),
expirationLength = uploadFile.options.ExpirationLength
} });
}
});
}
}
return Json(new { error = new { message = "Unable to upload file" } });

1
Teknik/Teknik.csproj
View File

@ -81,6 +81,7 @@
<ItemGroup>
<ProjectReference Include="..\Configuration\Configuration.csproj" />
<ProjectReference Include="..\ContentScanningService\ContentScanningService.csproj" />
<ProjectReference Include="..\GitService\GitService.csproj" />
<ProjectReference Include="..\Logging\Logging.csproj" />
<ProjectReference Include="..\MailService\MailService.csproj" />

33
Utilities/Cryptography/SHA1.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
namespace Teknik.Utilities.Cryptography
{
public class SHA1
{
public static string Hash(string text)
{
using (var ms = new MemoryStream(Encoding.UTF8.GetBytes(text)))
{
return Hash(ms);
}
}
public static string Hash(Stream stream)
{
var hash = default(string);
using (var algo = System.Security.Cryptography.SHA1.Create())
{
var hashBytes = algo.ComputeHash(stream);
// Return as hexadecimal string
hash = hashBytes.ToHex();
}
return hash;
}
}
}

9
Utilities/Cryptography/SHA256.cs
View File

@ -30,7 +30,6 @@ namespace Teknik.Utilities.Cryptography
public static string Hash(string value, string salt1, string salt2)
{
SHA256Managed hash = new SHA256Managed();
SHA1 sha1 = new SHA1Managed();
// gen salt2 hash
byte[] dataSalt2 = Encoding.UTF8.GetBytes(salt2);
byte[] salt2Bytes = hash.ComputeHash(dataSalt2);
@ -40,13 +39,7 @@ namespace Teknik.Utilities.Cryptography
salt2Str += String.Format("{0:x2}", x);
}
string dataStr = salt1 + value + salt2Str;
byte[] dataStrBytes = Encoding.UTF8.GetBytes(dataStr);
byte[] shaBytes = sha1.ComputeHash(dataStrBytes);
string sha1Str = string.Empty;
foreach (byte x in shaBytes)
{
sha1Str += String.Format("{0:x2}", x);
}
string sha1Str = SHA1.Hash(dataStr);
byte[] sha1Bytes = Encoding.UTF8.GetBytes(sha1Str);
byte[] valueBytes = hash.ComputeHash(sha1Bytes);
string hashString = string.Empty;