Mirrors/invoiceninja
1
0
Fork 0
mirror of https://github.com/invoiceninja/invoiceninja.git synced 2024-11-08 20:22:42 +01:00
Code Issues Releases Activity

Minor refactors for inbound email processing

Browse Source
This commit is contained in:
David Bomba 2024-08-30 10:57:43 +10:00
parent 8f88c408f7
commit 93c382eae1
9 changed files with 162 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
app/Http/Controllers/MailgunController.php
View File

@ -11,9 +11,11 @@
namespace App\Http\Controllers;
use App\Jobs\Mailgun\ProcessMailgunInboundWebhook;
use App\Jobs\Mailgun\ProcessMailgunWebhook;
use App\Models\Company;
use App\Libraries\MultiDB;
use Illuminate\Http\Request;
use App\Jobs\Mailgun\ProcessMailgunWebhook;
use App\Jobs\Mailgun\ProcessMailgunInboundWebhook;
/**
* Class MailgunController.
@ -126,9 +128,15 @@ class MailgunController extends BaseController
$authorizedByHash = \hash_equals(\hash_hmac('sha256', $input['timestamp'] . $input['token'], config('services.mailgun.webhook_signing_key')), $input['signature']);
$authorizedByToken = $request->has('token') && $request->get('token') == config('ninja.inbound_mailbox.inbound_webhook_token');
if (!$authorizedByHash && !$authorizedByToken)
return response()->json(['message' => 'Unauthorized'], 403);
return response()->json(['message' => 'Unauthorized'], 403);
ProcessMailgunInboundWebhook::dispatch($input["sender"], $input["recipient"], $input["message-url"])->delay(rand(2, 10));
/** @var \App\Models\Company $company */
$company = MultiDB::findAndSetDbByExpenseMailbox($input["recipient"]);
if(!$company)
return response()->json(['message' => 'Ok'], 200); // Fail gracefully
ProcessMailgunInboundWebhook::dispatch($input["sender"], $input["recipient"], $input["message-url"], $company)->delay(rand(2, 10));
return response()->json(['message' => 'Success.'], 200);
}

17
app/Http/Controllers/PostMarkController.php
View File

@ -280,20 +280,21 @@ class PostMarkController extends BaseController
nlog('Failed: Message could not be parsed, because required parameters are missing.');
return response()->json(['message' => 'Failed. Missing/Invalid Parameters.'], 400);
}
$company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]);
$inboundEngine = new InboundMailEngine();
if (!$company) {
nlog('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $input["To"]);
// $inboundEngine->saveMeta($input["From"], $input["To"], true); // important to save this, to protect from spam
return response()->json(['message' => 'Ok'], 200);
}
$inboundEngine = new InboundMailEngine($company);
if ($inboundEngine->isInvalidOrBlocked($input["From"], $input["To"])) {
return response()->json(['message' => 'Blocked.'], 403);
}
$company = MultiDB::findAndSetDbByExpenseMailbox($input["To"]);
if (!$company) {
nlog('[PostmarkInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from postmark: ' . $input["To"]);
$inboundEngine->saveMeta($input["From"], $input["To"], true); // important to save this, to protect from spam
return;
}
try { // important to save meta if something fails here to prevent spam
// prepare data for ingresEngine

3
app/Http/ValidationRules/Company/ValidExpenseMailbox.php
View File

@ -23,10 +23,7 @@ class ValidExpenseMailbox implements Rule
{
private $validated_schema = false;
private $isEnterprise = false;
private array $endings;
private bool $hasCompanyKey;
private array $enterprise_endings;
public function __construct()
{

47
app/Jobs/Brevo/ProcessBrevoInboundWebhook.php
View File

@ -11,19 +11,20 @@
namespace App\Jobs\Brevo;
use App\Libraries\MultiDB;
use App\Services\InboundMail\InboundMail;
use App\Services\InboundMail\InboundMailEngine;
use App\Utils\TempFile;
use Brevo\Client\Api\InboundParsingApi;
use Brevo\Client\Configuration;
use Illuminate\Support\Carbon;
use App\Libraries\MultiDB;
use Illuminate\Bus\Queueable;
use Illuminate\Support\Carbon;
use Brevo\Client\Configuration;
use Illuminate\Http\UploadedFile;
use Illuminate\Queue\SerializesModels;
use Brevo\Client\Api\InboundParsingApi;
use Illuminate\Queue\InteractsWithQueue;
use App\Services\InboundMail\InboundMail;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Http\UploadedFile;
use App\Services\InboundMail\InboundMailEngine;
use Illuminate\Queue\Middleware\WithoutOverlapping;
class ProcessBrevoInboundWebhook implements ShouldQueue
{
@ -111,7 +112,6 @@ class ProcessBrevoInboundWebhook implements ShouldQueue
*/
public function __construct(private array $input)
{
$this->engine = new InboundMailEngine();
}
/**
@ -134,18 +134,24 @@ class ProcessBrevoInboundWebhook implements ShouldQueue
// match company
$company = MultiDB::findAndSetDbByExpenseMailbox($recipient);
if (!$company) {
nlog('[ProcessBrevoInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from brevo: ' . $recipient);
continue;
}
$this->engine = new InboundMailEngine($company);
$foundOneRecipient = true;
try { // important to save meta if something fails here to prevent spam
$company_brevo_secret = $company->settings?->email_sending_method === 'client_brevo' && $company->settings?->brevo_secret ? $company->settings?->brevo_secret : null;
if (empty($company_brevo_secret) && empty(config('services.brevo.secret')))
if(strlen($company->getSetting('brevo_secret') ?? '') < 2 && empty(config('services.brevo.secret'))){
nlog("No Brevo Configuration available for this company");
throw new \Error("[ProcessBrevoInboundWebhook] no brevo credenitals found, we cannot get the attachement");
}
$company_brevo_secret = strlen($company->getSetting('brevo_secret') ?? '') < 2 ? $company->getSetting('brevo_secret') : config('services.brevo.secret');
// prepare data for ingresEngine
$inboundMail = new InboundMail();
@ -160,8 +166,10 @@ class ProcessBrevoInboundWebhook implements ShouldQueue
// parse documents as UploadedFile from webhook-data
foreach ($this->input["Attachments"] as $attachment) {
// @todo - i think this allows switching between client configured brevo AND system configured brevo
// download file and save to tmp dir
if (!empty($company_brevo_secret)) {
if (!empty($company_brevo_secret))
{
try {
@ -220,4 +228,17 @@ class ProcessBrevoInboundWebhook implements ShouldQueue
$this->engine->saveMeta($this->input["From"]["Address"], $recipient, true);
}
}
public function middleware()
{
return [new WithoutOverlapping($this->input["From"]["Address"])];
}
public function failed($exception)
{
nlog("BREVO:: Ingest Exception:: => ".$exception->getMessage());
config(['queue.failed.driver' => null]);
}
}

34
app/Jobs/Mailgun/ProcessMailgunInboundWebhook.php
View File

@ -11,16 +11,17 @@
namespace App\Jobs\Mailgun;
use App\Libraries\MultiDB;
use App\Services\InboundMail\InboundMail;
use App\Services\InboundMail\InboundMailEngine;
use App\Models\Company;
use App\Utils\TempFile;
use Illuminate\Support\Carbon;
use App\Libraries\MultiDB;
use Illuminate\Bus\Queueable;
use Illuminate\Support\Carbon;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use App\Services\InboundMail\InboundMail;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Services\InboundMail\InboundMailEngine;
class ProcessMailgunInboundWebhook implements ShouldQueue
{
@ -34,9 +35,9 @@ class ProcessMailgunInboundWebhook implements ShouldQueue
* Create a new job instance.
* $input consists of 3 informations: sender/from|recipient/to|messageUrl
*/
public function __construct(private string $sender, private string $recipient, private string $message_url)
public function __construct(private string $sender, private string $recipient, private string $message_url, private Company $company)
{
$this->engine = new InboundMailEngine();
$this->engine = new InboundMailEngine($company);
}
/**
@ -176,19 +177,20 @@ class ProcessMailgunInboundWebhook implements ShouldQueue
return;
}
// lets assess this at a higher level to ensure that only valid email inboxes are processed.
// match company
$company = MultiDB::findAndSetDbByExpenseMailbox($to);
if (!$company) {
nlog('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to);
$this->engine->saveMeta($from, $to, true); // important to save this, to protect from spam
return;
}
// $company = MultiDB::findAndSetDbByExpenseMailbox($to);
// if (!$company) {
// nlog('[ProcessMailgunInboundWebhook] unknown Expense Mailbox occured while handling an inbound email from mailgun: ' . $to);
// $this->engine->saveMeta($from, $to, true); // important to save this, to protect from spam
// return;
// }
try { // important to save meta if something fails here to prevent spam
// fetch message from mailgun-api
$company_mailgun_domain = $company->getSetting('email_sending_method') == 'client_mailgun' && strlen($company->getSetting('mailgun_domain') ?? '') > 2 ? $company->getSetting('mailgun_domain') : null;
$company_mailgun_secret = $company->getSetting('email_sending_method') == 'client_mailgun' && strlen($company->getSetting('mailgun_secret') ?? '') > 2 ? $company->getSetting('mailgun_secret') : null;
$company_mailgun_domain = $this->company->getSetting('email_sending_method') == 'client_mailgun' && strlen($this->company->getSetting('mailgun_domain') ?? '') > 2 ? $this->company->getSetting('mailgun_domain') : null;
$company_mailgun_secret = $this->company->getSetting('email_sending_method') == 'client_mailgun' && strlen($this->company->getSetting('mailgun_secret') ?? '') > 2 ? $this->company->getSetting('mailgun_secret') : null;
if (!($company_mailgun_domain && $company_mailgun_secret) && !(config('services.mailgun.domain') && config('services.mailgun.secret')))
throw new \Error("[ProcessMailgunInboundWebhook] no mailgun credentials found, we cannot get the attachements and files");

53
app/Services/EDocument/Imports/MindeeEDocument.php
View File

@ -47,8 +47,10 @@ class MindeeEDocument extends AbstractService
public function run(): Expense
{
$api_key = config('services.mindee.api_key');
if (!$api_key)
throw new Exception('Mindee API key not configured');
$this->checkLimits();
// perform parsing
@ -69,42 +71,52 @@ class MindeeEDocument extends AbstractService
$invoiceCurrency = $prediction->locale->currency;
$country = $prediction->locale->country;
$expense = Expense::where('amount', $grandTotalAmount)->where("transaction_reference", $documentno)->whereDate("date", $documentdate)->first();
if (empty($expense)) {
$expense = Expense::query()
->where('company_id', $this->company->id)
->where('amount', $grandTotalAmount)
->where("transaction_reference", $documentno)
->whereDate("date", $documentdate)
->first();
if (!$expense) {
// The document does not exist as an expense
// Handle accordingly
/** @var \App\Models\Currency $currency */
$currency = app('currencies')->first(function ($c) use ($invoiceCurrency){
/** @var \App\Models\Currency $c */
return $c->code == $invoiceCurrency;
});
$expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id);
$expense->date = $documentdate;
$expense->user_id = $this->company->owner()->id;
$expense->company_id = $this->company->id;
$expense->public_notes = $documentno;
$expense->currency_id = Currency::whereCode($invoiceCurrency)->first()?->id || $this->company->settings->currency_id;
$expense->currency_id = $currency ? $currency->id : $this->company->settings->currency_id;
$expense->save();
$this->saveDocuments([
$this->file,
TempFile::UploadedFileFromRaw(strval($result->document), $documentno . "_mindee_orc_result.txt", "text/plain")
], $expense);
$expense->saveQuietly();
// $expense->saveQuietly();
$expense->uses_inclusive_taxes = True;
$expense->amount = $grandTotalAmount;
$counter = 1;
foreach ($prediction->taxes as $taxesElem) {
$expense->{"tax_amount$counter"} = $taxesElem->amount;
$expense->{"tax_rate$counter"} = $taxesElem->rate;
$expense->{"tax_amount{$counter}"} = $taxesElem->amount;
$expense->{"tax_rate{$counter}"} = $taxesElem->rate;
$counter++;
}
$vendor = null;
$vendor_contact = VendorContact::where("company_id", $this->company->id)->where("email", $prediction->supplierEmail)->first();
if ($vendor_contact)
$vendor = $vendor_contact->vendor;
if (!$vendor)
$vendor = Vendor::where("company_id", $this->company->id)->where("name", $prediction->supplierName)->first();
/** @var \App\Models\VendorContact $vendor_contact */
$vendor_contact = VendorContact::query()->where("company_id", $this->company->id)->where("email", $prediction->supplierEmail)->first();
/** @var \App\Models\Vendor|null $vendor */
$vendor = $vendor_contact ? $vendor_contact->vendor : Vendor::query()->where("company_id", $this->company->id)->where("name", $prediction->supplierName)->first();
if ($vendor) {
// Vendor found
$expense->vendor_id = $vendor->id;
} else {
$vendor = VendorFactory::create($this->company->id, $this->company->owner()->id);
@ -116,15 +128,19 @@ class MindeeEDocument extends AbstractService
// $vendor->address2 = $address_2;
// $vendor->city = $city;
// $vendor->postal_code = $postcode;
/** @var ?\App\Models\Country $country */
$country = app('countries')->first(function ($c) use ($country) {
/** @var \App\Models\Country $c */
return $c->iso_3166_2 == $country || $c->iso_3166_3 == $country;
});
if ($country)
$vendor->country_id = $country->id;
$vendor->save();
if ($prediction->supplierEmail) {
if (strlen($prediction->supplierEmail ?? '') > 2) {
$vendor_contact = VendorContactFactory::create($this->company->id, $this->company->owner()->id);
$vendor_contact->vendor_id = $vendor->id;
$vendor_contact->email = $prediction->supplierEmail;
@ -138,8 +154,9 @@ class MindeeEDocument extends AbstractService
// The document exists as an expense
// Handle accordingly
nlog("Mindee: Document already exists");
$expense->private_notes = $expense->private_notes . ctrans("texts.edocument_import_already_exists", ["date" => time()]);
$expense->private_notes = $expense->private_notes . ctrans("texts.edocument_import_already_exists", ["date" => now()->format('Y-m-d')]);
}
$expense->save();
return $expense;
}

23
app/Services/EDocument/Imports/ZugferdEDocument.php
View File

@ -102,17 +102,20 @@ class ZugferdEDocument extends AbstractService
if (array_key_exists("VA", $taxtype)) {
$taxid = $taxtype["VA"];
}
$vendor = Vendor::where("company_id", $user->company()->id)->where('vat_number', $taxid)->first();
if (!$vendor) {
$vendor_contact = VendorContact::where("company_id", $user->company()->id)->where("email", $contact_email)->first();
if ($vendor_contact)
$vendor = $vendor_contact->vendor;
}
if (!$vendor)
$vendor = Vendor::where("company_id", $user->company()->id)->where("name", $person_name)->first();
if (!empty($vendor)) {
// Vendor found
$vendor = Vendor::query()
->where("company_id", $user->company()->id)
->where(function ($q) use($taxid, $person_name, $contact_email){
$q->when(!is_null($taxid), function ($when_query) use($taxid){
$when_query->orWhere('vat_number', $taxid);
})
->orWhere("name", $person_name)
->orWhereHas('contacts', function ($qq) use ($contact_email){
$qq->where("email", $contact_email);
});
})->first();
if ($vendor) {
$expense->vendor_id = $vendor->id;
} else {
$vendor = VendorFactory::create($this->company->id, $user->id);

78
app/Services/InboundMail/InboundMailEngine.php
View File

@ -35,7 +35,7 @@ class InboundMailEngine
private array $globalBlacklist;
private array $globalWhitelist; // only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders
public function __construct()
public function __construct(private Company $company)
{
// only for global validation, not for allowing to send something into the company, should be used to disabled blocking for mass-senders
@ -52,19 +52,19 @@ class InboundMailEngine
return;
// Expense Mailbox => will create an expense
$company = MultiDB::findAndSetDbByExpenseMailbox($email->to);
if (!$company) {
$this->saveMeta($email->from, $email->to, true);
return;
}
// $company = MultiDB::findAndSetDbByExpenseMailbox($email->to);
// if (!$company) {
// $this->saveMeta($email->from, $email->to, true);
// return;
// }
// check if company plan matches requirements
if (Ninja::isHosted() && !($company->account->isPaid() && $company->account->plan == 'enterprise')) {
if (Ninja::isHosted() && !($this->company->account->isPaid() && $this->company->account->plan == 'enterprise')) {
$this->saveMeta($email->from, $email->to);
return;
}
$this->createExpenses($company, $email);
$this->createExpenses($email);
$this->saveMeta($email->from, $email->to);
}
@ -145,6 +145,8 @@ class InboundMailEngine
// TODO: ignore, when known sender (for heavy email-usage mostly on isHosted())
// TODO: handle external blocking
}
//@todo - refactor
public function saveMeta(string $from, string $to, bool $isUnknownRecipent = false)
{
// save cache
@ -161,24 +163,24 @@ class InboundMailEngine
}
// MAIN-PROCESSORS
protected function createExpenses(Company $company, InboundMail $email)
protected function createExpenses(InboundMail $email)
{
// Skipping executions: will not result in not saving Metadata to prevent usage of these conditions, to spam
if (!($company?->expense_mailbox_active ?: false)) {
$this->logBlocked($company, 'mailbox not active for this company. from: ' . $email->from);
if (!$this->company->expense_mailbox_active) {
$this->logBlocked($this->company, 'mailbox not active for this company. from: ' . $email->from);
return;
}
if (!$this->validateExpenseSender($company, $email)) {
$this->logBlocked($company, 'invalid sender of an ingest email for this company. from: ' . $email->from);
if (!$this->validateExpenseSender($email)) {
$this->logBlocked($this->company, 'invalid sender of an ingest email for this company. from: ' . $email->from);
return;
}
if (sizeOf($email->documents) == 0) {
$this->logBlocked($company, 'email does not contain any attachments and is likly not an expense. from: ' . $email->from);
$this->logBlocked($this->company, 'email does not contain any attachments and is likly not an expense. from: ' . $email->from);
return;
}
// prepare data
$expense_vendor = $this->getVendor($company, $email);
$expense_vendor = $this->getVendor($email);
$this->processHtmlBodyToDocument($email);
$parsed_expense_ids = []; // used to check if an expense was already matched within this job
@ -192,7 +194,7 @@ class InboundMailEngine
// check if document can be parsed to an expense
try {
$expense = (new ParseEDocument($document, $company))->run();
$expense = (new ParseEDocument($document, $this->company))->run();
// check if expense was already matched within this job and skip if true
if (array_search($expense->id, $parsed_expense_ids))
@ -213,7 +215,7 @@ class InboundMailEngine
// populate missing data with data from email
if (!$expense)
$expense = ExpenseFactory::create($company->id, $company->owner()->id);
$expense = ExpenseFactory::create($this->company->id, $this->company->owner()->id);
$is_imported_by_parser = array_search($expense->id, $parsed_expense_ids);
@ -256,61 +258,61 @@ class InboundMailEngine
$email->body_document = TempFile::UploadedFileFromRaw($email->body, "E-Mail.html", "text/html");
}
private function validateExpenseSender(Company $company, InboundMail $email)
private function validateExpenseSender(InboundMail $email)
{
$parts = explode('@', $email->from);
$domain = array_pop($parts);
// whitelists
$whitelist = explode(",", $company->inbound_mailbox_whitelist);
$whitelist = explode(",", $this->company->inbound_mailbox_whitelist);
if (in_array($email->from, $whitelist))
return true;
if (in_array($domain, $whitelist))
return true;
$blacklist = explode(",", $company->inbound_mailbox_blacklist);
$blacklist = explode(",", $this->company->inbound_mailbox_blacklist);
if (in_array($email->from, $blacklist))
return false;
if (in_array($domain, $blacklist))
return false;
// allow unknown
if ($company->inbound_mailbox_allow_unknown)
if ($this->company->inbound_mailbox_allow_unknown)
return true;
// own users
if ($company->inbound_mailbox_allow_company_users && $company->users()->where("email", $email->from)->exists())
if ($this->company->inbound_mailbox_allow_company_users && $this->company->users()->where("email", $email->from)->exists())
return true;
// from vendors
if ($company->inbound_mailbox_allow_vendors && VendorContact::where("company_id", $company->id)->where("email", $email->from)->exists())
if ($this->company->inbound_mailbox_allow_vendors && VendorContact::where("company_id", $this->company->id)->where("email", $email->from)->exists())
return true;
// from clients
if ($company->inbound_mailbox_allow_clients && ClientContact::where("company_id", $company->id)->where("email", $email->from)->exists())
if ($this->company->inbound_mailbox_allow_clients && ClientContact::where("company_id", $this->company->id)->where("email", $email->from)->exists())
return true;
// denie
return false;
}
private function getClient(Company $company, InboundMail $email)
{
$clientContact = ClientContact::where("company_id", $company->id)->where("email", $email->from)->first();
if (!$clientContact)
return null;
return $clientContact->client();
}
private function getVendor(Company $company, InboundMail $email)
{
$vendorContact = VendorContact::where("company_id", $company->id)->where("email", $email->from)->first();
if (!$vendorContact)
return null;
// private function getClient(InboundMail $email)
// {
// $clientContact = ClientContact::where("company_id", $this->company->id)->where("email", $email->from)->first();
// if (!$clientContact)
// return null;
return $vendorContact->vendor();
// return $clientContact->client();
// }
private function getVendor(InboundMail $email)
{
$vendorContact = VendorContact::with('vendor')->where("company_id", $this->company->id)->where("email", $email->from)->first();
return $vendorContact ? $vendorContact->vendor : null;
}
private function logBlocked(Company $company, string $data)
{
nlog("[InboundMailEngine][company:" . $company->company_key . "] " . $data);
nlog("[InboundMailEngine][company:" . $this->company->company_key . "] " . $data);
(
new SystemLogger(

2
database/migrations/2023_12_10_110951_inbound_mail_parsing.php
View File

@ -11,7 +11,7 @@ return new class extends Migration {
public function up(): void
{
Schema::table('companies', function (Blueprint $table) {
$table->boolean("expense_mailbox_active")->default(true);
$table->boolean("expense_mailbox_active")->default(false);
$table->string("expense_mailbox")->nullable();
$table->boolean("inbound_mailbox_allow_company_users")->default(false);
$table->boolean("inbound_mailbox_allow_vendors")->default(false);