Mirrors/invoiceninja
1
0
Fork 0
mirror of https://github.com/invoiceninja/invoiceninja.git synced 2024-09-20 16:31:33 +02:00
Code Issues Releases Activity

Fixes for Webhooks + retries in Jobs

Browse Source
This commit is contained in:
David Bomba 2023-01-29 16:19:20 +11:00
parent 94a84dfb15
commit c981de29ab
7 changed files with 283 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/Http/Requests/Webhook/StoreWebhookRequest.php
View File

@ -30,7 +30,7 @@ class StoreWebhookRequest extends Request
return [
'target_url' => 'bail|required|url',
'event_id' => 'bail|required',
'headers' => 'bail|sometimes|json',
// 'headers' => 'bail|sometimes|json',
'rest_method' => 'required|in:post,put'
];
}
@ -39,8 +39,8 @@ class StoreWebhookRequest extends Request
{
$input = $this->all();
if(isset($input['headers']) && count($input['headers']) == 0)
$input['headers'] = null;
// if(isset($input['headers']) && count($input['headers']) == 0)
// $input['headers'] = null;
$this->replace($input);
}

6
app/Http/Requests/Webhook/UpdateWebhookRequest.php
View File

@ -36,7 +36,7 @@ class UpdateWebhookRequest extends Request
'target_url' => 'bail|required|url',
'event_id' => 'bail|required',
'rest_method' => 'required|in:post,put',
'headers' => 'bail|sometimes|json',
// 'headers' => 'bail|sometimes|json',
];
}
@ -44,8 +44,8 @@ class UpdateWebhookRequest extends Request
{
$input = $this->all();
if(isset($input['headers']) && count($input['headers']) == 0)
$input['headers'] = null;
// if(isset($input['headers']) && count($input['headers']) == 0)
// $input['headers'] = null;
$this->replace($input);
}

20
app/Jobs/Mail/NinjaMailerJob.php
View File

@ -58,6 +58,7 @@ class NinjaMailerJob implements ShouldQueue
public $override;
/* @var Company $company*/
public $company;
private $mailer;
@ -130,7 +131,7 @@ class NinjaMailerJob implements ShouldQueue
//send email
try {
nlog("trying to send to {$this->nmo->to_user->email} ". now()->toDateTimeString());
nlog("Trying to send to {$this->nmo->to_user->email} ". now()->toDateTimeString());
nlog("Using mailer => ". $this->mailer);
$mailer = Mail::mailer($this->mailer);
@ -156,7 +157,22 @@ class NinjaMailerJob implements ShouldQueue
$this->nmo = null;
$this->company = null;
} catch (\Exception | \RuntimeException | \Google\Service\Exception $e) {
}
catch(\Symfony\Component\Mime\Exception\RfcComplianceException $e) {
nlog("Mailer failed with a Logic Exception {$e->getMessage()}");
$this->fail();
$this->cleanUpMailers();
$this->logMailError($e->getMessage(), $this->company->clients()->first());
return;
}
catch(\Symfony\Component\Mime\Exception\LogicException $e){
nlog("Mailer failed with a Logic Exception {$e->getMessage()}");
$this->fail();
$this->cleanUpMailers();
$this->logMailError($e->getMessage(), $this->company->clients()->first());
return;
}
catch (\Exception | \Google\Service\Exception $e) {
nlog("Mailer failed with {$e->getMessage()}");
$message = $e->getMessage();

1
app/Jobs/Mail/NinjaMailerObject.php
View File

@ -18,6 +18,7 @@ class NinjaMailerObject
{
public $mailable;
/* @var Company $company */
public $company;
public $from_user; //not yet used

101
app/Jobs/Util/WebhookHandler.php
View File

@ -12,6 +12,7 @@
namespace App\Jobs\Util;
use App\Jobs\Util\SystemLogger;
use App\Jobs\Util\WebhookSingle;
use App\Libraries\MultiDB;
use App\Models\Client as ClientModel;
use App\Models\SystemLog;
@ -38,7 +39,7 @@ class WebhookHandler implements ShouldQueue
private $company;
public $tries = 4; //number of retries
public $tries = 1; //number of retries
public $deleteWhenMissingModels = true;
@ -58,11 +59,6 @@ class WebhookHandler implements ShouldQueue
$this->includes = $includes;
}
public function backoff()
{
return [10, 30, 60, 180];
}
/**
* Execute the job.
*
@ -83,104 +79,17 @@ class WebhookHandler implements ShouldQueue
->cursor()
->each(function ($subscription) {
$this->process($subscription);
// $this->process($subscription);
WebhookSingle::dispatch($subscription->id, $this->entity, $this->company->db, $this->includes);
});
}
private function process($subscription)
{
$this->entity->refresh();
// generate JSON data
$manager = new Manager();
$manager->setSerializer(new ArraySerializer());
$manager->parseIncludes($this->includes);
$class = sprintf('App\\Transformers\\%sTransformer', class_basename($this->entity));
$transformer = new $class();
$resource = new Item($this->entity, $transformer, $this->entity->getEntityType());
$data = $manager->createData($resource)->toArray();
$this->postData($subscription, $data, []);
}
private function postData($subscription, $data, $headers = [])
{
$base_headers = [
'Content-Length' => strlen(json_encode($data)),
'Accept' => 'application/json',
];
$client = new Client(['headers' => array_merge($base_headers, $headers)]);
try {
$response = $client->post($subscription->target_url, [
RequestOptions::JSON => $data, // or 'json' => [...]
]);
SystemLogger::dispatch(
array_merge((array) $response, $data),
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_SUCCESS,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company
);
if ($response->getStatusCode() == 429){
nlog("429 retry after 10 seconds internally");
sleep(10);
$response = $client->post($subscription->target_url, [
RequestOptions::JSON => $data, // or 'json' => [...]
]);
SystemLogger::dispatch(
array_merge((array) $response, $data),
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_SUCCESS,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company
);
}
} catch (\Exception $e) {
nlog("429 occurred in the exception handler");
nlog($e->getMessage());
SystemLogger::dispatch(
$e->getMessage(),
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_FAILURE,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company,
);
}
}
private function resolveClient()
{
//make sure it isn't an instance of the Client Model
if ((! $this->entity instanceof ClientModel) && $this->entity->client()->exists()) {
return $this->entity->client;
}
return $this->company->clients()->first();
}
public function failed($exception)
{
sleep(2);
nlog(print_r($exception->getMessage(), 1));
}

253
app/Jobs/Util/WebhookSingle.php Normal file
View File

@ -0,0 +1,253 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Jobs\Util;
use App\Jobs\Util\SystemLogger;
use App\Libraries\MultiDB;
use App\Models\Client as ClientModel;
use App\Models\Company;
use App\Models\SystemLog;
use App\Models\Webhook;
use App\Transformers\ArraySerializer;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\BadResponseException;
use GuzzleHttp\RequestOptions;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use League\Fractal\Manager;
use League\Fractal\Resource\Item;
use GuzzleHttp\Exception\ClientException;
use GuzzleHttp\Exception\ServerException;
class WebhookSingle implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
private $entity;
private string $db;
private int $subscription_id;
public $tries = 5; //number of retries
public $deleteWhenMissingModels = true;
private string $includes;
private Company $company;
/**
* Create a new job instance.
*
* @param $event_id
* @param $entity
*/
public function __construct($subscription_id, $entity, $db, $includes = '')
{
$this->entity = $entity;
$this->db = $db;
$this->includes = $includes;
$this->subscription_id = $subscription_id;
}
public function backoff()
{
return [10, 30, 60, 180, 3600];
}
/**
* Execute the job.
*
* @return bool
*/
public function handle()
{
nlog($this->attempts());
MultiDB::setDb($this->db);
$subscription = Webhook::with('company')->find($this->subscription_id);
$this->company = $subscription->company;
$this->entity->refresh();
// generate JSON data
$manager = new Manager();
$manager->setSerializer(new ArraySerializer());
$manager->parseIncludes($this->includes);
$class = sprintf('App\\Transformers\\%sTransformer', class_basename($this->entity));
$transformer = new $class();
$resource = new Item($this->entity, $transformer, $this->entity->getEntityType());
$data = $manager->createData($resource)->toArray();
$this->postData($subscription, $data, []);
}
private function postData($subscription, $data, $headers = [])
{
$base_headers = [
'Content-Length' => strlen(json_encode($data)),
'Accept' => 'application/json',
];
$client = new Client(['headers' => array_merge($base_headers, $headers)]);
nlog("attempting ". $subscription->target_url);
try {
$response = $client->post($subscription->target_url, [
RequestOptions::JSON => $data, // or 'json' => [...]
]);
nlog($response->getStatusCode());
SystemLogger::dispatch(
array_merge((array) $response, $data),
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_SUCCESS,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company
);
}
catch(\GuzzleHttp\Exception\ConnectException $e){
nlog("connection problem");
nlog($e->getCode());
nlog($e->getMessage());
SystemLogger::dispatch(
['message' => "Error connecting to ". $subscription->target_url],
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_FAILURE,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company
);
}
catch (BadResponseException $e) {
if ($e->getResponse()->getStatusCode() >= 400 && $e->getResponse()->getStatusCode() < 500){
$message = "Server encountered a problem when connecting to {$subscription->target_url} => status code ". $e->getResponse()->getStatusCode(). " scheduling retry.";
nlog($message);
SystemLogger::dispatch(
['message' => $message],
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_FAILURE,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company
);
$this->release($this->backoff()[$this->attempts()-1]);
}
if($e->getResponse()->getStatusCode() >= 500){
nlog("endpoint returned a 500, failing");
$message = "Server encountered a problem when connecting to {$subscription->target_url} => status code ". $e->getResponse()->getStatusCode(). " no retry attempted.";
SystemLogger::dispatch(
['message' => $message],
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_FAILURE,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company
);
$this->fail();
return;
}
}
catch (ServerException $e) {
nlog("Server exception");
$error = json_decode($e->getResponse()->getBody()->getContents());
SystemLogger::dispatch(
['message' => $error],
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_FAILURE,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company
);
}
catch (ClientException $e) {
nlog("Client exception");
$error = json_decode($e->getResponse()->getBody()->getContents());
SystemLogger::dispatch(
['message' => $error],
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_FAILURE,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company
);
}
catch (\Exception $e) {
nlog("Exception handler => " . $e->getMessage());
nlog($e->getCode());
SystemLogger::dispatch(
$e->getMessage(),
SystemLog::CATEGORY_WEBHOOK,
SystemLog::EVENT_WEBHOOK_FAILURE,
SystemLog::TYPE_WEBHOOK_RESPONSE,
$this->resolveClient(),
$this->company,
);
$this->release($this->backoff()[$this->attempts()-1]);
}
}
private function resolveClient()
{
//make sure it isn't an instance of the Client Model
if ((! $this->entity instanceof ClientModel) && $this->entity->client()->exists()) {
return $this->entity->client;
}
return $this->company->clients()->first();
}
public function failed($exception)
{
nlog(print_r($exception->getMessage(), 1));
}
}

1
routes/api.php
View File

@ -375,5 +375,4 @@ Route::post('api/v1/yodlee/data_updates', [YodleeController::class, 'dataUpdates
Route::post('api/v1/yodlee/refresh_updates', [YodleeController::class, 'refreshUpdatesWebhook'])->middleware('throttle:100,1');
Route::post('api/v1/yodlee/balance', [YodleeController::class, 'balanceWebhook'])->middleware('throttle:100,1');
Route::fallback([BaseController::class, 'notFound']);