Fixes for Webhooks + retries in Jobs

app/Http/Requests/Webhook/StoreWebhookRequest.php

@@ -30,7 +30,7 @@ class StoreWebhookRequest extends Request
         return [
             'target_url' => 'bail|required|url',
             'event_id' => 'bail|required',
-            'headers' => 'bail|sometimes|json',
+            // 'headers' => 'bail|sometimes|json',
             'rest_method' => 'required|in:post,put'
         ];
     }
@@ -39,8 +39,8 @@ class StoreWebhookRequest extends Request
     {
         $input = $this->all();
 
-            if(isset($input['headers']) && count($input['headers']) == 0)
+            // if(isset($input['headers']) && count($input['headers']) == 0)
-                $input['headers'] = null;
+                // $input['headers'] = null;
 
         $this->replace($input);
     }

app/Http/Requests/Webhook/UpdateWebhookRequest.php

@@ -36,7 +36,7 @@ class UpdateWebhookRequest extends Request
             'target_url' => 'bail|required|url',
             'event_id' => 'bail|required',
             'rest_method' => 'required|in:post,put',
-            'headers' => 'bail|sometimes|json',
+            // 'headers' => 'bail|sometimes|json',
         ];
     }
 
@@ -44,8 +44,8 @@ class UpdateWebhookRequest extends Request
     {
         $input = $this->all();
 
-            if(isset($input['headers']) && count($input['headers']) == 0)
+            // if(isset($input['headers']) && count($input['headers']) == 0)
-                $input['headers'] = null;
+            //     $input['headers'] = null;
 
         $this->replace($input);
     }

app/Jobs/Mail/NinjaMailerJob.php

@@ -58,6 +58,7 @@ class NinjaMailerJob implements ShouldQueue
 
     public $override;
 
+    /* @var Company $company*/
     public $company;
 
     private $mailer;
@@ -130,7 +131,7 @@ class NinjaMailerJob implements ShouldQueue
 
         //send email
         try {
-            nlog("trying to send to {$this->nmo->to_user->email} ". now()->toDateTimeString());
+            nlog("Trying to send to {$this->nmo->to_user->email} ". now()->toDateTimeString());
             nlog("Using mailer => ". $this->mailer);
 
             $mailer = Mail::mailer($this->mailer);
@@ -156,7 +157,22 @@ class NinjaMailerJob implements ShouldQueue
             $this->nmo = null;
             $this->company = null;
     
-        } catch (\Exception | \RuntimeException | \Google\Service\Exception $e) {
+        }
+        catch(\Symfony\Component\Mime\Exception\RfcComplianceException $e) {
+                nlog("Mailer failed with a Logic Exception {$e->getMessage()}");
+                $this->fail();
+                $this->cleanUpMailers();
+                $this->logMailError($e->getMessage(), $this->company->clients()->first());
+                return;
+        }
+        catch(\Symfony\Component\Mime\Exception\LogicException $e){
+                nlog("Mailer failed with a Logic Exception {$e->getMessage()}");
+                $this->fail();
+                $this->cleanUpMailers();
+                $this->logMailError($e->getMessage(), $this->company->clients()->first());
+                return;
+        }
+        catch (\Exception | \Google\Service\Exception $e) {
             
             nlog("Mailer failed with {$e->getMessage()}");
             $message = $e->getMessage();

app/Jobs/Mail/NinjaMailerObject.php

@@ -18,6 +18,7 @@ class NinjaMailerObject
 {
     public $mailable;
 
+    /* @var Company $company */
     public $company;
 
     public $from_user; //not yet used

app/Jobs/Util/WebhookHandler.php

@@ -12,6 +12,7 @@
 namespace App\Jobs\Util;
 
 use App\Jobs\Util\SystemLogger;
+use App\Jobs\Util\WebhookSingle;
 use App\Libraries\MultiDB;
 use App\Models\Client as ClientModel;
 use App\Models\SystemLog;
@@ -38,7 +39,7 @@ class WebhookHandler implements ShouldQueue
 
     private $company;
 
-    public $tries = 4; //number of retries
+    public $tries = 1; //number of retries
 
     public $deleteWhenMissingModels = true;
 
@@ -58,11 +59,6 @@ class WebhookHandler implements ShouldQueue
         $this->includes = $includes;
     }
 
-    public function backoff()
-    {
-        return [10, 30, 60, 180];
-    }
-
     /**
      * Execute the job.
      *
@@ -83,104 +79,17 @@ class WebhookHandler implements ShouldQueue
                                     ->cursor()
                                     ->each(function ($subscription) {
 
-            $this->process($subscription);
+            // $this->process($subscription);
+
+            WebhookSingle::dispatch($subscription->id, $this->entity, $this->company->db, $this->includes);
 
         });
 
     }
 
-    private function process($subscription)
-    {
-        $this->entity->refresh();
-
-        // generate JSON data
-        $manager = new Manager();
-        $manager->setSerializer(new ArraySerializer());
-        $manager->parseIncludes($this->includes);
-
-        $class = sprintf('App\\Transformers\\%sTransformer', class_basename($this->entity));
-
-        $transformer = new $class();
-
-        $resource = new Item($this->entity, $transformer, $this->entity->getEntityType());
-        $data = $manager->createData($resource)->toArray();
-
-        $this->postData($subscription, $data, []);
-    }
-
-    private function postData($subscription, $data, $headers = [])
-    {
-        $base_headers = [
-            'Content-Length' => strlen(json_encode($data)),
-            'Accept'         => 'application/json',
-        ];
-
-        $client = new Client(['headers' => array_merge($base_headers, $headers)]);
-
-        try {
-            $response = $client->post($subscription->target_url, [
-                RequestOptions::JSON => $data, // or 'json' => [...]
-            ]);
-
-            SystemLogger::dispatch(
-                array_merge((array) $response, $data),
-                SystemLog::CATEGORY_WEBHOOK,
-                SystemLog::EVENT_WEBHOOK_SUCCESS,
-                SystemLog::TYPE_WEBHOOK_RESPONSE,
-                $this->resolveClient(),
-                $this->company
-            );
-
-            if ($response->getStatusCode() == 429){
-                nlog("429 retry after 10 seconds internally");
-                sleep(10);
-
-                $response = $client->post($subscription->target_url, [
-                    RequestOptions::JSON => $data, // or 'json' => [...]
-                ]);
-
-                SystemLogger::dispatch(
-                    array_merge((array) $response, $data),
-                    SystemLog::CATEGORY_WEBHOOK,
-                    SystemLog::EVENT_WEBHOOK_SUCCESS,
-                    SystemLog::TYPE_WEBHOOK_RESPONSE,
-                    $this->resolveClient(),
-                    $this->company
-                );
-
-            }
-
-        } catch (\Exception $e) {
-            
-            nlog("429 occurred in the exception handler");
-            nlog($e->getMessage());
-
-            SystemLogger::dispatch(
-                $e->getMessage(),
-                SystemLog::CATEGORY_WEBHOOK,
-                SystemLog::EVENT_WEBHOOK_FAILURE,
-                SystemLog::TYPE_WEBHOOK_RESPONSE,
-                $this->resolveClient(),
-                $this->company,
-            );
-        }
-    }
-
-    private function resolveClient()
-    {
-        //make sure it isn't an instance of the Client Model
-        if ((! $this->entity instanceof ClientModel) && $this->entity->client()->exists()) {
-            return $this->entity->client;
-        }
-
-        return $this->company->clients()->first();
-    }
-
     public function failed($exception)
     {
         
-        sleep(2);
-
         nlog(print_r($exception->getMessage(), 1));
 
     }

app/Jobs/Util/WebhookSingle.php

@@ -0,0 +1,253 @@
+<?php
+/**
+ * Invoice Ninja (https://invoiceninja.com).
+ *
+ * @link https://github.com/invoiceninja/invoiceninja source repository
+ *
+ * @copyright Copyright (c) 2023. Invoice Ninja LLC (https://invoiceninja.com)
+ *
+ * @license https://www.elastic.co/licensing/elastic-license
+ */
+
+namespace App\Jobs\Util;
+
+use App\Jobs\Util\SystemLogger;
+use App\Libraries\MultiDB;
+use App\Models\Client as ClientModel;
+use App\Models\Company;
+use App\Models\SystemLog;
+use App\Models\Webhook;
+use App\Transformers\ArraySerializer;
+use GuzzleHttp\Client;
+use GuzzleHttp\Exception\BadResponseException;
+use GuzzleHttp\RequestOptions;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use League\Fractal\Manager;
+use League\Fractal\Resource\Item;
+use GuzzleHttp\Exception\ClientException;
+use GuzzleHttp\Exception\ServerException;
+
+class WebhookSingle implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    private $entity;
+
+    private string $db;
+
+    private int $subscription_id;
+
+    public $tries = 5; //number of retries
+
+    public $deleteWhenMissingModels = true;
+
+    private string $includes;
+
+    private Company $company;
+    /**
+     * Create a new job instance.
+     *
+     * @param $event_id
+     * @param $entity
+     */
+    public function __construct($subscription_id, $entity, $db, $includes = '')
+    {
+        $this->entity = $entity;
+        $this->db = $db;
+        $this->includes = $includes;
+        $this->subscription_id = $subscription_id;
+    }
+
+    public function backoff()
+    {
+        return [10, 30, 60, 180, 3600];
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return bool
+     */
+    public function handle()
+    {
+nlog($this->attempts());
+
+        MultiDB::setDb($this->db);
+
+        $subscription = Webhook::with('company')->find($this->subscription_id);
+        
+        $this->company = $subscription->company;
+
+        $this->entity->refresh();
+
+        // generate JSON data
+        $manager = new Manager();
+        $manager->setSerializer(new ArraySerializer());
+        $manager->parseIncludes($this->includes);
+
+        $class = sprintf('App\\Transformers\\%sTransformer', class_basename($this->entity));
+
+        $transformer = new $class();
+
+        $resource = new Item($this->entity, $transformer, $this->entity->getEntityType());
+        $data = $manager->createData($resource)->toArray();
+
+        $this->postData($subscription, $data, []);
+    }
+
+    private function postData($subscription, $data, $headers = [])
+    {
+        $base_headers = [
+            'Content-Length' => strlen(json_encode($data)),
+            'Accept'         => 'application/json',
+        ];
+
+        $client = new Client(['headers' => array_merge($base_headers, $headers)]);
+
+nlog("attempting ". $subscription->target_url);
+
+        try {
+            $response = $client->post($subscription->target_url, [
+                RequestOptions::JSON => $data, // or 'json' => [...]
+            ]);
+
+nlog($response->getStatusCode());
+
+            SystemLogger::dispatch(
+                array_merge((array) $response, $data),
+                SystemLog::CATEGORY_WEBHOOK,
+                SystemLog::EVENT_WEBHOOK_SUCCESS,
+                SystemLog::TYPE_WEBHOOK_RESPONSE,
+                $this->resolveClient(),
+                $this->company
+            );
+
+        } 
+        catch(\GuzzleHttp\Exception\ConnectException $e){
+
+            nlog("connection problem");
+            nlog($e->getCode());
+            nlog($e->getMessage());
+
+            SystemLogger::dispatch(
+                ['message' => "Error connecting to ". $subscription->target_url],
+                SystemLog::CATEGORY_WEBHOOK,
+                SystemLog::EVENT_WEBHOOK_FAILURE,
+                SystemLog::TYPE_WEBHOOK_RESPONSE,
+                $this->resolveClient(),
+                $this->company
+            );
+
+        }
+        catch (BadResponseException $e) {
+
+            if ($e->getResponse()->getStatusCode() >= 400 && $e->getResponse()->getStatusCode() < 500){
+
+                $message = "Server encountered a problem when connecting to {$subscription->target_url} => status code ". $e->getResponse()->getStatusCode(). " scheduling retry.";
+                
+                nlog($message);
+
+                SystemLogger::dispatch(
+                    ['message' => $message],
+                    SystemLog::CATEGORY_WEBHOOK,
+                    SystemLog::EVENT_WEBHOOK_FAILURE,
+                    SystemLog::TYPE_WEBHOOK_RESPONSE,
+                    $this->resolveClient(),
+                    $this->company
+                );
+
+                $this->release($this->backoff()[$this->attempts()-1]);
+
+            }
+
+            if($e->getResponse()->getStatusCode() >= 500){
+                nlog("endpoint returned a 500, failing");
+
+                $message = "Server encountered a problem when connecting to {$subscription->target_url} => status code ". $e->getResponse()->getStatusCode(). " no retry attempted.";
+
+                SystemLogger::dispatch(
+                    ['message' => $message],
+                    SystemLog::CATEGORY_WEBHOOK,
+                    SystemLog::EVENT_WEBHOOK_FAILURE,
+                    SystemLog::TYPE_WEBHOOK_RESPONSE,
+                    $this->resolveClient(),
+                    $this->company
+                );
+
+                $this->fail();
+                return;
+            }
+
+
+        }
+        catch (ServerException $e) {
+
+            nlog("Server exception");
+            $error = json_decode($e->getResponse()->getBody()->getContents());
+
+            SystemLogger::dispatch(
+                ['message' => $error],
+                SystemLog::CATEGORY_WEBHOOK,
+                SystemLog::EVENT_WEBHOOK_FAILURE,
+                SystemLog::TYPE_WEBHOOK_RESPONSE,
+                $this->resolveClient(),
+                $this->company
+            );
+            
+        }
+        catch (ClientException $e) {
+
+            nlog("Client exception");
+            $error = json_decode($e->getResponse()->getBody()->getContents());
+
+            SystemLogger::dispatch(
+                ['message' => $error],
+                SystemLog::CATEGORY_WEBHOOK,
+                SystemLog::EVENT_WEBHOOK_FAILURE,
+                SystemLog::TYPE_WEBHOOK_RESPONSE,
+                $this->resolveClient(),
+                $this->company
+            );
+            
+
+        }
+        catch (\Exception $e) {
+            
+            nlog("Exception handler => " . $e->getMessage());
+            nlog($e->getCode());
+
+            SystemLogger::dispatch(
+                $e->getMessage(),
+                SystemLog::CATEGORY_WEBHOOK,
+                SystemLog::EVENT_WEBHOOK_FAILURE,
+                SystemLog::TYPE_WEBHOOK_RESPONSE,
+                $this->resolveClient(),
+                $this->company,
+            );
+
+            $this->release($this->backoff()[$this->attempts()-1]);
+
+        }
+    }
+
+    private function resolveClient()
+    {
+        //make sure it isn't an instance of the Client Model
+        if ((! $this->entity instanceof ClientModel) && $this->entity->client()->exists()) {
+            return $this->entity->client;
+        }
+
+        return $this->company->clients()->first();
+    }
+
+    public function failed($exception)
+    {
+        
+        nlog(print_r($exception->getMessage(), 1));
+
+    }
+}

routes/api.php

@@ -375,5 +375,4 @@ Route::post('api/v1/yodlee/data_updates', [YodleeController::class, 'dataUpdates
 Route::post('api/v1/yodlee/refresh_updates', [YodleeController::class, 'refreshUpdatesWebhook'])->middleware('throttle:100,1');
 Route::post('api/v1/yodlee/balance', [YodleeController::class, 'balanceWebhook'])->middleware('throttle:100,1');
 
-
 Route::fallback([BaseController::class, 'notFound']);