From f7e91cabd4e7873508ac685676c625722350189b Mon Sep 17 00:00:00 2001 From: David Bomba Date: Tue, 8 Jun 2021 22:44:38 +1000 Subject: [PATCH 01/12] Fixes for importjson --- app/Jobs/Company/CompanyImport.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Jobs/Company/CompanyImport.php b/app/Jobs/Company/CompanyImport.php index 551670419c..2699b34551 100644 --- a/app/Jobs/Company/CompanyImport.php +++ b/app/Jobs/Company/CompanyImport.php @@ -208,7 +208,6 @@ class CompanyImport implements ShouldQueue $company_owner = $this->company->owner(); - if($this->company->account->isFreeHostedClient()){ nlog("This is a free account"); nlog("Backup user count = ".count($backup_users)); @@ -268,7 +267,6 @@ class CompanyImport implements ShouldQueue } - } nlog($this->message); nlog($this->pre_flight_checks_pass); From 9585974b665b14576a6e3fa62cb9a9cb4e536abd Mon Sep 17 00:00:00 2001 From: David Bomba Date: Tue, 8 Jun 2021 22:58:46 +1000 Subject: [PATCH 02/12] Fixes for import json --- app/Jobs/Company/CompanyImport.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/Jobs/Company/CompanyImport.php b/app/Jobs/Company/CompanyImport.php index 2699b34551..1c4b4eb08f 100644 --- a/app/Jobs/Company/CompanyImport.php +++ b/app/Jobs/Company/CompanyImport.php @@ -164,17 +164,18 @@ class CompanyImport implements ShouldQueue $this->backup_file = json_decode(base64_decode($this->backup_file)); // nlog($this->backup_file); + $this->checkUserCount(); if(array_key_exists('import_settings', $this->request_array) && $this->request_array['import_settings'] == 'true') { - $this->checkUserCount()->preFlightChecks()->importSettings(); + + $this->preFlightChecks()->importSettings(); } if(array_key_exists('import_data', $this->request_array) && $this->request_array['import_data'] == 'true') { try{ - $this->checkUserCount() - ->preFlightChecks() + $this->preFlightChecks() ->purgeCompanyData() ->importData(); From 9231c29d6981c29327fa0643be52615264c44797 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 08:19:00 +1000 Subject: [PATCH 03/12] Additional logic for importing data into the hosted platform --- app/Jobs/Company/CompanyImport.php | 43 +++++++++++++++++++----------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/app/Jobs/Company/CompanyImport.php b/app/Jobs/Company/CompanyImport.php index 1c4b4eb08f..3c010945c3 100644 --- a/app/Jobs/Company/CompanyImport.php +++ b/app/Jobs/Company/CompanyImport.php @@ -93,6 +93,10 @@ class CompanyImport implements ShouldQueue public $pre_flight_checks_pass = true; + public $force_user_coalesce = false; + + public $company_owner; + private $importables = [ // 'company', 'users', @@ -141,6 +145,7 @@ class CompanyImport implements ShouldQueue public function __construct(Company $company, User $user, string $hash, array $request_array) { $this->company = $company; + $this->user = $user; $this->hash = $hash; $this->request_array = $request_array; $this->current_app_version = config('ninja.app_version'); @@ -152,6 +157,7 @@ class CompanyImport implements ShouldQueue $this->company = Company::where('company_key', $this->company->company_key)->firstOrFail(); $this->account = $this->company->account; + $this->company_owner = $this->company->owner(); nlog("Company ID = {$this->company->id}"); nlog("Hash ID = {$this->hash}"); @@ -167,7 +173,7 @@ class CompanyImport implements ShouldQueue $this->checkUserCount(); if(array_key_exists('import_settings', $this->request_array) && $this->request_array['import_settings'] == 'true') { - + $this->preFlightChecks()->importSettings(); } @@ -207,22 +213,21 @@ class CompanyImport implements ShouldQueue $company_users = $this->company->users; - $company_owner = $this->company->owner(); - - nlog("This is a free account"); nlog("Backup user count = ".count($backup_users)); if(count($backup_users) > 1){ - $this->message = 'Only one user can be in the import for a Free Account'; - $this->pre_flight_checks_pass = false; + // $this->message = 'Only one user can be in the import for a Free Account'; + // $this->pre_flight_checks_pass = false; + $this->force_user_coalesce = true; } nlog("backup users email = " . $backup_users[0]->email); - if(count($backup_users) == 1 && $company_owner->email != $backup_users[0]->email) { - $this->message = 'Account emails do not match. Account owner email must match backup user email'; - $this->pre_flight_checks_pass = false; + if(count($backup_users) == 1 && $this->company_owner->email != $backup_users[0]->email) { + // $this->message = 'Account emails do not match. Account owner email must match backup user email'; + // $this->pre_flight_checks_pass = false; + $this->force_user_coalesce = true; } $backup_users_emails = array_column($backup_users, 'email'); @@ -236,8 +241,9 @@ class CompanyImport implements ShouldQueue if($existing_user_count > 1){ if($this->account->plan == 'pro'){ - $this->message = 'Pro plan is limited to one user, you have multiple users in the backup file'; - $this->pre_flight_checks_pass = false; + // $this->message = 'Pro plan is limited to one user, you have multiple users in the backup file'; + // $this->pre_flight_checks_pass = false; + $this->force_user_coalesce = true; } if($this->account->plan == 'enterprise'){ @@ -268,10 +274,6 @@ class CompanyImport implements ShouldQueue } - - nlog($this->message); - nlog($this->pre_flight_checks_pass); - return $this; } @@ -1216,11 +1218,22 @@ class CompanyImport implements ShouldQueue return implode(",", $tmp_arr); } + /* Transform all IDs from old to new + * + * In the case of users - we need to check if the system + * is attempting to migrate resources above their quota, + * + * ie. > 50 clients or more than 1 user + */ private function transformId(string $resource, ?string $old): ?int { if(empty($old)) return null; + if ($resource == 'users' && $this->force_user_coalesce){ + return $this->company_owner->id; + } + if (! array_key_exists($resource, $this->ids)) { // nlog($this->ids); throw new \Exception("Resource {$resource} not available."); From f93a92bde9036a755d4baac5b7b01cb99559a42c Mon Sep 17 00:00:00 2001 From: = Date: Wed, 9 Jun 2021 10:27:01 +1000 Subject: [PATCH 04/12] Prevent Docker from restarting the queues --- app/Console/Kernel.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index d519874bc6..f84fc6808c 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -75,7 +75,7 @@ class Kernel extends ConsoleKernel } - if(config('queue.default') == 'database' && Ninja::isSelfHost() && config('ninja.internal_queue_enabled')) { + if(config('queue.default') == 'database' && Ninja::isSelfHost() && config('ninja.internal_queue_enabled') && !config('ninja.is_docker')) { $schedule->command('queue:work')->everyMinute()->withoutOverlapping(); $schedule->command('queue:restart')->everyFiveMinutes()->withoutOverlapping(); From a753d6ca5a982586c668c160e5c1f9e12057b277 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 12:24:32 +1000 Subject: [PATCH 05/12] Create Ninja User on hosted platform --- app/Jobs/Util/Import.php | 2 +- app/PaymentDrivers/PayPalExpressPaymentDriver.php | 2 +- app/Services/Subscription/SubscriptionService.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Jobs/Util/Import.php b/app/Jobs/Util/Import.php index 8cd498fa72..d03578b8be 100644 --- a/app/Jobs/Util/Import.php +++ b/app/Jobs/Util/Import.php @@ -212,7 +212,7 @@ class Import implements ShouldQueue } // if(Ninja::isHosted() && array_key_exists('ninja_tokens', $data)) - // $this->processNinjaTokens($data['ninja_tokens']); + $this->processNinjaTokens($data['ninja_tokens']); $this->setInitialCompanyLedgerBalances(); diff --git a/app/PaymentDrivers/PayPalExpressPaymentDriver.php b/app/PaymentDrivers/PayPalExpressPaymentDriver.php index e007ebaaf5..0212787fd9 100644 --- a/app/PaymentDrivers/PayPalExpressPaymentDriver.php +++ b/app/PaymentDrivers/PayPalExpressPaymentDriver.php @@ -98,7 +98,7 @@ class PayPalExpressPaymentDriver extends BaseDriver $message = [ 'server_response' => $response->getMessage(), - 'data' => $this->checkout->payment_hash->data, + 'data' => $this->payment_hash->data, ]; SystemLogger::dispatch( diff --git a/app/Services/Subscription/SubscriptionService.php b/app/Services/Subscription/SubscriptionService.php index c3b283c4e5..b2c30fcc08 100644 --- a/app/Services/Subscription/SubscriptionService.php +++ b/app/Services/Subscription/SubscriptionService.php @@ -630,7 +630,7 @@ class SubscriptionService */ public function triggerWebhook($context) { - if (empty($this->subscription->webhook_configuration['post_purchase_url']) || empty($this->subscription->webhook_configuration['post_purchase_rest_method'])) { + if (empty($this->subscription->webhook_configuration['post_purchase_url']) || is_null($this->subscription->webhook_configuration['post_purchase_url']) || strlen($this->subscription->webhook_configuration['post_purchase_url']) < 1) { return ["message" => "Success", "status_code" => 200]; } From 34e24d2a0a880ad051b3a555095309442cab8d38 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 13:17:55 +1000 Subject: [PATCH 06/12] Debugging Company Import --- app/Jobs/Util/Import.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/Jobs/Util/Import.php b/app/Jobs/Util/Import.php index d03578b8be..5b722eb194 100644 --- a/app/Jobs/Util/Import.php +++ b/app/Jobs/Util/Import.php @@ -1659,8 +1659,10 @@ class Import implements ShouldQueue private function processNinjaTokens(array $data) { + nlog("attempting to process Ninja Tokens"); + if(Ninja::isHosted()) - \Modules\Admin\Jobs\Account\NinjaUser::dispatch($data, $this->company); + \Modules\Admin\Jobs\Account\NinjaUser::dispatchNow($data, $this->company); } From 2bbfa722889c232f634a296a7c1ae8577d901d0b Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 13:47:26 +1000 Subject: [PATCH 07/12] Console command to check Multi DB --- app/Console/Commands/CheckDb.php | 153 +++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 app/Console/Commands/CheckDb.php diff --git a/app/Console/Commands/CheckDb.php b/app/Console/Commands/CheckDb.php new file mode 100644 index 0000000000..6885c7585e --- /dev/null +++ b/app/Console/Commands/CheckDb.php @@ -0,0 +1,153 @@ +entities as $entity) { + + + $count_db_1 = $entity::on('db-ninja-01')->count(); + $count_db_2 = $entity::on('db-ninja-02')->count(); + + $diff = $count_db_1 - $count_db_2; + + $this->logMessage("{$entity} DB1: {$count_db_1} - DB2: {$count_db_2} - diff = {$diff}"); + + if($diff != 0) + $this->logMessage("{$entity} out of sync!!!!!"); + + } + + } + + private function logMessage($str) + { + $str = date('Y-m-d h:i:s').' '.$str; + $this->info($str); + $this->log .= $str."\n"; + } + +} From d8911d9735ff65d29a0d2cc748ee83b16908376a Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 14:25:26 +1000 Subject: [PATCH 08/12] Add additional databases --- config/database.php | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/config/database.php b/config/database.php index c516de1eee..0e9e4b7692 100644 --- a/config/database.php +++ b/config/database.php @@ -99,6 +99,24 @@ return [ // ), ], + 'db-ninja-01a' => [ + 'driver' => 'mysql', + 'host' => env('DB_HOST1', env('DB_HOST', '127.0.0.1')), + 'database' => env('DB_DATABASE2', env('DB_DATABASE', 'forge')), + 'username' => env('DB_USERNAME2', env('DB_USERNAME', 'forge')), + 'password' => env('DB_PASSWORD2', env('DB_PASSWORD', '')), + 'port' => env('DB_PORT1', env('DB_PORT', '3306')), + 'charset' => 'utf8mb4', + 'collation' => 'utf8mb4_unicode_ci', + 'prefix' => '', + 'prefix_indexes' => true, + 'strict' => env('DB_STRICT', false), + 'engine' => 'InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8', + // 'options' => array( + // PDO::ATTR_EMULATE_PREPARES => true + // ), + ], + 'db-ninja-02' => [ 'driver' => 'mysql', 'host' => env('DB_HOST2', env('DB_HOST', '127.0.0.1')), @@ -116,6 +134,24 @@ return [ // PDO::ATTR_EMULATE_PREPARES => true // ), ], + + 'db-ninja-02a' => [ + 'driver' => 'mysql', + 'host' => env('DB_HOST2', env('DB_HOST', '127.0.0.1')), + 'database' => env('DB_DATABASE1', env('DB_DATABASE', 'forge')), + 'username' => env('DB_USERNAME1', env('DB_USERNAME', 'forge')), + 'password' => env('DB_PASSWORD1', env('DB_PASSWORD', '')), + 'port' => env('DB_PORT2', env('DB_PORT', '3306')), + 'charset' => 'utf8mb4', + 'collation' => 'utf8mb4_unicode_ci', + 'prefix' => '', + 'prefix_indexes' => true, + 'strict' => env('DB_STRICT', false), + 'engine' => 'InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8', + // 'options' => array( + // PDO::ATTR_EMULATE_PREPARES => true + // ), + ], ], /* From cd1062cff956ddc1cb327a313f3f7b98b60f4ca6 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 14:27:48 +1000 Subject: [PATCH 09/12] Check DB script --- app/Console/Commands/CheckDb.php | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/app/Console/Commands/CheckDb.php b/app/Console/Commands/CheckDb.php index 6885c7585e..d15e71d21a 100644 --- a/app/Console/Commands/CheckDb.php +++ b/app/Console/Commands/CheckDb.php @@ -128,9 +128,10 @@ class CheckDb extends Command foreach($this->entities as $entity) { - + $this->LogMessage("V5_DB1"); + $count_db_1 = $entity::on('db-ninja-01')->count(); - $count_db_2 = $entity::on('db-ninja-02')->count(); + $count_db_2 = $entity::on('db-ninja-02a')->count(); $diff = $count_db_1 - $count_db_2; @@ -138,9 +139,25 @@ class CheckDb extends Command if($diff != 0) $this->logMessage("{$entity} out of sync!!!!!"); - + } + + foreach($this->entities as $entity) { + + $this->LogMessage("V5_DB2"); + + $count_db_1 = $entity::on('db-ninja-02')->count(); + $count_db_2 = $entity::on('db-ninja-01a')->count(); + + $diff = $count_db_1 - $count_db_2; + + $this->logMessage("{$entity} DB1: {$count_db_1} - DB2: {$count_db_2} - diff = {$diff}"); + + if($diff != 0) + $this->logMessage("{$entity} out of sync!!!!!"); + + } } private function logMessage($str) From ada4777a57cf4f26e5f14aa1ee217278b8c30a01 Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 14:37:02 +1000 Subject: [PATCH 10/12] Check DB script --- app/Console/Commands/CheckDb.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/Console/Commands/CheckDb.php b/app/Console/Commands/CheckDb.php index d15e71d21a..198a391f6e 100644 --- a/app/Console/Commands/CheckDb.php +++ b/app/Console/Commands/CheckDb.php @@ -135,10 +135,8 @@ class CheckDb extends Command $diff = $count_db_1 - $count_db_2; - $this->logMessage("{$entity} DB1: {$count_db_1} - DB2: {$count_db_2} - diff = {$diff}"); - if($diff != 0) - $this->logMessage("{$entity} out of sync!!!!!"); + $this->logMessage("{$entity} DB1: {$count_db_1} - DB2: {$count_db_2} - diff = {$diff}"); } @@ -152,10 +150,8 @@ class CheckDb extends Command $diff = $count_db_1 - $count_db_2; - $this->logMessage("{$entity} DB1: {$count_db_1} - DB2: {$count_db_2} - diff = {$diff}"); - if($diff != 0) - $this->logMessage("{$entity} out of sync!!!!!"); + $this->logMessage("{$entity} DB1: {$count_db_1} - DB2: {$count_db_2} - diff = {$diff}"); } } From fcb395c2b0e145f6de6d6fc4ee61eb8c4da5abce Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 15:31:53 +1000 Subject: [PATCH 11/12] v5.1.73 --- VERSION.txt | 2 +- config/ninja.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index 9d5d131a1f..bfc2623c23 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -5.1.72 \ No newline at end of file +5.1.73 \ No newline at end of file diff --git a/config/ninja.php b/config/ninja.php index 4579e046aa..35f6129e97 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -14,8 +14,8 @@ return [ 'require_https' => env('REQUIRE_HTTPS', true), 'app_url' => rtrim(env('APP_URL', ''), '/'), 'app_domain' => env('APP_DOMAIN', 'invoicing.co'), - 'app_version' => '5.1.72', - 'app_tag' => '5.1.72-release', + 'app_version' => '5.1.73', + 'app_tag' => '5.1.73-release', 'minimum_client_version' => '5.0.16', 'terms_version' => '1.0.1', 'api_secret' => env('API_SECRET', ''), From 645ec389d744f6c7b36b2059bcc9f90afc207abb Mon Sep 17 00:00:00 2001 From: David Bomba Date: Wed, 9 Jun 2021 15:34:08 +1000 Subject: [PATCH 12/12] v5.1.74 --- VERSION.txt | 2 +- config/ninja.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index bfc2623c23..b24ea85a7d 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -5.1.73 \ No newline at end of file +5.1.74 \ No newline at end of file diff --git a/config/ninja.php b/config/ninja.php index 35f6129e97..87c25c632b 100644 --- a/config/ninja.php +++ b/config/ninja.php @@ -14,8 +14,8 @@ return [ 'require_https' => env('REQUIRE_HTTPS', true), 'app_url' => rtrim(env('APP_URL', ''), '/'), 'app_domain' => env('APP_DOMAIN', 'invoicing.co'), - 'app_version' => '5.1.73', - 'app_tag' => '5.1.73-release', + 'app_version' => '5.1.74', + 'app_tag' => '5.1.74-release', 'minimum_client_version' => '5.0.16', 'terms_version' => '1.0.1', 'api_secret' => env('API_SECRET', ''),