invoiceninja/app/Jobs/Util/Import.php

<?php
/**
 * Invoice Ninja (https://invoiceninja.com).
 *
 * @link https://github.com/invoiceninja/invoiceninja source repository
 *
 * @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
 *
 * @license https://www.elastic.co/licensing/elastic-license
 */

namespace App\Jobs\Util;

use App\DataMapper\Analytics\MigrationFailure;
use App\DataMapper\CompanySettings;
use App\Exceptions\MigrationValidatorFailed;
use App\Exceptions\ProcessingMigrationArchiveFailed;
use App\Exceptions\ResourceDependencyMissing;
use App\Exceptions\ResourceNotAvailableForMigration;
use App\Factory\ClientContactFactory;
use App\Factory\ClientFactory;
use App\Factory\CompanyLedgerFactory;
use App\Factory\CreditFactory;
use App\Factory\InvoiceFactory;
use App\Factory\PaymentFactory;
use App\Factory\ProductFactory;
use App\Factory\QuoteFactory;
use App\Factory\RecurringInvoiceFactory;
use App\Factory\TaxRateFactory;
use App\Factory\UserFactory;
use App\Factory\VendorFactory;
use App\Http\Requests\Company\UpdateCompanyRequest;
use App\Http\ValidationRules\User\AttachableUser;
use App\Http\ValidationRules\ValidCompanyGatewayFeesAndLimitsRule;
use App\Http\ValidationRules\ValidUserForCompany;
use App\Jobs\Company\CreateCompanyTaskStatuses;
use App\Jobs\Company\CreateCompanyToken;
use App\Jobs\Ninja\CheckCompanyData;
use App\Jobs\Ninja\CompanySizeCheck;
use App\Jobs\Util\VersionCheck;
use App\Libraries\MultiDB;
use App\Mail\MigrationCompleted;
use App\Models\Activity;
use App\Models\Client;
use App\Models\ClientContact;
use App\Models\ClientGatewayToken;
use App\Models\Company;
use App\Models\CompanyGateway;
use App\Models\Credit;
use App\Models\Document;
use App\Models\Expense;
use App\Models\ExpenseCategory;
use App\Models\Invoice;
use App\Models\Payment;
use App\Models\PaymentTerm;
use App\Models\Product;
use App\Models\Project;
use App\Models\Quote;
use App\Models\RecurringInvoice;
use App\Models\Task;
use App\Models\TaskStatus;
use App\Models\TaxRate;
use App\Models\User;
use App\Models\Vendor;
use App\Repositories\ClientContactRepository;
use App\Repositories\ClientRepository;
use App\Repositories\CompanyRepository;
use App\Repositories\CreditRepository;
use App\Repositories\Migration\InvoiceMigrationRepository;
use App\Repositories\Migration\PaymentMigrationRepository;
use App\Repositories\ProductRepository;
use App\Repositories\QuoteRepository;
use App\Repositories\UserRepository;
use App\Repositories\VendorContactRepository;
use App\Repositories\VendorRepository;
use App\Utils\Ninja;
use App\Utils\Traits\CleanLineItems;
use App\Utils\Traits\CompanyGatewayFeesAndLimitsSaver;
use App\Utils\Traits\MakesHash;
use App\Utils\Traits\SavesDocuments;
use App\Utils\Traits\Uploadable;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Http\UploadedFile;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Mail;
use Illuminate\Support\Facades\Validator;
use Illuminate\Support\Str;
use Turbo124\Beacon\Facades\LightLogs;

class Import implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
    use CompanyGatewayFeesAndLimitsSaver;
    use MakesHash;
    use CleanLineItems;
    use Uploadable;
    use SavesDocuments;
    /**
     * @var array
     */
    private $file_path; //the file path - using a different JSON parser here.

    /**
     * @var Company
     */
    private $company;

    private $token;

    /**
     * @var array
     */
    private $available_imports = [
        'account',
        'company',
        'users',
        'payment_terms',
        'tax_rates',
        'clients',
        'company_gateways',
        'client_gateway_tokens',
        'vendors',
        'projects',
        'products',
        'credits',
        'invoices',
        'recurring_invoices',
        'quotes',
        'payments',
        'expense_categories',
        'task_statuses',
        'expenses',
        'tasks',
        'documents',
    ];

    /**
     * @var User
     */
    private $user;

    /**
     * Custom list of resources to be imported.
     *
     * @var array
     */
    private $resources;

    /**
     * Local state manager for ids.
     *
     * @var array
     */
    private $ids = [];

    public $tries = 1;

    public $timeout = 0;

    // public $backoff = 86430;

    //  public $maxExceptions = 2;
    /**
     * Create a new job instance.
     *
     * @param array $data
     * @param Company $company
     * @param User $user
     * @param array $resources
     */
    public function __construct(string $file_path, Company $company, User $user, array $resources = [])
    {
        $this->file_path = $file_path;
        $this->company = $company;
        $this->user = $user;
        $this->resources = $resources;
    }

    /**
     * Execute the job.
     *
     * @return bool
     */
    public function handle()
    {
        set_time_limit(0);

        nlog("Starting Migration");
        nlog($this->user->email);
        info("Starting Migration");
        info($this->user->email);
        
        auth()->login($this->user, false);
        auth()->user()->setCompany($this->company);

        //   $jsonStream = \JsonMachine\JsonMachine::fromFile($this->file_path, "/data");
        $array = json_decode(file_get_contents($this->file_path), 1);
        $data = $array['data'];

        foreach ($this->available_imports as $import) {
            if (! array_key_exists($import, $data)) {
                //throw new ResourceNotAvailableForMigration("Resource {$key} is not available for migration.");
                info("Resource {$import} is not available for migration.");
                continue;
            }

            $method = sprintf('process%s', Str::ucfirst(Str::camel($import)));

            info("Importing {$import}");

            $this->{$method}($data[$import]);
        }

        // if(Ninja::isHosted() && array_key_exists('ninja_tokens', $data))
        $this->processNinjaTokens($data['ninja_tokens']);

        // $this->fixData();

        $this->setInitialCompanyLedgerBalances();
        
        // $this->fixClientBalances();
        $check_data = CheckCompanyData::dispatchNow($this->company, md5(time()));
        
        try{
            Mail::to($this->user->email, $this->user->name())
                ->send(new MigrationCompleted($this->company, implode("<br>",$check_data)));
        }
        catch(\Exception $e) {

            nlog($e->getMessage());
        }
        
        /*After a migration first some basic jobs to ensure the system is up to date*/
        VersionCheck::dispatch();
        
            //company size check
            if ($this->company->invoices()->count() > 1000 || $this->company->products()->count() > 1000 || $this->company->clients()->count() > 1000) {
                $this->company->is_large = true;
                $this->company->save();
            }

        // CreateCompanyPaymentTerms::dispatchNow($sp035a66, $spaa9f78);
        CreateCompanyTaskStatuses::dispatchNow($this->company, $this->user);

        info('Completed🚀🚀🚀🚀🚀 at '.now());

        unlink($this->file_path);
    }

    private function fixData()
    {

        $this->company->clients()->withTrashed()->where('is_deleted', 0)->cursor()->each(function ($client) {
            $total_invoice_payments = 0;
            $credit_total_applied = 0;

            foreach ($client->invoices()->where('is_deleted', false)->where('status_id', '>', 1)->get() as $invoice) {

                $total_amount = $invoice->payments()->where('is_deleted', false)->whereIn('status_id', [Payment::STATUS_COMPLETED, Payment:: STATUS_PENDING, Payment::STATUS_PARTIALLY_REFUNDED, Payment::STATUS_REFUNDED])->get()->sum('pivot.amount');
                $total_refund = $invoice->payments()->where('is_deleted', false)->whereIn('status_id', [Payment::STATUS_COMPLETED, Payment:: STATUS_PENDING, Payment::STATUS_PARTIALLY_REFUNDED, Payment::STATUS_REFUNDED])->get()->sum('pivot.refunded');

                $total_invoice_payments += ($total_amount - $total_refund);
            }

            // 10/02/21
            foreach ($client->payments as $payment) {
                $credit_total_applied += $payment->paymentables()->where('paymentable_type', App\Models\Credit::class)->get()->sum(\DB::raw('amount'));
            }

            if ($credit_total_applied < 0) {
                $total_invoice_payments += $credit_total_applied;
            } 


            if (round($total_invoice_payments, 2) != round($client->paid_to_date, 2)) {

                $client->paid_to_date = $total_invoice_payments;
                $client->save();
                
            }
        });

    }

    private function setInitialCompanyLedgerBalances()
    {
        Client::cursor()->each(function ($client) {

            $invoice_balances = $client->invoices->where('is_deleted', false)->where('status_id', '>', 1)->sum('balance');

            $company_ledger = CompanyLedgerFactory::create($client->company_id, $client->user_id);
            $company_ledger->client_id = $client->id;
            $company_ledger->adjustment = $invoice_balances;
            $company_ledger->notes = 'Migrated Client Balance';
            $company_ledger->balance = $invoice_balances;
            $company_ledger->activity_id = Activity::CREATE_CLIENT;
            $company_ledger->save();

            $client->company_ledger()->save($company_ledger);

            $client->balance = $invoice_balances;
            $client->save();

        });
    }

    private function processAccount(array $data) :void
    {
        if(array_key_exists('token', $data)){
            $this->token = $data['token'];
            unset($data['token']);
        }

        $account = $this->company->account;
        $account->fill($data);
        $account->save();

        //Prevent hosted users being pushed into a trial
        if(Ninja::isHosted() && $account->plan != ''){
            $account->trial_plan = '';
            $account->save();
        }
    }

    /**
     * @param array $data
     * @throws Exception
     */
    private function processCompany(array $data): void
    {
        Company::unguard();

        if (
            $data['settings']['invoice_design_id'] > 9 ||
            $data['settings']['invoice_design_id'] > "9"
        ) {
            $data['settings']['invoice_design_id'] = 1;
        }

        $data = $this->transformCompanyData($data);

        if(Ninja::isHosted()) {

            if(!MultiDB::checkDomainAvailable($data['subdomain']))
                $data['subdomain'] = MultiDB::randomSubdomainGenerator();

            if(strlen($data['subdomain']) == 0)
                $data['subdomain'] = MultiDB::randomSubdomainGenerator();

        }

        $rules = (new UpdateCompanyRequest())->rules();

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) 
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        
        if (isset($data['account_id'])) 
            unset($data['account_id']);
        
        if(isset($data['version']))
            unset($data['version']);

        if (isset($data['referral_code'])) {
            $account = $this->company->account;
            $account->referral_code = $data['referral_code'];
            $account->save();

            unset($data['referral_code']);
        }

        if (isset($data['custom_fields']) && is_array($data['custom_fields'])) {
            $data['custom_fields'] = $this->parseCustomFields($data['custom_fields']);
        }

        $company_repository = new CompanyRepository();
        $company_repository->save($data, $this->company);

        if (isset($data['settings']->company_logo) && strlen($data['settings']->company_logo) > 0) {
            try {
                $tempImage = tempnam(sys_get_temp_dir(), basename($data['settings']->company_logo));
                copy($data['settings']->company_logo, $tempImage);
                $this->uploadLogo($tempImage, $this->company, $this->company);
            } catch (\Exception $e) {
            }
        }

        Company::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $rules = null;
        $validator = null;
        $company_repository = null;
    }

    private function parseCustomFields($fields) :array
    {
        
        if(array_key_exists('account1', $fields))
            $fields['company1'] = $fields['account1'];

        if(array_key_exists('account2', $fields))
            $fields['company2'] = $fields['account2'];

        if(array_key_exists('invoice1', $fields))
            $fields['surcharge1'] = $fields['invoice1'];

        if(array_key_exists('invoice2', $fields))
            $fields['surcharge2'] = $fields['invoice2'];

        if(array_key_exists('invoice_text1', $fields))
            $fields['invoice1'] = $fields['invoice_text1'];

        if(array_key_exists('invoice_text2', $fields))
            $fields['invoice2'] = $fields['invoice_text2'];

        foreach ($fields as &$value) {
            $value = (string) $value;
        }

        return $fields;
    }

    private function transformCompanyData(array $data): array
    {

        $company_settings = CompanySettings::defaults();

        if (array_key_exists('settings', $data)) {
            foreach ($data['settings'] as $key => $value) {
                if ($key == 'invoice_design_id' || $key == 'quote_design_id' || $key == 'credit_design_id') {
                    $value = $this->encodePrimaryKey($value);

                    if(!$value)
                        $value = $this->encodePrimaryKey(1);
                    
                }

                if ($key == 'payment_terms' && $key = '') {
                    $value = -1;
                }

                $company_settings->{$key} = $value;
            }

            $data['settings'] = $company_settings;
        }


        return $data;
    }

    /**
     * @param array $data
     * @throws Exception
     */
    private function processTaxRates(array $data): void
    {
        TaxRate::unguard();

        $rules = [
            '*.name' => 'required',
            //'*.name' => 'required|distinct|unique:tax_rates,name,null,null,company_id,' . $this->company->id,
            '*.rate' => 'required|numeric',
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        foreach ($data as $resource) {
            $modified = $resource;
            $company_id = $this->company->id;
            $user_id = $this->processUserId($resource);

            if (isset($resource['user_id'])) {
                unset($resource['user_id']);
            }

            if (isset($resource['company_id'])) {
                unset($resource['company_id']);
            }

            $tax_rate = TaxRateFactory::create($this->company->id, $user_id);
            $tax_rate->fill($resource);

            $tax_rate->save();
        }

        TaxRate::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $rules = null;
        $validator = null;
    }

    /**
     * @param array $data
     * @throws Exception
     */
    private function processUsers(array $data): void
    {
        User::unguard();

        $rules = [
            '*.first_name' => ['string'],
            '*.last_name' => ['string'],
            //'*.email' => ['distinct'],
            '*.email' => ['distinct', 'email', new ValidUserForCompany()],
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        $user_repository = new UserRepository();

        foreach ($data as $resource) {
            $modified = $resource;
            unset($modified['id']);
            unset($modified['password']); //cant import passwords.

            $user = $user_repository->save($modified, $this->fetchUser($resource['email']), true, true);
            $user->email_verified_at = now();
            // $user->confirmation_code = '';

            if($modified['deleted_at'])
                $user->deleted_at = now();
            
            $user->save();
            
            $user_agent = array_key_exists('token_name', $resource) ?: request()->server('HTTP_USER_AGENT');

            CreateCompanyToken::dispatchNow($this->company, $user, $user_agent);

            $key = "users_{$resource['id']}";

            $this->ids['users'][$key] = [
                'old' => $resource['id'],
                'new' => $user->id,
            ];
        }

        User::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $rules = null;
        $validator = null;
        $user_repository = null;
    }

    private function checkUniqueConstraint($model, $column, $value)
    {
        $value = trim($value);

        $model_query = $model::where($column, $value)
                             ->where('company_id', $this->company->id)
                             ->withTrashed()
                             ->exists();

        if($model_query)
            return $value . '_' . Str::random(5);

        return $value;
    }

    /**
     * @param array $data
     * @throws Exception
     */
    private function processClients(array $data): void
    {
        Client::unguard();

        $contact_repository = new ClientContactRepository();
        $client_repository = new ClientRepository($contact_repository);

        foreach ($data as $key => $resource) {
            $modified = $resource;
            $modified['company_id'] = $this->company->id;
            $modified['user_id'] = $this->processUserId($resource);
            $modified['balance'] = $modified['balance'] ?: 0;
            $modified['paid_to_date'] = $modified['paid_to_date'] ?: 0;
            $modified['number'] = $this->checkUniqueConstraint(Client::class, 'number', $modified['number']);

            unset($modified['id']);
            unset($modified['contacts']);

            $client = $client_repository->save(
                $modified,
                ClientFactory::create(
                    $this->company->id,
                    $modified['user_id']
                )
            );

            if(array_key_exists('created_at', $modified))
                $client->created_at = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $client->updated_at = Carbon::parse($modified['updated_at']);

            $client->save(['timestamps' => false]);

            $client->contacts()->forceDelete();

            if (array_key_exists('contacts', $resource)) { // need to remove after importing new migration.json
                $modified_contacts = $resource['contacts'];

                foreach ($modified_contacts as $key => $client_contacts) {
                    $modified_contacts[$key]['company_id'] = $this->company->id;
                    $modified_contacts[$key]['user_id'] = $this->processUserId($resource);
                    $modified_contacts[$key]['client_id'] = $client->id;
                    $modified_contacts[$key]['password'] = 'mysuperpassword'; // @todo, and clean up the code..
                    unset($modified_contacts[$key]['id']);
                }

                $saveable_contacts['contacts'] = $modified_contacts;

                $contact_repository->save($saveable_contacts, $client);

                //link contact ids

                foreach ($resource['contacts'] as $key => $old_contact) {
                    
                    $contact_match = ClientContact::where('contact_key', $old_contact['contact_key'])
                                                 ->where('company_id', $this->company->id)
                                                 ->where('client_id', $client->id)
                                                 ->withTrashed()
                                                 ->first();

                    if ($contact_match) {
                        
                        $this->ids['client_contacts']['client_contacts_'.$old_contact['id']] = [
                            'old' => $old_contact['id'],
                            'new' => $contact_match->id,
                        ];
                        
                    }
                }
            }

            $key = "clients_{$resource['id']}";

            $this->ids['clients'][$key] = [
                'old' => $resource['id'],
                'new' => $client->id,
            ];
        }

        Client::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $contact_repository = null;
        $client_repository = null;
    }

    /**
     * @param array $data
     * @throws Exception
     */
    private function processVendors(array $data): void
    {
        Vendor::unguard();

        $contact_repository = new VendorContactRepository();
        $vendor_repository = new VendorRepository($contact_repository);

        foreach ($data as $key => $resource) {
            $modified = $resource;
            $modified['company_id'] = $this->company->id;
            $modified['user_id'] = $this->processUserId($resource);

            unset($modified['id']);
            unset($modified['contacts']);

            if(array_key_exists('created_at', $modified))
                $modified['created_at'] = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $modified['updated_at'] = Carbon::parse($modified['updated_at']);

            $vendor = $vendor_repository->save(
                $modified,
                VendorFactory::create(
                    $this->company->id,
                    $modified['user_id']
                )
            );

            $vendor->contacts()->forceDelete();

            if (array_key_exists('contacts', $resource)) { // need to remove after importing new migration.json
                $modified_contacts = $resource['contacts'];

                foreach ($modified_contacts as $key => $vendor_contacts) {
                    $modified_contacts[$key]['company_id'] = $this->company->id;
                    $modified_contacts[$key]['user_id'] = $this->processUserId($resource);
                    $modified_contacts[$key]['vendor_id'] = $vendor->id;
                    $modified_contacts[$key]['password'] = 'mysuperpassword'; // @todo, and clean up the code..
                    unset($modified_contacts[$key]['id']);
                }

                $saveable_contacts['contacts'] = $modified_contacts;

                $contact_repository->save($saveable_contacts, $vendor);
            }

            $key = "vendors_{$resource['id']}";

            $this->ids['vendors'][$key] = [
                'old' => $resource['id'],
                'new' => $vendor->id,
            ];
        }

        Vendor::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $contact_repository = null;
        $client_repository = null;
    }


    private function processProducts(array $data): void
    {
        Product::unguard();

        $rules = [
            //'*.product_key' => 'required|distinct|unique:products,product_key,null,null,company_id,' . $this->company->id,
            '*.cost' => 'numeric',
            '*.price' => 'numeric',
            '*.quantity' => 'numeric',
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        $product_repository = new ProductRepository();

        foreach ($data as $resource) {
            $modified = $resource;
            $modified['company_id'] = $this->company->id;
            $modified['user_id'] = $this->processUserId($resource);

            if(array_key_exists('created_at', $modified))
                $modified['created_at'] = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $modified['updated_at'] = Carbon::parse($modified['updated_at']);

            unset($modified['id']);

            $product_repository->save(
                $modified,
                ProductFactory::create(
                    $this->company->id,
                    $modified['user_id']
                )
            );
        }

        Product::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $product_repository = null;
    }

    private function processRecurringInvoices(array $data) :void
    {
        RecurringInvoice::unguard();

        $rules = [
            '*.client_id' => ['required'],
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        $invoice_repository = new InvoiceMigrationRepository();

        foreach ($data as $key => $resource) {
            $modified = $resource;

            if (array_key_exists('client_id', $resource) && ! array_key_exists('clients', $this->ids)) {
                throw new ResourceDependencyMissing('Processing invoices failed, because of missing dependency - clients.');
            }

            $modified['client_id'] = $this->transformId('clients', $resource['client_id']);
            $modified['user_id'] = $this->processUserId($resource);
            $modified['company_id'] = $this->company->id;
            $modified['line_items'] = $this->cleanItems($modified['line_items']);

            if(array_key_exists('created_at', $modified))
                $modified['created_at'] = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $modified['updated_at'] = Carbon::parse($modified['updated_at']);

            unset($modified['id']);

            if (array_key_exists('invitations', $resource)) {
                foreach ($resource['invitations'] as $key => $invite) {
                    $resource['invitations'][$key]['client_contact_id'] = $this->transformId('client_contacts', $invite['client_contact_id']);
                    $resource['invitations'][$key]['user_id'] = $modified['user_id'];
                    $resource['invitations'][$key]['company_id'] = $this->company->id;
                    $resource['invitations'][$key]['email_status'] = '';

                    unset($resource['invitations'][$key]['recurring_invoice_id']);
                    unset($resource['invitations'][$key]['id']);

                }
            
                $modified['invitations'] = $this->deDuplicateInvitations($resource['invitations']);

            }
            
            $invoice = $invoice_repository->save(
                $modified,
                RecurringInvoiceFactory::create($this->company->id, $modified['user_id'])
            );

            $key = "recurring_invoices_{$resource['id']}";

            $this->ids['recurring_invoices'][$key] = [
                'old' => $resource['id'],
                'new' => $invoice->id,
            ];
        }

        RecurringInvoice::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $invoice_repository = null;
    }

    private function processInvoices(array $data): void
    {
        Invoice::unguard();

        $rules = [
            '*.client_id' => ['required'],
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        $invoice_repository = new InvoiceMigrationRepository();

        foreach ($data as $key => $resource) {
            $modified = $resource;

            if (array_key_exists('client_id', $resource) && ! array_key_exists('clients', $this->ids)) {
                throw new ResourceDependencyMissing('Processing invoices failed, because of missing dependency - clients.');
            }

            $modified['client_id'] = $this->transformId('clients', $resource['client_id']);
            $modified['user_id'] = $this->processUserId($resource);
            $modified['company_id'] = $this->company->id;
            $modified['line_items'] = $this->cleanItems($modified['line_items']);

            unset($modified['id']);
                
            if (array_key_exists('invitations', $resource)) {
                foreach ($resource['invitations'] as $key => $invite) {
                    $resource['invitations'][$key]['client_contact_id'] = $this->transformId('client_contacts', $invite['client_contact_id']);
                    $resource['invitations'][$key]['user_id'] = $modified['user_id'];
                    $resource['invitations'][$key]['company_id'] = $this->company->id;
                    $resource['invitations'][$key]['email_status'] = '';
                    unset($resource['invitations'][$key]['invoice_id']);
                    unset($resource['invitations'][$key]['id']);
                }

                $modified['invitations'] = $this->deDuplicateInvitations($resource['invitations']);

            }

            $invoice = $invoice_repository->save(
                $modified,
                InvoiceFactory::create($this->company->id, $modified['user_id'])
            );

            $key = "invoices_{$resource['id']}";

            $this->ids['invoices'][$key] = [
                'old' => $resource['id'],
                'new' => $invoice->id,
            ];
        }

        Invoice::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $invoice_repository = null;
    }


    /* Prevent edge case where V4 has inserted multiple invitations for a resource for a client contact */
    private function deDuplicateInvitations($invitations)
    {        
        return  array_intersect_key($invitations, array_unique(array_column($invitations, 'client_contact_id')));
    }

    private function processCredits(array $data): void
    {
        Credit::unguard();

        $rules = [
            '*.client_id' => ['required'],
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        $credit_repository = new CreditRepository();

        foreach ($data as $resource) {
            $modified = $resource;

            if (array_key_exists('client_id', $resource) && ! array_key_exists('clients', $this->ids)) {
                throw new ResourceDependencyMissing('Processing credits failed, because of missing dependency - clients.');
            }

            $modified['client_id'] = $this->transformId('clients', $resource['client_id']);
            $modified['user_id'] = $this->processUserId($resource);
            $modified['company_id'] = $this->company->id;

            if(array_key_exists('created_at', $modified))
                $modified['created_at'] = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $modified['updated_at'] = Carbon::parse($modified['updated_at']);

            unset($modified['id']);

            $credit = $credit_repository->save(
                $modified,
                CreditFactory::create($this->company->id, $modified['user_id'])
            );

            //remove credit balance from ledger
            if($credit->balance > 0 && $credit->client->balance > 0){
                $client = $credit->client;
                $client->balance -= $credit->balance;
                $client->save();
            }


            $key = "credits_{$resource['id']}";

            $this->ids['credits'][$key] = [
                'old' => $resource['id'],
                'new' => $credit->id,
            ];
        }

        Credit::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $credit_repository = null;

    }

    private function processQuotes(array $data): void
    {
        Quote::unguard();

        $rules = [
            '*.client_id' => ['required'],
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        $quote_repository = new QuoteRepository();

        foreach ($data as $resource) {
            $modified = $resource;

            if (array_key_exists('client_id', $resource) && ! array_key_exists('clients', $this->ids)) {
                throw new ResourceDependencyMissing('Processing quotes failed, because of missing dependency - clients.');
            }

            $modified['client_id'] = $this->transformId('clients', $resource['client_id']);

            if(array_key_exists('invoice_id', $resource) && isset($resource['invoice_id']) && $this->tryTransformingId('invoices', $resource['invoice_id']))
                $modified['invoice_id'] = $this->transformId('invoices', $resource['invoice_id']);

            $modified['user_id'] = $this->processUserId($resource);

            $modified['company_id'] = $this->company->id;

            if(array_key_exists('created_at', $modified))
                $modified['created_at'] = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $modified['updated_at'] = Carbon::parse($modified['updated_at']);
            
            if(array_key_exists('tax_rate1', $modified) && is_null($modified['tax_rate1']))
                $modified['tax_rate1'] = 0;

            if(array_key_exists('tax_rate2', $modified) && is_null($modified['tax_rate2']))
                $modified['tax_rate2'] = 0;

            unset($modified['id']);


            if (array_key_exists('invitations', $resource)) {
                foreach ($resource['invitations'] as $key => $invite) {
                    $resource['invitations'][$key]['client_contact_id'] = $this->transformId('client_contacts', $invite['client_contact_id']);
                    $resource['invitations'][$key]['user_id'] = $modified['user_id'];
                    $resource['invitations'][$key]['company_id'] = $this->company->id;
                    $resource['invitations'][$key]['email_status'] = '';
                    unset($resource['invitations'][$key]['invoice_id']);
                    unset($resource['invitations'][$key]['id']);
                }

                $modified['invitations'] = $this->deDuplicateInvitations($resource['invitations']);

            }

            $quote = $quote_repository->save(
                $modified,
                QuoteFactory::create($this->company->id, $modified['user_id'])
            );

            if(array_key_exists('created_at', $modified))
                $quote->created_at = $modified['created_at'];

            if(array_key_exists('updated_at', $modified))
                $quote->updated_at = $modified['updated_at'];

            $quote->save(['timestamps' => false]);

            $old_user_key = array_key_exists('user_id', $resource) ?? $this->user->id;

            $key = "quotes_{$resource['id']}";

            $this->ids['quotes'][$key] = [
                'old' => $resource['id'],
                'new' => $quote->id,
            ];
        }

        Quote::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $quote_repository = null;
    }

    private function processPayments(array $data): void
    {
        Payment::reguard();

        $rules = [
            '*.amount' => ['required'],
            '*.client_id' => ['required'],
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        $payment_repository = new PaymentMigrationRepository(new CreditRepository());

        foreach ($data as $resource) {
            $modified = $resource;

            if (array_key_exists('client_id', $resource) && ! array_key_exists('clients', $this->ids)) {
                throw new ResourceDependencyMissing('Processing payments failed, because of missing dependency - clients.');
            }

            $modified['client_id'] = $this->transformId('clients', $resource['client_id']);
            $modified['user_id'] = $this->processUserId($resource);
            $modified['company_id'] = $this->company->id;

            unset($modified['invoice_id']);

            if (isset($modified['invoices'])) {
                foreach ($modified['invoices'] as $key => $invoice) {
                    if ($this->tryTransformingId('invoices', $invoice['invoice_id'])) {
                        $modified['invoices'][$key]['invoice_id'] = $this->transformId('invoices', $invoice['invoice_id']);
                    } else {
                       nlog($modified['invoices']);
                       unset($modified['invoices']);
                       //if the transformation didn't work - you _must_ unset this data as it will be incorrect!
                    }
                }
            }

            $payment = $payment_repository->save(
                $modified,
                PaymentFactory::create($this->company->id, $modified['user_id'])
            );

            if(array_key_exists('created_at', $modified))
                $payment->created_at = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $payment->updated_at = Carbon::parse($modified['updated_at']);

            $payment->save(['timestamps' => false]);

            if (array_key_exists('company_gateway_id', $resource) && isset($resource['company_gateway_id']) && $resource['company_gateway_id'] != 'NULL') {
                $payment->company_gateway_id = $this->transformId('company_gateways', $resource['company_gateway_id']);
                $payment->save();
            }
            

            $old_user_key = array_key_exists('user_id', $resource) ?? $this->user->id;

            $this->ids['payments'] = [
                "payments_{$old_user_key}" => [
                    'old' => $old_user_key,
                    'new' => $payment->id,
                ],
            ];

            if(in_array($payment->status_id, [Payment::STATUS_REFUNDED, Payment::STATUS_PARTIALLY_REFUNDED])) {
                $this->processPaymentRefund($payment);
            }


        }

        Payment::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
        $payment_repository = null;
    }

    private function processPaymentRefund($payment)
    {
        $invoices = $payment->invoices()->get();

        $invoices->each(function ($invoice) use($payment) {
            
            if ($payment->refunded > 0 && in_array($invoice->status_id, [Invoice::STATUS_SENT])) {

                $invoice->service()
                        ->updateBalance($payment->refunded)
                        ->updatePaidToDate($payment->refunded*-1)
                        ->updateStatus()
                        ->save();
            }

        });
    }

    private function updatePaymentForStatus($payment, $status_id) :Payment
    {
        // define('PAYMENT_STATUS_PENDING', 1);
        // define('PAYMENT_STATUS_VOIDED', 2);
        // define('PAYMENT_STATUS_FAILED', 3);
        // define('PAYMENT_STATUS_COMPLETED', 4);
        // define('PAYMENT_STATUS_PARTIALLY_REFUNDED', 5);
        // define('PAYMENT_STATUS_REFUNDED', 6);
            
        switch ($status_id) {
            case 1:
                return $payment;
                break;
            case 2:
                return $payment->service()->deletePayment();
                break;
            case 3:
                return $payment->service()->deletePayment();
                break;
            case 4:
                return $payment;
                break;
            case 5:
                $payment->status_id = Payment::STATUS_PARTIALLY_REFUNDED;
                $payment->save();
                return $payment;
                break;
            case 6:
                $payment->status_id = Payment::STATUS_REFUNDED;
                $payment->save();
                return $payment;
                break;

            default:
                return $payment;
                break;
        }
    }

    private function processDocuments(array $data): void
    {
        // Document::unguard();
        /* No validators since data provided by database is already valid. */

        foreach ($data as $resource) {

            $modified = $resource;

            if (array_key_exists('invoice_id', $resource) && $resource['invoice_id'] && ! array_key_exists('invoices', $this->ids)) {
                return;
                //throw new ResourceDependencyMissing('Processing documents failed, because of missing dependency - invoices.');
            }

            if (array_key_exists('expense_id', $resource) && $resource['expense_id'] && ! array_key_exists('expenses', $this->ids)) {
                return;
                //throw new ResourceDependencyMissing('Processing documents failed, because of missing dependency - expenses.');
            }

            if (array_key_exists('invoice_id', $resource) && $resource['invoice_id'] && array_key_exists('invoices', $this->ids)) {

                $try_quote = false;
                $exception = false;
                $entity = false;
                
                try{
                    $invoice_id = $this->transformId('invoices', $resource['invoice_id']);
                    $entity = Invoice::where('id', $invoice_id)->withTrashed()->first();
                }
                catch(\Exception $e){
                    nlog("i couldn't find the invoice document {$resource['invoice_id']}, perhaps it is a quote?");
                    nlog($e->getMessage());

                    $try_quote = true;
                }

                if($try_quote && array_key_exists('quotes', $this->ids) ) {
                    
                    try{
                        $quote_id = $this->transformId('quotes', $resource['invoice_id']);
                        $entity = Quote::where('id', $quote_id)->withTrashed()->first();
                    }
                    catch(\Exception $e){
                        nlog("i couldn't find the quote document {$resource['invoice_id']}, perhaps it is a quote?");
                        nlog($e->getMessage());
                    }
                }
                
                if(!$entity)
                    throw new Exception("Resource invoice/quote document not available.");


            }


            if (array_key_exists('expense_id', $resource) && $resource['expense_id'] && array_key_exists('expenses', $this->ids)) {
                $expense_id = $this->transformId('expenses', $resource['expense_id']);
                $entity = Expense::where('id', $expense_id)->withTrashed()->first();
            }

            $file_url = $resource['url'];
            $file_name = $resource['name'];
            $file_path = sys_get_temp_dir().'/'.$file_name;

            try {
                file_put_contents($file_path, $this->curlGet($file_url));
                $finfo = new \finfo(FILEINFO_MIME_TYPE);
                $file_info = $finfo->file($file_path);

                $uploaded_file = new UploadedFile(
                                $file_path,
                                $file_name,
                                $file_info,
                                filesize($file_path),
                                0,
                                false
                            );

                $this->saveDocument($uploaded_file, $entity, $is_public = true);
            }
            catch(\Exception $e) {

                //do nothing, gracefully :)
                
            }

        }

    }

    private function processPaymentTerms(array $data) :void
    {
        PaymentTerm::unguard();

        $modified = collect($data)->map(function ($item) {
            $item['user_id'] = $this->user->id;
            $item['company_id'] = $this->company->id;
            $item['is_deleted'] = isset($item['is_deleted']) ? $item['is_deleted'] : 0;
            
            return $item;
        })->toArray();

        PaymentTerm::insert($modified);

        PaymentTerm::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
    }

    private function processCompanyGateways(array $data) :void
    {
        CompanyGateway::unguard();

        $rules = [
            '*.gateway_key' => 'required',
            '*.fees_and_limits' => new ValidCompanyGatewayFeesAndLimitsRule(),
        ];

        $validator = Validator::make($data, $rules);

        if ($validator->fails()) {
            throw new MigrationValidatorFailed(json_encode($validator->errors()));
        }

        foreach ($data as $resource) {
            $modified = $resource;

            $modified['user_id'] = $this->processUserId($resource);
            $modified['company_id'] = $this->company->id;

            unset($modified['id']);

            if (isset($modified['config'])) {
                $modified['config'] = encrypt($modified['config']);
            }

            if (isset($modified['fees_and_limits'])) {
                $modified['fees_and_limits'] = $this->cleanFeesAndLimits($modified['fees_and_limits']);
            }

            if(Ninja::isHosted() && $modified['gateway_key'] == 'd14dd26a37cecc30fdd65700bfb55b23'){
                $modified['gateway_key'] = 'd14dd26a47cecc30fdd65700bfb67b34';
                $modified['fees_and_limits'] = [];
            }

            $company_gateway = CompanyGateway::create($modified);

            $key = "company_gateways_{$resource['id']}";

            $this->ids['company_gateways'][$key] = [
                    'old' => $resource['id'],
                    'new' => $company_gateway->id,
            ];
        }

        CompanyGateway::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
    }

    private function processClientGatewayTokens(array $data) :void
    {
        ClientGatewayToken::unguard();

        foreach ($data as $resource) {
            $modified = $resource;

            unset($modified['id']);

            $modified['company_id'] = $this->company->id;
            $modified['client_id'] = $this->transformId('clients', $resource['client_id']);
            $modified['company_gateway_id'] = $this->transformId('company_gateways', $resource['company_gateway_id']);
            
            //$modified['user_id'] = $this->processUserId($resource);

            $cgt = ClientGatewayToken::Create($modified);

            $key = "client_gateway_tokens_{$resource['id']}";

            $this->ids['client_gateway_tokens'][$key] = [
                'old' => $resource['id'],
                'new' => $cgt->id,
            ];
        }

        ClientGatewayToken::reguard();

        /*Improve memory handling by setting everything to null when we have finished*/
        $data = null;
    }

    private function processTaskStatuses(array $data) :void
    {
        info('in task statuses');
        TaskStatus::unguard();

        foreach ($data as $resource) {
            $modified = $resource;

            unset($modified['id']);

            $modified['company_id'] = $this->company->id;
            $modified['user_id'] = $this->processUserId($resource);

            $task_status = TaskStatus::Create($modified);

            $key = "task_statuses_{$resource['id']}";

            $this->ids['task_statuses'][$key] = [
                'old' => $resource['id'],
                'new' => $task_status->id,
            ];
        }

        TaskStatus::reguard();

        $data = null;
        info('finished task statuses');
    }

    private function processExpenseCategories(array $data) :void
    {
        ExpenseCategory::unguard();

        foreach ($data as $resource) {
            $modified = $resource;

            unset($modified['id']);

            $modified['company_id'] = $this->company->id;
            $modified['user_id'] = $this->processUserId($resource);

            $expense_category = ExpenseCategory::Create($modified);

            $old_user_key = array_key_exists('user_id', $resource) ?? $this->user->id;

            $key = "expense_categories_{$resource['id']}";

            $this->ids['expense_categories'][$key] = [
                'old' => $resource['id'],
                'new' => $expense_category->id,
            ];

            // $this->ids['expense_categories'] = [
            //     "expense_categories_{$old_user_key}" => [
            //         'old' => $resource['id'],
            //         'new' => $expense_category->id,
            //     ],
            // ];
        }
        
        ExpenseCategory::reguard();

        $data = null;
    }

    private function processTasks(array $data) :void
    {
        Task::unguard();

        foreach ($data as $resource) {
            $modified = $resource;

            unset($modified['id']);

            $modified['company_id'] = $this->company->id;
            $modified['user_id'] = $this->processUserId($resource);

            if (isset($modified['client_id'])) {
                $modified['client_id'] = $this->transformId('clients', $resource['client_id']);
            }

            if (isset($modified['invoice_id'])) {
                $modified['invoice_id'] = $this->transformId('invoices', $resource['invoice_id']);
            }
            
            if (isset($modified['project_id'])) {
                $modified['project_id'] = $this->transformId('projects', $resource['project_id']);
            }
            
            if (isset($modified['status_id'])) {
                $modified['status_id'] = $this->transformId('task_statuses', $resource['status_id']);
            }

            $task = Task::Create($modified);

            if(array_key_exists('created_at', $modified))
                $task->created_at = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $task->updated_at = Carbon::parse($modified['updated_at']);



            $task->save(['timestamps' => false]);

            $old_user_key = array_key_exists('user_id', $resource) ?? $this->user->id;

            $this->ids['tasks'] = [
                "tasks_{$old_user_key}" => [
                    'old' => $resource['id'],
                    'new' => $task->id,
                ],
            ];
        }
        
        Task::reguard();

        $data = null;
    }

    private function processProjects(array $data) :void
    {
        Project::unguard();

        foreach ($data as $resource) {
            $modified = $resource;

            unset($modified['id']);

            $modified['company_id'] = $this->company->id;
            $modified['user_id'] = $this->processUserId($resource);

            if (isset($modified['client_id'])) {
                $modified['client_id'] = $this->transformId('clients', $resource['client_id']);
            }

            $project = Project::Create($modified);

            $key = "projects_{$resource['id']}";

            $this->ids['projects'][$key] = [
                'old' => $resource['id'],
                'new' => $project->id,
            ];
        }

        Project::reguard();

        $data = null;
    }

    private function processExpenses(array $data) :void
    {
        Expense::unguard();

        foreach ($data as $resource) {
            $modified = $resource;

            unset($modified['id']);

            $modified['company_id'] = $this->company->id;
            $modified['user_id'] = $this->processUserId($resource);

            if (isset($resource['client_id'])) {
                $modified['client_id'] = $this->transformId('clients', $resource['client_id']);
            }
            
            if (isset($resource['category_id'])) {
                $modified['category_id'] = $this->transformId('expense_categories', $resource['category_id']);
            }
            
            if (isset($resource['invoice_id'])) {
                $modified['invoice_id'] = $this->transformId('invoices', $resource['invoice_id']);
            }

            if (isset($resource['project_id'])) {
                $modified['project_id'] = $this->transformId('projects', $resource['project_id']);
            }

            if (isset($resource['vendor_id'])) {
                $modified['vendor_id'] = $this->transformId('vendors', $resource['vendor_id']);
            }

            $expense = Expense::Create($modified);

            if(array_key_exists('created_at', $modified))
                $expense->created_at = Carbon::parse($modified['created_at']);

            if(array_key_exists('updated_at', $modified))
                $expense->updated_at = Carbon::parse($modified['updated_at']);



            $expense->save(['timestamps' => false]);
            
            $old_user_key = array_key_exists('user_id', $resource) ?? $this->user->id;

            $key = "expenses_{$resource['id']}";

            $this->ids['expenses'][$key] = [
                'old' => $resource['id'],
                'new' => $expense->id,
            ];

        }

        Expense::reguard();

        $data = null;
    }
    /**
     * |--------------------------------------------------------------------------
     * | Additional migration methods.
     * |--------------------------------------------------------------------------
     * |
     * | These methods aren't initialized automatically, so they don't depend on
     * | the migration data.
     */

    /**
     * Cloned from App\Http\Requests\User\StoreUserRequest.
     *
     * @param string $data
     * @return User
     */
    public function fetchUser(string $data): User
    {
        $user = MultiDB::hasUser(['email' => $data]);

        if (! $user) {
            $user = UserFactory::create($this->company->account->id);
        }
        
        return $user;
    }

    /**
     * @param string $resource
     * @param string $old
     * @return int
     * @throws Exception
     */
    public function transformId($resource, string $old): int
    {

        if (! array_key_exists($resource, $this->ids)) {
            info(print_r($resource, 1));
            throw new Exception("Resource {$resource} not available.");
        }

        if (! array_key_exists("{$resource}_{$old}", $this->ids[$resource])) {
            throw new Exception("Missing resource key: {$resource}_{$old}");
        }

        return $this->ids[$resource]["{$resource}_{$old}"]['new'];
    }

    private function tryTransformingId($resource, string $old): ?int
    {
        if (! array_key_exists($resource, $this->ids)) {
            return false;
        }

        if (! array_key_exists("{$resource}_{$old}", $this->ids[$resource])) {
            return false;
        }

        return $this->ids[$resource]["{$resource}_{$old}"]['new'];
    }

    /**
     * Process & handle user_id.
     *
     * @param array $resource
     * @return int|mixed
     * @throws Exception
     */
    public function processUserId(array $resource)
    {
        if (! array_key_exists('user_id', $resource)) {
            return $this->user->id;
        }

        if (array_key_exists('user_id', $resource) && ! array_key_exists('users', $this->ids)) {
            return $this->user->id;
        }

        return $this->transformId('users', $resource['user_id']);
    }

    public function failed($exception = null)
    {
        info('the job failed');

        $job_failure = new MigrationFailure();
        $job_failure->string_metric5 = get_class($this);
        $job_failure->string_metric6 = $exception->getMessage();

        LightLogs::create($job_failure)
                 ->batch();

        info(print_r($exception->getMessage(), 1));

        if(Ninja::isHosted())
            app('sentry')->captureException($exception);
    }


    public function curlGet($url, $headers = false)
    {

        return $this->exec('GET', $url, null);
    }

    public function exec($method, $url, $data)
    {

        $client =  new \GuzzleHttp\Client(['headers' => 
            [ 
            'X-Ninja-Token' => $this->token,        
            ]
        ]);

        $response = $client->request('GET', $url);

        return $response->getBody();
    }



    private function processNinjaTokens(array $data)
    {
        nlog("attempting to process Ninja Tokens");

        if(Ninja::isHosted())
            \Modules\Admin\Jobs\Account\NinjaUser::dispatchNow($data, $this->company);

    }


    /* In V4 we use negative invoices (credits) and add then into the client balance. In V5, these sit off ledger and are applied later.
     This next section will check for credit balances and reduce the client balance so that the V5 balances are correct
    */
    // private function fixClientBalances()
    // {
       
    //     Client::cursor()->each(function ($client) {

    //         $credit_balance = $client->credits->where('is_deleted', false)->sum('balance');

    //         if($credit_balance > 0){
    //             $client->balance += $credit_balance;
    //             $client->save();
    //         }

    //     });

    // }
}