1
0
mirror of https://github.com/invoiceninja/invoiceninja.git synced 2024-11-10 05:02:36 +01:00

Working on csv import refactor

This commit is contained in:
David Bomba 2022-02-01 17:14:27 +11:00
parent c432ee2693
commit 03d43470fb
14 changed files with 888 additions and 3 deletions

View File

@ -0,0 +1,166 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Providers;
use App\Import\ImportException;
use App\Models\Company;
use App\Models\User;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Validator;
use Illuminate\Support\Str;
use League\Csv\Reader;
use League\Csv\Statement;
use Symfony\Component\HttpFoundation\ParameterBag;
class BaseImport {
public Company $company;
public array $request;
public array $error_array = [];
public $request_name;
public $repository_name;
public $factory_name;
public $repository;
public $transformer;
public function __construct( array $request, Company $company ) {
$this->company = $company;
$this->request = $request;
$this->hash = $request['hash'];
$this->import_type = $request['import_type'];
$this->skip_header = $request['skip_header'] ?? null;
$this->column_map =
! empty( $request['column_map'] ) ?
array_combine( array_keys( $request['column_map'] ), array_column( $request['column_map'], 'mapping' ) ) : null;
auth()->login( $this->company->owner(), true );
auth()->user()->setCompany($this->company);
}
protected function findUser( $user_hash ) {
$user = User::where( 'account_id', $this->company->account_id )
->where( DB::raw( 'CONCAT_WS(" ", first_name, last_name)' ), 'like', '%' . $user_hash . '%' )
->first();
if ( $user ) {
return $user->id;
} else {
return $this->company->owner()->id;
}
}
protected function getCsvData( $entity_type ) {
$base64_encoded_csv = Cache::pull( $this->hash . '-' . $entity_type );
if ( empty( $base64_encoded_csv ) ) {
return null;
}
$csv = base64_decode( $base64_encoded_csv );
$csv = Reader::createFromString( $csv );
$stmt = new Statement();
$data = iterator_to_array( $stmt->process( $csv ) );
if ( count( $data ) > 0 ) {
$headers = $data[0];
// Remove Invoice Ninja headers
if ( count( $headers ) && count( $data ) > 4 && $this->import_type === 'csv' ) {
$first_cell = $headers[0];
if ( strstr( $first_cell, config( 'ninja.app_name' ) ) ) {
array_shift( $data ); // Invoice Ninja...
array_shift( $data ); // <blank line>
array_shift( $data ); // Enitty Type Header
}
}
}
return $data;
}
public function mapCSVHeaderToKeys( $csvData ) {
$keys = array_shift( $csvData );
return array_map( function ( $values ) use ( $keys ) {
return array_combine( $keys, $values );
}, $csvData );
}
private function groupInvoices( $csvData, $key ) {
// Group by invoice.
$grouped = [];
foreach ( $csvData as $line_item ) {
if ( empty( $line_item[ $key ] ) ) {
$this->error_array['invoice'][] = [ 'invoice' => $line_item, 'error' => 'No invoice number' ];
} else {
$grouped[ $line_item[ $key ] ][] = $line_item;
}
}
return $grouped;
}
public function getErrors()
{
return $this->error_array;
}
public function ingest($data, $entity_type)
{
foreach ( $data as $record ) {
try {
$entity = $this->transformer->transform( $record );
/** @var \App\Http\Requests\Request $request */
$request = new $this->request_name();
// Pass entity data to request so it can be validated
$request->query = $request->request = new ParameterBag( $entity );
$validator = Validator::make( $entity, $request->rules() );
if ( $validator->fails() ) {
$this->error_array[ $entity_type ][] =
[ $entity_type => $record, 'error' => $validator->errors()->all() ];
} else {
$entity =
$this->repository->save(
array_diff_key( $entity, [ 'user_id' => false ] ),
$this->factory_name::create( $this->company->id, $this->getUserIDForRecord( $entity ) ) );
$entity->saveQuietly();
}
} catch ( \Exception $ex ) {
if ( $ex instanceof ImportException ) {
$message = $ex->getMessage();
} else {
report( $ex );
$message = 'Unknown error';
}
$this->error_array[ $entity_type ][] = [ $entity_type => $record, 'error' => $message ];
}
}
}
}

View File

@ -0,0 +1,94 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Providers;
use App\Factory\ClientFactory;
use App\Http\Requests\Client\StoreClientRequest;
use App\Import\ImportException;
use App\Import\Providers\BaseImport;
use App\Import\Providers\ImportInterface;
use App\Import\Transformer\Csv\ClientTransformer;
use App\Repositories\ClientRepository;
use Illuminate\Support\Facades\Validator;
use Symfony\Component\HttpFoundation\ParameterBag;
class Csv extends BaseImport implements ImportInterface
{
public function import(string $entity)
{
if(in_array($entity, [ 'client', 'product', 'invoice', 'payment', 'vendor', 'expense' ]))
$this->{$entity};
}
private function client()
{
$entity_type = 'client';
$data = $this->getCsvData($entity_type);
$data = $this->preTransform($data);
if(empty($data))
return;
$this->request_name = StoreClientRequest::class;
$this->repository_name = ClientRepository::class;
$this->factory_name = ClientFactory::class;
$this->repository = app()->make( $this->repository_name );
$this->repository->import_mode = true;
$this->transformer = new ClientTransformer($this->company);
$this->ingest($data, $entity_type);
}
public function preTransform(array $data)
{
if ( empty( $this->column_map[ 'client' ] ) ) {
return false;
}
if ( $this->skip_header ) {
array_shift( $data );
}
//sort the array by key
$keys = $this->column_map[ 'client' ];
ksort( $keys );
$data = array_map( function ( $row ) use ( $keys ) {
return array_combine( $keys, array_intersect_key( $row, $keys ) );
}, $data );
return $data;
}
public function transform(array $data)
{
}
}

View File

@ -0,0 +1,16 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Providers;
class Freshbooks extends BaseImport
{
}

View File

@ -0,0 +1,21 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Providers;
interface ImportInterface
{
public function import(string $entity);
public function preTransform(array $data);
public function transform(array $data);
}

View File

@ -0,0 +1,16 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Providers;
class Invoice2Go extends BaseImport
{
}

View File

@ -0,0 +1,16 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Providers;
class Invoicely extends BaseImport
{
}

View File

@ -0,0 +1,16 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Providers;
class Wave extends BaseImport
{
}

View File

@ -0,0 +1,16 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Providers;
class Zoho extends BaseImport
{
}

View File

@ -0,0 +1,344 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Transformer;
use App\Models\ClientContact;
use App\Models\Country;
use App\Models\PaymentType;
use App\Utils\Number;
use Exception;
use Illuminate\Support\Carbon;
/**
* Class BaseTransformer.
*/
class BaseTransformer
{
protected $company;
public function __construct($company)
{
$this->company = $company;
}
public function getString($data, $field)
{
return (isset($data[$field]) && $data[$field]) ? $data[$field] : '';
}
public function getCurrencyByCode( $data, $key = 'client.currency_id' )
{
$code = array_key_exists( $key, $data ) ? $data[ $key ] : false;
return $this->maps['currencies'][ $code ] ?? $this->company->settings->currency_id;
}
public function getClient($client_name, $client_email) {
$clients = $this->company->clients();
$client_id_search = $clients->where( 'id_number', $client_name );
if ( $client_id_search->count() >= 1 ) {
return $client_id_search->first()->id;
nlog("found via id number");
}
$client_name_search = $clients->where( 'name', $client_name );
if ( $client_name_search->count() >= 1 ) {
return $client_name_search->first()->id;
nlog("found via name");
}
if ( ! empty( $client_email ) ) {
$contacts = ClientContact::where( 'company_id', $this->company->id )
->where( 'email', $client_email );
if ( $contacts->count() >= 1 ) {
return $contacts->first()->client_id;
nlog("found via contact");
}
}
nlog("did not find client");
return null;
}
///////////////////////////////////////////////////////////////////////////////////
/**
* @param $name
*
* @return bool
*/
public function hasClient($name)
{
return $this->company->clients()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->exists();
}
/**
* @param $name
*
* @return bool
*/
public function hasVendor($name)
{
return $this->company->vendors()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->exists();
}
/**
* @param $key
*
* @return bool
*/
public function hasProduct($key)
{
return $this->company->products()->whereRaw("LOWER(REPLACE(`product_key`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $key))])->exists();
}
/**
* @param $data
* @param $field
*
* @return float
*/
public function getFloat($data, $field)
{
if (array_key_exists($field, $data)) {
$number = preg_replace('/[^0-9-.]+/', '', $data[$field]);
} else {
$number = 0;
}
return Number::parseFloat($number);
}
/**
* @param $name
*
* @return int|null
*/
public function getClientId($name)
{
$client = $this->company->clients()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first();
return $client ? $client->id : null;
}
/**
* @param $name
*
* @return string
*/
public function getProduct($data, $key, $field, $default = false)
{
$product = $this->company->products()->whereRaw("LOWER(REPLACE(`product_key`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $data->{$key}))])->first();
if($product)
return $product->{$field} ?: $default;
return $default;
}
/**
* @param $email
*
* @return ?Contact
*/
public function getContact($email)
{
$contact = $this->company->client_contacts()->whereRaw("LOWER(REPLACE(`email`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $email))])->first();
if(!$contact)
return null;
return $contact;
}
/**
* @param $name
*
* @return int|null
*/
public function getCountryId($name)
{
if(strlen($name) == 2)
return $this->getCountryIdBy2($name);
$country = Country::whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first();
return $country ? $country->id : null;
}
/**
* @param $name
*
* @return int|null
*/
public function getCountryIdBy2($name)
{
return Country::where('iso_3166_2', $name)->exists() ? Country::where('iso_3166_2', $name)->first()->id : null;
}
/**
* @param $name
*
* @return int
*/
public function getTaxRate($name)
{
$name = strtolower(trim($name));
$tax_rate = $this->company->tax_rates()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first();
return $tax_rate ? $tax_rate->rate : 0;
}
/**
* @param $name
*
* @return string
*/
public function getTaxName($name)
{
$name = strtolower(trim($name));
$tax_rate = $this->company->tax_rates()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first();
return $tax_rate ? $tax_rate->name : '';
}
/**
* @param $date
* @param string $format
* @param mixed $data
* @param mixed $field
*
* @return null
*/
public function getDate($data, $field)
{
if ($date = data_get($data, $field)) {
try {
$date = new Carbon($date);
} catch (\Exception $e) {
// if we fail to parse return blank
$date = false;
}
}
return $date ? $date->format('Y-m-d') : null;
}
/**
* @param $number
*
* @return ?string
*/
public function getInvoiceNumber($number)
{
return $number ? ltrim( trim( $number ), '0' ) : null;
}
/**
* @param $invoice_number
*
* @return int|null
*/
public function getInvoiceId($invoice_number)
{
$invoice = $this->company->invoices()->whereRaw("LOWER(REPLACE(`number`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $invoice_number))])->first();
return $invoice ? $invoice->id : null;
}
/**
* @param $invoice_number
*
* @return bool
*/
public function hasInvoice($invoice_number)
{
return $this->company->invoices()->whereRaw("LOWER(REPLACE(`number`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $invoice_number))])->exists();
}
/**
* @param $invoice_number
*
* @return int|null
*/
public function getInvoiceClientId($invoice_number)
{
$invoice = $this->company->invoices()->whereRaw("LOWER(REPLACE(`number`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $invoice_number))])->first();
return $invoice ? $invoice->client_id : null;
}
/**
* @param $name
*
* @return int|null
*/
public function getVendorId($name)
{
$vendor = $this->company->vendors()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first();
return $vendor ? $vendor->id : null;
}
/**
* @param $name
*
* @return int|null
*/
public function getExpenseCategoryId( $name ) {
$ec = $this->company->expense_categories()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first();
return $ec ? $ec->id : null;
}
/**
* @param $name
*
* @return int|null
*/
public function getProjectId( $name ) {
$project = $this->company->projects()->whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first();
return $project ? $project->id : null;
}
/**
* @param $name
*
* @return int|null
*/
public function getPaymentTypeId( $name ) {
$pt = PaymentType::whereRaw("LOWER(REPLACE(`name`, ' ' ,'')) = ?", [strtolower(str_replace(' ', '', $name))])->first();
return $pt ? $pt->id : null;
}
}

View File

@ -0,0 +1,80 @@
<?php
/**
* client Ninja (https://clientninja.com).
*
* @link https://github.com/clientninja/clientninja source repository
*
* @copyright Copyright (c) 2021. client Ninja LLC (https://clientninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Import\Transformer\Csv;
use App\Import\ImportException;
use App\Import\Transformer\BaseTransformer;
use Illuminate\Support\Str;
/**
* Class ClientTransformer.
*/
class ClientTransformer extends BaseTransformer
{
/**
* @param $data
*
* @return array|bool
*/
public function transform($data)
{
if (isset($data->name) && $this->hasClient($data->name)) {
throw new ImportException('Client already exists');
}
$settings = new \stdClass;
$settings->currency_id = (string)$this->getCurrencyByCode($data);
return [
'company_id' => $this->company->id,
'name' => $this->getString( $data, 'client.name' ),
'work_phone' => $this->getString( $data, 'client.phone' ),
'address1' => $this->getString( $data, 'client.address1' ),
'address2' => $this->getString( $data, 'client.address2' ),
'postal_code' => $this->getString( $data, 'client.postal_code'),
'city' => $this->getString( $data, 'client.city' ),
'state' => $this->getString( $data, 'client.state' ),
'shipping_address1' => $this->getString( $data, 'client.shipping_address1' ),
'shipping_address2' => $this->getString( $data, 'client.shipping_address2' ),
'shipping_city' => $this->getString( $data, 'client.shipping_city' ),
'shipping_state' => $this->getString( $data, 'client.shipping_state' ),
'shipping_postal_code' => $this->getString( $data, 'client.shipping_postal_code' ),
'public_notes' => $this->getString( $data, 'client.public_notes' ),
'private_notes' => $this->getString( $data, 'client.private_notes' ),
'website' => $this->getString( $data, 'client.website' ),
'vat_number' => $this->getString( $data, 'client.vat_number' ),
'id_number' => $this->getString( $data, 'client.id_number' ),
'custom_value1' => $this->getString( $data, 'client.custom_value1' ),
'custom_value2' => $this->getString( $data, 'client.custom_value2' ),
'custom_value3' => $this->getString( $data, 'client.custom_value3' ),
'custom_value4' => $this->getString( $data, 'client.custom_value4' ),
'balance' => preg_replace( '/[^0-9,.]+/', '', $this->getFloat( $data, 'client.balance' ) ),
'paid_to_date' => preg_replace( '/[^0-9,.]+/', '', $this->getFloat( $data, 'client.paid_to_date' ) ),
'credit_balance' => 0,
'settings' => $settings,
'client_hash' => Str::random( 40 ),
'contacts' => [
[
'first_name' => $this->getString( $data, 'contact.first_name' ),
'last_name' => $this->getString( $data, 'contact.last_name' ),
'email' => $this->getString( $data, 'contact.email' ),
'phone' => $this->getString( $data, 'contact.phone' ),
'custom_value1' => $this->getString( $data, 'contact.custom_value1' ),
'custom_value2' => $this->getString( $data, 'contact.custom_value2' ),
'custom_value3' => $this->getString( $data, 'contact.custom_value3' ),
'custom_value4' => $this->getString( $data, 'contact.custom_value4' ),
],
],
'country_id' => isset( $data['client.country'] ) ? $this->getCountryId( $data['client.country']) : null,
'shipping_country_id' => isset($data['client.shipping_country'] ) ? $this->getCountryId( $data['client.shipping_country'] ) : null,
];
}
}

View File

@ -0,0 +1,99 @@
<?php
/**
* Invoice Ninja (https://invoiceninja.com).
*
* @link https://github.com/invoiceninja/invoiceninja source repository
*
* @copyright Copyright (c) 2021. Invoice Ninja LLC (https://invoiceninja.com)
*
* @license https://www.elastic.co/licensing/elastic-license
*/
namespace App\Jobs\Import;
use App\Import\Providers\Csv;
use App\Import\Providers\Freshbooks;
use App\Import\Providers\Invoice2Go;
use App\Import\Providers\Invoicely;
use App\Import\Providers\Wave;
use App\Import\Providers\Zoho;
use App\Libraries\MultiDB;
use App\Models\Company;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class CSVIngest implements ShouldQueue {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public Company $company;
public string $hash;
public string $import_type;
public ?string $skip_header;
public array $column_map;
public function __construct( array $request, Company $company ) {
$this->company = $company;
$this->request = $request;
$this->hash = $request['hash'];
$this->import_type = $request['import_type'];
$this->skip_header = $request['skip_header'] ?? null;
$this->column_map =
! empty( $request['column_map'] ) ?
array_combine( array_keys( $request['column_map'] ), array_column( $request['column_map'], 'mapping' ) ) : null;
}
/**
* Execute the job.
*
*
* @return void
*/
public function handle() {
MultiDB::setDb( $this->company->db );
$engine = $this->bootEngine($this->import_type);
foreach ( [ 'client', 'product', 'invoice', 'payment', 'vendor', 'expense' ] as $entity ) {
$engine->import($entity);
}
}
private function bootEngine(string $import_type)
{
switch ($import_type) {
case 'csv':
return new Csv( $this->request, $this->company);
break;
case 'waveaccounting':
return new Wave( $this->request, $this->company);
break;
case 'invoicely':
return new Invoicely( $this->request, $this->company);
break;
case 'invoice2go':
return new Invoice2Go( $this->request, $this->company);
break;
case 'zoho':
return new Zoho( $this->request, $this->company);
break;
case 'freshbooks':
return new Freshbooks( $this->request, $this->company);
break;
default:
// code...
break;
}
}
}

View File

@ -119,7 +119,8 @@ class PaymentIntentWebhook implements ShouldQueue
$payment_hash = PaymentHash::where('hash', $hash)->first();
nlog("no payment found");
if(!$payment_hash)
return;
if(optional($this->stripe_request['object']['charges']['data'][0]['metadata']['payment_hash']) && in_array('card', $this->stripe_request['object']['allowed_source_types']))
{

View File

@ -91,7 +91,7 @@ class SubscriptionService
'invoice' => $this->encodePrimaryKey($payment_hash->fee_invoice_id),
'client' => $recurring_invoice->client->hashed_id,
'subscription' => $this->subscription->hashed_id,
'contact' => auth('contact')->user()->hashed_id,
'contact' => auth('contact')->user() ? auth('contact')->user()->hashed_id : $recurring_invoice->client->contacts()->first()->hashed_id,
'account_key' => $recurring_invoice->client->custom_value2,
];

View File

@ -212,7 +212,7 @@ Route::group(['middleware' => ['throttle:300,1', 'api_db', 'token_auth', 'locale
Route::resource('subscriptions', 'SubscriptionController');
Route::post('subscriptions/bulk', 'SubscriptionController@bulk')->name('subscriptions.bulk');
Route::get('statics', 'StaticController');
Route::post('apple_pay/upload_file','ApplyPayController@upload');
// Route::post('apple_pay/upload_file','ApplyPayController@upload');
});