fractal = $manager; $this->fractal->setSerializer(new ArraySerializer()); $this->clientRepo = $clientRepo; $this->customerRepo = $customerRepo; $this->invoiceRepo = $invoiceRepo; $this->paymentRepo = $paymentRepo; $this->contactRepo = $contactRepo; $this->productRepo = $productRepo; $this->expenseRepo = $expenseRepo; $this->vendorRepo = $vendorRepo; $this->expenseCategoryRepo = $expenseCategoryRepo; $this->taxRateRepository = $taxRateRepository; } /** * @param $file * * @throws Exception * * @return array */ public function importJSON($fileName, $includeData, $includeSettings) { $this->initMaps(); $this->checkForFile($fileName); $file = file_get_contents($fileName); $json = json_decode($file, true); $json = $this->removeIdFields($json); $transformer = new BaseTransformer($this->maps); $this->checkClientCount(count($json['clients'])); if ($includeSettings) { // remove blank id values $settings = []; foreach ($json as $field => $value) { if (strstr($field, '_id') && ! $value) { // continue; } else { $settings[$field] = $value; } } $account = Auth::user()->account; $account->fill($settings); $account->save(); $emailSettings = $account->account_email_settings; $emailSettings->fill(isset($settings['account_email_settings']) ? $settings['account_email_settings'] : $settings); $emailSettings->save(); } if ($includeData) { foreach ($json['products'] as $jsonProduct) { if ($transformer->hasProduct($jsonProduct['product_key'])) { continue; } $productValidate = EntityModel::validate($jsonProduct, ENTITY_PRODUCT); if ($productValidate === true) { $product = $this->productRepo->save($jsonProduct); $this->addProductToMaps($product); $this->addSuccess($product); } else { $jsonProduct['type'] = ENTITY_PRODUCT; $jsonProduct['error'] = $productValidate; $this->addFailure(ENTITY_PRODUCT, $jsonProduct); continue; } } foreach ($json['clients'] as $jsonClient) { $clientValidate = EntityModel::validate($jsonClient, ENTITY_CLIENT); if ($clientValidate === true) { $client = $this->clientRepo->save($jsonClient); $this->addClientToMaps($client); $this->addSuccess($client); } else { $jsonClient['type'] = ENTITY_CLIENT; $jsonClient['error'] = $clientValidate; $this->addFailure(ENTITY_CLIENT, $jsonClient); continue; } foreach ($jsonClient['invoices'] as $jsonInvoice) { $jsonInvoice['client_id'] = $client->id; $invoiceValidate = EntityModel::validate($jsonInvoice, ENTITY_INVOICE); if ($invoiceValidate === true) { $invoice = $this->invoiceRepo->save($jsonInvoice); $this->addInvoiceToMaps($invoice); $this->addSuccess($invoice); } else { $jsonInvoice['type'] = ENTITY_INVOICE; $jsonInvoice['error'] = $invoiceValidate; $this->addFailure(ENTITY_INVOICE, $jsonInvoice); continue; } foreach ($jsonInvoice['payments'] as $jsonPayment) { $jsonPayment['invoice_id'] = $invoice->public_id; $paymentValidate = EntityModel::validate($jsonPayment, ENTITY_PAYMENT); if ($paymentValidate === true) { $jsonPayment['client_id'] = $client->id; $jsonPayment['invoice_id'] = $invoice->id; $payment = $this->paymentRepo->save($jsonPayment); $this->addSuccess($payment); } else { $jsonPayment['type'] = ENTITY_PAYMENT; $jsonPayment['error'] = $paymentValidate; $this->addFailure(ENTITY_PAYMENT, $jsonPayment); continue; } } } } } File::delete($fileName); return $this->results; } /** * @param $array * * @return mixed */ public function removeIdFields($array) { foreach ($array as $key => $val) { if (is_array($val)) { $array[$key] = $this->removeIdFields($val); } elseif ($key === 'id') { unset($array[$key]); } } return $array; } /** * @param $source * @param $files * * @return array */ public function importFiles($source, $files) { $results = []; $imported_files = null; $this->initMaps(); foreach ($files as $entityType => $file) { $results[$entityType] = $this->execute($source, $entityType, $file); } return $results; } /** * @param $source * @param $entityType * @param $file * * @return array */ private function execute($source, $entityType, $fileName) { $results = [ RESULT_SUCCESS => [], RESULT_FAILURE => [], ]; // Convert the data $row_list = []; $this->checkForFile($fileName); Excel::load($fileName, function ($reader) use ($source, $entityType, &$row_list, &$results) { $this->checkData($entityType, count($reader->all())); $reader->each(function ($row) use ($source, $entityType, &$row_list, &$results) { if ($this->isRowEmpty($row)) { return; } $data_index = $this->transformRow($source, $entityType, $row); if ($data_index !== false) { if ($data_index !== true) { // Wasn't merged with another row $row_list[] = ['row' => $row, 'data_index' => $data_index]; } } else { $results[RESULT_FAILURE][] = $row; } }); }); // Save the data foreach ($row_list as $row_data) { $result = $this->saveData($source, $entityType, $row_data['row'], $row_data['data_index']); if ($result) { $results[RESULT_SUCCESS][] = $result; } else { $results[RESULT_FAILURE][] = $row_data['row']; } } File::delete($fileName); return $results; } /** * @param $source * @param $entityType * @param $row * * @return bool|mixed */ private function transformRow($source, $entityType, $row) { $transformer = $this->getTransformer($source, $entityType, $this->maps); $resource = $transformer->transform($row); if (! $resource) { return false; } $data = $this->fractal->createData($resource)->toArray(); // Create expesnse category if ($entityType == ENTITY_EXPENSE) { if (! empty($row->expense_category)) { $categoryId = $transformer->getExpenseCategoryId($row->expense_category); if (! $categoryId) { $category = $this->expenseCategoryRepo->save(['name' => $row->expense_category]); $this->addExpenseCategoryToMaps($category); $data['expense_category_id'] = $category->id; } } if (! empty($row->vendor) && ($vendorName = trim($row->vendor))) { if (! $transformer->getVendorId($vendorName)) { $vendor = $this->vendorRepo->save(['name' => $vendorName, 'vendor_contact' => []]); $this->addVendorToMaps($vendor); $data['vendor_id'] = $vendor->id; } } } /* // if the invoice number is blank we'll assign it if ($entityType == ENTITY_INVOICE && ! $data['invoice_number']) { $account = Auth::user()->account; $invoice = Invoice::createNew(); $data['invoice_number'] = $account->getNextNumber($invoice); } */ if (EntityModel::validate($data, $entityType) !== true) { return false; } if ($entityType == ENTITY_INVOICE) { if (empty($this->processedRows[$data['invoice_number']])) { $this->processedRows[$data['invoice_number']] = $data; } else { // Merge invoice items $this->processedRows[$data['invoice_number']]['invoice_items'] = array_merge($this->processedRows[$data['invoice_number']]['invoice_items'], $data['invoice_items']); return true; } } else { $this->processedRows[] = $data; } end($this->processedRows); return key($this->processedRows); } /** * @param $source * @param $entityType * @param $row * @param $data_index * * @return mixed */ private function saveData($source, $entityType, $row, $data_index) { $data = $this->processedRows[$data_index]; if ($entityType == ENTITY_INVOICE) { $data['is_public'] = true; } $entity = $this->{"{$entityType}Repo"}->save($data); // update the entity maps if ($entityType != ENTITY_CUSTOMER) { $mapFunction = 'add' . ucwords($entity->getEntityType()) . 'ToMaps'; if (method_exists($this, $mapFunction)) { $this->$mapFunction($entity); } } // if the invoice is paid we'll also create a payment record if ($entityType === ENTITY_INVOICE && isset($data['paid']) && $data['paid'] > 0) { $this->createPayment($source, $row, $data['client_id'], $entity->id, $entity->public_id); } return $entity; } /** * @param $entityType * @param $count * * @throws Exception */ private function checkData($entityType, $count) { if (Utils::isNinja() && $count > MAX_IMPORT_ROWS) { throw new Exception(trans('texts.limit_import_rows', ['count' => MAX_IMPORT_ROWS])); } if ($entityType === ENTITY_CLIENT) { $this->checkClientCount($count); } } /** * @param $count * * @throws Exception */ private function checkClientCount($count) { $totalClients = $count + Client::scope()->withTrashed()->count(); if ($totalClients > Auth::user()->getMaxNumClients()) { throw new Exception(trans('texts.limit_clients', ['count' => Auth::user()->getMaxNumClients()])); } } /** * @param $source * @param $entityType * * @return string */ public static function getTransformerClassName($source, $entityType) { return 'App\\Ninja\\Import\\'.$source.'\\'.ucwords($entityType).'Transformer'; } /** * @param $source * @param $entityType * @param $maps * * @return mixed */ public static function getTransformer($source, $entityType, $maps) { $className = self::getTransformerClassName($source, $entityType); return new $className($maps); } /** * @param $source * @param $data * @param $clientId * @param $invoiceId */ private function createPayment($source, $row, $clientId, $invoiceId, $invoicePublicId) { $paymentTransformer = $this->getTransformer($source, ENTITY_PAYMENT, $this->maps); $row->client_id = $clientId; $row->invoice_id = $invoiceId; if ($resource = $paymentTransformer->transform($row)) { $data = $this->fractal->createData($resource)->toArray(); $data['invoice_id'] = $invoicePublicId; if (Payment::validate($data) === true) { $data['invoice_id'] = $invoiceId; $this->paymentRepo->save($data); } } } /** * @param array $files * * @throws Exception * * @return array */ public function mapCSV(array $files) { $data = []; foreach ($files as $entityType => $filename) { $class = 'App\\Models\\' . ucwords($entityType); $columns = $class::getImportColumns(); $map = $class::getImportMap(); // Lookup field translations foreach ($columns as $key => $value) { unset($columns[$key]); $label = $value; // disambiguate some of the labels if ($entityType == ENTITY_INVOICE) { if ($label == 'name') { $label = 'client_name'; } elseif ($label == 'notes') { $label = 'product_notes'; } elseif ($label == 'terms') { $label = 'invoice_terms'; } } $columns[$value] = trans("texts.{$label}"); } array_unshift($columns, ' '); $data[$entityType] = $this->mapFile($entityType, $filename, $columns, $map); if ($entityType === ENTITY_CLIENT) { if (count($data[$entityType]['data']) + Client::scope()->count() > Auth::user()->getMaxNumClients()) { throw new Exception(trans('texts.limit_clients', ['count' => Auth::user()->getMaxNumClients()])); } } } return $data; } /** * @param $entityType * @param $filename * @param $columns * @param $map * * @return array */ public function mapFile($entityType, $filename, $columns, $map) { $data = $this->getCsvData($filename); $headers = false; $hasHeaders = false; $mapped = []; if (count($data) > 0) { $headers = $data[0]; foreach ($headers as $title) { if (strpos(strtolower($title), 'name') > 0) { $hasHeaders = true; break; } } for ($i = 0; $i < count($headers); $i++) { $title = strtolower($headers[$i]); $mapped[$i] = ''; foreach ($map as $search => $column) { if ($this->checkForMatch($title, $search)) { $hasHeaders = true; $mapped[$i] = $column; break; } } } } $data = [ 'entityType' => $entityType, 'data' => $data, 'headers' => $headers, 'hasHeaders' => $hasHeaders, 'columns' => $columns, 'mapped' => $mapped, 'warning' => false, ]; // check that dates are valid if (count($data['data']) > 1) { $row = $data['data'][1]; foreach ($mapped as $index => $field) { if (! strstr($field, 'date')) { continue; } try { $date = new Carbon($row[$index]); } catch(Exception $e) { $data['warning'] = 'invalid_date'; } } } return $data; } private function getCsvData($fileName) { $this->checkForFile($fileName); $file = file_get_contents($fileName); $data = array_map("str_getcsv", preg_split('/\r*\n+|\r+/', $file)); if (count($data) > 0) { $headers = $data[0]; // Remove Invoice Ninja headers if (count($headers) && count($data) > 4) { $firstCell = $headers[0]; if (strstr($firstCell, APP_NAME)) { array_shift($data); // Invoice Ninja... array_shift($data); // array_shift($data); // Enitty Type Header } } } return $data; } /** * @param $column * @param $pattern * * @return bool */ private function checkForMatch($column, $pattern) { if (strpos($column, 'sec') === 0) { return false; } if (strpos($pattern, '^')) { list($include, $exclude) = explode('^', $pattern); $includes = explode('|', $include); $excludes = explode('|', $exclude); } else { $includes = explode('|', $pattern); $excludes = []; } foreach ($includes as $string) { if (strpos($column, $string) !== false) { $excluded = false; foreach ($excludes as $exclude) { if (strpos($column, $exclude) !== false) { $excluded = true; break; } } if (! $excluded) { return true; } } } return false; } /** * @param array $maps * @param $headers * * @return array */ public function importCSV(array $maps, $headers, $timestamp) { $results = []; foreach ($maps as $entityType => $map) { $results[$entityType] = $this->executeCSV($entityType, $map, $headers[$entityType], $timestamp); } return $results; } /** * @param $entityType * @param $map * @param $hasHeaders * * @return array */ private function executeCSV($entityType, $map, $hasHeaders, $timestamp) { $results = [ RESULT_SUCCESS => [], RESULT_FAILURE => [], ]; $source = IMPORT_CSV; $path = env('FILE_IMPORT_PATH') ?: storage_path() . '/import'; $fileName = sprintf('%s/%s_%s_%s.csv', $path, Auth::user()->account_id, $timestamp, $entityType); $data = $this->getCsvData($fileName); $this->checkData($entityType, count($data)); $this->initMaps(); // Convert the data $row_list = []; foreach ($data as $row) { if ($hasHeaders) { $hasHeaders = false; continue; } $row = $this->convertToObject($entityType, $row, $map); if ($this->isRowEmpty($row)) { continue; } $data_index = $this->transformRow($source, $entityType, $row); if ($data_index !== false) { if ($data_index !== true) { // Wasn't merged with another row $row_list[] = ['row' => $row, 'data_index' => $data_index]; } } else { $results[RESULT_FAILURE][] = $row; } } // Save the data foreach ($row_list as $row_data) { $result = $this->saveData($source, $entityType, $row_data['row'], $row_data['data_index']); if ($result) { $results[RESULT_SUCCESS][] = $result; } else { $results[RESULT_FAILURE][] = $row; } } File::delete($fileName); return $results; } /** * @param $entityType * @param $data * @param $map * * @return stdClass */ private function convertToObject($entityType, $data, $map) { $obj = new stdClass(); $class = 'App\\Models\\' . ucwords($entityType); $columns = $class::getImportColumns(); foreach ($columns as $column) { $obj->$column = false; } foreach ($map as $index => $field) { if (! $field) { continue; } if (isset($obj->$field) && $obj->$field) { continue; } if (isset($data[$index])) { $obj->$field = $data[$index]; } } return $obj; } /** * @param $entity */ private function addSuccess($entity) { $this->results[$entity->getEntityType()][RESULT_SUCCESS][] = $entity; } /** * @param $entityType * @param $data */ private function addFailure($entityType, $data) { $this->results[$entityType][RESULT_FAILURE][] = $data; } private function init() { EntityModel::$notifySubscriptions = false; foreach ([ENTITY_CLIENT, ENTITY_INVOICE, ENTITY_PAYMENT, ENTITY_QUOTE, ENTITY_PRODUCT] as $entityType) { $this->results[$entityType] = [ RESULT_SUCCESS => [], RESULT_FAILURE => [], ]; } } private function initMaps() { $this->init(); $this->maps = [ 'client' => [], 'contact' => [], 'customer' => [], 'invoice' => [], 'invoice_client' => [], 'product' => [], 'countries' => [], 'countries2' => [], 'currencies' => [], 'client_ids' => [], 'invoice_ids' => [], 'vendors' => [], 'expense_categories' => [], 'tax_rates' => [], 'tax_names' => [], ]; $clients = $this->clientRepo->all(); foreach ($clients as $client) { $this->addClientToMaps($client); } $customers = $this->customerRepo->all(); foreach ($customers as $customer) { $this->addCustomerToMaps($customer); } $contacts = $this->contactRepo->all(); foreach ($contacts as $contact) { $this->addContactToMaps($contact); } $invoices = $this->invoiceRepo->all(); foreach ($invoices as $invoice) { $this->addInvoiceToMaps($invoice); } $products = $this->productRepo->all(); foreach ($products as $product) { $this->addProductToMaps($product); } $countries = Cache::get('countries'); foreach ($countries as $country) { $this->maps['countries'][strtolower($country->name)] = $country->id; $this->maps['countries2'][strtolower($country->iso_3166_2)] = $country->id; } $currencies = Cache::get('currencies'); foreach ($currencies as $currency) { $this->maps['currencies'][strtolower($currency->code)] = $currency->id; } $vendors = $this->vendorRepo->all(); foreach ($vendors as $vendor) { $this->addVendorToMaps($vendor); } $expenseCaegories = $this->expenseCategoryRepo->all(); foreach ($expenseCaegories as $category) { $this->addExpenseCategoryToMaps($category); } $taxRates = $this->taxRateRepository->all(); foreach ($taxRates as $taxRate) { $name = trim(strtolower($taxRate->name)); $this->maps['tax_rates'][$name] = $taxRate->rate; $this->maps['tax_names'][$name] = $taxRate->name; } } /** * @param Invoice $invoice */ private function addInvoiceToMaps(Invoice $invoice) { if ($number = strtolower(trim($invoice->invoice_number))) { $this->maps['invoices'][$number] = $invoice; $this->maps['invoice'][$number] = $invoice->id; $this->maps['invoice_client'][$number] = $invoice->client_id; $this->maps['invoice_ids'][$invoice->public_id] = $invoice->id; } } /** * @param Client $client */ private function addClientToMaps(Client $client) { if ($name = strtolower(trim($client->name))) { $this->maps['client'][$name] = $client->id; $this->maps['client_ids'][$client->public_id] = $client->id; } if ($client->contacts->count() && $name = strtolower(trim($client->contacts[0]->email))) { $this->maps['client'][$name] = $client->id; $this->maps['client_ids'][$client->public_id] = $client->id; } } /** * @param Customer $customer */ private function addCustomerToMaps(AccountGatewayToken $customer) { $this->maps['customer'][$customer->token] = $customer; $this->maps['customer'][$customer->contact->email] = $customer; } /** * @param Product $product */ private function addContactToMaps(Contact $contact) { if ($key = strtolower(trim($contact->email))) { $this->maps['contact'][$key] = $contact; } } /** * @param Product $product */ private function addProductToMaps(Product $product) { if ($key = strtolower(trim($product->product_key))) { $this->maps['product'][$key] = $product; } } private function addExpenseToMaps(Expense $expense) { // do nothing } private function addVendorToMaps(Vendor $vendor) { $this->maps['vendor'][strtolower($vendor->name)] = $vendor->id; } private function addExpenseCategoryToMaps(ExpenseCategory $category) { if ($name = strtolower($category->name)) { $this->maps['expense_category'][$name] = $category->id; } } private function isRowEmpty($row) { $isEmpty = true; foreach ($row as $key => $val) { if (trim($val)) { $isEmpty = false; } } return $isEmpty; } public function presentResults($results, $includeSettings = false) { $message = ''; $skipped = []; if ($includeSettings) { $message = trans('texts.imported_settings') . '
'; } foreach ($results as $entityType => $entityResults) { if ($count = count($entityResults[RESULT_SUCCESS])) { $message .= trans("texts.created_{$entityType}s", ['count' => $count]) . '
'; } if (count($entityResults[RESULT_FAILURE])) { $skipped = array_merge($skipped, $entityResults[RESULT_FAILURE]); } } if (count($skipped)) { $message .= '

' . trans('texts.failed_to_import') . '
'; foreach ($skipped as $skip) { $message .= json_encode($skip) . '
'; } } return $message; } private function checkForFile($fileName) { $counter = 0; while (! file_exists($fileName)) { $counter++; if ($counter > 60) { throw new Exception('File not found: ' . $fileName); } sleep(2); } return true; } }