$hash, 'mappings' => [], ]; /** @var UploadedFile $file */ foreach ($request->files->get('files') as $entityType => $file) { $contents = file_get_contents($file->getPathname()); $contents = $this->convertEncoding($contents); // Store the csv in cache with an expiry of 10 minutes Cache::put($hash.'-'.$entityType, base64_encode($contents), 600); // Parse CSV $csv_array = $this->getCsvData($contents); $class_map = $this->getEntityMap($entityType); $hints = $this->setImportHints($entityType, $class_map::importable(), $csv_array[0]); $data['mappings'][$entityType] = [ 'available' => $class_map::importable(), 'headers' => array_slice($csv_array, 0, 2), 'hints' => $hints, ]; } return response()->json($data); } private function setImportHints($entity_type, $available_keys, $headers): array { $hints = []; $translated_keys = collect($available_keys)->map(function ($value, $key) { $parts = explode(".", $value); $index = $parts[0]; $label = $parts[1] ?? $parts[0]; return ['key' => $key, 'index' => ctrans("texts.{$index}"), 'label' => ctrans("texts.{$label}")]; })->toArray(); foreach($headers as $key => $value) { foreach($translated_keys as $tkey => $tvalue) { if($this->testMatch($value, $tvalue['label'])) { $hit = $tvalue['key']; $hints[$key] = $hit; unset($translated_keys[$tkey]); break; } else { $hints[$key] = null; } } } //second pass using the index of the translation here foreach($headers as $key => $value) { if(isset($hints[$key])) { continue; } foreach($translated_keys as $tkey => $tvalue) { if($this->testMatch($value, $tvalue['index'])) { $hit = $tvalue['key']; $hints[$key] = $hit; unset($translated_keys[$tkey]); break; } else { $hints[$key] = null; } } } return $hints; } private function testMatch($haystack, $needle): bool { return stripos($haystack, $needle) !== false; } private function convertEncoding($data) { // $enc = mb_detect_encoding($data, mb_list_encodings(), true); // if($enc !== false) { // $data = mb_convert_encoding($data, "UTF-8", $enc); // } return $data; } public function import(ImportRequest $request) { /** @var \App\Models\User $user */ $user = auth()->user(); $data = $request->all(); if (empty($data['hash'])) { // Create a reference $data['hash'] = $hash = Str::random(32); /** @var UploadedFile $file */ foreach ($request->files->get('files') as $entityType => $file) { $contents = file_get_contents($file->getPathname()); // Store the csv in cache with an expiry of 10 minutes Cache::put($hash.'-'.$entityType, base64_encode($contents), 600); } } unset($data['files']); CSVIngest::dispatch($data, $user->company()); return response()->json(['message' => ctrans('texts.import_started')], 200); } private function getEntityMap($entity_type) { return sprintf('App\\Import\\Definitions\%sMap', ucfirst(Str::camel($entity_type))); } private function getCsvData($csvfile) { if (! ini_get('auto_detect_line_endings')) { ini_set('auto_detect_line_endings', '1'); } $csv = Reader::createFromString($csvfile); $csvdelimiter = self::detectDelimiter($csvfile); $csv->setDelimiter($csvdelimiter); $stmt = new Statement(); $data = iterator_to_array($stmt->process($csv)); if (count($data) > 0) { $headers = $data[0]; // Remove Invoice Ninja headers if (count($headers) && count($data) > 4) { $firstCell = $headers[0]; if (strstr($firstCell, (string) config('ninja.app_name'))) { array_shift($data); // Invoice Ninja... array_shift($data); // array_shift($data); // Entity Type Header } } } return $this->convertData($data); } private function convertData(array $data): array { // List of encodings to check against $encodings = [ 'UTF-8', 'ISO-8859-1', // Latin-1 'ISO-8859-2', // Latin-2 'WINDOWS-1252', // CP1252 'SHIFT-JIS', 'EUC-JP', 'GB2312', 'GBK', 'BIG5', 'ISO-2022-JP', 'KOI8-R', 'KOI8-U', 'WINDOWS-1251', // CP1251 'UTF-16', 'UTF-32', 'ASCII' ]; foreach ($data as $key => $value) { // Only process strings if (is_string($value)) { // Detect the encoding of the string $detectedEncoding = mb_detect_encoding($value, $encodings, true); // If encoding is detected and it's not UTF-8, convert it to UTF-8 if ($detectedEncoding && $detectedEncoding !== 'UTF-8') { $array[$key] = mb_convert_encoding($value, 'UTF-8', $detectedEncoding); } } } return $data; } /** * Returns the best delimiter * * @param string $csvfile * @return string */ public function detectDelimiter($csvfile): string { $delimiters = [',', '.', ';', '|']; $bestDelimiter = ','; $count = 0; // 10-01-2024 - A better way to resolve the csv file delimiter. $csvfile = substr($csvfile, 0, strpos($csvfile, "\n")); foreach ($delimiters as $delimiter) { if (substr_count(strstr($csvfile, "\n", true), $delimiter) >= $count) { $count = substr_count($csvfile, $delimiter); $bestDelimiter = $delimiter; } } /** @phpstan-ignore-next-line **/ return $bestDelimiter ?? ','; } }