From 28b5b5f9c248c75e6046128a52a44d6c267d7646 Mon Sep 17 00:00:00 2001 From: Troels Liebe Bentsen Date: Mon, 6 Oct 2014 20:42:07 +0200 Subject: [PATCH] Refactor code and do more work on import function --- app/commands/ImportTimesheetData.php | 326 ++++++++---------- ...p => 2014_10_06_103529_add_timesheets.php} | 17 +- app/models/TimesheetEvent.php | 3 + app/tests/ExampleTest.php | 3 +- app/tests/TimesheetUtilsTest.php | 26 ++ 5 files changed, 189 insertions(+), 186 deletions(-) rename app/database/migrations/{2014_09_30_103529_add_timesheets.php => 2014_10_06_103529_add_timesheets.php} (92%) create mode 100644 app/tests/TimesheetUtilsTest.php diff --git a/app/commands/ImportTimesheetData.php b/app/commands/ImportTimesheetData.php index 0291519df1..d493250f80 100644 --- a/app/commands/ImportTimesheetData.php +++ b/app/commands/ImportTimesheetData.php @@ -6,39 +6,31 @@ use Symfony\Component\Console\Input\InputArgument; class ImportTimesheetData extends Command { - protected $name = 'ninja:import-timesheet-data'; - protected $description = 'Import timesheet data'; + protected $name = 'ninja:import-timesheet-data'; + protected $description = 'Import timesheet data'; - public function fire() - { - $this->info(date('Y-m-d') . ' Running ImportTimesheetData...'); + public function fire() { + $this->info(date('Y-m-d') . ' Running ImportTimesheetData...'); - /* try { - $dt = new DateTime("now"); - var_dump($dt); - echo "1:".$dt."\n"; - echo $dt->getTimestamp()."\n"; - } catch (Exception $ex) { - echo $ex->getMessage(); - echo $ex->getTraceAsString(); - } - exit(0); */ - - - - - - + $dt = new DateTime("now"); + var_dump($dt); + echo "1:".$dt."\n"; + echo $dt->getTimestamp()."\n"; + } catch (Exception $ex) { + echo $ex->getMessage(); + echo $ex->getTraceAsString(); + } + exit(0); */ + // Create some initial sources we can test with $user = User::first(); if (!$user) { $this->error("Error: please create user account by logging in"); return; } - + // TODO: Populate with own test data until test data has been created - // Truncate the tables $this->info("Truncate tables"); DB::statement('SET FOREIGN_KEY_CHECKS=0;'); @@ -55,36 +47,37 @@ class ImportTimesheetData extends Command { $project = Project::createNew($user); $project->name = $options['description']; $project->save(); - + $code = ProjectCode::createNew($user); $code->name = $name; $project->codes()->save($code); } #Project::createNew($user); } - + if (!TimesheetEventSource::find(1)) { $this->info("Import old event sources"); - + $oldevent_sources = json_decode(file_get_contents("/home/tlb/git/itktime/employes.json"), true); //array_shift($oldevent_sources); //array_pop($oldevent_sources); - + foreach ($oldevent_sources as $source) { $event_source = TimesheetEventSource::createNew($user); $event_source->name = $source['name']; $event_source->url = $source['url']; $event_source->owner = $source['owner']; $event_source->type = 'ical'; + //$event_source->from_date = new DateTime("2009-01-01"); $event_source->save(); } } - + // Add all URL's to Curl $this->info("Download ICAL feeds"); $event_sources = TimesheetEventSource::all(); // TODO: Filter based on ical feeds - + $urls = []; $event_sources->map(function($item) use(&$urls) { $urls[] = $item->url; @@ -92,20 +85,20 @@ class ImportTimesheetData extends Command { $results = $this->curlGetUrls($urls); // Fetch all codes so we can do a quick lookup - $codes = array(); + $codes = array(); ProjectCode::all()->map(function($item) use(&$codes) { $codes[$item->name] = $item; }); - + //FIXME: Make sure we keep track of duplicate UID's so we don't fail when inserting them to the database $this->info("Start parsing ICAL files"); - foreach($event_sources as $i => $event_source) { - if(!is_array($results[$i])) { - $this->info("Find events in ".$event_source->name); - file_put_contents("/tmp/".$event_source->name.".ical", $results[$i]); - if(preg_match_all('/BEGIN:VEVENT\r?\n(.+?)\r?\nEND:VEVENT/s', $results[$i], $icalmatches)) { + foreach ($event_sources as $i => $event_source) { + if (!is_array($results[$i])) { + $this->info("Find events in " . $event_source->name); + file_put_contents("/tmp/" . $event_source->name . ".ical", $results[$i]); + if (preg_match_all('/BEGIN:VEVENT\r?\n(.+?)\r?\nEND:VEVENT/s', $results[$i], $icalmatches)) { $uids = []; - foreach($icalmatches[1] as $eventstr) { + foreach ($icalmatches[1] as $eventstr) { //print "---\n"; //print $eventstr."\n"; //print "---\n"; @@ -113,137 +106,116 @@ class ImportTimesheetData extends Command { # Fix lines broken by 76 char limit $eventstr = preg_replace('/\r?\n\s/s', '', $eventstr); //$this->info("Parse data"); - if(preg_match_all('/(?:^|\r?\n)([^;:]+)[;:]([^\r\n]+)/s', $eventstr, $eventmatches)) { - // Build ICAL event array - $data = ['summary' => '']; - foreach($eventmatches[1] as $i => $key) { - # Convert escaped linebreakes to linebreak - $value = preg_replace("/\r?\n\s/", "", $eventmatches[2][$i]); - # Unescape , and ; - $value = preg_replace('/\\\\([,;])/s', '$1', $value); - $data[strtolower($key)] = $value; - } - + $data = TimesheetUtils::parseICALEvent($eventstr); + if ($data) { // Extract code for summary so we only import events we use - //$this->info("Match summary"); - if(preg_match('/^\s*([^\s:\/]+)(?:\/([^:]+))?\s*:\s*(.*?)\s*$/s', $data['summary'], $matches)) { - $codename = strtoupper($matches[1]); - $tags = strtolower($matches[2]); - $title = $matches[3]; - - //$this->info("Check code"); - if(isset($codes[$codename])) { - //var_dump($data); - $code = $codes[$codename]; - $event = TimesheetEvent::createNew($user); - $event->summary = $title; - $event->description = $title; - $event->owner = $event_source->owner; - $event->timesheet_event_source_id = $event_source->id; - $event->project_id = $code->project_id; - $event->project_code_id = $code->id; - $event->uid = $data['uid']; - - # Add RECURRENCE-ID to the UID to make sure the event is unique - if(isset($data['recurrence-id'])) { - $event->uid .= $data['recurrence-id']; + list($codename, $tags, $title) = TimesheetUtils::parseEventSummary($data['summary']); + if ($codename != null) { + $event = TimesheetEvent::createNew($user); + $event->uid = $data['uid']; + + # Add RECURRENCE-ID to the UID to make sure the event is unique + if (isset($data['recurrence-id'])) { + $event->uid .= $data['recurrence-id']; + } + + // Check for duplicate events in the feed + if (isset($uids[$event->uid])) { + echo "Duplicate event found:"; + echo "org:\n"; + var_dump($uids[$event->uid]); + echo "new:\n"; + var_dump($data); + continue; + } + $uids[$event->uid] = $data; + + //TODO: Bail on RRULE as we don't support that + // Convert to DateTime objects + foreach (['dtstart', 'dtend', 'created', 'last-modified'] as $key) { + // Parse and create DataTime object from ICAL format + list($dt, $timezone) = TimesheetUtils::parseICALDate($data[$key]); + + // Handle bad dates in created and last-modified + if ($dt == null) { + if ($key == 'created' || $key == 'last-modified') { + $dt = new DateTime('1970-01-01T00:00:00', new DateTimeZone("UTC")); // Default to UNIX epoc + echo "Could not parse date for $key: '" . $data[$key] . "' so default to UNIX Epoc\n"; // TODO write to error table + } else { + echo "Could not parse date for $key: '" . $data[$key] . "'\n"; // TODO write to error table + exit(255); // TODO: Bail on this event + } } - - // Check for duplicate events in the feed - if (isset($uids[$event->uid])) { - echo "Duplicate event found:"; - echo "org:\n"; - var_dump($uids[$event->uid]); - echo "new:\n"; - var_dump($data); + + // Assign DateTime object to + switch ($key) { + case 'dtstart': + $event->start_date = $dt; + $event->org_start_date_timezone = $timezone; + break; + case 'dtend': + $event->end_date = $dt; + $event->org_end_date_timezone = $timezone; + break; + case 'created': $event->org_created_at = $dt; + break; + case 'last-modified': $event->org_updated_at = $dt; + break; + } + } + + // Check that we are witin the range + if ($event_source->from_date != null) { + $from_date = new DateTime($event_source->from_date, new DateTimeZone('UTC')); + if ($from_date > $event->end_date) { + // Skip this event + echo "Skiped: $codename: $title\n"; continue; } - $uids[$event->uid] = $data; + } - //TODO: Bail on RRULE as we don't support that - - //$event->org_data = $eventstr; - - if(isset($data['location'])) { - $event->location = $data['location']; - } + // Calculate number of hours + $di = $event->end_date->diff($event->start_date); + $event->hours = $di->h + $di->i / 60; - foreach (['dtstart', 'dtend', 'created', 'last-modified'] as $key) { - // Parse and create DataTime object from ICAL format - $dt = null; - $timezone = null; - if (preg_match('/^TZID=(.+?):([12]\d\d\d)(\d\d)(\d\d)T(\d\d)(\d\d)(\d\d)$/', $data[$key], $m)) { - $timezone = $m[1]; - $dt = new DateTime("{$m[2]}-{$m[3]}-{$m[4]}T{$m[5]}:{$m[6]}:{$m[7]}", new DateTimeZone($m[1])); - } else if (preg_match('/^VALUE=DATE:([12]\d\d\d)(\d\d)(\d\d)$/', $data[$key], $m)) { - $dt = new DateTime("{$m[1]}-{$m[2]}-{$m[3]}T00:00:00", new DateTimeZone("UTC")); - } else if (preg_match('/^([12]\d\d\d)(\d\d)(\d\d)T(\d\d)(\d\d)(\d\d)Z$/', $data[$key], $m)) { - $dt = new DateTime("{$m[1]}-{$m[2]}-{$m[3]}T{$m[4]}:{$m[5]}:{$m[6]}", new DateTimeZone("UTC")); - } else if($key == 'created' || $key == 'last-modified') { - $dt = new DateTime('1970-01-01T00:00:00', new DateTimeZone("UTC")); // Default to UNIX epoc - echo "Could not parse date for $key: '".$data[$key]."' so default to UNIX Epoc\n"; // TODO write to error table - } else { - echo "Could not parse date for $key: '".$data[$key]."'\n"; // TODO write to error table - exit(255); // TODO: Bail onthis event - } - - // Assign DateTime object to - switch ($key) { - case 'dtstart': - $event->start_date = $dt; - $event->org_start_date_timezone = $timezone; - break; - case 'dtend': - $event->end_date = $dt; - $event->org_end_date_timezone = $timezone; - break; - case 'created': $event->org_created_at = $dt; break; - case 'last-modified': $event->org_updated_at = $dt; break; - } - } - - // Calculate number of hours - $di = $event->end_date->diff($event->start_date); - $event->hours = $di->h + $di->i / 60; - - /*var_dump($event); - exit();*/ - + // Copy data to new object + $event->org_data = $eventstr; + $event->summary = $title; + $event->description = $title; + $event->org_code = $code; + $event->owner = $event_source->owner; + $event->timesheet_event_source_id = $event_source->id; + if (isset($codes[$codename])) { + $event->project_id = $codes[$codename]->project_id; + $event->project_code_id = $codes[$codename]->id; + } + if (isset($data['location'])) { + $event->location = $data['location']; + } + + try { // Save event - - //if(!preg_match("/forbered møde med Peter Pietras - nyt sjovt projekt./", $event->summary)) { - - try { - //$event->start_date = new DateTime(""); - $event->save(); - } catch (Exception $ex) { - echo "'".$event->summary."'\n"; - var_dump($data); - echo $ex->getMessage(); - echo $ex->getTraceAsString(); - //exit(); - } - //} - - } else { - //TODO: Add to error table so we can show user - echo "Code not found: $codename\n"; + $event->save(); + } catch (Exception $ex) { + echo "'" . $event->summary . "'\n"; + var_dump($data); + echo $ex->getMessage(); + echo $ex->getTraceAsString(); + //exit(); } } } } - } else { // Parse error } - } else { // Curl Error } } - + $this->info('Done'); - } + } private function curlGetUrls($urls = [], $timeout = 30) { // Create muxer @@ -256,10 +228,10 @@ class ImportTimesheetData extends Command { // Create new handle and add to muxer $ch = curl_init($url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_ENCODING , "gzip"); - curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout); + curl_setopt($ch, CURLOPT_ENCODING, "gzip"); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout); curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); //timeout in seconds - + curl_multi_add_handle($multi, $ch); $handles[(int) $ch] = $ch; $ch2idx[(int) $ch] = $i; @@ -267,41 +239,37 @@ class ImportTimesheetData extends Command { // Do initial connect $still_running = true; - while($still_running) { + while ($still_running) { // Do curl stuff while (($mrc = curl_multi_exec($multi, $still_running)) === CURLM_CALL_MULTI_PERFORM); - if ($mrc !== CURLM_OK) { break; } - + if ($mrc !== CURLM_OK) { + break; + } + // Try to read from handles that are ready while ($info = curl_multi_info_read($multi)) { if ($info["result"] == CURLE_OK) { $results[$ch2idx[(int) $info["handle"]]] = curl_multi_getcontent($info["handle"]); - } else { - if(CURLE_UNSUPPORTED_PROTOCOL == $info["result"]) { + if (CURLE_UNSUPPORTED_PROTOCOL == $info["result"]) { $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Unsupported protocol"]; - - } else if(CURLE_URL_MALFORMAT == $info["result"]) { + } else if (CURLE_URL_MALFORMAT == $info["result"]) { $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Malform url"]; - - } else if(CURLE_COULDNT_RESOLVE_HOST == $info["result"]){ + } else if (CURLE_COULDNT_RESOLVE_HOST == $info["result"]) { $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Could not resolve host"]; - - } else if(CURLE_OPERATION_TIMEDOUT == $info["result"]){ + } else if (CURLE_OPERATION_TIMEDOUT == $info["result"]) { $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Timed out waiting for operations to finish"]; - } else { $results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Unknown curl error code"]; } } } - + // Sleep until if (($rs = curl_multi_select($multi)) === -1) { usleep(20); // select failed for some reason, so we sleep for 20ms and run some more curl stuff } } - } finally { foreach ($handles as $chi => $ch) { curl_multi_remove_handle($multi, $ch); @@ -309,22 +277,18 @@ class ImportTimesheetData extends Command { curl_multi_close($multi); } - + return $results; } - - protected function getArguments() - { - return array( - - ); - } - protected function getOptions() - { - return array( - - ); - } + protected function getArguments() { + return array( + ); + } -} \ No newline at end of file + protected function getOptions() { + return array( + ); + } + +} diff --git a/app/database/migrations/2014_09_30_103529_add_timesheets.php b/app/database/migrations/2014_10_06_103529_add_timesheets.php similarity index 92% rename from app/database/migrations/2014_09_30_103529_add_timesheets.php rename to app/database/migrations/2014_10_06_103529_add_timesheets.php index 6bf31ce962..951041697d 100644 --- a/app/database/migrations/2014_09_30_103529_add_timesheets.php +++ b/app/database/migrations/2014_10_06_103529_add_timesheets.php @@ -79,6 +79,9 @@ class AddTimesheets extends Migration { $t->string('url'); $t->enum('type', array('ical', 'googlejson')); + $t->dateTime('from_date')->nullable(); + $t->dateTime('to_date')->nullable(); + $t->foreign('account_id')->references('id')->on('accounts'); $t->foreign('user_id')->references('id')->on('users')->onDelete('cascade'); }); @@ -94,23 +97,31 @@ class AddTimesheets extends Migration { $t->timestamps(); $t->softDeletes(); - $t->string('summary'); + // Basic fields $t->string('uid'); + $t->string('summary'); $t->text('description'); $t->string('location'); $t->string('owner'); - $t->dateTime('start_date'); $t->dateTime('end_date'); + + # Calculated values $t->decimal('hours'); $t->float('discount'); + // Original data + $t->string('org_code'); $t->timeStamp('org_created_at'); $t->timeStamp('org_updated_at'); $t->string('org_start_date_timezone')->nullable(); $t->string('org_end_date_timezone')->nullable(); $t->text('org_data'); - + + // Error and merge handling + $t->string('import_error')->nullable(); + $t->text('updated_data')->nullable(); + $t->foreign('account_id')->references('id')->on('accounts'); $t->foreign('user_id')->references('id')->on('users')->onDelete('cascade'); $t->foreign('timesheet_event_source_id')->references('id')->on('timesheet_event_sources')->onDelete('cascade'); diff --git a/app/models/TimesheetEvent.php b/app/models/TimesheetEvent.php index 259466460b..0fb78e73b9 100644 --- a/app/models/TimesheetEvent.php +++ b/app/models/TimesheetEvent.php @@ -35,6 +35,9 @@ class TimesheetEvent extends Eloquent return $this->belongsTo('ProjectCode'); } + /** + * @return TimesheetEvent + */ public static function createNew($parent = false) { $className = get_called_class(); diff --git a/app/tests/ExampleTest.php b/app/tests/ExampleTest.php index ead53e07d4..990a8389a7 100755 --- a/app/tests/ExampleTest.php +++ b/app/tests/ExampleTest.php @@ -10,8 +10,7 @@ class ExampleTest extends TestCase { public function testBasicExample() { $crawler = $this->client->request('GET', '/'); - - $this->assertTrue($this->client->getResponse()->isOk()); + $this->assertTrue($this->client->getResponse()->isRedirect()); } } \ No newline at end of file diff --git a/app/tests/TimesheetUtilsTest.php b/app/tests/TimesheetUtilsTest.php new file mode 100644 index 0000000000..ccfefef428 --- /dev/null +++ b/app/tests/TimesheetUtilsTest.php @@ -0,0 +1,26 @@ +assertSame(null, $code); + + list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test:'); + $this->assertSame("TEST", $code); + + list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test: '); + $this->assertSame("TEST", $code); + + list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test::'); + $this->assertSame("TEST", $code); + + list($code, $tags, $title) = TimesheetUtils::parseEventSummary('TEST: Hello :)'); + $this->assertSame("TEST", $code); + + list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test/tags: '); + $this->assertSame('TEST', $code); + $this->assertSame('tags', $tags); + } + +}