mirror of
https://github.com/invoiceninja/invoiceninja.git
synced 2024-09-20 00:11:35 +02:00
Refactor code and do more work on import function
This commit is contained in:
parent
56a296f103
commit
28b5b5f9c2
@ -6,29 +6,22 @@ use Symfony\Component\Console\Input\InputArgument;
|
||||
|
||||
class ImportTimesheetData extends Command {
|
||||
|
||||
protected $name = 'ninja:import-timesheet-data';
|
||||
protected $description = 'Import timesheet data';
|
||||
|
||||
public function fire()
|
||||
{
|
||||
$this->info(date('Y-m-d') . ' Running ImportTimesheetData...');
|
||||
protected $name = 'ninja:import-timesheet-data';
|
||||
protected $description = 'Import timesheet data';
|
||||
|
||||
public function fire() {
|
||||
$this->info(date('Y-m-d') . ' Running ImportTimesheetData...');
|
||||
|
||||
/* try {
|
||||
$dt = new DateTime("now");
|
||||
var_dump($dt);
|
||||
echo "1:".$dt."\n";
|
||||
echo $dt->getTimestamp()."\n";
|
||||
} catch (Exception $ex) {
|
||||
echo $ex->getMessage();
|
||||
echo $ex->getTraceAsString();
|
||||
}
|
||||
exit(0); */
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
$dt = new DateTime("now");
|
||||
var_dump($dt);
|
||||
echo "1:".$dt."\n";
|
||||
echo $dt->getTimestamp()."\n";
|
||||
} catch (Exception $ex) {
|
||||
echo $ex->getMessage();
|
||||
echo $ex->getTraceAsString();
|
||||
}
|
||||
exit(0); */
|
||||
|
||||
// Create some initial sources we can test with
|
||||
$user = User::first();
|
||||
@ -38,7 +31,6 @@ class ImportTimesheetData extends Command {
|
||||
}
|
||||
|
||||
// TODO: Populate with own test data until test data has been created
|
||||
|
||||
// Truncate the tables
|
||||
$this->info("Truncate tables");
|
||||
DB::statement('SET FOREIGN_KEY_CHECKS=0;');
|
||||
@ -77,6 +69,7 @@ class ImportTimesheetData extends Command {
|
||||
$event_source->url = $source['url'];
|
||||
$event_source->owner = $source['owner'];
|
||||
$event_source->type = 'ical';
|
||||
//$event_source->from_date = new DateTime("2009-01-01");
|
||||
$event_source->save();
|
||||
}
|
||||
}
|
||||
@ -99,13 +92,13 @@ class ImportTimesheetData extends Command {
|
||||
|
||||
//FIXME: Make sure we keep track of duplicate UID's so we don't fail when inserting them to the database
|
||||
$this->info("Start parsing ICAL files");
|
||||
foreach($event_sources as $i => $event_source) {
|
||||
if(!is_array($results[$i])) {
|
||||
$this->info("Find events in ".$event_source->name);
|
||||
file_put_contents("/tmp/".$event_source->name.".ical", $results[$i]);
|
||||
if(preg_match_all('/BEGIN:VEVENT\r?\n(.+?)\r?\nEND:VEVENT/s', $results[$i], $icalmatches)) {
|
||||
foreach ($event_sources as $i => $event_source) {
|
||||
if (!is_array($results[$i])) {
|
||||
$this->info("Find events in " . $event_source->name);
|
||||
file_put_contents("/tmp/" . $event_source->name . ".ical", $results[$i]);
|
||||
if (preg_match_all('/BEGIN:VEVENT\r?\n(.+?)\r?\nEND:VEVENT/s', $results[$i], $icalmatches)) {
|
||||
$uids = [];
|
||||
foreach($icalmatches[1] as $eventstr) {
|
||||
foreach ($icalmatches[1] as $eventstr) {
|
||||
//print "---\n";
|
||||
//print $eventstr."\n";
|
||||
//print "---\n";
|
||||
@ -113,137 +106,116 @@ class ImportTimesheetData extends Command {
|
||||
# Fix lines broken by 76 char limit
|
||||
$eventstr = preg_replace('/\r?\n\s/s', '', $eventstr);
|
||||
//$this->info("Parse data");
|
||||
if(preg_match_all('/(?:^|\r?\n)([^;:]+)[;:]([^\r\n]+)/s', $eventstr, $eventmatches)) {
|
||||
// Build ICAL event array
|
||||
$data = ['summary' => ''];
|
||||
foreach($eventmatches[1] as $i => $key) {
|
||||
# Convert escaped linebreakes to linebreak
|
||||
$value = preg_replace("/\r?\n\s/", "", $eventmatches[2][$i]);
|
||||
# Unescape , and ;
|
||||
$value = preg_replace('/\\\\([,;])/s', '$1', $value);
|
||||
$data[strtolower($key)] = $value;
|
||||
}
|
||||
|
||||
$data = TimesheetUtils::parseICALEvent($eventstr);
|
||||
if ($data) {
|
||||
// Extract code for summary so we only import events we use
|
||||
//$this->info("Match summary");
|
||||
if(preg_match('/^\s*([^\s:\/]+)(?:\/([^:]+))?\s*:\s*(.*?)\s*$/s', $data['summary'], $matches)) {
|
||||
$codename = strtoupper($matches[1]);
|
||||
$tags = strtolower($matches[2]);
|
||||
$title = $matches[3];
|
||||
list($codename, $tags, $title) = TimesheetUtils::parseEventSummary($data['summary']);
|
||||
if ($codename != null) {
|
||||
$event = TimesheetEvent::createNew($user);
|
||||
$event->uid = $data['uid'];
|
||||
|
||||
//$this->info("Check code");
|
||||
if(isset($codes[$codename])) {
|
||||
//var_dump($data);
|
||||
$code = $codes[$codename];
|
||||
$event = TimesheetEvent::createNew($user);
|
||||
$event->summary = $title;
|
||||
$event->description = $title;
|
||||
$event->owner = $event_source->owner;
|
||||
$event->timesheet_event_source_id = $event_source->id;
|
||||
$event->project_id = $code->project_id;
|
||||
$event->project_code_id = $code->id;
|
||||
$event->uid = $data['uid'];
|
||||
# Add RECURRENCE-ID to the UID to make sure the event is unique
|
||||
if (isset($data['recurrence-id'])) {
|
||||
$event->uid .= $data['recurrence-id'];
|
||||
}
|
||||
|
||||
# Add RECURRENCE-ID to the UID to make sure the event is unique
|
||||
if(isset($data['recurrence-id'])) {
|
||||
$event->uid .= $data['recurrence-id'];
|
||||
// Check for duplicate events in the feed
|
||||
if (isset($uids[$event->uid])) {
|
||||
echo "Duplicate event found:";
|
||||
echo "org:\n";
|
||||
var_dump($uids[$event->uid]);
|
||||
echo "new:\n";
|
||||
var_dump($data);
|
||||
continue;
|
||||
}
|
||||
$uids[$event->uid] = $data;
|
||||
|
||||
//TODO: Bail on RRULE as we don't support that
|
||||
// Convert to DateTime objects
|
||||
foreach (['dtstart', 'dtend', 'created', 'last-modified'] as $key) {
|
||||
// Parse and create DataTime object from ICAL format
|
||||
list($dt, $timezone) = TimesheetUtils::parseICALDate($data[$key]);
|
||||
|
||||
// Handle bad dates in created and last-modified
|
||||
if ($dt == null) {
|
||||
if ($key == 'created' || $key == 'last-modified') {
|
||||
$dt = new DateTime('1970-01-01T00:00:00', new DateTimeZone("UTC")); // Default to UNIX epoc
|
||||
echo "Could not parse date for $key: '" . $data[$key] . "' so default to UNIX Epoc\n"; // TODO write to error table
|
||||
} else {
|
||||
echo "Could not parse date for $key: '" . $data[$key] . "'\n"; // TODO write to error table
|
||||
exit(255); // TODO: Bail on this event
|
||||
}
|
||||
}
|
||||
|
||||
// Check for duplicate events in the feed
|
||||
if (isset($uids[$event->uid])) {
|
||||
echo "Duplicate event found:";
|
||||
echo "org:\n";
|
||||
var_dump($uids[$event->uid]);
|
||||
echo "new:\n";
|
||||
var_dump($data);
|
||||
// Assign DateTime object to
|
||||
switch ($key) {
|
||||
case 'dtstart':
|
||||
$event->start_date = $dt;
|
||||
$event->org_start_date_timezone = $timezone;
|
||||
break;
|
||||
case 'dtend':
|
||||
$event->end_date = $dt;
|
||||
$event->org_end_date_timezone = $timezone;
|
||||
break;
|
||||
case 'created': $event->org_created_at = $dt;
|
||||
break;
|
||||
case 'last-modified': $event->org_updated_at = $dt;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Check that we are witin the range
|
||||
if ($event_source->from_date != null) {
|
||||
$from_date = new DateTime($event_source->from_date, new DateTimeZone('UTC'));
|
||||
if ($from_date > $event->end_date) {
|
||||
// Skip this event
|
||||
echo "Skiped: $codename: $title\n";
|
||||
continue;
|
||||
}
|
||||
$uids[$event->uid] = $data;
|
||||
}
|
||||
|
||||
//TODO: Bail on RRULE as we don't support that
|
||||
// Calculate number of hours
|
||||
$di = $event->end_date->diff($event->start_date);
|
||||
$event->hours = $di->h + $di->i / 60;
|
||||
|
||||
//$event->org_data = $eventstr;
|
||||
|
||||
if(isset($data['location'])) {
|
||||
$event->location = $data['location'];
|
||||
}
|
||||
|
||||
foreach (['dtstart', 'dtend', 'created', 'last-modified'] as $key) {
|
||||
// Parse and create DataTime object from ICAL format
|
||||
$dt = null;
|
||||
$timezone = null;
|
||||
if (preg_match('/^TZID=(.+?):([12]\d\d\d)(\d\d)(\d\d)T(\d\d)(\d\d)(\d\d)$/', $data[$key], $m)) {
|
||||
$timezone = $m[1];
|
||||
$dt = new DateTime("{$m[2]}-{$m[3]}-{$m[4]}T{$m[5]}:{$m[6]}:{$m[7]}", new DateTimeZone($m[1]));
|
||||
} else if (preg_match('/^VALUE=DATE:([12]\d\d\d)(\d\d)(\d\d)$/', $data[$key], $m)) {
|
||||
$dt = new DateTime("{$m[1]}-{$m[2]}-{$m[3]}T00:00:00", new DateTimeZone("UTC"));
|
||||
} else if (preg_match('/^([12]\d\d\d)(\d\d)(\d\d)T(\d\d)(\d\d)(\d\d)Z$/', $data[$key], $m)) {
|
||||
$dt = new DateTime("{$m[1]}-{$m[2]}-{$m[3]}T{$m[4]}:{$m[5]}:{$m[6]}", new DateTimeZone("UTC"));
|
||||
} else if($key == 'created' || $key == 'last-modified') {
|
||||
$dt = new DateTime('1970-01-01T00:00:00', new DateTimeZone("UTC")); // Default to UNIX epoc
|
||||
echo "Could not parse date for $key: '".$data[$key]."' so default to UNIX Epoc\n"; // TODO write to error table
|
||||
} else {
|
||||
echo "Could not parse date for $key: '".$data[$key]."'\n"; // TODO write to error table
|
||||
exit(255); // TODO: Bail onthis event
|
||||
}
|
||||
|
||||
// Assign DateTime object to
|
||||
switch ($key) {
|
||||
case 'dtstart':
|
||||
$event->start_date = $dt;
|
||||
$event->org_start_date_timezone = $timezone;
|
||||
break;
|
||||
case 'dtend':
|
||||
$event->end_date = $dt;
|
||||
$event->org_end_date_timezone = $timezone;
|
||||
break;
|
||||
case 'created': $event->org_created_at = $dt; break;
|
||||
case 'last-modified': $event->org_updated_at = $dt; break;
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate number of hours
|
||||
$di = $event->end_date->diff($event->start_date);
|
||||
$event->hours = $di->h + $di->i / 60;
|
||||
|
||||
/*var_dump($event);
|
||||
exit();*/
|
||||
// Copy data to new object
|
||||
$event->org_data = $eventstr;
|
||||
$event->summary = $title;
|
||||
$event->description = $title;
|
||||
$event->org_code = $code;
|
||||
$event->owner = $event_source->owner;
|
||||
$event->timesheet_event_source_id = $event_source->id;
|
||||
if (isset($codes[$codename])) {
|
||||
$event->project_id = $codes[$codename]->project_id;
|
||||
$event->project_code_id = $codes[$codename]->id;
|
||||
}
|
||||
if (isset($data['location'])) {
|
||||
$event->location = $data['location'];
|
||||
}
|
||||
|
||||
try {
|
||||
// Save event
|
||||
|
||||
//if(!preg_match("/forbered møde med Peter Pietras - nyt sjovt projekt./", $event->summary)) {
|
||||
|
||||
try {
|
||||
//$event->start_date = new DateTime("");
|
||||
$event->save();
|
||||
} catch (Exception $ex) {
|
||||
echo "'".$event->summary."'\n";
|
||||
var_dump($data);
|
||||
echo $ex->getMessage();
|
||||
echo $ex->getTraceAsString();
|
||||
//exit();
|
||||
}
|
||||
//}
|
||||
|
||||
} else {
|
||||
//TODO: Add to error table so we can show user
|
||||
echo "Code not found: $codename\n";
|
||||
$event->save();
|
||||
} catch (Exception $ex) {
|
||||
echo "'" . $event->summary . "'\n";
|
||||
var_dump($data);
|
||||
echo $ex->getMessage();
|
||||
echo $ex->getTraceAsString();
|
||||
//exit();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
// Parse error
|
||||
}
|
||||
|
||||
} else {
|
||||
// Curl Error
|
||||
}
|
||||
}
|
||||
|
||||
$this->info('Done');
|
||||
}
|
||||
}
|
||||
|
||||
private function curlGetUrls($urls = [], $timeout = 30) {
|
||||
// Create muxer
|
||||
@ -256,7 +228,7 @@ class ImportTimesheetData extends Command {
|
||||
// Create new handle and add to muxer
|
||||
$ch = curl_init($url);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_ENCODING , "gzip");
|
||||
curl_setopt($ch, CURLOPT_ENCODING, "gzip");
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); //timeout in seconds
|
||||
|
||||
@ -267,29 +239,26 @@ class ImportTimesheetData extends Command {
|
||||
|
||||
// Do initial connect
|
||||
$still_running = true;
|
||||
while($still_running) {
|
||||
while ($still_running) {
|
||||
// Do curl stuff
|
||||
while (($mrc = curl_multi_exec($multi, $still_running)) === CURLM_CALL_MULTI_PERFORM);
|
||||
if ($mrc !== CURLM_OK) { break; }
|
||||
if ($mrc !== CURLM_OK) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Try to read from handles that are ready
|
||||
while ($info = curl_multi_info_read($multi)) {
|
||||
if ($info["result"] == CURLE_OK) {
|
||||
$results[$ch2idx[(int) $info["handle"]]] = curl_multi_getcontent($info["handle"]);
|
||||
|
||||
} else {
|
||||
if(CURLE_UNSUPPORTED_PROTOCOL == $info["result"]) {
|
||||
if (CURLE_UNSUPPORTED_PROTOCOL == $info["result"]) {
|
||||
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Unsupported protocol"];
|
||||
|
||||
} else if(CURLE_URL_MALFORMAT == $info["result"]) {
|
||||
} else if (CURLE_URL_MALFORMAT == $info["result"]) {
|
||||
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Malform url"];
|
||||
|
||||
} else if(CURLE_COULDNT_RESOLVE_HOST == $info["result"]){
|
||||
} else if (CURLE_COULDNT_RESOLVE_HOST == $info["result"]) {
|
||||
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Could not resolve host"];
|
||||
|
||||
} else if(CURLE_OPERATION_TIMEDOUT == $info["result"]){
|
||||
} else if (CURLE_OPERATION_TIMEDOUT == $info["result"]) {
|
||||
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Timed out waiting for operations to finish"];
|
||||
|
||||
} else {
|
||||
$results[$ch2idx[(int) $info["handle"]]] = [$info["result"], "Unknown curl error code"];
|
||||
}
|
||||
@ -301,7 +270,6 @@ class ImportTimesheetData extends Command {
|
||||
usleep(20); // select failed for some reason, so we sleep for 20ms and run some more curl stuff
|
||||
}
|
||||
}
|
||||
|
||||
} finally {
|
||||
foreach ($handles as $chi => $ch) {
|
||||
curl_multi_remove_handle($multi, $ch);
|
||||
@ -313,18 +281,14 @@ class ImportTimesheetData extends Command {
|
||||
return $results;
|
||||
}
|
||||
|
||||
protected function getArguments()
|
||||
{
|
||||
return array(
|
||||
protected function getArguments() {
|
||||
return array(
|
||||
);
|
||||
}
|
||||
|
||||
);
|
||||
}
|
||||
|
||||
protected function getOptions()
|
||||
{
|
||||
return array(
|
||||
|
||||
);
|
||||
}
|
||||
protected function getOptions() {
|
||||
return array(
|
||||
);
|
||||
}
|
||||
|
||||
}
|
@ -79,6 +79,9 @@ class AddTimesheets extends Migration {
|
||||
$t->string('url');
|
||||
$t->enum('type', array('ical', 'googlejson'));
|
||||
|
||||
$t->dateTime('from_date')->nullable();
|
||||
$t->dateTime('to_date')->nullable();
|
||||
|
||||
$t->foreign('account_id')->references('id')->on('accounts');
|
||||
$t->foreign('user_id')->references('id')->on('users')->onDelete('cascade');
|
||||
});
|
||||
@ -94,23 +97,31 @@ class AddTimesheets extends Migration {
|
||||
$t->timestamps();
|
||||
$t->softDeletes();
|
||||
|
||||
$t->string('summary');
|
||||
// Basic fields
|
||||
$t->string('uid');
|
||||
$t->string('summary');
|
||||
$t->text('description');
|
||||
$t->string('location');
|
||||
$t->string('owner');
|
||||
|
||||
$t->dateTime('start_date');
|
||||
$t->dateTime('end_date');
|
||||
|
||||
# Calculated values
|
||||
$t->decimal('hours');
|
||||
$t->float('discount');
|
||||
|
||||
// Original data
|
||||
$t->string('org_code');
|
||||
$t->timeStamp('org_created_at');
|
||||
$t->timeStamp('org_updated_at');
|
||||
$t->string('org_start_date_timezone')->nullable();
|
||||
$t->string('org_end_date_timezone')->nullable();
|
||||
$t->text('org_data');
|
||||
|
||||
// Error and merge handling
|
||||
$t->string('import_error')->nullable();
|
||||
$t->text('updated_data')->nullable();
|
||||
|
||||
$t->foreign('account_id')->references('id')->on('accounts');
|
||||
$t->foreign('user_id')->references('id')->on('users')->onDelete('cascade');
|
||||
$t->foreign('timesheet_event_source_id')->references('id')->on('timesheet_event_sources')->onDelete('cascade');
|
@ -35,6 +35,9 @@ class TimesheetEvent extends Eloquent
|
||||
return $this->belongsTo('ProjectCode');
|
||||
}
|
||||
|
||||
/**
|
||||
* @return TimesheetEvent
|
||||
*/
|
||||
public static function createNew($parent = false)
|
||||
{
|
||||
$className = get_called_class();
|
||||
|
@ -10,8 +10,7 @@ class ExampleTest extends TestCase {
|
||||
public function testBasicExample()
|
||||
{
|
||||
$crawler = $this->client->request('GET', '/');
|
||||
|
||||
$this->assertTrue($this->client->getResponse()->isOk());
|
||||
$this->assertTrue($this->client->getResponse()->isRedirect());
|
||||
}
|
||||
|
||||
}
|
26
app/tests/TimesheetUtilsTest.php
Normal file
26
app/tests/TimesheetUtilsTest.php
Normal file
@ -0,0 +1,26 @@
|
||||
<?php
|
||||
|
||||
class TimesheetUtilTest extends \PHPUnit_Framework_TestCase {
|
||||
|
||||
public function testParseEventSummary() {
|
||||
list($code, $codes, $title) = TimesheetUtils::parseEventSummary('Riga :)');
|
||||
$this->assertSame(null, $code);
|
||||
|
||||
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test:');
|
||||
$this->assertSame("TEST", $code);
|
||||
|
||||
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test: ');
|
||||
$this->assertSame("TEST", $code);
|
||||
|
||||
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test::');
|
||||
$this->assertSame("TEST", $code);
|
||||
|
||||
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('TEST: Hello :)');
|
||||
$this->assertSame("TEST", $code);
|
||||
|
||||
list($code, $tags, $title) = TimesheetUtils::parseEventSummary('Test/tags: ');
|
||||
$this->assertSame('TEST', $code);
|
||||
$this->assertSame('tags', $tags);
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in New Issue
Block a user