diff --git a/CREDITS b/CREDITS index ee5b552..975d8fc 100644 --- a/CREDITS +++ b/CREDITS @@ -4,6 +4,8 @@ Robert Terzi Seems to have the mbox vs Maildir/ problem and gave the --regexmess 's/\AFrom \w .*\n//' solution. +Had the "Word too large" bug with old UW and +big folder. Made me add --split1 and --split2 options. Subbarao & Prasanthi Punnamaraju diff --git a/ChangeLog b/ChangeLog index 2a79b9d..323e718 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,15 +1,22 @@ RCS file: RCS/imapsync,v Working file: imapsync -head: 1.163 +head: 1.164 branch: locks: strict access list: symbolic names: keyword substitution: kv -total revisions: 163; selected revisions: 163 +total revisions: 164; selected revisions: 164 description: ---------------------------- +revision 1.164 +date: 2006/03/26 03:17:48; author: gilles; state: Exp; lines: +97 -9 +Fixed the bug "Word too large" by spliting requests on +demand with --split1 and --split2 options. + +Added Split() fetch_hash2() in lib +---------------------------- revision 1.163 date: 2006/03/24 04:18:58; author: gilles; state: Exp; lines: +61 -51 Added ask_for_password() function diff --git a/README b/README index 93abe62..8749708 100644 --- a/README +++ b/README @@ -2,7 +2,7 @@ NAME imapsync - IMAP synchronization, copy or migration tool. Synchronize mailboxes between two imap servers. Good at IMAP migration. - $Revision: 1.163 $ + $Revision: 1.164 $ INSTALL imapsync works fine under any Unix OS. @@ -58,6 +58,7 @@ SYNOPSIS [--dry] [--debug] [--debugimap] [--timeout ] [--fast] + [--split1] [--split2] [--version] [--help] DESCRIPTION @@ -286,5 +287,5 @@ AUTHOR teaching free open and gratis softwares. Don't hesitate to pay him for that services. - $Id: imapsync,v 1.163 2006/03/24 04:18:58 gilles Exp $ + $Id: imapsync,v 1.164 2006/03/26 03:17:48 gilles Exp $ diff --git a/TODO b/TODO index 6c47c72..80680b6 100644 --- a/TODO +++ b/TODO @@ -31,6 +31,8 @@ http://asg.web.cmu.edu/cyrus/download/imapd/altnamespace.html Explain expunge behavior. +DONE. Fix the "word too large" or "Command line too long" + bug with some imap servers. DONE. Add features from Herman (patch against 1.139 and again with 1.156 by Kjetil Torgrim Homme) diff --git a/VERSION b/VERSION index 3ded366..fa954c7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.163 +1.164 diff --git a/aaa b/aaa deleted file mode 100644 index 545119e..0000000 --- a/aaa +++ /dev/null @@ -1,93 +0,0 @@ -get options: [1] -$RCSfile: imapsync,v $ $Revision: 1.162 $ $Date: 2006/03/22 03:45:05 $ -Mail::IMAPClient version used here is 2.2.9 -will try to use CRAM-MD5 authentication on host1 -will try to use CRAM-MD5 authentication on host2 -From imap server [loul] port [143] user [tata] -To imap server [plume] port [143] user [tata@est.belle] -Banner : * OK louloutte Cyrus IMAP4 v1.5.19 server ready -loul: no support for AUTHENTICATE CRAM-MD5, using LOGIN -Banner : * OK [CAPABILITY IMAP4rev1 UIDPLUS CHILDREN NAMESPACE THREAD=ORDEREDSUBJECT THREAD=REFERENCES SORT QUOTA AUTH=PLAIN CRAM-MD5 CRAM-SHA1 IDLE ACL ACL2=UNION STARTTLS] Courier-IMAP ready. Copyright 1998-2004 Double Precision, Inc. See COPYING for distribution information. -From Buffer I/O : 4096 -To Buffer I/O : 4096 -From capability : QUOTA X-NETSCAPE NAMESPACE X-NON-HIERARCHICAL-RENAME ACL UNSELECT LITERAL+ NO_ATOMIC_RENAME UIDPLUS IMAP4 IMAP4REV1 -To capability : QUOTA STARTTLS NAMESPACE CRAM-SHA1 IDLE AUTH=PLAIN THREAD=ORDEREDSUBJECT ACL SORT UIDPLUS CHILDREN ACL2=UNION CRAM-MD5 IMAP4REV1 THREAD=REFERENCES -From state Authenticated -To state Authenticated -Getting separators -Calling namespace capability -Calling namespace capability -Getting prefix namespace -Calling namespace capability -Getting prefix namespace -Calling namespace capability -From separator and prefix : [.][INBOX.] -To separator and prefix : [.][INBOX.] -From folders : [INBOX.yop.yap] -To folders : [INBOX] [INBOX.2005-INBOX] [INBOX.Draft] [INBOX.Drafts] [INBOX.Sent] [INBOX.Trash] [INBOX.ppp] [INBOX.qqq] [INBOX.yop] [INBOX.yop.yap] [INBOX.yop.yup] [INBOX.yopX] [INBOX.yopX.yap] [INBOX.yopX.yup] [INBOX.zz] -From subscribed folders : [INBOX.yop.yap] -From Folder [INBOX.yop.yap] -removed source prefix : [yop.yap] -inverted separators : [yop.yap] -added target prefix : [INBOX.yop.yap] -To Folder [INBOX.yop.yap] -LIST FROM : 1 messages [1] -LIST TO : 1 messages [7] -++++ From [INBOX.yop.yap] Parse 1 ++++ -Time headers: 0 s -Time sizes : 0 s -Head NUM:10 -FH Date:Fri, 7 Mar 2003 17:00:01 +0100 (CET) -FH From:root@louloutte.dyndns.org (Cron Daemon) -FH Message-Id:<20030307160001.69D18FEEF@louloutte.dyndns.org> -FH Received:by louloutte.dyndns.org (Postfix, from userid 0) id 69D18FEEF; Fri, 7 Mar 2003 17:00:01 +0100 (CET) -FH Return-Path: -FH Subject:Cron /usr/bin/poff libertynico -FH To:root@louloutte.dyndns.org -FH X-Cron-Env: -FH X-Cron-Env: -FH X-Cron-Env: -FH X-Cron-Env: -FH X-Spam-Level:* -FH X-Spam-Status:No, hits=1.6 required=5.0 tests=NO_MX_FOR_FROM,AWL version=2.20 -F msg 1:ri4E1CAmNkh3rP3Mq15HSw:714 -Time headers: 0 s -++++ To [INBOX.yop.yap] Parse 1 ++++ -Time headers: 0 s -Time sizes : 0 s -Head NUM:10 -TH Date:Fri, 7 Mar 2003 17:00:01 +0100 (CET) -TH From:root@louloutte.dyndns.org (Cron Daemon) -TH Message-Id:<20030307160001.69D18FEEF@louloutte.dyndns.org> -TH Received:by louloutte.dyndns.org (Postfix, from userid 0) id 69D18FEEF; Fri, 7 Mar 2003 17:00:01 +0100 (CET) -TH Return-Path: -TH Subject:Cron /usr/bin/poff libertynico -TH To:root@louloutte.dyndns.org -TH X-Cron-Env: -TH X-Cron-Env: -TH X-Cron-Env: -TH X-Cron-Env: -TH X-Spam-Level:* -TH X-Spam-Status:No, hits=1.6 required=5.0 tests=NO_MX_FOR_FROM,AWL version=2.20 -T msg 7:ri4E1CAmNkh3rP3Mq15HSw:714 -Time headers: 0 s -++++ Verifying [INBOX.yop.yap] -> [INBOX.yop.yap] ++++ -+ key ri4E1CAmNkh3rP3Mq15HSw:714 #1 -Message id [ri4E1CAmNkh3rP3Mq15HSw:714] found in t:INBOX.yop.yap -Setting flags -flags from : \Seen -flags to : \Seen -Looking dates -idate from : 22-Aug-2003 19:16:01 +0200 -idate to : 21-May-2005 12:39:11 +0200 -Message ri4E1CAmNkh3rP3Mq15HSw:714 SZ_GOOD f:1:714 t:7:714 -Time : 0 s -++++ Statistics ++++ -Time : 0 sec -Messages transfered : 0 -Messages skipped : 1 -Total bytes transfered : 0 -Total bytes skipped : 714 -Total bytes error : 0 -Detected 0 errors -Please, rate imapsync at http://freshmeat.net/projects/imapsync/ diff --git a/aaa2 b/aaa2 deleted file mode 100644 index 1e57c69..0000000 --- a/aaa2 +++ /dev/null @@ -1,93 +0,0 @@ -get options: [1] -$RCSfile: imapsync,v $ $Revision: 1.162 $ $Date: 2006/03/22 03:45:05 $ -Mail::IMAPClient version used here is 2.2.9 -will try to use CRAM-MD5 authentication on host1 -will try to use CRAM-MD5 authentication on host2 -From imap server [loul] port [143] user [tata] -To imap server [plume] port [143] user [tata@est.belle] -Banner : * OK louloutte Cyrus IMAP4 v1.5.19 server ready -loul: no support for AUTHENTICATE CRAM-MD5, using LOGIN -Banner : * OK [CAPABILITY IMAP4rev1 UIDPLUS CHILDREN NAMESPACE THREAD=ORDEREDSUBJECT THREAD=REFERENCES SORT QUOTA AUTH=PLAIN CRAM-MD5 CRAM-SHA1 IDLE ACL ACL2=UNION STARTTLS] Courier-IMAP ready. Copyright 1998-2004 Double Precision, Inc. See COPYING for distribution information. -From Buffer I/O : 4096 -To Buffer I/O : 4096 -From capability : QUOTA X-NETSCAPE NAMESPACE X-NON-HIERARCHICAL-RENAME ACL UNSELECT LITERAL+ NO_ATOMIC_RENAME UIDPLUS IMAP4 IMAP4REV1 -To capability : QUOTA STARTTLS NAMESPACE CRAM-SHA1 IDLE AUTH=PLAIN THREAD=ORDEREDSUBJECT ACL SORT UIDPLUS CHILDREN ACL2=UNION CRAM-MD5 IMAP4REV1 THREAD=REFERENCES -From state Authenticated -To state Authenticated -Getting separators -Calling namespace capability -Calling namespace capability -Getting prefix namespace -Calling namespace capability -Getting prefix namespace -Calling namespace capability -From separator and prefix : [.][INBOX.] -To separator and prefix : [.][INBOX.] -From folders : [INBOX.yop.yap] -To folders : [INBOX] [INBOX.2005-INBOX] [INBOX.Draft] [INBOX.Drafts] [INBOX.Sent] [INBOX.Trash] [INBOX.ppp] [INBOX.qqq] [INBOX.yop] [INBOX.yop.yap] [INBOX.yop.yup] [INBOX.yopX] [INBOX.yopX.yap] [INBOX.yopX.yup] [INBOX.zz] -From subscribed folders : [INBOX.yop.yap] -From Folder [INBOX.yop.yap] -removed source prefix : [yop.yap] -inverted separators : [yop.yap] -added target prefix : [INBOX.yop.yap] -To Folder [INBOX.yop.yap] -LIST FROM : 1 messages [1] -LIST TO : 1 messages [7] -++++ From [INBOX.yop.yap] Parse 1 ++++ -Time headers: 1 s -Time fir : 0 s -Head NUM:10 -FH Date:Fri, 7 Mar 2003 17:00:01 +0100 (CET) -FH From:root@louloutte.dyndns.org (Cron Daemon) -FH Message-Id:<20030307160001.69D18FEEF@louloutte.dyndns.org> -FH Received:by louloutte.dyndns.org (Postfix, from userid 0) id 69D18FEEF; Fri, 7 Mar 2003 17:00:01 +0100 (CET) -FH Return-Path: -FH Subject:Cron /usr/bin/poff libertynico -FH To:root@louloutte.dyndns.org -FH X-Cron-Env: -FH X-Cron-Env: -FH X-Cron-Env: -FH X-Cron-Env: -FH X-Spam-Level:* -FH X-Spam-Status:No, hits=1.6 required=5.0 tests=NO_MX_FOR_FROM,AWL version=2.20 -F msg 1:ri4E1CAmNkh3rP3Mq15HSw:714 -Time headers: 0 s -++++ To [INBOX.yop.yap] Parse 1 ++++ -Time headers: 0 s -Time fir : 0 s -Head NUM:10 -TH Date:Fri, 7 Mar 2003 17:00:01 +0100 (CET) -TH From:root@louloutte.dyndns.org (Cron Daemon) -TH Message-Id:<20030307160001.69D18FEEF@louloutte.dyndns.org> -TH Received:by louloutte.dyndns.org (Postfix, from userid 0) id 69D18FEEF; Fri, 7 Mar 2003 17:00:01 +0100 (CET) -TH Return-Path: -TH Subject:Cron /usr/bin/poff libertynico -TH To:root@louloutte.dyndns.org -TH X-Cron-Env: -TH X-Cron-Env: -TH X-Cron-Env: -TH X-Cron-Env: -TH X-Spam-Level:* -TH X-Spam-Status:No, hits=1.6 required=5.0 tests=NO_MX_FOR_FROM,AWL version=2.20 -T msg 7:ri4E1CAmNkh3rP3Mq15HSw:714 -Time headers: 0 s -++++ Verifying [INBOX.yop.yap] -> [INBOX.yop.yap] ++++ -+ key ri4E1CAmNkh3rP3Mq15HSw:714 #1 -Message id [ri4E1CAmNkh3rP3Mq15HSw:714] found in t:INBOX.yop.yap -Setting flags -flags from : \Seen -flags to : \Seen -Looking dates -idate from : 22-Aug-2003 19:16:01 +0200 -idate to : 21-May-2005 12:39:11 +0200 -Message ri4E1CAmNkh3rP3Mq15HSw:714 SZ_GOOD f:1:714 t:7:714 -Time : 0 s -++++ Statistics ++++ -Time : 1 sec -Messages transfered : 0 -Messages skipped : 1 -Total bytes transfered : 0 -Total bytes skipped : 714 -Total bytes error : 0 -Detected 0 errors -Please, rate imapsync at http://freshmeat.net/projects/imapsync/ diff --git a/freshmeat_submition b/freshmeat_submition index 608b24f..a2e317d 100644 --- a/freshmeat_submition +++ b/freshmeat_submition @@ -1,8 +1,10 @@ Project: imapsync -Version: 1.161 -Release-Focus: Major bugfixes +Version: 1.163 +Release-Focus: Minor feature enhancements Hide: Y Home-Page-URL: http://www.linux-france.org/prj/imapsync/ Gzipped-Tar-URL: http://www.linux-france.org/prj/imapsync/dist/ -Code cleanup. +Fixed the bug "Word too large" by spliting requests on +demand with --split1 and --split2 options. + diff --git a/imapsync b/imapsync index b724f2c..d4a3842 100755 --- a/imapsync +++ b/imapsync @@ -6,7 +6,7 @@ imapsync - IMAP synchronization, copy or migration tool. Synchronize mailboxes between two imap servers. Good at IMAP migration. -$Revision: 1.163 $ +$Revision: 1.164 $ =head1 INSTALL @@ -64,6 +64,7 @@ $Revision: 1.163 $ [--dry] [--debug] [--debugimap] [--timeout ] [--fast] + [--split1] [--split2] [--version] [--help] =cut @@ -336,7 +337,7 @@ Gilles LAMIRAL earn his living writing, installing, configuring and teaching free open and gratis softwares. Don't hesitate to pay him for that services. -$Id: imapsync,v 1.163 2006/03/24 04:18:58 gilles Exp $ +$Id: imapsync,v 1.164 2006/03/26 03:17:48 gilles Exp $ =cut @@ -385,12 +386,13 @@ my( $ssl1, $ssl2, $authuser1, $authuser2, $authmech1, $authmech2, + $split1, $split2, ); use vars qw ($opt_G); # missing code for this will be option. -$rcs = ' $Id: imapsync,v 1.163 2006/03/24 04:18:58 gilles Exp $ '; +$rcs = ' $Id: imapsync,v 1.164 2006/03/26 03:17:48 gilles Exp $ '; $rcs =~ m/,v (\d+\.\d+)/; $VERSION = ($1) ? $1 : "UNKNOWN"; @@ -427,8 +429,8 @@ $error=0; my $banner = join("", '$RCSfile: imapsync,v $ ', - '$Revision: 1.163 $ ', - '$Date: 2006/03/24 04:18:58 $ ', + '$Revision: 1.164 $ ', + '$Date: 2006/03/26 03:17:48 $ ', "\n", "Mail::IMAPClient version used here is ", $VERSION_IMAPClient,"\n" @@ -641,6 +643,10 @@ print "From state Authenticated\n"; die unless $to->IsAuthenticated(); print "To state Authenticated\n"; +$split1 and $from->Split($split1); +$split2 and $to->Split($split2); + + my (@f_folders, @t_folders, %fs_folders); # Make a hash of subscribed folders in source server. @@ -771,7 +777,7 @@ sub foldersizes { $smess = $imap->message_count(); unless ($smess == 0) { #$imap->Ranges(1); - $imap->fetch_hash("RFC822.SIZE",$hashref) or die "$@"; + $imap->fetch_hash2("RFC822.SIZE",$hashref) or die "$@"; #$imap->Ranges(0); #print map {$hashref->{$_}->{"RFC822.SIZE"}, " "} keys %$hashref; map {$stot += $hashref->{$_}->{"RFC822.SIZE"}} keys %$hashref; @@ -944,7 +950,7 @@ FOLDER: foreach my $f_fold (@f_folders) { my $f_heads = $from->parse_headers($from->Range([@f_msgs]),@useheader) if (@f_msgs) ; $debug and print "Time headers: ", timenext(), " s\n"; - my $f_fir = $from->fetch_hash("FLAGS", + my $f_fir = $from->fetch_hash2("FLAGS", "INTERNALDATE", "RFC822.SIZE") if (@f_msgs); $debug and print "Time fir : ", timenext(), " s\n"; @@ -958,7 +964,7 @@ FOLDER: foreach my $f_fold (@f_folders) { my $t_heads = $to->parse_headers($to->Range([@t_msgs]),@useheader) if (@t_msgs); $debug and print "Time headers: ", timenext(), " s\n"; - my $t_fir = $to->fetch_hash("FLAGS", + my $t_fir = $to->fetch_hash2("FLAGS", "INTERNALDATE", "RFC822.SIZE") if (@t_msgs); $debug and print "Time fir : ", timenext(), " s\n"; @@ -1221,6 +1227,8 @@ sub get_options "authmech2=s" => \$authmech2, "authuser1=s" => \$authuser1, "authuser2=s" => \$authuser2, + "split1=i" => \$split1, + "split2=i" => \$split2, ); $debug and print "get options: [$opt_ret]\n"; @@ -1408,7 +1416,9 @@ Several options are mandatory. --justconnect : just connect to both servers and print useful information. Need only --host1 and --host2 options. --justfolders : just do things about folders (ignore messages). ---fast : be faster (does not sync flags). +--fast : be faster (just does not sync flags). +--split1 : split the requests in several parts on source server. +--split2 : same thing on the "destination" server. --nofastio1 : don't use fastio with the "from" server. --nofastio2 : don't use fastio with the "destination" server. --timeout : imap connect timeout. @@ -1439,3 +1449,81 @@ sub Authuser { if (@_) { $self->{AUTHUSER} = shift } return $self->{AUTHUSER}; } + + +sub Split { + my $self = shift; + + if (@_) { $self->{SPLIT} = shift } + return $self->{SPLIT}; +} + + + +sub fetch_hash2 { + # taken from original lib, + # just added split code. + my $self = shift; + my $hash = ref($_[-1]) ? pop @_ : {}; + my @words = @_; + for (@words) { + s/([\( ])FAST([\) ])/${1}FLAGS INTERNALDATE RFC822\.SIZE$2/i ; + s/([\( ])FULL([\) ])/${1}FLAGS INTERNALDATE RFC822\.SIZE ENVELOPE BODY$2/i ; + } + my $msgref_all = scalar($self->messages); + my $split = $self->Split() || scalar(@$msgref_all); + while(my @msgs = splice(@$msgref_all, 0, $split)) { + #print "SPLIT: @msgs\n"; + my $msgref = \@msgs; + my $output = scalar($self->fetch($msgref,"(" . join(" ",@_) . ")")) + ; # unless grep(/\b(?:FAST|FULL)\b/i,@words); + my $x; + for ($x = 0; $x <= $#$output ; $x++) { + my $entry = {}; + my $l = $output->[$x]; + if ($self->Uid) { + my($uid) = $l =~ /\((?:.* )?UID (\d+).*\)/i; + next unless $uid; + if ( exists $hash->{$uid} ) { + $entry = $hash->{$uid} ; + } else { + $hash->{$uid} ||= $entry; + } + } else { + my($mid) = $l =~ /^\* (\d+) FETCH/i; + next unless $mid; + if ( exists $hash->{$mid} ) { + $entry = $hash->{$mid} ; + } else { + $hash->{$mid} ||= $entry; + } + } + + foreach my $w (@words) { + if ( $l =~ /\Q$w\E\s*$/i ) { + $entry->{$w} = $output->[$x+1]; + $entry->{$w} =~ s/(?:\x0a?\x0d)+$//g; + chomp $entry->{$w}; + } else { + $l =~ /\( # open paren followed by ... + (?:.*\s)? # ...optional stuff and a space + \Q$w\E\s # escaped fetch field + (?:" # then: a dbl-quote + (\\.| # then bslashed anychar(s) or ... + [^"]+) # ... nonquote char(s) + "| # then closing quote; or ... + \( # ...an open paren + (\\.| # then bslashed anychar or ... + [^\)]+) # ... non-close-paren char + \)| # then closing paren; or ... + (\S+)) # unquoted string + (?:\s.*)? # possibly followed by space-stuff + \) # close paren + /xi; + $entry->{$w}=defined($1)?$1:defined($2)?$2:$3; + } + } + } +} + return wantarray ? %$hash : $hash; +} diff --git a/learn/splice_fetch b/learn/splice_fetch new file mode 100644 index 0000000..93a0cb2 --- /dev/null +++ b/learn/splice_fetch @@ -0,0 +1,12 @@ +#!/usr/bin/perl -w + + + +$rt_big = [1..1000]; + + +while (@t_small = splice(@$rt_big, 0, 33)) { + $rt_small = \@t_small; + print "@{$rt_small}", "\n"; + +}