From 091ae4a2e5443a4e486049cc3e455398bfdf8efa Mon Sep 17 00:00:00 2001 From: Nick Bebout Date: Tue, 14 Jun 2011 06:04:24 -0500 Subject: [PATCH] 1.446 --- CREDITS | 9 +- ChangeLog | 58 ++++- FAQ | 41 +++- Makefile | 50 +++- README | 4 +- TIME | 2 + TODO | 4 +- VERSION | 2 +- VERSION_EXE | 2 +- build_exe.bat | 15 +- imapsync | 456 +++++++++++++++++++++++++----------- index.shtml | 25 +- learn/rpm/imapsync.spec | 185 +++++++++++---- paypal_return.shtml | 22 +- paypal_return_support.shtml | 4 +- test2.bat | 11 +- tests.sh | 101 +++++++- 17 files changed, 740 insertions(+), 251 deletions(-) diff --git a/CREDITS b/CREDITS index f2391a8..af4c221 100644 --- a/CREDITS +++ b/CREDITS @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: CREDITS,v 1.157 2011/05/07 02:30:05 gilles Exp gilles $ +# $Id: CREDITS,v 1.159 2011/05/30 21:58:46 gilles Exp gilles $ If you want to make a donation to the author, Gilles LAMIRAL, use any of the following ways: @@ -30,6 +30,13 @@ I thank very much all of these people. I thank also very much all people who bought imapsync from the homepage but I don't cite them here. +Dex Kelson. +Contributed by his patch for a better good_date() with --idatefromheader over 100000 messages. + +Unknow (@baccari.it) +Contributed by giving the book +32.65 "Metamagical Themas: Questing For The Essence Of Mind And Pattern" + Unknow Contributed by giving the book 20.31 "Fluid Concepts And Creative Analogies: Computer Models Of The Fundamental Mechanisms Of Thought" diff --git a/ChangeLog b/ChangeLog index 55c09f4..00af703 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,17 +1,67 @@ RCS file: RCS/imapsync,v Working file: imapsync -head: 1.434 +head: 1.446 branch: locks: strict - gilles: 1.434 + gilles: 1.446 access list: symbolic names: keyword substitution: kv -total revisions: 434; selected revisions: 434 +total revisions: 446; selected revisions: 446 description: ---------------------------- -revision 1.434 locked by: gilles; +revision 1.446 locked by: gilles; +date: 2011/05/31 09:11:18; author: gilles; state: Exp; lines: +17 -17 +Bugfix. Try to handle Markus bug in foldersizes() when select_msgs() returns a list of undef. +---------------------------- +revision 1.445 +date: 2011/05/31 08:00:45; author: gilles; state: Exp; lines: +53 -46 +Check if uidexpunge is supported at the beginning of execution, not when needed. +Set --uidexpunge2 if --delete2 or --expunge2 if uidexpunge not supported. +Changed all warn() calls (STDERR) to print calls (STDOUT) +---------------------------- +revision 1.444 +date: 2011/05/30 15:16:46; author: gilles; state: Exp; lines: +10 -7 +good_date() "24 Aug 77" -> "24-Aug-1977" +---------------------------- +revision 1.443 +date: 2011/05/28 16:50:27; author: gilles; state: Exp; lines: +125 -41 + Patched tests_good_date() and good_date() with Dax Kelson patches. +---------------------------- +revision 1.442 +date: 2011/05/28 16:14:31; author: gilles; state: Exp; lines: +28 -8 +Started code to deal with epoch of messages. +---------------------------- +revision 1.441 +date: 2011/05/26 01:01:25; author: gilles; state: Exp; lines: +14 -11 +Handle better folder creation, not a failure when folder "already exists" during its creation. +---------------------------- +revision 1.440 +date: 2011/05/26 00:40:51; author: gilles; state: Exp; lines: +28 -28 +Replaced default setting. Now --delete2 sets --uidexpunge2 instead of --expunge2 (unless --nouidexpunge2 is set) +---------------------------- +revision 1.439 +date: 2011/05/25 03:11:41; author: gilles; state: Exp; lines: +65 -9 +Added epoch() routine to prepare the safe bidirectional sync (maybe...) +---------------------------- +revision 1.438 +date: 2011/05/25 00:47:27; author: gilles; state: Exp; lines: +9 -8 +Adapted the usage output multiline character to Unix or Win, \ or ^ +---------------------------- +revision 1.437 +date: 2011/05/25 00:29:04; author: gilles; state: Exp; lines: +8 -7 +Bugfix. Avoid a "no number" warning when size is null. +---------------------------- +revision 1.436 +date: 2011/05/23 23:30:20; author: gilles; state: Exp; lines: +8 -12 +Added "Date" in the default --useheader list. ("Message-Id", "Message-ID", "Date") +---------------------------- +revision 1.435 +date: 2011/05/23 23:06:31; author: gilles; state: Exp; lines: +29 -12 +Bugfix. Bad header beginning with a blank character. +---------------------------- +revision 1.434 date: 2011/05/16 07:16:19; author: gilles; state: Exp; lines: +142 -57 Bugfix. Made --usecache work with --maxage or --maxsize or --min* ---------------------------- diff --git a/FAQ b/FAQ index 16d7d52..b25bcee 100644 --- a/FAQ +++ b/FAQ @@ -1,10 +1,34 @@ #!/bin/cat -# $Id: FAQ,v 1.86 2011/05/16 16:43:12 gilles Exp gilles $ +# $Id: FAQ,v 1.88 2011/05/26 00:53:26 gilles Exp gilles $ +------------------+ | FAQ for imapsync | +------------------+ +Unix versus Windows syntax. +On Unix shells you can write a single command on multiple lines +by using the escape character \ at the end of each line +(except the last one). + +./imapsync \ + --host1 imap.truc.org --user1 foo --password1 secret1 \ + --host2 imap.trac.org --user2 bar --password2 secret2 + + +On Windows this character is ^ + +imapsync ^ + --host1 imap.truc.org --user1 foo --password1 secret1 ^ + --host2 imap.trac.org --user2 bar --password2 secret2 + + +Of course you can write the command on one only line without +characters \ nor ^. I use them because the output is +better, no truncation, pretty print. It's just sugar. + +In this FAQ I use \ for examples. Transcript to ^ if +you're on a Windows system. + ======================================================================= Q. How to install imapsync? @@ -519,12 +543,11 @@ Q. I want the --folder 'MyFolder' option be recursive. Two solutions: -R. Use +R1. Use --folderrec 'MyFolder' -R. Do not use the --folder option. - Instead, use --include '^MyFolder' +R2. Use --include '^MyFolder' Then the folder "MyFolder" and all its subfolders will be handled and only them. @@ -1192,6 +1215,10 @@ R. imapsync ... \ ====================================================================== Q: How can I write an .rpm with imapsync -R: I don't know but Neil Brown wrote one rpm package and you'll find - his .spec file here : - http://www.linux-france.org/prj/imapsync/learn/rpm/ + +R. You'll find an RPM imapsync.spec file in the directory learn/rpm/ + It has been downloaded from + https://svn.fysik.dtu.dk/projects/rpmbuild/trunk/SPECS/imapsync.spec + It has been tested with imapsync 1.434 (May 2011) on CentOS5 + and RedHat RHEL5 Linux. (Thanks to Ole Holm Nielsen). + This imapsync.spec is derivated from Neil Brown work in 2007. diff --git a/Makefile b/Makefile index e2801a6..326ed8f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -# $Id: Makefile,v 1.74 2011/05/16 17:25:22 gilles Exp gilles $ +# $Id: Makefile,v 1.79 2011/05/31 21:32:16 gilles Exp gilles $ .PHONY: help usage all @@ -25,6 +25,12 @@ DIST_NAME=imapsync-$(VERSION) DIST_FILE=$(DIST_NAME).tgz DEB_FILE=$(DIST_NAME).deb VERSION=$(shell perl -I./Mail-IMAPClient-2.2.9 ./imapsync --version) +VERSION_EXE=$(shell cat ./VERSION_EXE) + +HELLO=$(shell date;uname -a) + +hello: + echo "$(HELLO)" all: ChangeLog README VERSION @@ -48,7 +54,7 @@ VERSION: imapsync clean: clean_tilde clean_man clean_test: - rm -f .test .test_3xx .test_229 + rm -f .test_3xx .test_229 clean_tilde: rm -f *~ @@ -93,8 +99,9 @@ test_quick_229: imapsync tests.sh test_quick_3xx: imapsync tests.sh CMD_PERL='perl -I./Mail-IMAPClient-3.28/lib' /usr/bin/time sh -x tests.sh locallocal -testv: - sh -x tests.sh +testv2: + CMD_PERL='perl -I./Mail-IMAPClient-2.2.9' /usr/bin/time sh tests.sh + touch .test_229 testv3: CMD_PERL='perl -I./Mail-IMAPClient-3.28/lib' sh -x tests.sh @@ -155,6 +162,7 @@ imapsync.exe: imapsync build_exe.bat test_exe.bat .dosify_bat ssh Admin@c 'C:/msys/1.0/home/Admin/imapsync/test_exe.bat' scp Admin@c:'C:/msys/1.0/home/Admin/imapsync/imapsync.exe' . ssh Admin@c 'C:/msys/1.0/home/Admin/imapsync/imapsync.exe --version' > VERSION_EXE + dos2unix VERSION_EXE (date "+%s"| tr "\n" " "; echo -n "END " $(VERSION) ": "; date) >> .BUILD_EXE_TIME @@ -191,7 +199,7 @@ imapsync_elf_x86.bin: imapsync lfo: cidone niouze_lfo upload_lfo -dist: cidone test clean all INSTALL tarball +dist: cidone test clean all INSTALL dist_prepa dist_prepa_exe tarball: cidone all echo making tarball $(DIST_FILE) @@ -205,6 +213,36 @@ tarball: cidone all cd ../prepa_dist && md5sum -c $(DIST_FILE).md5.txt ls -l ../prepa_dist/$(DIST_FILE) + +DO_IT := $(shell test -f ./dist/path_$(VERSION).txt || makepasswd --chars 4 > ./dist/path_$(VERSION).txt) +DIST_SECRET := $(shell cat ./dist/path_$(VERSION).txt) +DIST_PATH := ./dist/$(DIST_SECRET) + +lalala: + echo $(DIST_SECRET) + +dist_prepa: tarball dist_dir + ln -f ../prepa_dist/$(DIST_FILE) $(DIST_PATH)/ + #cd $(DIST_PATH)/ && md5sum $(DIST_FILE) > $(DIST_FILE).md5.txt + #cd $(DIST_PATH)/ && md5sum -c $(DIST_FILE).md5.txt + ls -l $(DIST_PATH)/ + +dist_dir: + @echo $(DIST_SECRET) + @echo $(DIST_PATH) + mkdir -p $(DIST_PATH) + ln -f ./dist/path_$(VERSION).txt ./dist/path_last.txt + + +dist_prepa_exe: imapsync.exe + mkdir -p $(DIST_PATH) + ln -f ./imapsync.exe $(DIST_PATH)/ + #cd $(DIST_PATH)/ && md5sum ./imapsync.exe > ./imapsync.exe.md5.txt + #cd $(DIST_PATH)/ && md5sum -c ./imapsync.exe.md5.txt + + + + ks: rsync -avz --delete . imapsync@ks.lamiral.info:public_html/imapsync { cd /g/var/paypal_reply/ &&\ @@ -221,7 +259,7 @@ PUBLIC_FILES = ./ChangeLog ./COPYING ./CREDITS ./FAQ \ upload_ks: rsync -lptvHz $(PUBLIC_FILES) \ root@ks.lamiral.info:/var/www/imapsync/ - rsync -lptvHz ./dist/index.shtml \ + rsync -lptvHzr ./dist/ \ root@ks.lamiral.info:/var/www/imapsync/dist/ upload_lfo: diff --git a/README b/README index 04153c8..8304429 100644 --- a/README +++ b/README @@ -3,7 +3,7 @@ NAME Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 36 different IMAP server softwares supported with success. - $Revision: 1.434 $ + $Revision: 1.446 $ SYNOPSIS To synchronise imap account "foo" on "imap.truc.org" to imap account @@ -424,5 +424,5 @@ SIMILAR SOFTWARES Feedback (good or bad) will often be welcome. - $Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ + $Id: imapsync,v 1.446 2011/05/31 09:11:18 gilles Exp gilles $ diff --git a/TIME b/TIME index 3023723..768afef 100644 --- a/TIME +++ b/TIME @@ -1,3 +1,5 @@ + 30 Patched tests_good_date() and good_date() with Dax Kelson patches. +120 Added a good reply for buying support. 540 (1.434) (1.433) (1.432) 180 Tests of mkpath very long path > 300 char. Win32 fails on them. (1.431) Added special case for Inbox vs INBOX bug creation. (1.430) diff --git a/TODO b/TODO index 7c4518c..5913d99 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: TODO,v 1.97 2011/05/16 16:39:38 gilles Exp gilles $ +# $Id: TODO,v 1.98 2011/05/30 21:59:06 gilles Exp gilles $ TODO file for imapsync ---------------------- @@ -23,6 +23,8 @@ Evaluate http://www.rackspace.com/apps/email_hosting/migrations http://www.yippiemove.com/ +Find a way to avoid passwords in --debugimap unless needed. + Fix long path over than 256 character on Win32. Think about Digest::SHA or Digest::SHA::PurePerl. diff --git a/VERSION b/VERSION index d18240f..126917f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.434 +1.446 diff --git a/VERSION_EXE b/VERSION_EXE index c31093c..126917f 100644 --- a/VERSION_EXE +++ b/VERSION_EXE @@ -1 +1 @@ -1.434 +1.446 diff --git a/build_exe.bat b/build_exe.bat index 1598c26..ba3e82d 100755 --- a/build_exe.bat +++ b/build_exe.bat @@ -1,10 +1,19 @@ -REM $Id: build_exe.bat,v 1.8 2010/11/09 01:22:29 gilles Exp gilles $ +REM $Id: build_exe.bat,v 1.9 2011/05/31 08:28:29 gilles Exp gilles $ echo Building imapsync.exe cd C:\msys\1.0\home\Admin\imapsync -perl -mMail::IMAPClient -mDigest::MD5 -mTerm::ReadKey -mIO::Socket::SSL -mFile::Spec -mDigest::HMAC_MD5 -mAuthen::NTLM -e '' +perl -mMail::IMAPClient -mIO::Socket -mIO::Socket::SSL ^ + -mDigest::MD5 -mDigest::HMAC_MD5 ^ + -mTerm::ReadKey -mFile::Spec -mAuthen::NTLM ^ + -mTime::Local ^ + -e '' -pp -o imapsync.exe --link libeay32_.dll --link libssl32_.dll -M Mail::IMAPClient -M IO::Socket -M IO::Socket::SSL -M Digest::MD5 -M Digest::HMAC_MD5 -M Term::ReadKey -M Authen::NTLM imapsync +pp -o imapsync.exe --link libeay32_.dll --link libssl32_.dll ^ + -M Mail::IMAPClient -M IO::Socket -M IO::Socket::SSL ^ + -M Digest::MD5 -M Digest::HMAC_MD5 ^ + -M Term::ReadKey -M Authen::NTLM ^ + -M Time::Local ^ + imapsync echo Done building imapsync.exe diff --git a/imapsync b/imapsync index 9055ab3..8f08011 100755 --- a/imapsync +++ b/imapsync @@ -20,7 +20,7 @@ Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 36 different IMAP server softwares supported with success. -$Revision: 1.434 $ +$Revision: 1.446 $ =head1 SYNOPSIS @@ -498,7 +498,7 @@ Entries for imapsync: Feedback (good or bad) will often be welcome. -$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ +$Id: imapsync,v 1.446 2011/05/31 09:11:18 gilles Exp gilles $ =cut @@ -525,6 +525,7 @@ use IO::Socket qw(:crlf SOL_SOCKET SO_KEEPALIVE); use Errno qw(EAGAIN EPIPE ECONNRESET); use File::Glob qw( :glob ) ; use IO::File; +use Time::Local ; use Test::More 'no_plan'; @@ -610,7 +611,7 @@ my( # global variables initialisation -$rcs = '$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ '; +$rcs = '$Id: imapsync,v 1.446 2011/05/31 09:11:18 gilles Exp gilles $ '; $total_bytes_transferred = 0; $total_bytes_skipped = 0; @@ -625,11 +626,28 @@ $h1_total_bytes_duplicate = $h2_total_bytes_duplicate = 0; $nb_errors = 0; $max_msg_size_in_bytes = 0; +my %month_abrev = ( + Jan => 0, + Feb => 1, + Mar => 2, + Apr => 3, + May => 4, + Jun => 5, + Jul => 6, + Aug => 7, + Sep => 8, + Oct => 9, + Nov => 10, + Dec => 11, +); + unless(defined(&_SYSEXITS_H)) { # 64 on my linux box. eval 'sub EX_USAGE () {64;}' unless defined(&EX_USAGE); } + + # @ARGV will be eat by get_options() my @argv_copy = @ARGV; @@ -731,15 +749,24 @@ if ($delete) { } } -if ( $delete2 ) { - if ( ! defined( $expunge2 ) ) { - $expunge2 = 1 ; - } +if ( $uidexpunge2 and ! Mail::IMAPClient->can( 'uidexpunge' ) ) { + print "Failure: uidexpunge not supported (IMAPClient release < 3.17), use --expunge2 instead\n" ; + exit_clean( 3 ) ; +} + +if ( $delete2 and ! defined( $uidexpunge2 ) ) { + if ( Mail::IMAPClient->can( 'uidexpunge' ) ) { + print "Info: will act as --uidexpunge2\n" ; + $uidexpunge2 = 1 ; + }elsif ( ! defined( $expunge2 ) ) { + print "Info: will act as --expunge2 (no uidexpunge support)\n" ; + $expunge2 = 1 ; + } } if ( $delete and $delete2 ) { print "Warning: using --delete and --delete2 is almost always a bad idea, exiting imapsync\n" ; - exit_clean( 0 ) ; + exit_clean( 4 ) ; } if ($idatefromheader) { @@ -749,10 +776,10 @@ if ($idatefromheader) { } if ($syncinternaldates) { - print "Turned ON syncinternaldates, ", + print "Info: turned ON syncinternaldates, ", "will set the internal dates (arrival dates) on host2 same as host1.\n"; }else{ - print "Turned OFF syncinternaldates\n"; + print "Info: turned OFF syncinternaldates\n"; } if (defined($authmd5) and ($authmd5)) { @@ -788,8 +815,8 @@ if (defined $proxyauth2 && !$authuser2) { $authuser1 ||= $user1; $authuser2 ||= $user2; -print "Will try to use $authmech1 authentication on host1\n"; -print "Will try to use $authmech2 authentication on host2\n"; +print "Info: will try to use $authmech1 authentication on host1\n"; +print "Info: will try to use $authmech2 authentication on host2\n"; $syncacls = (defined($syncacls)) ? $syncacls : 0; $foldersizes = (defined($foldersizes)) ? $foldersizes : 1; @@ -800,16 +827,13 @@ $fastio2 = (defined($fastio2)) ? $fastio2 : 0; $reconnectretry1 = (defined($reconnectretry1)) ? $reconnectretry1 : 3; $reconnectretry2 = (defined($reconnectretry2)) ? $reconnectretry2 : 3; -@useheader = ( "Message-Id", "Message-ID" ) unless ( @useheader ) ; +@useheader = ( "Message-Id", "Message-ID", "Date" ) unless ( @useheader ) ; my %useheader ; # Make a hash %useheader of each --useheader 'key' in uppercase @useheader{ map( { uc( $_ ) } @useheader ) } = ( ) ; -my %useheaderclassic ; -@useheaderclassic{ qw(MESSAGE-ID DATE) } = ( ) ; - #require Data::Dumper ; #print Data::Dumper->Dump( [ \%useheader ] ) ; @@ -1084,7 +1108,6 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { $debug and print '[', map ( { "$_->$cache_1_2_ref->{$_} " } keys %$cache_1_2_ref ), " ]\n"; } - #sleep 4 ; my %h1_hash = (); my %h2_hash = (); @@ -1096,33 +1119,33 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { my @h1_msgs_in_cache = sort { $a <=> $b } keys %$cache_1_2_ref ; my @h2_msgs_in_cache = keys %$cache_2_1_ref ; - my ( %h1_msgs_no_cache, %h2_msgs_no_cache ) ; - %h1_msgs_no_cache = %h1_msgs ; - %h2_msgs_no_cache = %h2_msgs ; - delete @h1_msgs_no_cache{ @h1_msgs_in_cache } ; - delete @h2_msgs_no_cache{ @h2_msgs_in_cache } ; + my ( %h1_msgs_not_in_cache, %h2_msgs_not_in_cache ) ; + %h1_msgs_not_in_cache = %h1_msgs ; + %h2_msgs_not_in_cache = %h2_msgs ; + delete @h1_msgs_not_in_cache{ @h1_msgs_in_cache } ; + delete @h2_msgs_not_in_cache{ @h2_msgs_in_cache } ; - my @h1_msgs_no_cache = keys %h1_msgs_no_cache ; - #print "h1_msgs_no_cache: [@h1_msgs_no_cache]\n" ; - my @h2_msgs_no_cache = keys %h2_msgs_no_cache ; + my @h1_msgs_not_in_cache = keys %h1_msgs_not_in_cache ; + #print "h1_msgs_not_in_cache: [@h1_msgs_not_in_cache]\n" ; + my @h2_msgs_not_in_cache = keys %h2_msgs_not_in_cache ; - my @h2_msgs_delete2_no_cache = () ; + my @h2_msgs_delete2_not_in_cache = () ; %h1_msgs_copy_by_uid = ( ) ; if ( $useuid ) { # use uid so we have to avoid getting header - @h1_msgs_copy_by_uid{ @h1_msgs_no_cache } = ( ) ; - @h2_msgs_delete2_no_cache = @h2_msgs_no_cache if $usecache ; - @h1_msgs_no_cache = ( ) ; - @h2_msgs_no_cache = ( ) ; + @h1_msgs_copy_by_uid{ @h1_msgs_not_in_cache } = ( ) ; + @h2_msgs_delete2_not_in_cache = @h2_msgs_not_in_cache if $usecache ; + @h1_msgs_not_in_cache = ( ) ; + @h2_msgs_not_in_cache = ( ) ; - #print "delete2: @h2_msgs_delete2_no_cache\n"; + #print "delete2: @h2_msgs_delete2_not_in_cache\n"; } $debug and print "Host1 parsing headers of folder [$h1_fold]\n"; my ($h1_heads_ref, $h1_fir_ref) = ({}, {}); - $h1_heads_ref = $imap1->parse_headers([@h1_msgs_no_cache], @useheader) if (@h1_msgs_no_cache); + $h1_heads_ref = $imap1->parse_headers([@h1_msgs_not_in_cache], @useheader) if (@h1_msgs_not_in_cache); $debug and print "Host1 parsing headers of folder [$h1_fold] took ", timenext(), " s\n"; @$h1_fir_ref{@h1_msgs} = (undef); @@ -1132,7 +1155,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { if (@h1_msgs); $debug and print "Host1 getting flags idate and sizes of folder [$h1_fold] took ", timenext(), " s\n"; unless ($h1_fir_ref) { - warn + print "Host1 folder $h1_fold: Could not fetch_hash_2 ", scalar(@h1_msgs), " msgs: ", $imap1->LastError, "\n"; $nb_errors++; @@ -1140,7 +1163,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { } my @h1_msgs_duplicate; - foreach my $m (@h1_msgs_no_cache) { + foreach my $m (@h1_msgs_not_in_cache) { my $rc = parse_header_msg($imap1, $m, $h1_heads_ref, $h1_fir_ref, 'Host1', \%h1_hash); if (! defined($rc)) { my $h1_size = $h1_fir_ref->{$m}->{"RFC822.SIZE"} || 0; @@ -1164,7 +1187,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { $debug and print "Host2 parsing headers of folder [$h2_fold]\n"; my ($h2_heads_ref, $h2_fir_ref) = ( {}, {} ); - $h2_heads_ref = $imap2->parse_headers([@h2_msgs_no_cache], @useheader) if (@h2_msgs_no_cache); + $h2_heads_ref = $imap2->parse_headers([@h2_msgs_not_in_cache], @useheader) if (@h2_msgs_not_in_cache); $debug and print "Host2 parsing headers of folder [$h2_fold] took ", timenext(), " s\n" ; $debug and print "Host2 getting flags idate and sizes of folder [$h2_fold]\n" ; @@ -1174,7 +1197,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { $debug and print "Host2 getting flags idate and sizes of folder [$h2_fold] took ", timenext(), " s\n" ; my @h2_msgs_duplicate; - foreach my $m (@h2_msgs_no_cache) { + foreach my $m (@h2_msgs_not_in_cache) { my $rc = parse_header_msg($imap2, $m, $h2_heads_ref, $h2_fir_ref, 'Host2', \%h2_hash); my $h2_size = $h2_fir_ref->{$m}->{"RFC822.SIZE"} || 0; if (! defined($rc)) { @@ -1200,7 +1223,19 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { my @h2_hash_keys_sorted_by_uid = sort {$h2_hash{$a}{'m'} <=> $h2_hash{$b}{'m'}} keys(%h2_hash); - + if ( 0 ) { + # hashes, keys are uid, values are the internaldates in epoch (best format to compare dates) + my %h1_epoch ; + my %h2_epoch ; + @h1_epoch{ @h1_msgs } = map( { epoch( $h1_fir_ref->{ $_ }->{ 'INTERNALDATE' } ) } @h1_msgs ) ; + @h2_epoch{ @h2_msgs } = map( { epoch( $h2_fir_ref->{ $_ }->{ 'INTERNALDATE' } ) } @h2_msgs ) ; + #print keyval( %h1_epoch ) ; + #print keyval( %h2_epoch ) ; + my $h1_greatest_epoch = max( values %h1_epoch ) ; + print "h1_greatest_epoch $h1_greatest_epoch\n" ; + } + + #next FOLDER ; if($delete2) { my @h2_expunge; foreach my $m_id (@h2_hash_keys_sorted_by_uid) { @@ -1226,24 +1261,22 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { $h2_nb_msg_deleted += 1; } } - foreach my $h2_msg ( @h2_msgs_delete2_no_cache ) { + foreach my $h2_msg ( @h2_msgs_delete2_not_in_cache ) { print "msg $h2_fold/$h2_msg marked \\Deleted [not in cache] on host2\n"; + push(@h2_expunge, $h2_msg) if $uidexpunge2; unless ($dry) { $imap2->delete_message($h2_msg); $h2_nb_msg_deleted += 1; } } - my $cnt = scalar @h2_expunge; - if(@h2_expunge and !$imap2->can("uidexpunge")) { - warn "uidexpunge not supported (< IMAPClient 3.17)\n"; - } - elsif(@h2_expunge) { - print "uidexpunge $cnt message(s)\n"; - $imap2->uidexpunge(\@h2_expunge) if !$dry; + my $cnt = scalar @h2_expunge ; + if( @h2_expunge ) { + print "uidexpunge $cnt message(s)\n" ; + $imap2->uidexpunge( \@h2_expunge ) if ! $dry ; } if ($expunge2){ - print "Expunging host2 folder $h2_fold\n"; - unless($dry) { $imap2->expunge() }; + print "Expunging host2 folder $h2_fold\n" ; + $imap2->expunge( ) if ! $dry ; } } @@ -1319,7 +1352,7 @@ $debug and print "Time: ", timenext(), " s\n"; } sub size_filtered_flag { - my( $h1_size ) = @_ ; + my $h1_size = shift ; if (defined $maxsize and $h1_size >= $maxsize) { return( 1 ) ; @@ -1362,7 +1395,7 @@ sub sync_flags { # we need most of the time. if ( ! $dry and $diff and ! $imap2->store( $h2_msg, "FLAGS.SILENT (@h1_flags)" ) ) { - warn "- msg $h2_fold/$h2_msg could not add flags [@h1_flags]: ", + print "- msg $h2_fold/$h2_msg could not add flags [@h1_flags]: ", $imap2->LastError, "\n"; #$nb_errors++; } @@ -1403,6 +1436,13 @@ sub tests_max { #ok(100 == max(100, "haha", 1), "max 100 42 1"); } +sub keyval { + my %hash = @_ ; + return( join( " ", map( { "$_ => " . $hash{ $_ } } keys %hash ) ) . "\n" ) ; +} + + + sub check_lib_version { $debug and print "IMAPClient $Mail::IMAPClient::VERSION\n"; if ($Mail::IMAPClient::VERSION eq '2.2.9') { @@ -1681,22 +1721,22 @@ sub login_imap { #$imap->connect() myconnect($imap) - or die_clean("Can not open imap connection on [$host] with user [$user]: $@\n"); + or die_clean("Failure: can not open imap connection on [$host] with user [$user]: $@\n"); print "Banner: ", server_banner($imap); if ($imap->has_capability("AUTH=$authmech") or $imap->has_capability($authmech) ) { - printf("Host %s says it has CAPABILITY for AUTHENTICATE %s\n", + printf("Info: host %s says it has CAPABILITY for AUTHENTICATE %s\n", $imap->Server, $authmech); } else { - printf("Host %s says it has NO CAPABILITY for AUTHENTICATE %s\n", + printf("Info: host %s says it has NO CAPABILITY for AUTHENTICATE %s\n", $imap->Server, $authmech); if ($authmech eq 'PLAIN') { - print "Frequently PLAIN is only supported with SSL, ", - "try --ssl1 or --ssl2 option\n"; + print "Info: frequently PLAIN is only supported with SSL, ", + "try --ssl or --tls options\n"; } } @@ -1722,13 +1762,13 @@ sub login_imap { } unless ($imap->login()) { - my $info = "Error login: [$host] with user [$user] auth"; + my $info = "Failure: error login [$host] with user [$user] auth"; my $einfo = $imap->LastError || @{$imap->History}[-1]; chomp($einfo); my $error = "$info [$authmech]: $einfo\n"; print $error; # note: duplicating error on stdout/stderr die_clean($error) if ($authmech eq 'LOGIN' or $imap->IsUnconnected() or $authuser); - print "Trying LOGIN Auth mechanism on [$host] with user [$user]\n"; + print "Info: trying LOGIN Auth mechanism on [$host] with user [$user]\n"; $imap->Authmechanism(""); $imap->login() or die_clean("$info [LOGIN]: ", $imap->LastError, "\n"); @@ -1736,7 +1776,7 @@ sub login_imap { $proxyauth && $imap->proxyauth($user); $split and $imap->Split( $split ) ; - print "Success login on [$host] with user [$user] auth [$authmech]\n"; + print "Info: success login on [$host] with user [$user] auth [$authmech]\n"; return($imap); } @@ -1763,8 +1803,8 @@ sub banner_imapsync { my @argv_copy = @_; my $banner_imapsync = join("", '$RCSfile: imapsync,v $ ', - '$Revision: 1.434 $ ', - '$Date: 2011/05/16 07:16:19 $ ', + '$Revision: 1.446 $ ', + '$Date: 2011/05/31 09:11:18 $ ', "\n",localhost_info(), "\n", "Command line used:\n", "$0 ", command_line_nopassword(@argv_copy), "\n", @@ -1786,10 +1826,10 @@ sub write_pidfile { print "PID file is $pidfile\n"; if (-e $pidfile) { - warn "$pidfile already exists, overwriting it\n"; + print "$pidfile already exists, overwriting it\n"; } open(PIDFILE, ">$pidfile") or do { - warn "Could not open $pidfile for writing"; + print "Could not open $pidfile for writing"; return undef; }; @@ -1820,7 +1860,7 @@ sub missing_option { sub select_folder { my ($imap, $folder, $hostside) = @_; if ( ! $imap->select($folder)) { - warn + print "$hostside folder $folder: Could not select: ", $imap->LastError, "\n"; $nb_errors++; @@ -1842,18 +1882,21 @@ sub create_folder { return( 1 ) ; } if ( ! $dry ){ - if ( ! $imap2->create($h2_fold)){ - warn( "Couldn't create folder [$h2_fold] from [$h1_fold]: ", - $imap2->LastError(), "\n" ); + if ( ! $imap2->create( $h2_fold ) ) { + print( "Couldn't create folder [$h2_fold] from [$h1_fold]: ", + $imap2->LastError( ), "\n" ); $nb_errors++; - return(0); + # success if folder exists ("already exists" error) + return( 1 ) if $imap2->exists( $h2_fold ) ; + # failure since create failed + return( 0 ); }else{ #create succeeded - return(1); + return( 1 ); } }else{ # dry mode, no folder so many imap will fail, assuming failure - return(0); + return( 0 ); } } @@ -2149,14 +2192,14 @@ sub imap2_folder_name { sub foldersizes { my ($side, $imap, @folders) = @_; - my $tot = 0; - my $tmess = 0; + my $total_size = 0; + my $total_nb = 0; my $biggest = 0 ; print "++++ Calculating sizes\n"; foreach my $folder (@folders) { my $stot = 0; - my $smess = 0; + my $nb_msgs = 0; printf("$side folder %-35s", "[$folder]"); if ( 'Host2' eq $side and ! exists( $h2_folders_all{ $folder } ) ) { print(" does not exist yet\n") ; @@ -2168,7 +2211,7 @@ sub foldersizes { } unless ($imap->examine($folder)) { - warn + print "$side Folder $folder: Could not examine: ", $imap->LastError, "\n"; $nb_errors++; @@ -2177,10 +2220,10 @@ sub foldersizes { my $hash_ref = {}; my @msgs = select_msgs($imap); - $smess = scalar(@msgs); + $nb_msgs = scalar(@msgs); my $smax = 0 ; - @$hash_ref{@msgs} = (undef); - unless ($smess == 0) { + @$hash_ref{@msgs} = (undef) if @msgs ; + if ( $nb_msgs > 0 and @msgs ) { $imap->fetch_hash_2("RFC822.SIZE",$hash_ref) or die_clean("$@"); #print map {$hash_ref->{$_}->{"RFC822.SIZE"}, " "} keys %$hash_ref; map {$stot += $hash_ref->{$_}->{"RFC822.SIZE"}} keys %$hash_ref ; @@ -2189,13 +2232,13 @@ sub foldersizes { } printf(" Size: %9s", $stot); - printf(" Messages: %5s", $smess); + printf(" Messages: %5s", $nb_msgs); printf(" Biggest: %9s\n", $smax); - $tot += $stot; - $tmess += $smess; + $total_size += $stot; + $total_nb += $nb_msgs; } - printf ("Nb messages: %11s\n", $tmess ) ; - printf ("Total size: %11s bytes\n", $tot ) ; + printf ("Nb messages: %11s\n", $total_nb ) ; + printf ("Total size: %11s bytes\n", $total_size ) ; printf ("Biggest message: %11s bytes\n", $biggest ) ; printf ("Time: %11s secondes\n", timenext( ) ) ; } @@ -2328,9 +2371,9 @@ sub acls_sync { my($h1_fold, $h2_fold) = @_; if ($syncacls) { my $h1_hash = $imap1->getacl($h1_fold) - or warn "Could not getacl for $h1_fold: $@\n"; + or print "Could not getacl for $h1_fold: $@\n"; my $h2_hash = $imap2->getacl($h2_fold) - or warn "Could not getacl for $h2_fold: $@\n"; + or print "Could not getacl for $h2_fold: $@\n"; my %users = map({ ($_, 1) } (keys(%$h1_hash), keys(%$h2_hash))); foreach my $user (sort(keys(%users))) { my $acl = $h1_hash->{$user} || "none"; @@ -2340,7 +2383,7 @@ sub acls_sync { unless ($dry) { print "setting acl $h2_fold $user $acl\n"; $imap2->setacl($h2_fold, $user, $acl) - or warn "Could not set acl: $@\n"; + or print "Could not set acl: $@\n"; } } } @@ -2520,6 +2563,7 @@ sub lastuid { sub size_filtered { my( $h1_size, $h1_msg, $h1_fold, $h2_fold ) = @_ ; + $h1_size = 0 if ( ! $h1_size ) ; # null if empty or undef if (defined $maxsize and $h1_size >= $maxsize) { print "msg $h1_fold/$h1_msg skipped ($h1_size exceeds maxsize limit $maxsize bytes)\n"; $total_bytes_skipped += $h1_size; @@ -2558,7 +2602,7 @@ sub copy_message { my $string_len = defined( $string ) ? length( $string ) : '' ; # length or undef #print "- msg $h1_fold/$h1_msg {$string_len}\n" ; unless ( defined( $string ) and $string_len ) { # undef or 0 length - warn + print "- msg $h1_fold/$h1_msg {$string_len} S[$h1_size] F[$h1_flags] I[$h1_idate] could not be fetched: ", $imap1->LastError, "\n" ; $nb_errors++ ; @@ -2607,7 +2651,7 @@ sub copy_message { $new_id = $imap2->append_string($h2_fold, $string, $h1_flags, $h1_date); unless($new_id){ no warnings 'uninitialized'; - warn "- msg $h1_fold/$h1_msg {$string_len} couldn't append (Subject:[". + print "- msg $h1_fold/$h1_msg {$string_len} couldn't append (Subject:[". $imap1->subject($h1_msg)."]) to folder $h2_fold: ", $imap2->LastError, "\n"; $nb_errors++; @@ -2900,7 +2944,7 @@ sub clean_cache { or ( ! exists( $h2_msgs_all_hash_ref->{ $uid2 } ) ) ) { $debugcache and print "remove $file\n" ; - unlink( $file ) or warn "$!" ; + unlink( $file ) or print "$!" ; } } @@ -3057,6 +3101,8 @@ sub cache_folder { my $h2_fold_slash = convert_sep_to_slash( $h2_fold, $sep2 ); return( "$cache_dir/$h1_fold_slash/$h2_fold_slash" ) ; + + } sub convert_sep_to_slash { @@ -3393,8 +3439,7 @@ sub parse_header_msg { foreach my $h (sort keys(%$head)){ next if ( ! exists( $useheader{ uc( $h ) } ) - and ! exists( $useheader{ 'ALL' } ) - and ! exists( $useheaderclassic{ uc( $h ) } ) + and ! exists( $useheader{ 'ALL' } ) ) ; foreach my $val (sort @{$head->{$h}}) { # no 8-bit data in headers ! @@ -3506,7 +3551,7 @@ sub check_last_release { } sub imapsync_version { - my $rcs = '$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ '; + my $rcs = '$Id: imapsync,v 1.446 2011/05/31 09:11:18 gilles Exp gilles $ '; $rcs =~ m/,v (\d+\.\d+)/; my $VERSION = ($1) ? $1: "UNKNOWN"; return($VERSION); @@ -3562,7 +3607,7 @@ sub not_long { POSIX::sigaction(SIGALRM, POSIX::SigAction->new(sub { die "alarm" })) - or warn "Error setting SIGALRM handler: $!\n"; + or print "Error setting SIGALRM handler: $!\n"; } eval { @@ -3613,6 +3658,7 @@ sub usage { my $thank = thank_author(); my $warn_release =''; $warn_release = check_last_release() if (not defined($releasecheck)); + my $escape_char = ( 'MSWin32' eq $OSNAME ) ? '^' : '\\'; print < return same string - return($d); - } - - $d = qq("$d"); - return($d); -} - sub memory_consumption { # memory consumed by imapsync until now in bytes return((memory_consumption_of_pids())[0]); @@ -3938,6 +3949,109 @@ sub tests_memory_consumption { ok(print memory_consumption(), "\n"); } +sub good_date { + # two incoming formats: + # header Tue, 24 Aug 2010 16:00:00 +0200 + # internal 24-Aug-2010 16:00:00 +0200 + + # outgoing format: internal date format + # 24-Aug-2010 16:00:00 +0200 + + my $d = shift ; + return ('') if not defined($d); + + if ( $d =~ m{(\d?)(\d-...-\d{4})( \d{2}:\d{2}:\d{2})( (?:\+|-)\d{4})?}o ) { + #print "internal: [$1][$2][$3][$4]\n"; + my ($day_1, $date_rest, $hour, $zone) = ($1,$2,$3,$4); + $day_1 = '0' if ($day_1 eq ''); + $zone = ' +0000' if not defined($zone); + $d = $day_1 . $date_rest . $hour . $zone; + }elsif ($d =~ m{(?:\w{3,}, )?(\d{1,2}),?\s+(\w{3,})\s+(\d{2,4})\s+(\d{1,2})(?::|\.)(\d{1,2})(?:(?::|\.)(\d{1,2}))?\s*((?:\+|-)\d{4})?}o ) { + # Handles any combination of following formats + # Tue, 24 Aug 2010 16:00:00 +0200 -- Standard + # 24 Aug 2010 16:00:00 +0200 -- Missing Day of Week + # Tue, 24 Aug 97 16:00:00 +0200 -- Two digit year + # Tue, 24 Aug 1997 16.00.00 +0200 -- Periods instead of colons + # Tue, 24 Aug 1997 16:00:00 +0200 -- Extra whitespace between year and hour + # Tue, 24 Aug 1997 6:5:2 +0200 -- Single digit hour, min, or second + # Tue, 24, Aug 1997 16:00:00 +0200 -- Extra comma + + #print "header: [$1][$2][$3][$4][$5][$6][$7][$8]\n"; + my ($day, $month, $year, $hour, $min, $sec, $zone) = ($1,$2,$3,$4,$5,$6,$7,$8); + $year = '19' . $year if length($year) == 2 && $year =~ /^[789]/; + $year = '20' . $year if length($year) == 2; + + $month = substr $month, 0, 3 if length($month) > 4; + $day = sprintf("%02d", $day); + $hour = sprintf("%02d", $hour); + $min = sprintf("%02d", $min); + $sec = '00' if not defined($sec); + $sec = sprintf("%02d", $sec); + $zone = '+0000' if not defined($zone); + $d = "$day-$month-$year $hour:$min:$sec $zone"; + + }elsif ($d =~ m{(?:.{3}) (...)\s+(\d{1,2}) (\d{1,2}):(\d{1,2}):(\d{1,2}) (?:\w{3})?\s?(\d{4})}o ) { + # Handles any combination of following formats + # Sun Aug 20 11:55:09 2006 + # Wed Jan 24 11:58:38 MST 2007 + # Wed Jan 2 08:40:57 2008 + + #print "header: [$1][$2][$3][$4][$5][$6]\n"; + my ($month, $day, $hour, $min, $sec, $year) = ($1,$2,$3,$4,$5,$6); + $day = sprintf("%02d", $day); + $hour = sprintf("%02d", $hour); + $min = sprintf("%02d", $min); + $sec = sprintf("%02d", $sec); + $d = "$day-$month-$year $hour:$min:$sec +0000"; + + }elsif ($d =~ m{(\d{2})/(\d{2})/(\d{2}) (\d{2}):(\d{2}):(\d{2})}o ) { + # Handles the following format + # 02/06/09 22:18:08 -- Generated by AVTECH TemPageR devices + + #print "header: [$1][$2][$3][$4][$5][$6]\n"; + my ($month, $day, $year, $hour, $min, $sec) = ($1,$2,$3,$4,$5,$6); + $year = '20' . $year; + my %num2mon = qw(01 Jan 02 Feb 03 Mar 04 Apr 05 May 06 Jun 07 Jul 08 Aug 09 Sep 10 Oct 11 Nov 12 Dec); + $month = $num2mon{$month}; + $d = "$day-$month-$year $hour:$min:$sec +0000"; + + }elsif ($d =~ m{\w{6,}, (\w{3})\w+\s+(\d{1,2}), (\d{4}) (\d{2}):(\d{2}) (AM|PM)}o ) { + # Handles the following format + # Saturday, December 14, 2002 05:00 PM - KBtoys.com order confirmations + + my ($month, $day, $year, $hour, $min, $apm) = ($1,$2,$3,$4,$5,$6); + + $hour += 12 if $apm eq 'PM'; + $day = sprintf("%02d", $day); + $d = "$day-$month-$year $hour:$min:00 +0000"; + + }elsif ($d =~ m{(\w{3}) (\d{1,2}) (\d{4}) (\d{2}):(\d{2}):(\d{2}) ((?:\+|-)\d{4})}o ) { + # Handles the following format + # Saturday, December 14, 2002 05:00 PM - jr.com order confirmations + + my ($month, $day, $year, $hour, $min, $sec, $zone) = ($1,$2,$3,$4,$5,$6,$7); + + $day = sprintf("%02d", $day); + $d = "$day-$month-$year $hour:$min:$sec $zone"; + + }elsif ($d =~ m{(\d{1,2})-(\w{3})-(\d{4})}o ) { + # Handles the following format + # 21-Jun-2001 - register.com domain transfer email circa 2001 + + my ($day, $month, $year) = ($1,$2,$3); + $day = sprintf("%02d", $day); + $d = "$day-$month-$year 11:11:11 +0000"; + + }else{ + # unknown/unmatch => return same string + return($d); + } + + $d = qq("$d"); + return($d); +} + + sub tests_good_date { ok('' eq good_date(), 'good_date no arg'); @@ -3948,6 +4062,25 @@ sub tests_good_date { ok('"01-Sep-2010 16:00:00 +0000"' eq good_date('Wed, 1 Sep 2010 16:00:00'), 'good_date header SP 1digit zone'); ok('"01-Sep-2010 16:00:00 +0200"' eq good_date('Wed, 1 Sep 2010 16:00:00 +0200'), 'good_date header SP 1digit zone'); ok('"01-Sep-2010 16:00:00 +0200"' eq good_date('Wed, 1 Sep 2010 16:00:00 +0200 (CEST)'), 'good_date header SP 1digit zone'); + ok('"06-Feb-2009 22:18:08 +0000"' eq good_date('02/06/09 22:18:08'), 'good_date header TemPageR'); + ok('"02-Jan-2008 08:40:57 +0000"' eq good_date('Wed Jan 2 08:40:57 2008'), 'good_date header dice.com support 1digit day'); + ok('"20-Aug-2006 11:55:09 +0000"' eq good_date('Sun Aug 20 11:55:09 2006'), 'good_date header dice.com support 2digit day'); + ok('"24-Jan-2007 11:58:38 +0000"' eq good_date('Wed Jan 24 11:58:38 MST 2007'), 'good_date header status-now.com'); + ok('"24-Aug-2010 16:00:00 +0200"' eq good_date('24 Aug 2010 16:00:00 +0200'), 'good_date header missing date of week'); + ok('"24-Aug-2067 16:00:00 +0200"' eq good_date('Tue, 24 Aug 67 16:00:00 +0200'), 'good_date header 2digit year'); + ok('"24-Aug-1977 16:00:00 +0200"' eq good_date('Tue, 24 Aug 77 16:00:00 +0200'), 'good_date header 2digit year'); + ok('"24-Aug-1987 16:00:00 +0200"' eq good_date('Tue, 24 Aug 87 16:00:00 +0200'), 'good_date header 2digit year'); + ok('"24-Aug-1997 16:00:00 +0200"' eq good_date('Tue, 24 Aug 97 16:00:00 +0200'), 'good_date header 2digit year'); + ok('"24-Aug-2004 16:00:00 +0200"' eq good_date('Tue, 24 Aug 04 16:00:00 +0200'), 'good_date header 2digit year'); + ok('"24-Aug-1997 16:00:00 +0200"' eq good_date('Tue, 24 Aug 1997 16.00.00 +0200'), 'good_date header period time sep'); + ok('"24-Aug-1997 16:00:00 +0200"' eq good_date('Tue, 24 Aug 1997 16:00:00 +0200'), 'good_date header extra white space type1'); + ok('"24-Aug-1997 05:06:02 +0200"' eq good_date('Tue, 24 Aug 1997 5:6:2 +0200'), 'good_date header 1digit time vals'); + ok('"24-Aug-1997 05:06:02 +0200"' eq good_date('Tue, 24, Aug 1997 05:06:02 +0200'), 'good_date header extra commas'); + ok('"01-Oct-2003 12:45:24 +0000"' eq good_date('Wednesday, 01 October 2003 12:45:24 CDT'), 'good_date header no abbrev'); + ok('"11-Jan-2005 17:58:27 -0500"' eq good_date('Tue, 11 Jan 2005 17:58:27 -0500'), 'good_date extra white space'); + ok('"18-Dec-2002 15:07:00 +0000"' eq good_date('Wednesday, December 18, 2002 03:07 PM'), 'good_date kbtoys.com orders'); + ok('"16-Dec-2004 02:01:49 -0500"' eq good_date('Dec 16 2004 02:01:49 -0500'), 'good_date jr.com orders'); + ok('"21-Jun-2001 11:11:11 +0000"' eq good_date('21-Jun-2001'), 'good_date register.com domain transfer'); } @@ -4054,7 +4187,7 @@ sub decompose_header{ }elsif( $line =~ m/^(\s+)(.*)/ ) { $val = $2 ; #print "DDD only [$val]\n" ; - @{ $header->{ $key } }[ -1 ] .= " $val" ; + @{ $header->{ $key } }[ -1 ] .= " $val" if $key ; }else{ next ; } @@ -4126,18 +4259,72 @@ Received: from plume [192.168.68.7] ok( 'from plume [192.168.68.7] by plume with POP3 (fetchmail-6.3.6) for (single-drop); Mon, 26 Nov 2007 10:39:06 +0100 (CET)' eq $header_dec->{ 'Received' }[1], 'decompose_header: 3' ) ; + +# Bad header beginning with a blank character + $header_dec = decompose_header( +' KEY_1: VAL_1 +KEY_2: VAL_2 + VAL_2_+ + VAL_2_++ +KEY_3: VAL_3 +KEY_1: VAL_1_other +' + ) ; + + ok( 'VAL_3' + eq $header_dec->{ 'KEY_3' }[0], 'decompose_header: Bad header VAL_3' ) ; + + ok( 'VAL_1_other' + eq $header_dec->{ 'KEY_1' }[0], 'decompose_header: Bad header VAL_1_other' ) ; + + ok( 'VAL_2 VAL_2_+ VAL_2_++' + eq $header_dec->{ 'KEY_2' }[0], 'decompose_header: Bad header VAL_2 VAL_2_+ VAL_2_++' ) ; } +sub epoch { + # incoming format: + # internal date 24-Aug-2010 16:00:00 +0200 + + # outgoing format: epoch + + + my $d = shift ; + return ('') if not defined($d); + + my ( $mday, $month, $year, $hour, $min, $sec, $sign, $zone_h, $zone_m ) ; + my $time ; + + if ( $d =~ m{(\d{2})-([A-Z][a-z]{2})-(\d{4}) (\d{2}):(\d{2}):(\d{2}) ((?:\+|-))(\d{2})(\d{2})}o ) { + #print "internal: [$1][$2][$3][$4][$5][$6][$7][$8][$9]\n" ; + ( $mday, $month, $year, $hour, $min, $sec, $sign, $zone_h, $zone_m ) + = ( $1, $2, $3, $4, $5, $6, $7, $8, $9 ) ; + #print "( $mday, $month, $year, $hour, $min, $sec, $sign, $zone_h, $zone_m )\n" ; + + $sign = +1 if ( '+' eq $sign ) ; + $sign = -1 if ( '-' eq $sign ) ; + + $time = timegm( $sec, $min, $hour, $mday, $month_abrev{$month}, $year ) + - $sign * ( 3600 * $zone_h + 60 * $zone_m ) ; + + #print( "$time ", scalar(localtime($time)), "\n"); + } + return( $time ) ; +} + +sub tests_epoch { + ok( '1282658400' eq epoch( '24-Aug-2010 16:00:00 +0200' ), 'epoch 24-Aug-2010 16:00:00 +0200 -> 1282658400' ) ; + ok( '1282658400' eq epoch( '24-Aug-2010 14:00:00 +0000' ), 'epoch 24-Aug-2010 14:00:00 +0000 -> 1282658400' ) ; + ok( '1282658400' eq epoch( '24-Aug-2010 12:00:00 -0200' ), 'epoch 24-Aug-2010 12:00:00 -0200 -> 1282658400' ) ; + ok( '1282658400' eq epoch( '24-Aug-2010 16:01:00 +0201' ), 'epoch 24-Aug-2010 16:01:00 +0201 -> 1282658400' ) ; + ok( '1282658400' eq epoch( '24-Aug-2010 14:01:00 +0001' ), 'epoch 24-Aug-2010 14:01:00 +0001 -> 1282658400' ) ; +} sub tests_debug { SKIP: { skip "No test in normal run" if ( not $tests_debug ); - tests_match_a_cache_file( ) ; - tests_cache_map( ) ; - tests_get_cache( ) ; - tests_clean_cache( ) ; - tests_clean_cache_2( ) ; + tests_good_date(); + tests_epoch( ) ; } } @@ -4172,6 +4359,7 @@ sub tests { tests_mkpath( ) ; tests_extract_header( ) ; tests_decompose_header( ) ; + tests_epoch( ) ; } } @@ -4617,7 +4805,7 @@ use constant NonFolderArg => 1; # Value to pass to Massage to if (! $self->Ignoresizeerrors ) { if ( length($string) != $expected_size ) { - warn "message_string: " . + print "message_string: " . "expected $expected_size bytes but received " . length($string) . "\n"; $self->LastError("message_string: expected ". diff --git a/index.shtml b/index.shtml index 0c96237..666a629 100644 --- a/index.shtml +++ b/index.shtml @@ -5,7 +5,7 @@ Imapsync: an IMAP migration tool ( release <!--#exec cmd="cat VERSION"--> ) - + @@ -68,9 +68,17 @@ where the user plays independently on both sides. Use offlineimap

See ChangeLog to know what's new in details since 2001.

-

New features or bugfixes since previous release 1.411:

+

New features or bugfixes since previous releases:

    +
  • 1.446
    • +
  • Bugfix: Better --idatefromheader behavior (thank to Dax Kelson patches).
    • +
  • Usability: Now --delete2 sets --uidexpunge2 instead of --expunge2 if possible.
    • +
  • Usability: Adapted the usage output multiline character to Unix or Win, \ or ^
    • +
  • Bugfix: Avoid a "not a number" warning when size is null.
    • +
  • Bugfix: Added "Date" in the default --useheader list. It is ("Message-Id", "Message-ID", "Date")
    • +
  • Bugfix: allows bad header beginning with a blank character.
    • +
  • 1.434
  • Bugfix: Changed the way imapsync knows whether a folder exists or not. Exchange might be happy and stop deconnecting for this reason.
  • Bugfix: Reconnections are well done in TLS mode now.
    • @@ -84,17 +92,6 @@ where the user plays independently on both sides. Use offlineimap
-
  • 1.422
    • -
  • Better default behavior: Option --delete2 implies --expunge2 now (unless --noexpunge2 is given.)
    • -
  • Better default behavior: Correct flags case to be RFC compliant on host2 if host1 is not (\SEEN -> \Seen)
    • -
  • Better debug: Added --debugcontent option to avoid debugging content (can be big) with --debug option.
    • -
  • Better debug: Added --debugflags to permit flag debugging only.
    • -
  • Bugfix: The APPEND error then the FETCH 0 byte error is fixed
    • -
  • Bugfix: Options --maxsize --minsize now work with --useuid
    • -
  • Bugfix: Flag sync of already transfered messages now take care of --maxsize --minsize options
    • -
  • Bugfix: Added 0 length message tracking when fetching host1 (to avoid frequently "APPEND {0}" recent issues).
    • -
  • Bugfix: Avoid Inbox vs INBOX case problem ("already exists").
    • -
  • Bugfix: --proxyauth2 was setting proxyauth1 instead of proxyauth2
    • This document last modified on -($Id: index.shtml,v 1.69 2011/05/16 17:10:13 gilles Exp gilles $) +($Id: index.shtml,v 1.70 2011/05/31 16:43:38 gilles Exp gilles $)

    diff --git a/learn/rpm/imapsync.spec b/learn/rpm/imapsync.spec index 507b9e0..b372827 100644 --- a/learn/rpm/imapsync.spec +++ b/learn/rpm/imapsync.spec @@ -1,52 +1,133 @@ -Summary: imapsync a tool to migrate across IMAP servers -URL: http://freshmeat.net/projects/imapsync/ -Name: imapsync -Version: 1.217 -Release: 1 -License: GPL -Group: DICE/Utils -Source: http://www.linux-france.org/prj/imapsync/dist/imapsync-1.217.tgz -Source99: filter-requires-imapsync.sh -BuildArch: noarch -BuildRoot: /var/tmp/%{name}-build -Packager: Neil Brown -Requires: perl(Mail::IMAPClient), perl(Net::SSLeay), perl(IO::Socket::SSL) - -# Working around perl dependency problem, its wrongly matching -# on "use --prefix" in the docs embeded in the code -%define __perl_requires %{SOURCE99} - -%description -imapsync is a tool for facilitating incremental recursive IMAP -transfers from one mailbox to another. It is useful for mailbox -migration, and reduces the amount of data transferred by only copying -messages that are not present on both servers. Read, unread, and -deleted flags are preserved, and the process can be stopped and -resumed. The original messages can optionally be deleted after a -successful transfer. - -%prep -%setup - -%build - -%install -make DESTDIR=$RPM_BUILD_ROOT install - -%files -%defattr(-,root,root) -%doc ChangeLog README INSTALL FAQ CREDITS TODO GPL -/usr/bin/imapsync -/usr/share/man - -%clean -rm -rf $RPM_BUILD_ROOT - -%changelog -* Mon Mar 19 2007 Neil Brown -- Packaged up source tarball into the RPM. Had to add a fix -- to stop the perl_requires script wrongly matching on "use --prefix" -- in the docs as a genuine perl "use module;" - - - +# The source cannot be distributed: +%{!?nosrc: %define nosrc 1} +# to include the source use: +# rpm -bs --define 'nosrc 0' + +%{?!imapsyncver: %define imapsyncver 1.434} + +Summary: Tool to migrate across IMAP servers +Name: imapsync +Version: %{imapsyncver} +Release: 1%{?dist} +License: WTFPL +Group: Applications/Internet +URL: http://www.linux-france.org/prj/imapsync/ + +Source: http://www.linux-france.org/prj/imapsync/dist/imapsync-%{version}.tgz +# The source cannot be distributed: +%if %{nosrc} +NoSource: 0 +%endif + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root + +BuildArch: noarch +BuildRequires: make +BuildRequires: perl(Mail::IMAPClient) >= 3.19 +BuildRequires: perl(Test::More) +Requires: perl(Date::Manip) +Requires: perl(Digest::MD5) +Requires: perl(IO::Socket::SSL) +Requires: perl(Mail::IMAPClient) >= 3.19 +Requires: perl(Term::ReadKey) +Requires: perl(Digest::HMAC_MD5) +#Requires: perl(Digest::MD5::M4p) +#Requires: perl(Net::SSLeay) + +# http://fedoraproject.org/wiki/Packaging:AutoProvidesAndRequiresFiltering +%{?filter_setup: +%filter_from_requires /^perl(--prefix2)/d +%filter_setup +} +%{!?filter_setup: +# filter_setup undefined +%define __perl_requires %{_builddir}/%{buildsubdir}/filter-requires-imapsync.sh +} + +%description +imapsync is a tool for facilitating incremental recursive IMAP +transfers from one mailbox to another. It is useful for mailbox +migration, and reduces the amount of data transferred by only copying +messages that are not present on both servers. Read, unread, and +deleted flags are preserved, and the process can be stopped and +resumed. The original messages can optionally be deleted after a +successful transfer. + +%prep +%setup -q + +%{!?filter_setup: +%{__cat} <<'EOF' >filter-requires-imapsync.sh +#!/bin/sh +/usr/lib/rpm/perl.req $* | sed -e '/perl(--prefix2)/d' +EOF +%{__chmod} a+x filter-requires-imapsync.sh +} + +%build + +%install +%{__rm} -rf %{buildroot} +%{__make} install DESTDIR="%{buildroot}" + +%files +%defattr(-, root, root, 0755) +%doc ChangeLog COPYING CREDITS FAQ INSTALL README TODO +%doc %{_mandir}/man1/imapsync.1* +%{_bindir}/imapsync + +%clean +%{__rm} -rf %{buildroot} + +%changelog +* Fri Mar 25 2011 Marcin Dulak - 1.440-1 +- Updated to release 1.440. +- introduced nosrc variable: source must not be distributed +- license is WTFPL: see ChangeLog +- use filter-requires-imapsync.sh when filter_setup undefined +- removed Authority: dag + +* Tue Sep 07 2010 Dag Wieers - 1.350-1 +- Updated to release 1.350. + +* Wed Jan 13 2010 Steve Huff - 1.293-1 +- Updated to version 1.293. + +* Sun Dec 20 2009 Steve Huff - 1.286-2 +- Added missing Perl dependencies (reported by John Thomas). + +* Thu Sep 10 2009 Dag Wieers - 1.286-1 +- Updated to release 1.286. + +* Thu Jul 09 2009 Christoph Maser - 1.285-1 +- Updated to release 1.285. + +* Mon Jun 30 2008 Dag Wieers - 1.255-1 +- Updated to release 1.255. + +* Fri May 09 2008 Dag Wieers - 1.252-1 +- Updated to release 1.252. + +* Sun Apr 27 2008 Dag Wieers - 1.250-1 +- Updated to release 1.250. + +* Wed Mar 26 2008 Dag Wieers - 1.249-1 +- Updated to release 1.249. + +* Mon Feb 11 2008 Dag Wieers - 1.241-1 +- Updated to release 1.241. + +* Thu Nov 22 2007 Dag Wieers - 1.233-1 +- Updated to release 1.233. + +* Thu Sep 13 2007 Dag Wieers - 1.223-1 +- Updated to release 1.223. + +* Thu Aug 16 2007 Fabian Arrotin - 1.219-1 +- Update to 1.219. +- Cosmetic changes for Requires: specific to RHEL/CentOS. + +* Mon Mar 19 2007 Neil Brown +- Packaged up source tarball into the RPM. Had to add a fix + to stop the perl_requires script wrongly matching on "use --prefix" + in the docs as a genuine perl "use module;" diff --git a/paypal_return.shtml b/paypal_return.shtml index 2acf084..8c91dee 100644 --- a/paypal_return.shtml +++ b/paypal_return.shtml @@ -5,7 +5,7 @@ imapsync download - + @@ -42,12 +42,9 @@ You may log into your account at www.paypal.com to view details of this transaction.

    -

    You will find the latest imapsync source code release 1.422 at the following link:
    -    http://www.linux-france.org/depot/2011_05_09/EocZFt/ -

    - -

    You will find the latest imapsync.exe binary release 1.422 at the following link:
    -http://www.linux-france.org/depot/2011_05_09/XhVbYj/ +

    You will find the latest imapsync.exe binary release
    +and the latest imapsync source code release at this +/">download page.

    You will receive an invoice soon.

    @@ -55,15 +52,16 @@ to view details of this transaction.

    Next imapsync releases will be available for one year without extra payment.
    I will send you a message explaining how to get them

    -

    To avoid loosing time, explain your specific needs, find best solutions
    -and then succeed your migration you can buy professionnal support at the link
    -http://www.linux-france.org/prj/imapsync/#buy_support +

    To explain your specific needs, find best solutions for them, avoid loosing time,
    +and then succeed your migration quickly you can buy +professionnal support.

    I thank you again for buying and using imapsync,
    I wish you successful imap transfers!

    -

    imapsync homepage

    +

    Back to imapsync homepage. +

    Gilles LAMIRAL
    gilles.lamiral@laposte.net

    @@ -83,7 +81,7 @@ gilles.lamiral@laposte.net

    This document last modified on
    -($Id: paypal_return.shtml,v 1.7 2011/05/16 17:23:12 gilles Exp gilles $) +($Id: paypal_return.shtml,v 1.9 2011/05/31 08:28:00 gilles Exp gilles $)

    diff --git a/paypal_return_support.shtml b/paypal_return_support.shtml index 4d0ea17..efe8cbd 100644 --- a/paypal_return_support.shtml +++ b/paypal_return_support.shtml @@ -51,7 +51,7 @@ to view details of this transaction.

    Now you can contact me (Gilles LAMIRAL) by email or phone

    • Email address: gilles.lamiral@laposte.net.
      • -
    • Professionnal phone number: +33 9 51 84 42 42 (in France) I can call you back for free in many countries.
      • +
    • Professionnal phone number: +33 951 84 42 42 (in France) I can call you back for free in many countries.
    • Mobile phone number: +33 620 79 76 06 (in France).
    @@ -78,7 +78,7 @@ gilles.lamiral@laposte.net

    This document last modified on
    -($Id: paypal_return_support.shtml,v 1.2 2011/04/19 13:09:12 gilles Exp gilles $) +($Id: paypal_return_support.shtml,v 1.3 2011/05/20 12:32:25 gilles Exp gilles $)

    diff --git a/test2.bat b/test2.bat index f21a1c0..b6c55fb 100755 --- a/test2.bat +++ b/test2.bat @@ -1,5 +1,5 @@ -REM $Id: test2.bat,v 1.2 2011/05/16 16:41:47 gilles Exp gilles $ +REM $Id: test2.bat,v 1.3 2011/05/30 21:58:08 gilles Exp gilles $ cd C:\msys\1.0\home\Admin\imapsync REM perl ./imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --delete2 --expunge2 --folder INBOX @@ -14,8 +14,9 @@ REM imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 ti REM perl imapsync --version REM perl imapsync --tests_debug -imapsync.exe ^ - --host1 p --user1 big1 --passfile1 secret.big1 ^ - --host2 p --user2 big2 --passfile2 secret.big2 ^ - --folder INBOX.bigmail +REM imapsync.exe ^ +REM --host1 p --user1 big1 --passfile1 secret.big1 ^ +REM --host2 p --user2 big2 --passfile2 secret.big2 ^ +REM --folder INBOX.bigmail +perl imapsync diff --git a/tests.sh b/tests.sh index 557985e..dda382d 100644 --- a/tests.sh +++ b/tests.sh @@ -1,6 +1,6 @@ #!/bin/sh -# $Id: tests.sh,v 1.165 2011/05/16 16:41:11 gilles Exp gilles $ +# $Id: tests.sh,v 1.168 2011/05/31 16:42:38 gilles Exp gilles $ # Example 1: # CMD_PERL='perl -I./Mail-IMAPClient-3.25/lib' sh -x tests.sh @@ -297,6 +297,24 @@ ll_few_emails() { --folder INBOX.few_emails } +ll_few_emails_dev() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.few_emails --nofoldersizes +} + +ll_size_null() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.size_null +} + ll_noheader() { $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 tata \ @@ -336,9 +354,20 @@ ll_folderrec() { --passfile1 ../../var/pass/secret.tata \ --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ - --folderrec INBOX.yop + --folderrec INBOX.yop --debugimap --justfolders } +ll_folderrec_star() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folderrec 'INBOX.yop.*' --justfolders +} + + + ll_folderrec_blank_bug() { $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 tata \ @@ -712,6 +741,19 @@ ll_maxage() --maxage 1 } +ll_maxage_nonew() +{ + can_send && sendtestmessage + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --maxage 1 --nofoldersizes \ + --folder INBOX.few_emails +} + + ll_newmessage() { can_send && sendtestmessage @@ -1274,9 +1316,7 @@ ll_authmech_CRAMMD5() { } ll_delete2() { - if can_send; then - sendtestmessage titi - fi + can_send && sendtestmessage titi $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 tata \ --passfile1 ../../var/pass/secret.tata \ @@ -1286,6 +1326,55 @@ ll_delete2() { --delete2 --expunge2 } +ll_delete2_minage() { + can_send && sendtestmessage titi + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX \ + --delete2 --expunge2 --minage 1 +} + +ll_delete2_minage_useuid() { + can_send && sendtestmessage titi + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX \ + --delete2 --uidexpunge2 --minage 1 --useuid +} + +ll_delete2_uidexpunge2_implicit() { + can_send && sendtestmessage titi + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX \ + --delete2 --useuid +} + + + + +ll_delete2_dev() { + can_send && sendtestmessage titi + can_send && sendtestmessage + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX --nofoldersizes \ + --delete2 +} + + ll_delete() { if can_send; then sendtestmessage titi @@ -1301,7 +1390,7 @@ ll_delete() { ll_delete_delete2() { - $CMD_PERL ./imapsync \ + ! $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 titi \ --passfile1 ../../var/pass/secret.titi \ --host2 $HOST2 --user2 tata \