Sunil's transaction patches.

svn path=/trunk/; revision=3858

1 files changed, 134 insertions, 81 deletions

diff --git a/driver.c b/driver.c
index 6de0ffbd..efe74a9c 100644
--- a/driver.c
+++ b/driver.c
@@ -415,11 +415,55 @@ static void mark_oversized(struct query *ctl, int num, int size)
 }
 
 static int fetch_messages(int mailserver_socket, struct query *ctl, 
-			  int count, int *msgsizes, int *msgcodes, int maxfetch,
+			  int count, int *msgsizes, int maxfetch,
 			  int *fetches, int *dispatches, int *deletions)
 /* fetch messages in lockstep mode */
 {
-    int num, err, len;
+    flag force_retrieval;
+    int num, firstnum = 1, lastnum = 0, err, len;
+    int fetchsizelimit = ctl->fetchsizelimit;
+    int msgsize;
+
+    if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit))
+    {
+	/* for POP3, we can get the size of one mail only! Unfortunately, this
+	 * protocol specific test cannot be done elsewhere as the protocol
+	 * could be "auto". */
+	if (ctl->server.protocol == P_POP3)
+	    fetchsizelimit = 1;
+
+	/* Time to allocate memory to store the sizes */
+	xalloca(msgsizes, int *, sizeof(int) * fetchsizelimit);
+    }
+
+    /*
+     * What forces this code is that in POP2 and
+     * IMAP2bis you can't fetch a message without
+     * having it marked `seen'.  In POP3 and IMAP4, on the
+     * other hand, you can (peek_capable is set by 
+     * each driver module to convey this; it's not a
+     * method constant because of the difference between
+     * IMAP2bis and IMAP4, and because POP3 doesn't  peek
+     * if fetchall is on).
+     *
+     * The result of being unable to peek is that if there's
+     * any kind of transient error (DNS lookup failure, or
+     * sendmail refusing delivery due to process-table limits)
+     * the message will be marked "seen" on the server without
+     * having been delivered.  This is not a big problem if
+     * fetchmail is running in foreground, because the user
+     * will see a "skipped" message when it next runs and get
+     * clued in.
+     *
+     * But in daemon mode this leads to the message
+     * being silently ignored forever.  This is not
+     * acceptable.
+     *
+     * We compensate for this by checking the error
+     * count from the previous pass and forcing all
+     * messages to be considered new if it's nonzero.
+     */
+    force_retrieval = !peek_capable && (ctl->errcount > 0);
 
     for (num = 1; num <= count; num++)
     {
@@ -427,30 +471,82 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	flag suppress_forward = FALSE;
 	flag suppress_readbody = FALSE;
 	flag retained = FALSE;
+	int msgcode = MSGLEN_UNKNOWN;
 
-	if (msgcodes[num-1] < 0)
+	/* check if the message is old
+	 * Note: the size of the message may not be known here */
+	if (ctl->fetchall || force_retrieval)
+	    ;
+	else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num))
+	    msgcode = MSGLEN_OLD;
+	if (msgcode == MSGLEN_OLD)
 	{
-	    if ((msgcodes[num-1] == MSGLEN_TOOLARGE) && !check_only)
-	    {
-		mark_oversized(ctl, num, msgsizes[num-1]);
-		suppress_delete = TRUE;
-	    }
   		/* To avoid flooding the syslog when using --keep,
   		 * report "Skipped message" only when:
   		 *  1) --verbose is on, or
-  		 *  2) fetchmail does not use syslog, or
-  		 *  3) the message was skipped for some other
-  		 *     reason than being old.
+  		 *  2) fetchmail does not use syslog
   		 */
 	    if (   (outlevel >= O_VERBOSE) ||
-	           (outlevel > O_SILENT && (!run.use_syslog || msgcodes[num-1] != MSGLEN_OLD))
+	           (outlevel > O_SILENT && !run.use_syslog)
 	       )
 	    {
 		report_build(stdout, 
+			     GT_("skipping message %s@%s:%d"),
+			     ctl->remotename, ctl->server.truename, num);
+	    }
+
+	    goto flagthemail;
+	}
+
+	if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit) &&
+	    lastnum < num)
+	{
+	    /* Instead of getting the sizes of all mails at the start, we get
+	     * the sizes in blocks of fetchsizelimit. This leads to better
+	     * performance when there are too many mails (say, 10000) in
+	     * the mailbox and either we are not getting all the mails at
+	     * one go (--fetchlimit 100) or there is a frequent socket
+	     * error while just getting the sizes of all mails! */
+
+	    int i;
+	    int oldstage = stage;
+	    firstnum = num;
+	    lastnum = num + fetchsizelimit - 1;
+	    if (lastnum > count)
+		lastnum = count;
+	    for (i = 0; i < fetchsizelimit; i++)
+		msgsizes[i] = 0;
+
+	    stage = STAGE_GETSIZES;
+	    err = (ctl->server.base_protocol->getpartialsizes)(mailserver_socket, num, lastnum, msgsizes);
+	    if (err != 0)
+		return err;
+	    stage = oldstage;
+	}
+
+	msgsize = msgsizes ? msgsizes[num-firstnum] : 0;
+
+	/* check if the message is oversized */
+	if (NUM_NONZERO(ctl->limit) && (msgsize > ctl->limit))
+	    msgcode = MSGLEN_TOOLARGE;
+/*	else if (msgsize == 512)
+	    msgcode = MSGLEN_OLD;  (hmh) sample code to skip message */
+
+	if (msgcode < 0)
+	{
+	    if ((msgcode == MSGLEN_TOOLARGE) && !check_only)
+	    {
+		mark_oversized(ctl, num, msgsize);
+		suppress_delete = TRUE;
+	    }
+	    if (outlevel > O_SILENT)
+	    {
+		/* old messages are already handled above */
+		report_build(stdout, 
 			     GT_("skipping message %s@%s:%d (%d octets)"),
 			     ctl->remotename, ctl->server.truename, num,
-			     msgsizes[num-1]);
-		switch (msgcodes[num-1])
+			     msgsize);
+		switch (msgcode)
 		{
 		case MSGLEN_INVALID:
 		    /*
@@ -483,7 +579,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 		report(stdout,
 			     GT_("couldn't fetch headers, message %s@%s:%d (%d octets)\n"),
 			     ctl->remotename, ctl->server.truename, num,
-			     msgsizes[num-1]);
+			     msgsize);
 		continue;
 	    }
 	    else if (err != 0)
@@ -492,7 +588,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	    /* -1 means we didn't see a size in the response */
 	    if (len == -1)
 	    {
-		len = msgsizes[num - 1];
+		len = msgsize;
 		wholesize = TRUE;
 	    }
 
@@ -515,7 +611,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	     * Read the message headers and ship them to the
 	     * output sink.  
 	     */
-	    err = readheaders(mailserver_socket, len, msgsizes[num-1],
+	    err = readheaders(mailserver_socket, len, msgsize,
 			     ctl, num,
 			     /* pass the suppress_readbody flag only if the underlying
 			      * protocol does not fetch the body separately */
@@ -573,7 +669,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 		     * string.  This violates RFC2060.
 		     */
 		    if (len == -1)
-			len = msgsizes[num-1] - msgblk.msglen;
+			len = msgsize - msgblk.msglen;
 		    if (outlevel > O_SILENT && !wholesize)
 			report_complete(stdout,
 					GT_(" (%d body octets) "), len);
@@ -630,13 +726,13 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	     * QUALCOMM server (at least) seems to be
 	     * reporting the on-disk size correctly.
 	     */
-	    if (msgblk.msglen != msgsizes[num-1])
+	    if (msgblk.msglen != msgsize)
 	    {
 		if (outlevel >= O_DEBUG)
 		    report(stdout,
 			   GT_("message %s@%s:%d was not the expected length (%d actual != %d expected)\n"),
 			   ctl->remotename, ctl->server.truename, num,
-			   msgblk.msglen, msgsizes[num-1]);
+			   msgblk.msglen, msgsize);
 	    }
 
 	    /* end-of-message processing starts here */
@@ -649,6 +745,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 		(*fetches)++;
 	}
 
+flagthemail:
 	/*
 	 * At this point in flow of control, either
 	 * we've bombed on a protocol error or had
@@ -668,9 +765,8 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	}
 	else if (ctl->server.base_protocol->delete
 		 && !suppress_delete
-		 && ((msgcodes[num-1] >= 0 && !ctl->keep)
-		     || (msgcodes[num-1] == MSGLEN_OLD||
-			 msgcodes[num-1] == MSGLEN_TOOLARGE) && ctl->flush))
+		 && ((msgcode >= 0 && !ctl->keep)
+		     || (msgcode == MSGLEN_OLD && ctl->flush)))
 	{
 	    (*deletions)++;
 	    if (outlevel > O_SILENT) 
@@ -689,14 +785,14 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
          		 *  3) the message was skipped for some other
          		 *     reason than just being old.
          		 */
-	           (outlevel > O_SILENT && (!run.use_syslog || msgcodes[num-1] != MSGLEN_OLD))
+	           (outlevel > O_SILENT && (!run.use_syslog || msgcode != MSGLEN_OLD))
 	       )
 	    report_complete(stdout, GT_(" not flushed\n"));
 
 	    /* maybe we mark this message as seen now? */
 	    if (ctl->server.base_protocol->mark_seen
 		&& !suppress_delete
-		&& (msgcodes[num-1] >= 0 && ctl->keep))
+		&& (msgcode >= 0 && ctl->keep))
 	    {
 		err = (ctl->server.base_protocol->mark_seen)(mailserver_socket, ctl, num);
 		if (err != 0)
@@ -827,7 +923,6 @@ const int maxfetch;		/* maximum number of messages to fetch */
 	char buf[MSGBUFSIZE+1], *realhost;
 	int count, new, bytes, deletions = 0;
 	int *msgsizes = (int *)NULL;
-	int *msgcodes = (int *)NULL;
 #if INET6_ENABLE
 	int fetches, dispatches, oldphase;
 #else /* INET6_ENABLE */
@@ -1302,37 +1397,7 @@ is restored."));
 		}
 		else if (count > 0)
 		{    
-		    flag	force_retrieval;
-		    int		i, num;
-
-		    /*
-		     * What forces this code is that in POP2 and
-		     * IMAP2bis you can't fetch a message without
-		     * having it marked `seen'.  In POP3 and IMAP4, on the
-		     * other hand, you can (peek_capable is set by 
-		     * each driver module to convey this; it's not a
-		     * method constant because of the difference between
-		     * IMAP2bis and IMAP4, and because POP3 doesn't  peek
-		     * if fetchall is on).
-		     *
-		     * The result of being unable to peek is that if there's
-		     * any kind of transient error (DNS lookup failure, or
-		     * sendmail refusing delivery due to process-table limits)
-		     * the message will be marked "seen" on the server without
-		     * having been delivered.  This is not a big problem if
-		     * fetchmail is running in foreground, because the user
-		     * will see a "skipped" message when it next runs and get
-		     * clued in.
-		     *
-		     * But in daemon mode this leads to the message
-		     * being silently ignored forever.  This is not
-		     * acceptable.
-		     *
-		     * We compensate for this by checking the error
-		     * count from the previous pass and forcing all
-		     * messages to be considered new if it's nonzero.
-		     */
-		    force_retrieval = !peek_capable && (ctl->errcount > 0);
+		    int		i;
 
 		    /*
 		     * Don't trust the message count passed by the server.
@@ -1347,23 +1412,23 @@ is restored."));
 			return(PS_PROTOCOL);
 		    }
 
-		    /* OK, we're going to gather size info next */
-		    xalloca(msgsizes, int *, sizeof(int) * count);
-		    xalloca(msgcodes, int *, sizeof(int) * count);
-		    for (i = 0; i < count; i++) {
-			msgsizes[i] = 0;
-			msgcodes[i] = MSGLEN_UNKNOWN;
-		    }
-
 		    /* 
 		     * We need the size of each message before it's
 		     * loaded in order to pass it to the ESMTP SIZE
 		     * option.  If the protocol has a getsizes method,
 		     * we presume this means it doesn't get reliable
 		     * sizes from message fetch responses.
+		     *
+		     * If the protocol supports getting sizes of subset of
+		     * messages, we skip this step now.
 		     */
-		    if (proto->getsizes)
+		    if (proto->getsizes &&
+			!(proto->getpartialsizes && NUM_NONZERO(ctl->fetchsizelimit)))
 		    {
+			xalloca(msgsizes, int *, sizeof(int) * count);
+			for (i = 0; i < count; i++)
+			    msgsizes[i] = 0;
+
 			stage = STAGE_GETSIZES;
 			err = (proto->getsizes)(mailserver_socket, count, msgsizes);
 			if (err != 0)
@@ -1377,25 +1442,12 @@ is restored."));
 			}
 		    }
 
-		    /* mark some messages not to be retrieved */
-		    for (num = 1; num <= count; num++)
-		    {
-			if (NUM_NONZERO(ctl->limit) && (msgsizes[num-1] > ctl->limit))
-			    msgcodes[num-1] = MSGLEN_TOOLARGE;
-			else if (ctl->fetchall || force_retrieval)
-			    continue;
-			else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num))
-			    msgcodes[num-1] = MSGLEN_OLD;
-/*			else if (msgsizes[num-1] == 512)
-				msgcodes[num-1] = MSGLEN_OLD;  (hmh) sample code to skip message */
-		    }
-
 		    /* read, forward, and delete messages */
 		    stage = STAGE_FETCH;
 
 		    /* fetch in lockstep mode */
 		    err = fetch_messages(mailserver_socket, ctl, 
-					 count, msgsizes, msgcodes,
+					 count, msgsizes,
 					 maxfetch,
 					 &fetches, &dispatches, &deletions);
 		    if (err)
@@ -1565,7 +1617,8 @@ const struct method *proto;	/* protocol method table */
      * If no expunge limit or we do expunges within the driver,
      * then just do one session, passing in any fetchlimit.
      */
-    if (proto->retry || !NUM_SPECIFIED(ctl->expunge))
+    if ((ctl->keep && !ctl->flush) ||
+	proto->retry || !NUM_SPECIFIED(ctl->expunge))
 	return(do_session(ctl, proto, NUM_VALUE_OUT(ctl->fetchlimit)));
     /*
      * There's an expunge limit, and it isn't handled in the driver itself.