Sunil's transaction patches.

svn path=/trunk/; revision=3858

16 files changed, 539 insertions, 121 deletions

diff --git a/NEWS b/NEWS
index ddb45980..e745a571 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,7 @@
   option.
 * Smash all NULs out of headers right after the socket read.
 * Dup-killer code now keys on an MD5 hash of the raw headers.
+* Sunil Shetye's patches to break up fetching of sizes and UIDLs.
 
 fetchmail-6.2.4 (Wed Aug 13 04:27:35 EDT 2003), 22625 lines:
 
diff --git a/conf.c b/conf.c
index ef80a6e0..77a4e2ca 100644
--- a/conf.c
+++ b/conf.c
@@ -376,6 +376,8 @@ void dump_config(struct runctl *runp, struct query *querylist)
 	numdump("limit", ctl->limit);
 	numdump("warnings", ctl->warnings);
 	numdump("fetchlimit", ctl->fetchlimit);
+	numdump("fetchsizelimit", ctl->fetchsizelimit);
+	numdump("fastuidl", ctl->fastuidl);
 	numdump("batchlimit", ctl->batchlimit);
 #ifdef SSL_ENABLE
 	booldump("ssl", ctl->use_ssl);
diff --git a/driver.c b/driver.c
index 6de0ffbd..efe74a9c 100644
--- a/driver.c
+++ b/driver.c
@@ -415,11 +415,55 @@ static void mark_oversized(struct query *ctl, int num, int size)
 }
 
 static int fetch_messages(int mailserver_socket, struct query *ctl, 
-			  int count, int *msgsizes, int *msgcodes, int maxfetch,
+			  int count, int *msgsizes, int maxfetch,
 			  int *fetches, int *dispatches, int *deletions)
 /* fetch messages in lockstep mode */
 {
-    int num, err, len;
+    flag force_retrieval;
+    int num, firstnum = 1, lastnum = 0, err, len;
+    int fetchsizelimit = ctl->fetchsizelimit;
+    int msgsize;
+
+    if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit))
+    {
+	/* for POP3, we can get the size of one mail only! Unfortunately, this
+	 * protocol specific test cannot be done elsewhere as the protocol
+	 * could be "auto". */
+	if (ctl->server.protocol == P_POP3)
+	    fetchsizelimit = 1;
+
+	/* Time to allocate memory to store the sizes */
+	xalloca(msgsizes, int *, sizeof(int) * fetchsizelimit);
+    }
+
+    /*
+     * What forces this code is that in POP2 and
+     * IMAP2bis you can't fetch a message without
+     * having it marked `seen'.  In POP3 and IMAP4, on the
+     * other hand, you can (peek_capable is set by 
+     * each driver module to convey this; it's not a
+     * method constant because of the difference between
+     * IMAP2bis and IMAP4, and because POP3 doesn't  peek
+     * if fetchall is on).
+     *
+     * The result of being unable to peek is that if there's
+     * any kind of transient error (DNS lookup failure, or
+     * sendmail refusing delivery due to process-table limits)
+     * the message will be marked "seen" on the server without
+     * having been delivered.  This is not a big problem if
+     * fetchmail is running in foreground, because the user
+     * will see a "skipped" message when it next runs and get
+     * clued in.
+     *
+     * But in daemon mode this leads to the message
+     * being silently ignored forever.  This is not
+     * acceptable.
+     *
+     * We compensate for this by checking the error
+     * count from the previous pass and forcing all
+     * messages to be considered new if it's nonzero.
+     */
+    force_retrieval = !peek_capable && (ctl->errcount > 0);
 
     for (num = 1; num <= count; num++)
     {
@@ -427,30 +471,82 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	flag suppress_forward = FALSE;
 	flag suppress_readbody = FALSE;
 	flag retained = FALSE;
+	int msgcode = MSGLEN_UNKNOWN;
 
-	if (msgcodes[num-1] < 0)
+	/* check if the message is old
+	 * Note: the size of the message may not be known here */
+	if (ctl->fetchall || force_retrieval)
+	    ;
+	else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num))
+	    msgcode = MSGLEN_OLD;
+	if (msgcode == MSGLEN_OLD)
 	{
-	    if ((msgcodes[num-1] == MSGLEN_TOOLARGE) && !check_only)
-	    {
-		mark_oversized(ctl, num, msgsizes[num-1]);
-		suppress_delete = TRUE;
-	    }
   		/* To avoid flooding the syslog when using --keep,
   		 * report "Skipped message" only when:
   		 *  1) --verbose is on, or
-  		 *  2) fetchmail does not use syslog, or
-  		 *  3) the message was skipped for some other
-  		 *     reason than being old.
+  		 *  2) fetchmail does not use syslog
   		 */
 	    if (   (outlevel >= O_VERBOSE) ||
-	           (outlevel > O_SILENT && (!run.use_syslog || msgcodes[num-1] != MSGLEN_OLD))
+	           (outlevel > O_SILENT && !run.use_syslog)
 	       )
 	    {
 		report_build(stdout, 
+			     GT_("skipping message %s@%s:%d"),
+			     ctl->remotename, ctl->server.truename, num);
+	    }
+
+	    goto flagthemail;
+	}
+
+	if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit) &&
+	    lastnum < num)
+	{
+	    /* Instead of getting the sizes of all mails at the start, we get
+	     * the sizes in blocks of fetchsizelimit. This leads to better
+	     * performance when there are too many mails (say, 10000) in
+	     * the mailbox and either we are not getting all the mails at
+	     * one go (--fetchlimit 100) or there is a frequent socket
+	     * error while just getting the sizes of all mails! */
+
+	    int i;
+	    int oldstage = stage;
+	    firstnum = num;
+	    lastnum = num + fetchsizelimit - 1;
+	    if (lastnum > count)
+		lastnum = count;
+	    for (i = 0; i < fetchsizelimit; i++)
+		msgsizes[i] = 0;
+
+	    stage = STAGE_GETSIZES;
+	    err = (ctl->server.base_protocol->getpartialsizes)(mailserver_socket, num, lastnum, msgsizes);
+	    if (err != 0)
+		return err;
+	    stage = oldstage;
+	}
+
+	msgsize = msgsizes ? msgsizes[num-firstnum] : 0;
+
+	/* check if the message is oversized */
+	if (NUM_NONZERO(ctl->limit) && (msgsize > ctl->limit))
+	    msgcode = MSGLEN_TOOLARGE;
+/*	else if (msgsize == 512)
+	    msgcode = MSGLEN_OLD;  (hmh) sample code to skip message */
+
+	if (msgcode < 0)
+	{
+	    if ((msgcode == MSGLEN_TOOLARGE) && !check_only)
+	    {
+		mark_oversized(ctl, num, msgsize);
+		suppress_delete = TRUE;
+	    }
+	    if (outlevel > O_SILENT)
+	    {
+		/* old messages are already handled above */
+		report_build(stdout, 
 			     GT_("skipping message %s@%s:%d (%d octets)"),
 			     ctl->remotename, ctl->server.truename, num,
-			     msgsizes[num-1]);
-		switch (msgcodes[num-1])
+			     msgsize);
+		switch (msgcode)
 		{
 		case MSGLEN_INVALID:
 		    /*
@@ -483,7 +579,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 		report(stdout,
 			     GT_("couldn't fetch headers, message %s@%s:%d (%d octets)\n"),
 			     ctl->remotename, ctl->server.truename, num,
-			     msgsizes[num-1]);
+			     msgsize);
 		continue;
 	    }
 	    else if (err != 0)
@@ -492,7 +588,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	    /* -1 means we didn't see a size in the response */
 	    if (len == -1)
 	    {
-		len = msgsizes[num - 1];
+		len = msgsize;
 		wholesize = TRUE;
 	    }
 
@@ -515,7 +611,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	     * Read the message headers and ship them to the
 	     * output sink.  
 	     */
-	    err = readheaders(mailserver_socket, len, msgsizes[num-1],
+	    err = readheaders(mailserver_socket, len, msgsize,
 			     ctl, num,
 			     /* pass the suppress_readbody flag only if the underlying
 			      * protocol does not fetch the body separately */
@@ -573,7 +669,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 		     * string.  This violates RFC2060.
 		     */
 		    if (len == -1)
-			len = msgsizes[num-1] - msgblk.msglen;
+			len = msgsize - msgblk.msglen;
 		    if (outlevel > O_SILENT && !wholesize)
 			report_complete(stdout,
 					GT_(" (%d body octets) "), len);
@@ -630,13 +726,13 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	     * QUALCOMM server (at least) seems to be
 	     * reporting the on-disk size correctly.
 	     */
-	    if (msgblk.msglen != msgsizes[num-1])
+	    if (msgblk.msglen != msgsize)
 	    {
 		if (outlevel >= O_DEBUG)
 		    report(stdout,
 			   GT_("message %s@%s:%d was not the expected length (%d actual != %d expected)\n"),
 			   ctl->remotename, ctl->server.truename, num,
-			   msgblk.msglen, msgsizes[num-1]);
+			   msgblk.msglen, msgsize);
 	    }
 
 	    /* end-of-message processing starts here */
@@ -649,6 +745,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 		(*fetches)++;
 	}
 
+flagthemail:
 	/*
 	 * At this point in flow of control, either
 	 * we've bombed on a protocol error or had
@@ -668,9 +765,8 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 	}
 	else if (ctl->server.base_protocol->delete
 		 && !suppress_delete
-		 && ((msgcodes[num-1] >= 0 && !ctl->keep)
-		     || (msgcodes[num-1] == MSGLEN_OLD||
-			 msgcodes[num-1] == MSGLEN_TOOLARGE) && ctl->flush))
+		 && ((msgcode >= 0 && !ctl->keep)
+		     || (msgcode == MSGLEN_OLD && ctl->flush)))
 	{
 	    (*deletions)++;
 	    if (outlevel > O_SILENT) 
@@ -689,14 +785,14 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
          		 *  3) the message was skipped for some other
          		 *     reason than just being old.
          		 */
-	           (outlevel > O_SILENT && (!run.use_syslog || msgcodes[num-1] != MSGLEN_OLD))
+	           (outlevel > O_SILENT && (!run.use_syslog || msgcode != MSGLEN_OLD))
 	       )
 	    report_complete(stdout, GT_(" not flushed\n"));
 
 	    /* maybe we mark this message as seen now? */
 	    if (ctl->server.base_protocol->mark_seen
 		&& !suppress_delete
-		&& (msgcodes[num-1] >= 0 && ctl->keep))
+		&& (msgcode >= 0 && ctl->keep))
 	    {
 		err = (ctl->server.base_protocol->mark_seen)(mailserver_socket, ctl, num);
 		if (err != 0)
@@ -827,7 +923,6 @@ const int maxfetch;		/* maximum number of messages to fetch */
 	char buf[MSGBUFSIZE+1], *realhost;
 	int count, new, bytes, deletions = 0;
 	int *msgsizes = (int *)NULL;
-	int *msgcodes = (int *)NULL;
 #if INET6_ENABLE
 	int fetches, dispatches, oldphase;
 #else /* INET6_ENABLE */
@@ -1302,37 +1397,7 @@ is restored."));
 		}
 		else if (count > 0)
 		{    
-		    flag	force_retrieval;
-		    int		i, num;
-
-		    /*
-		     * What forces this code is that in POP2 and
-		     * IMAP2bis you can't fetch a message without
-		     * having it marked `seen'.  In POP3 and IMAP4, on the
-		     * other hand, you can (peek_capable is set by 
-		     * each driver module to convey this; it's not a
-		     * method constant because of the difference between
-		     * IMAP2bis and IMAP4, and because POP3 doesn't  peek
-		     * if fetchall is on).
-		     *
-		     * The result of being unable to peek is that if there's
-		     * any kind of transient error (DNS lookup failure, or
-		     * sendmail refusing delivery due to process-table limits)
-		     * the message will be marked "seen" on the server without
-		     * having been delivered.  This is not a big problem if
-		     * fetchmail is running in foreground, because the user
-		     * will see a "skipped" message when it next runs and get
-		     * clued in.
-		     *
-		     * But in daemon mode this leads to the message
-		     * being silently ignored forever.  This is not
-		     * acceptable.
-		     *
-		     * We compensate for this by checking the error
-		     * count from the previous pass and forcing all
-		     * messages to be considered new if it's nonzero.
-		     */
-		    force_retrieval = !peek_capable && (ctl->errcount > 0);
+		    int		i;
 
 		    /*
 		     * Don't trust the message count passed by the server.
@@ -1347,23 +1412,23 @@ is restored."));
 			return(PS_PROTOCOL);
 		    }
 
-		    /* OK, we're going to gather size info next */
-		    xalloca(msgsizes, int *, sizeof(int) * count);
-		    xalloca(msgcodes, int *, sizeof(int) * count);
-		    for (i = 0; i < count; i++) {
-			msgsizes[i] = 0;
-			msgcodes[i] = MSGLEN_UNKNOWN;
-		    }
-
 		    /* 
 		     * We need the size of each message before it's
 		     * loaded in order to pass it to the ESMTP SIZE
 		     * option.  If the protocol has a getsizes method,
 		     * we presume this means it doesn't get reliable
 		     * sizes from message fetch responses.
+		     *
+		     * If the protocol supports getting sizes of subset of
+		     * messages, we skip this step now.
 		     */
-		    if (proto->getsizes)
+		    if (proto->getsizes &&
+			!(proto->getpartialsizes && NUM_NONZERO(ctl->fetchsizelimit)))
 		    {
+			xalloca(msgsizes, int *, sizeof(int) * count);
+			for (i = 0; i < count; i++)
+			    msgsizes[i] = 0;
+
 			stage = STAGE_GETSIZES;
 			err = (proto->getsizes)(mailserver_socket, count, msgsizes);
 			if (err != 0)
@@ -1377,25 +1442,12 @@ is restored."));
 			}
 		    }
 
-		    /* mark some messages not to be retrieved */
-		    for (num = 1; num <= count; num++)
-		    {
-			if (NUM_NONZERO(ctl->limit) && (msgsizes[num-1] > ctl->limit))
-			    msgcodes[num-1] = MSGLEN_TOOLARGE;
-			else if (ctl->fetchall || force_retrieval)
-			    continue;
-			else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num))
-			    msgcodes[num-1] = MSGLEN_OLD;
-/*			else if (msgsizes[num-1] == 512)
-				msgcodes[num-1] = MSGLEN_OLD;  (hmh) sample code to skip message */
-		    }
-
 		    /* read, forward, and delete messages */
 		    stage = STAGE_FETCH;
 
 		    /* fetch in lockstep mode */
 		    err = fetch_messages(mailserver_socket, ctl, 
-					 count, msgsizes, msgcodes,
+					 count, msgsizes,
 					 maxfetch,
 					 &fetches, &dispatches, &deletions);
 		    if (err)
@@ -1565,7 +1617,8 @@ const struct method *proto;	/* protocol method table */
      * If no expunge limit or we do expunges within the driver,
      * then just do one session, passing in any fetchlimit.
      */
-    if (proto->retry || !NUM_SPECIFIED(ctl->expunge))
+    if ((ctl->keep && !ctl->flush) ||
+	proto->retry || !NUM_SPECIFIED(ctl->expunge))
 	return(do_session(ctl, proto, NUM_VALUE_OUT(ctl->fetchlimit)));
     /*
      * There's an expunge limit, and it isn't handled in the driver itself.
diff --git a/etrn.c b/etrn.c
index 2a25f942..f7d4d48a 100644
--- a/etrn.c
+++ b/etrn.c
@@ -135,6 +135,7 @@ const static struct method etrn =
     NULL,		/* no need to get authentication */
     etrn_getrange,	/* initialize message sending */
     NULL,		/* we cannot get a list of sizes */
+    NULL,		/* we cannot get a list of sizes of subsets */
     NULL,		/* how do we tell a message is old? */
     NULL,		/* no way to fetch headers */
     NULL,		/* no way to fetch body */
diff --git a/fetchmail.c b/fetchmail.c
index 6b26a6b2..2901bb67 100644
--- a/fetchmail.c
+++ b/fetchmail.c
@@ -641,13 +641,20 @@ int main(int argc, char **argv)
 			continue;
 #endif /* (defined(linux) && !INET6_ENABLE) || defined(__FreeBSD__) */
 
+		    dofastuidl = 0; /* this is reset in the driver if required */
+
 		    querystatus = query_host(ctl);
 
+		    if (NUM_NONZERO(ctl->fastuidl))
+			ctl->fastuidlcount = (ctl->fastuidlcount + 1) % ctl->fastuidl;
 #ifdef POP3_ENABLE
 		    /* leave the UIDL state alone if there have been any errors */
 		    if (!check_only &&
 				((querystatus==PS_SUCCESS) || (querystatus==PS_NOMAIL) || (querystatus==PS_MAXFETCH)))
 			uid_swap_lists(ctl);
+		    else
+			uid_discard_new_list(ctl);
+		    uid_reset_num(ctl);
 #endif  /* POP3_ENABLE */
 
 		    if (querystatus == PS_SUCCESS)
@@ -872,6 +879,8 @@ static void optmerge(struct query *h2, struct query *h1, int force)
     FLAG_MERGE(limit);
     FLAG_MERGE(warnings);
     FLAG_MERGE(fetchlimit);
+    FLAG_MERGE(fetchsizelimit);
+    FLAG_MERGE(fastuidl);
     FLAG_MERGE(batchlimit);
 #ifdef	SSL_ENABLE
     FLAG_MERGE(use_ssl);
@@ -910,6 +919,8 @@ static int load_params(int argc, char **argv, int optind)
     def_opts.warnings = WARNING_INTERVAL;
     def_opts.remotename = user;
     def_opts.listener = SMTP_MODE;
+    def_opts.fetchsizelimit = 100;
+    def_opts.fastuidl = 10;
 
     /* get the location of rcfile */
     rcfiledir[0] = 0;
@@ -1632,6 +1643,20 @@ static void dump_params (struct runctl *runp,
 		       ctl->fetchlimit, ctl->fetchlimit);
 	    else if (outlevel >= O_VERBOSE)
 		printf(GT_("  No received-message limit (--fetchlimit 0).\n"));
+	    if (NUM_NONZERO(ctl->fetchsizelimit))
+		printf(GT_("  Fetch message size limit is %d (--fetchsizelimit %d).\n"),
+		       ctl->fetchsizelimit, ctl->fetchsizelimit);
+	    else if (outlevel >= O_VERBOSE)
+		printf(GT_("  No fetch message size limit (--fetchsizelimit 0).\n"));
+	    if (NUM_NONZERO(ctl->fastuidl) && MAILBOX_PROTOCOL(ctl))
+	    {
+		if (ctl->fastuidl == 1)
+		    printf(GT_("  Do binary search of UIDs during each poll (--fastuidl 1).\n"));
+		else
+		    printf(GT_("  Do binary search of UIDs during %d out of %d polls (--fastuidl %d).\n"), ctl->fastuidl - 1, ctl->fastuidl, ctl->fastuidl);
+	    }
+	    else if (outlevel >= O_VERBOSE)
+		printf(GT_("   Do linear search of UIDs during each poll (--fastuidl 0).\n"));
 	    if (NUM_NONZERO(ctl->batchlimit))
 		printf(GT_("  SMTP message batch limit is %d.\n"), ctl->batchlimit);
 	    else if (outlevel >= O_VERBOSE)
diff --git a/fetchmail.h b/fetchmail.h
index 6665ab06..ac89f42c 100644
--- a/fetchmail.h
+++ b/fetchmail.h
@@ -147,7 +147,7 @@ struct idlist
     {
 	struct
 	{
-	    short	num;
+	    int		num;
 	    flag	mark;		/* UID-index information */
 #define UID_UNSEEN	0		/* hasn't been seen */
 #define UID_SEEN	1		/* seen, but not deleted */
@@ -182,6 +182,8 @@ struct method		/* describe methods for protocol state machine */
 				/* get message range to fetch */
     int (*getsizes)(int, int, int *);
 				/* get sizes of messages */
+    int (*getpartialsizes)(int, int, int, int *);
+				/* get sizes of subset of messages */
     int (*is_old)(int, struct query *, int);
 				/* check for old message */
     int (*fetch_headers)(int, struct query *, int, int *);
@@ -289,6 +291,9 @@ struct query
     int	limit;			/* limit size of retrieved messages */
     int warnings;		/* size warning interval */
     int	fetchlimit;		/* max # msgs to get in single poll */
+    int fetchsizelimit;		/* max # msg sizes to get in a request */
+    int fastuidl;		/* do binary search for new UIDLs? */
+    int fastuidlcount;		/* internal count for frequency of binary search */
     int	batchlimit;		/* max # msgs to pass in single SMTP session */
     int	expunge;		/* max # msgs to pass between expunges */
     flag use_ssl;		/* use SSL encrypted session */
@@ -511,6 +516,8 @@ unsigned char *reply_hack(unsigned char *, const unsigned char *, int *);
 unsigned char *nxtaddr(const unsigned char *);
 
 /* uid.c: UID support */
+extern int dofastuidl;
+
 void initialize_saved_lists(struct query *, const char *);
 struct idlist *save_str(struct idlist **, const char *, flag);
 void free_str_list(struct idlist **);
@@ -518,17 +525,20 @@ struct idlist *copy_str_list(struct idlist *idl);
 void save_str_pair(struct idlist **, const char *, const char *);
 void free_str_pair_list(struct idlist **);
 int delete_str(struct idlist **, long);
-int str_in_list(struct idlist **, const char *, const flag);
+struct idlist *str_in_list(struct idlist **, const char *, const flag);
 int str_nr_in_list(struct idlist **, const char *);
 int str_nr_last_in_list(struct idlist **, const char *);
 void str_set_mark( struct idlist **, const char *, const flag);
 int count_list( struct idlist **idl );
 char *str_from_nr_list( struct idlist **idl, long number );
 char *str_find(struct idlist **, long);
+struct idlist *id_find(struct idlist **idl, long);
 char *idpair_find(struct idlist **, const char *);
 void append_str_list(struct idlist **, struct idlist **);
 void expunge_uids(struct query *);
 void uid_swap_lists(struct query *);
+void uid_discard_new_list(struct query *ctl);
+void uid_reset_num(struct query *ctl);
 void write_saved_lists(struct query *, const char *);
 
 /* rcfile_y.y */
diff --git a/fetchmail.man b/fetchmail.man
index 06dc9b41..8d29fefb 100644
--- a/fetchmail.man
+++ b/fetchmail.man
@@ -487,6 +487,29 @@ poll.  By default there is no limit. An explicit --fetchlimit of 0
 overrides any limits set in your run control file.
 This option does not work with ETRN or ODMR.
 .TP
+.B \-\-fetchsizelimit <number>
+(Keyword: fetchsizelimit)
+Limit the number of sizes of messages accepted from a given server in
+a single transaction.  This option is useful in reducing the delay in
+downloading the first mail when there are too many mails in the
+mailbox.  By default, the limit is 100.  If set to 0, sizes of all
+messages are downloaded at the start.
+This option does not work with ETRN or ODMR.  For POP3, the only valid
+non-zero value is 1.
+.TP
+.B \-\-fastuidl <number>
+(Keyword: fastuidl)
+Do a binary instead of linear search for the first unseen UID. Binary
+search avoids downloading the UIDs of all mails. This saves time
+(especially in daemon mode) where downloading the same set of UIDs in
+each poll is a waste of bandwidth. The number `n' indicates how rarely
+a linear search should be done. In daemon mode, linear search is used
+once followed by binary searches in `n-1' polls if `n' is greater than
+1; binary search is always used if `n' is 1; linear search is always
+used if `n' is 0. In non-daemon mode, binary search is used if `n' is
+1; otherwise linear search is used.
+This option works with POP3 only.
+.TP
 .B \-e <count> | \-\-expunge <count>
 (keyword: expunge) 
 Arrange for deletions to be made final after a given number of
@@ -1479,6 +1502,12 @@ T}
 fetchlimit	-B	T{
 Max # messages to fetch in single connect
 T}
+fetchsizelimit	\&	T{
+Max # message sizes to fetch in single transaction
+T}
+fastuidl	\&	T{
+Use binary search for first unseen message (POP3 only)
+T}
 expunge 	-e	T{
 Perform an expunge on every #th message (IMAP and POP3 only)
 T}
diff --git a/fetchmailconf b/fetchmailconf
index 88564c56..ef41a5df 100755
--- a/fetchmailconf
+++ b/fetchmailconf
@@ -250,6 +250,8 @@ class User:
 	self.limit = 0		# Message size limit
 	self.warnings = 3600	# Size warning interval (see tunable.h)
 	self.fetchlimit = 0	# Max messages fetched per batch
+	self.fetchsizelimit = 100	# Max message sizes fetched per transaction
+	self.fastuidl = 10	# Do fast uidl 9 out of 10 times
 	self.batchlimit = 0	# Max message forwarded per batch
 	self.expunge = 0	# Interval between expunges (IMAP)
 	self.ssl = 0		# Enable Seccure Socket Layer
@@ -287,6 +289,8 @@ class User:
 	    ('limit',       'Int'),
 	    ('warnings',    'Int'),
 	    ('fetchlimit',  'Int'),
+	    ('fetchsizelimit',  'Int'),
+	    ('fastuidl',    'Int'),
 	    ('batchlimit',  'Int'),
 	    ('expunge',     'Int'),
 	    ('ssl',	 'Boolean'),
@@ -347,6 +351,10 @@ class User:
 	    res = res + " warnings " + `self.warnings`
 	if self.fetchlimit != UserDefaults.fetchlimit:
 	    res = res + " fetchlimit " + `self.fetchlimit`
+	if self.fetchsizelimit != UserDefaults.fetchsizelimit:
+	    res = res + " fetchsizelimit " + `self.fetchsizelimit`
+	if self.fastuidl != UserDefaults.fastuidl:
+	    res = res + " fastuidl " + `self.fastuidl`
 	if self.batchlimit != UserDefaults.batchlimit:
 	    res = res + " batchlimit " + `self.batchlimit`
 	if self.ssl and self.ssl != UserDefaults.ssl:
@@ -1701,6 +1709,11 @@ class UserEdit(Frame, MyWidget):
 		      self.warnings, '30').pack(side=TOP, fill=X)
 	    LabeledEntry(limwin, 'Max messages to fetch per poll:',
 		      self.fetchlimit, '30').pack(side=TOP, fill=X)
+	    LabeledEntry(limwin, 'Max message sizes to fetch per transaction:',
+		      self.fetchsizelimit, '30').pack(side=TOP, fill=X)
+	    if self.parent.server.protocol not in ('ETRN', 'ODMR'):
+		LabeledEntry(limwin, 'Use fast UIDL:',
+			self.fastuidl, '30').pack(side=TOP, fill=X)
 	    LabeledEntry(limwin, 'Max messages to forward per poll:',
 		      self.batchlimit, '30').pack(side=TOP, fill=X)
 	    if self.parent.server.protocol not in ('ETRN', 'ODMR'):
diff --git a/imap.c b/imap.c
index e3a6f4ec..13bc170d 100644
--- a/imap.c
+++ b/imap.c
@@ -785,8 +785,8 @@ static int imap_getrange(int sock,
     return(PS_SUCCESS);
 }
 
-static int imap_getsizes(int sock, int count, int *sizes)
-/* capture the sizes of all messages */
+static int imap_getpartialsizes(int sock, int first, int last, int *sizes)
+/* capture the sizes of messages #first-#last */
 {
     char buf [MSGBUFSIZE+1];
 
@@ -824,14 +824,15 @@ static int imap_getsizes(int sock, int count, int *sizes)
      * on the fact that the sizes array has been preinitialized with a
      * known-bad size value.
      */
-    /* if fetchall is specified, startcount is 1;
-     * else if there is new mail, startcount is first unseen message;
-     * else startcount is greater than count.
-     */
-    if (count == startcount)
-	gen_send(sock, "FETCH %d RFC822.SIZE", count);
-    else if (count > startcount)
-	gen_send(sock, "FETCH %d:%d RFC822.SIZE", startcount, count);
+
+    /* expunges change the fetch numbers */
+    first -= expunged;
+    last -= expunged;
+
+    if (last == first)
+	gen_send(sock, "FETCH %d RFC822.SIZE", last);
+    else if (last > first)
+	gen_send(sock, "FETCH %d:%d RFC822.SIZE", first, last);
     else /* no unseen messages! */
 	return(PS_SUCCESS);
     for (;;)
@@ -852,8 +853,8 @@ static int imap_getsizes(int sock, int count, int *sizes)
 	    break;
 	else if (sscanf(buf, "* %u FETCH (RFC822.SIZE %u)", &num, &size) == 2) 
 	{
-	    if (num > 0 && num <= count)
-	        sizes[num - 1] = size;
+	    if (num >= first && num <= last)
+	        sizes[num - first] = size;
 	    else
 		report(stderr, "Warning: ignoring bogus data for message sizes returned by the server.\n");
 	}
@@ -862,6 +863,12 @@ static int imap_getsizes(int sock, int count, int *sizes)
     return(PS_SUCCESS);
 }
 
+static int imap_getsizes(int sock, int count, int *sizes)
+/* capture the sizes of all messages */
+{
+    return imap_getpartialsizes(sock, 1, count, sizes);
+}
+
 static int imap_is_old(int sock, struct query *ctl, int number)
 /* is the given message old? */
 {
@@ -1119,6 +1126,7 @@ const static struct method imap =
     imap_getauth,	/* get authorization */
     imap_getrange,	/* query range of messages */
     imap_getsizes,	/* get sizes of messages (used for ESMTP SIZE option) */
+    imap_getpartialsizes,	/* get sizes of subset of messages (used for ESMTP SIZE option) */
     imap_is_old,	/* no UID check */
     imap_fetch_headers,	/* request given message headers */
     imap_fetch_body,	/* request given message body */
diff --git a/odmr.c b/odmr.c
index c007b185..4f4aeca9 100644
--- a/odmr.c
+++ b/odmr.c
@@ -224,6 +224,7 @@ const static struct method odmr =
     NULL,		/* no need to get authentication */
     odmr_getrange,	/* initialize message sending */
     NULL,		/* we cannot get a list of sizes */
+    NULL,		/* we cannot get a list of sizes of subsets */
     NULL,		/* how do we tell a message is old? */
     NULL,		/* no way to fetch headers */
     NULL,		/* no way to fetch body */
diff --git a/options.c b/options.c
index 8ab2adce..f52fd6ff 100644
--- a/options.c
+++ b/options.c
@@ -86,6 +86,8 @@
 #define LA_SSLFINGERPRINT	60
 #endif
 
+#define LA_FETCHSIZELIMIT	61
+#define LA_FASTUIDL	62
 
 /* options still left: CDgGhHjJoORwWxXYz */
 static const char *shortoptions = 
@@ -140,6 +142,8 @@ static const struct option longoptions[] = {
   
   {"batchlimit",required_argument, (int *) 0, LA_BATCHLIMIT  },
   {"fetchlimit",required_argument, (int *) 0, LA_FETCHLIMIT  },
+  {"fetchsizelimit",required_argument, (int *) 0, LA_FETCHSIZELIMIT  },
+  {"fastuidl",	required_argument, (int *) 0, LA_FASTUIDL    },
   {"expunge",	required_argument, (int *) 0, LA_EXPUNGE     },
   {"mda",	required_argument, (int *) 0, LA_MDA         },
   {"bsmtp",	required_argument, (int *) 0, LA_BSMTP       },
@@ -529,6 +533,14 @@ struct query *ctl;	/* option record to be initialized */
 	    c = xatoi(optarg, &errflag);
 	    ctl->fetchlimit = NUM_VALUE_IN(c);
 	    break;
+	case LA_FETCHSIZELIMIT:
+	    c = xatoi(optarg, &errflag);
+	    ctl->fetchsizelimit = NUM_VALUE_IN(c);
+	    break;
+	case LA_FASTUIDL:
+	    c = xatoi(optarg, &errflag);
+	    ctl->fastuidl = NUM_VALUE_IN(c);
+	    break;
 	case 'e':
 	case LA_EXPUNGE:
 	    c = xatoi(optarg, &errflag);
@@ -705,6 +717,8 @@ struct query *ctl;	/* option record to be initialized */
 	P(GT_("  -Z, --antispam,   set antispam response values\n"));
 	P(GT_("  -b, --batchlimit  set batch limit for SMTP connections\n"));
 	P(GT_("  -B, --fetchlimit  set fetch limit for server connections\n"));
+	P(GT_("      --fetchsizelimit set fetch message size limit\n"));
+	P(GT_("      --fastuidl    do a binary search for UIDLs\n"));
 	P(GT_("  -e, --expunge     set max deletions between expunges\n"));
         P(GT_("  -m, --mda         set MDA to use for forwarding\n"));
         P(GT_("      --bsmtp       set output BSMTP file\n"));
diff --git a/pop2.c b/pop2.c
index 5b4c2e2c..b6af74bc 100644
--- a/pop2.c
+++ b/pop2.c
@@ -141,6 +141,7 @@ const static struct method pop2 =
     pop2_getauth,			/* get authorization */
     pop2_getrange,			/* query range of messages */
     NULL,				/* no way to get sizes */
+    NULL,				/* no way to get sizes of subsets */
     NULL,				/* messages are always new */
     pop2_fetch,				/* request given message */
     NULL,				/* no way to fetch body alone */
diff --git a/pop3.c b/pop3.c
index c1c46f9d..2deb9699 100644
--- a/pop3.c
+++ b/pop3.c
@@ -613,6 +613,77 @@ static int pop3_gettopid( int sock, int num , char *id)
     return 0;
 }
 
+static int pop3_getuidl( int sock, int num , char *id)
+{
+    int ok;
+    char buf [POPBUFSIZE+1];
+    gen_send(sock, "UIDL %d", num);
+    if ((ok = pop3_ok(sock, buf)) != 0)
+	return(ok);
+    if (sscanf(buf, "%d %s", &num, id) != 2)
+	return(PS_PROTOCOL);
+    return(PS_SUCCESS);
+}
+
+static int pop3_fastuidl( int sock,  struct query *ctl, unsigned int count, int *newp)
+{
+    int ok;
+    unsigned int first_nr, last_nr, try_nr;
+    char id [IDLEN+1];
+
+    first_nr = 0;
+    last_nr = count + 1;
+    while (first_nr < last_nr - 1)
+    {
+	struct idlist	*new;
+
+	try_nr = (first_nr + last_nr) / 2;
+	if( (ok = pop3_getuidl( sock, try_nr, id )) != 0 )
+	    return ok;
+	if ((new = str_in_list(&ctl->oldsaved, id, FALSE)))
+	{
+	    flag mark = new->val.status.mark;
+	    if (mark == UID_DELETED || mark == UID_EXPUNGED)
+	    {
+		if (outlevel >= O_VERBOSE)
+		    report(stderr, GT_("id=%s (num=%d) was deleted, but is still present!\n"), id, try_nr);
+		/* just mark it as seen now! */
+		new->val.status.mark = mark = UID_SEEN;
+	    }
+
+	    /* narrow the search region! */
+	    if (mark == UID_UNSEEN)
+	    {
+		if (outlevel >= O_DEBUG)
+		    report(stdout, GT_("%u is unseen\n"), try_nr);
+		last_nr = try_nr;
+	    }
+	    else
+		first_nr = try_nr;
+
+	    /* save the number */
+	    new->val.status.num = try_nr;
+	}
+	else
+	{
+	    if (outlevel >= O_DEBUG)
+		report(stdout, GT_("%u is unseen\n"), try_nr);
+	    last_nr = try_nr;
+
+	    /* save it */
+	    new = save_str(&ctl->oldsaved, id, UID_UNSEEN);
+	    new->val.status.num = try_nr;
+	}
+    }
+    if (outlevel >= O_DEBUG && last_nr <= count)
+	report(stdout, GT_("%u is first unseen\n"), last_nr);
+
+    /* update last! */
+    *newp = count - first_nr;
+    last = first_nr;
+    return 0;
+}
+
 static int pop3_slowuidl( int sock,  struct query *ctl, int *countp, int *newp)
 {
     /* This approach tries to get the message headers from the
@@ -737,8 +808,23 @@ static int pop3_getrange(int sock,
     *newp = -1;
     if (*countp > 0 && !ctl->fetchall)
     {
+	int fastuidl;
 	char id [IDLEN+1];
 
+	/* should we do fast uidl this time? */
+	fastuidl = ctl->fastuidl;
+	if (*countp > 7 &&		/* linear search is better if there are few mails! */
+	    !ctl->flush &&		/* with flush, it is safer to disable fastuidl */
+	    NUM_NONZERO (fastuidl))
+	{
+	    if (fastuidl == 1)
+		dofastuidl = 1;
+	    else
+		dofastuidl = ctl->fastuidlcount != 0;
+	}
+	else
+	    dofastuidl = 0;
+
 	if (!ctl->server.uidl) {
 	    gen_send(sock, "LAST");
 	    ok = pop3_ok(sock, buf);
@@ -755,6 +841,8 @@ static int pop3_getrange(int sock,
 	}
  	else
  	{
+	    if (dofastuidl)
+		return(pop3_fastuidl( sock, ctl, *countp, newp));
 	    /* grab the mailbox's UID list */
 	    if ((ok = gen_transact(sock, "UIDL")) != 0)
 	    {
@@ -776,17 +864,41 @@ static int pop3_getrange(int sock,
  			break;
  		    else if (sscanf(buf, "%d %s", &num, id) == 2)
 		    {
- 			struct idlist	*new;
+ 			struct idlist	*old, *new;
 
 			new = save_str(&ctl->newsaved, id, UID_UNSEEN);
 			new->val.status.num = num;
 
-			if (str_in_list(&ctl->oldsaved, id, FALSE)) {
-			    new->val.status.mark = UID_SEEN;
-			    str_set_mark(&ctl->oldsaved, id, UID_SEEN);
+			if ((old = str_in_list(&ctl->oldsaved, id, FALSE)))
+			{
+			    flag mark = old->val.status.mark;
+			    if (mark == UID_DELETED || mark == UID_EXPUNGED)
+			    {
+				if (outlevel >= O_VERBOSE)
+				    report(stderr, GT_("id=%s (num=%d) was deleted, but is still present!\n"), id, num);
+				/* just mark it as seen now! */
+				old->val.status.mark = mark = UID_SEEN;
+			    }
+			    new->val.status.mark = mark;
+			    if (mark == UID_UNSEEN)
+			    {
+				(*newp)++;
+				if (outlevel >= O_DEBUG)
+				    report(stdout, GT_("%u is unseen\n"), num);
+			    }
 			}
 			else
+			{
 			    (*newp)++;
+			    if (outlevel >= O_DEBUG)
+				report(stdout, GT_("%u is unseen\n"), num);
+			    /* add it to oldsaved also! In case, we do not
+			     * swap the lists (say, due to socket error),
+			     * the same mail will not be downloaded again.
+			     */
+			    old = save_str(&ctl->oldsaved, id, UID_UNSEEN);
+			    old->val.status.num = num;
+			}
 		    }
  		}
  	    }
@@ -796,6 +908,32 @@ static int pop3_getrange(int sock,
     return(PS_SUCCESS);
 }
 
+static int pop3_getpartialsizes(int sock, int first, int last, int *sizes)
+/* capture the size of message #first */
+{
+    int	ok;
+    char buf [POPBUFSIZE+1];
+    unsigned int num, size;
+
+    /* for POP3, we can get the size of one mail only! */
+    if (first != last)
+    {
+	report(stderr, "cannot get a range of message sizes (%d-%d).\n", first, last);
+	return(PS_PROTOCOL);
+    }
+    gen_send(sock, "LIST %d", first);
+    if ((ok = pop3_ok(sock, buf)) != 0)
+	return(ok);
+    if (sscanf(buf, "%u %u", &num, &size) == 2) {
+	if (num == first)
+	    sizes[0] = size;
+	else
+	    /* warn about possible attempt to induce buffer overrun */
+	    report(stderr, "Warning: ignoring bogus data for message sizes returned by server.\n");
+    }
+    return(ok);
+}
+
 static int pop3_getsizes(int sock, int count, int *sizes)
 /* capture the sizes of all messages */
 {
@@ -829,11 +967,42 @@ static int pop3_getsizes(int sock, int count, int *sizes)
 static int pop3_is_old(int sock, struct query *ctl, int num)
 /* is the given message old? */
 {
+    struct idlist *new;
     if (!ctl->oldsaved)
 	return (num <= last);
+    else if (dofastuidl)
+    {
+	char id [IDLEN+1];
+
+	if (num <= last)
+	    return(TRUE);
+
+	/* in fast uidl, we manipulate the old list only! */
+
+	if ((new = id_find(&ctl->oldsaved, num)))
+	{
+	    /* we already have the id! */
+	    return(new->val.status.mark != UID_UNSEEN);
+	}
+
+	/* get the uidl first! */
+	if (pop3_getuidl(sock, num, id) != PS_SUCCESS)
+	    return(TRUE);
+
+	if ((new = str_in_list(&ctl->oldsaved, id, FALSE))) {
+	    /* we already have the id! */
+	    new->val.status.num = num;
+	    return(new->val.status.mark != UID_UNSEEN);
+	}
+
+	/* save it */
+	new = save_str(&ctl->oldsaved, id, UID_UNSEEN);
+	new->val.status.num = num;
+	return(FALSE);
+    }
     else
-        return (str_in_list(&ctl->oldsaved,
-			    str_find(&ctl->newsaved, num), FALSE));
+        return ((new = id_find(&ctl->newsaved, num)) != NULL &&
+	    new->val.status.mark != UID_UNSEEN);
 }
 
 #ifdef UNUSED
@@ -944,17 +1113,16 @@ static int pop3_fetch(int sock, struct query *ctl, int number, int *lenp)
 static void mark_uid_seen(struct query *ctl, int number)
 /* Tell the UID code we've seen this. */
 {
-    if (ctl->newsaved)
-    {
-	struct idlist	*sdp;
+    struct idlist	*sdp;
 
-	for (sdp = ctl->newsaved; sdp; sdp = sdp->next)
-	    if (sdp->val.status.num == number)
-	    {
-		sdp->val.status.mark = UID_SEEN;
-		save_str(&ctl->oldsaved, sdp->id,UID_SEEN);
-	    }
-    }
+    if ((sdp = id_find(&ctl->newsaved, number)))
+	sdp->val.status.mark = UID_SEEN;
+    /* mark it as seen in oldsaved also! In case, we do not swap the lists
+     * (say, due to socket error), the same mail will not be downloaded
+     * again.
+     */
+    if ((sdp = id_find(&ctl->oldsaved, number)))
+	sdp->val.status.mark = UID_SEEN;
 }
 
 static int pop3_delete(int sock, struct query *ctl, int number)
@@ -966,7 +1134,7 @@ static int pop3_delete(int sock, struct query *ctl, int number)
     ok = gen_transact(sock, "DELE %d", number);
     if (ok != PS_SUCCESS)
 	return(ok);
-    delete_str(&ctl->newsaved, number);
+    delete_str(dofastuidl ? &ctl->oldsaved : &ctl->newsaved, number);
     return(PS_SUCCESS);
 }
 
@@ -1023,6 +1191,7 @@ const static struct method pop3 =
     pop3_getauth,	/* get authorization */
     pop3_getrange,	/* query range of messages */
     pop3_getsizes,	/* we can get a list of sizes */
+    pop3_getpartialsizes,	/* we can get the size of 1 mail */
     pop3_is_old,	/* how do we tell a message is old? */
     pop3_fetch,		/* request given message */
     NULL,		/* no way to fetch body alone */
diff --git a/rcfile_l.l b/rcfile_l.l
index 3650ca5a..b50ac617 100644
--- a/rcfile_l.l
+++ b/rcfile_l.l
@@ -124,6 +124,8 @@ plugin		{ return PLUGIN; }
 plugout		{ return PLUGOUT; }
 batchlimit	{ return BATCHLIMIT; }
 fetchlimit	{ return FETCHLIMIT; }
+fetchsizelimit	{ return FETCHSIZELIMIT; }
+fastuidl	{ return FASTUIDL; }
 expunge		{ return EXPUNGE; }
 properties	{ return PROPERTIES; }
 
diff --git a/rcfile_y.y b/rcfile_y.y
index 48f75ee4..fc3970a5 100644
--- a/rcfile_y.y
+++ b/rcfile_y.y
@@ -67,7 +67,7 @@ extern char * yytext;
 %token SMTPADDRESS SMTPNAME SPAMRESPONSE PRECONNECT POSTCONNECT LIMIT WARNINGS
 %token NETSEC INTERFACE MONITOR PLUGIN PLUGOUT
 %token IS HERE THERE TO MAP WILDCARD
-%token BATCHLIMIT FETCHLIMIT EXPUNGE PROPERTIES
+%token BATCHLIMIT FETCHLIMIT FETCHSIZELIMIT FASTUIDL EXPUNGE PROPERTIES
 %token SET LOGFILE DAEMON SYSLOG IDFILE INVISIBLE POSTMASTER BOUNCEMAIL 
 %token SPAMBOUNCE SHOWDOTS
 %token <proto> PROTO AUTHTYPE
@@ -374,6 +374,8 @@ user_option	: TO localnames HERE
 		| LIMIT NUMBER		{current.limit       = NUM_VALUE_IN($2);}
 		| WARNINGS NUMBER	{current.warnings    = NUM_VALUE_IN($2);}
 		| FETCHLIMIT NUMBER	{current.fetchlimit  = NUM_VALUE_IN($2);}
+		| FETCHSIZELIMIT NUMBER	{current.fetchsizelimit = NUM_VALUE_IN($2);}
+		| FASTUIDL NUMBER	{current.fastuidl    = NUM_VALUE_IN($2);}
 		| BATCHLIMIT NUMBER	{current.batchlimit  = NUM_VALUE_IN($2);}
 		| EXPUNGE NUMBER	{current.expunge     = NUM_VALUE_IN($2);}
 
diff --git a/uid.c b/uid.c
index afc49678..91362418 100644
--- a/uid.c
+++ b/uid.c
@@ -63,9 +63,45 @@
  * be picked up by the next run.  If there are no un-expunged
  * messages, the file is deleted.
  *
+ * One disadvantage of UIDL is that all the UIDs have to be downloaded
+ * before a search for new messages can be done. Typically, new messages
+ * are appended to mailboxes. Hence, downloading all UIDs just to download
+ * a few new mails is a waste of bandwidth. If new messages are always at
+ * the end of the mailbox, fast UIDL will decrease the time required to
+ * download new mails.
+ *
+ * During fast UIDL, the UIDs of all messages are not downloaded! The first
+ * unseen message is searched for by using a binary search on UIDs. UIDs
+ * after the first unseen message are downloaded as and when needed.
+ *
+ * The advantages of fast UIDL are (this is noticeable only when the
+ * mailbox has too many mails):
+ *
+ * - There is no need to download the UIDs of all mails right at the start.
+ * - There is no need to save all the UIDs in memory separately in
+ * `newsaved' list.
+ * - There is no need to download the UIDs of seen mail (except for the
+ * first binary search).
+ * - The first new mail is downloaded considerably faster.
+ *
+ * The disadvantages are:
+ *
+ * - Since all UIDs are not downloaded, it is not possible to swap old and
+ * new list. The current state of the mailbox is essentially a merged state
+ * of old and new mails.
+ * - If an intermediate mail has been temporarily refused (say, due to 4xx
+ * code from the smtp server), this mail may not get downloaded.
+ * - If 'flush' is used, such intermediate mails will also get deleted.
+ *
+ * The first two disadvantages can be overcome by doing a linear search
+ * once in a while (say, every 10th poll). Also, with flush, fast UIDL
+ * should be disabled.
+ *
  * Note: some comparisons (those used for DNS address lists) are caseblind!
  */
 
+int dofastuidl = 0;
+
 /* UIDs associated with un-queried hosts */
 static struct idlist *scratchlist;
 
@@ -305,20 +341,20 @@ void free_str_pair_list(struct idlist **idl)
 }
 #endif
 
-int str_in_list(struct idlist **idl, const char *str, const flag caseblind)
+struct idlist *str_in_list(struct idlist **idl, const char *str, const flag caseblind)
 /* is a given ID in the given list? (comparison may be caseblind) */
 {
     struct idlist *walk;
     if (caseblind) {
 	for( walk = *idl; walk; walk = walk->next )
 	    if( strcasecmp( str, (char *)walk->id) == 0 )
-		return 1;
+		return walk;
     } else {
 	for( walk = *idl; walk; walk = walk->next )
 	    if( strcmp( str, (char *)walk->id) == 0 )
-		return 1;
+		return walk;
     }
-    return 0;
+    return NULL;
 }
 
 int str_nr_in_list( struct idlist **idl, const char *str )
@@ -389,6 +425,16 @@ char *str_find(struct idlist **idl, long number)
 	return(str_find(&(*idl)->next, number));
 }
 
+struct idlist *id_find(struct idlist **idl, long number)
+/* return the id of the given number in the given list. */
+{
+    struct idlist	*idp;
+    for (idp = *idl; idp; idp = idp->next)
+	if (idp->val.status.num == number)
+	    return(idp);
+    return(0);
+}
+
 char *idpair_find(struct idlist **idl, const char *id)
 /* return the id of the given id in the given list (caseblind comparison) */
 {
@@ -449,7 +495,7 @@ void expunge_uids(struct query *ctl)
 {
     struct idlist *idl;
 
-    for (idl = ctl->newsaved; idl; idl = idl->next)
+    for (idl = dofastuidl ? ctl->oldsaved : ctl->newsaved; idl; idl = idl->next)
 	if (idl->val.status.mark == UID_DELETED)
 	    idl->val.status.mark = UID_EXPUNGED;
 }
@@ -462,8 +508,11 @@ void uid_swap_lists(struct query *ctl)
     {
 	struct idlist *idp;
 
-	report_build(stdout, GT_("New UID list from %s:"), ctl->server.pollname);
-	for (idp = ctl->newsaved; idp; idp = idp->next)
+	if (dofastuidl)
+	    report_build(stdout, GT_("Merged UID list from %s:"), ctl->server.pollname);
+	else
+	    report_build(stdout, GT_("New UID list from %s:"), ctl->server.pollname);
+	for (idp = dofastuidl ? ctl->oldsaved : ctl->newsaved; idp; idp = idp->next)
 	    report_build(stdout, " %s = %d", idp->id, idp->val.status.mark);
 	if (!idp)
 	    report_build(stdout, GT_(" <empty>"));
@@ -495,10 +544,48 @@ void uid_swap_lists(struct query *ctl)
 	ctl->oldsaved = ctl->newsaved;
 	ctl->newsaved = (struct idlist *) NULL;
     }
-    else if (outlevel >= O_DEBUG)
+    /* in fast uidl, there is no need to swap lists: the old state of
+     * mailbox cannot be discarded! */
+    else if (outlevel >= O_DEBUG && !dofastuidl)
 	report(stdout, GT_("not swapping UID lists, no UIDs seen this query\n"));
 }
 
+void uid_discard_new_list(struct query *ctl)
+/* finish a query which had errors */
+{
+    /* debugging code */
+    if (ctl->server.uidl && outlevel >= O_DEBUG)
+    {
+	struct idlist *idp;
+
+	/* this is now a merged list! the mails which were seen in this
+	 * poll are marked here. */
+	report_build(stdout, GT_("Merged UID list from %s:"), ctl->server.pollname);
+	for (idp = ctl->oldsaved; idp; idp = idp->next)
+	    report_build(stdout, " %s = %d", idp->id, idp->val.status.mark);
+	if (!idp)
+	    report_build(stdout, GT_(" <empty>"));
+	report_complete(stdout, "\n");
+    }
+
+    if (ctl->newsaved)
+    {
+	/* new state of mailbox is not reliable */
+	if (outlevel >= O_DEBUG)
+	    report(stdout, GT_("discarding new UID list\n"));
+	free_str_list(&ctl->newsaved);
+	ctl->newsaved = (struct idlist *) NULL;
+    }
+}
+
+void uid_reset_num(struct query *ctl)
+/* reset the number associated with each id */
+{
+    struct idlist *idp;
+    for (idp = ctl->oldsaved; idp; idp = idp->next)
+	idp->val.status.num = 0;
+}
+
 void write_saved_lists(struct query *hostlist, const char *idfile)
 /* perform end-of-run write of seen-messages list */
 {