From 32cf48a357ed372b9135aff60ea794b3db61bae6 Mon Sep 17 00:00:00 2001
From: "Eric S. Raymond" <esr@thyrsus.com>
Date: Fri, 7 Apr 2000 10:07:06 +0000
Subject: speed improvement by usingh SEARCH UNSEEN.

svn path=/trunk/; revision=2863
---
 NEWS                    |   6 ++
 fetchmail-FAQ.html      |   7 +-
 fetchmail-features.html |  12 ++-
 imap.c                  | 225 +++++++++++++++---------------------------------
 4 files changed, 90 insertions(+), 160 deletions(-)

diff --git a/NEWS b/NEWS
index b6809491..c5c78d30 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,12 @@
 
 * Autoprobe now recognizes Domino IMAP servers and warns the user.
 * Add a needed pclose(3) call in interface.c (thank Matthias Andree).
+* In IMAP, use SEARCH UNSEEN instead of flag queries to count and identify
+  unseen messages (thanks to Alexis Lê-Quôc <alq666@hotmail.com>).
+* Added contrib/domino, a workaround for the Domino attachment problem.
+* Added contrib/toprocmail, which demonstrates how a plugin can
+  displaty selected mail headers.
+* French internationalization update from Guy Brand.
 
 fetchmail-5.3.5 (Wed Mar 29 18:17:19 EST 2000), 18840 lines:
 
diff --git a/fetchmail-FAQ.html b/fetchmail-FAQ.html
index aa31b6c3..c0d7f6f0 100644
--- a/fetchmail-FAQ.html
+++ b/fetchmail-FAQ.html
@@ -10,7 +10,7 @@
 <table width="100%" cellpadding=0><tr>
 <td width="30%">Back to <a href="index.html">Fetchmail Home Page</a>
 <td width="30%" align=center>To <a href="/~esr/sitemap.html">Site Map</a>
-<td width="30%" align=right>$Date: 2000/03/30 00:40:56 $
+<td width="30%" align=right>$Date: 2000/04/07 10:07:05 $
 </table>
 <HR>
 <H1>Frequently Asked Questions About Fetchmail</H1>
@@ -2531,7 +2531,8 @@ fetchmail uses FETCH RFC822.HEADER and FETCH RFC822.TEXT to get first
 the header and then the body, Domino generates different Boundary tags
 for each part, .e.g. one tag is declared in the Content-type header and
 another is used to separate the MIME parts in the body.  This doesn't
-work.<p>
+work. (I have heard a rumor that this bug is scheduled to be fixed 
+in Domino release 6; you can find a workaround at contrib/domino.)<p>
 
 Another rich source of attachment problems is Microsoft Exchange and
 Microsoft Outlook.  If you see unreadable attachments with a
@@ -2782,7 +2783,7 @@ terminate it.<p>
 <table width="100%" cellpadding=0><tr>
 <td width="30%">Back to <a href="index.html">Fetchmail Home Page</a>
 <td width="30%" align=center>To <a href="/~esr/sitemap.html">Site Map</a>
-<td width="30%" align=right>$Date: 2000/03/30 00:40:56 $
+<td width="30%" align=right>$Date: 2000/04/07 10:07:05 $
 </table>
 
 <P><ADDRESS>Eric S. Raymond <A HREF="mailto:esr@thyrsus.com">&lt;esr@snark.thyrsus.com&gt;</A></ADDRESS>
diff --git a/fetchmail-features.html b/fetchmail-features.html
index 4643ee50..f9082515 100644
--- a/fetchmail-features.html
+++ b/fetchmail-features.html
@@ -10,7 +10,7 @@
 <table width="100%" cellpadding=0><tr>
 <td width="30%">Back to <a href="index.html">Fetchmail Home Page</a>
 <td width="30%" align=center>To <a href="/~esr/sitemap.html">Site Map</a>
-<td width="30%" align=right>$Date: 2000/03/13 21:03:10 $
+<td width="30%" align=right>$Date: 2000/04/07 10:07:05 $
 </table>
 <HR>
 
@@ -18,10 +18,18 @@
 
 <H2>Since 5.0:</H2>
 <UL>
+<LI>
+In IMAP, unseen-message counting and indexing is now done by SEARCH
+UNSEEN at the beginning of each poll or re-poll (rather than with
+the UNSEEN and RECENT responses and FLAGS queries on individual
+messages). This significantly cuts down on traffic to and from the
+server, and gives more reliable results.<p>
+
 <LI>
 The aka option now matches hostname suffixes, so (for example) saying
 `aka netaxs.com' will match not just netaxs.com but also (say)
 pop3.netaxs.com and mail.netaxs.com.
+
 <LI>
 Fetchmail can optionally use the RFC 2177 IDLE extension on an IMAP 
 server that supports it.
@@ -200,7 +208,7 @@ get-mail, gwpop, pimp-1.0, pop-perl5-1.2, popc, popmail-1.6 and upop.<P>
 <table width="100%" cellpadding=0><tr>
 <td width="30%">Back to <a href="index.html">Fetchmail Home Page</a>
 <td width="30%" align=center>To <a href="/~esr/sitemap.html">Site Map</a>
-<td width="30%" align=right>$Date: 2000/03/13 21:03:10 $
+<td width="30%" align=right>$Date: 2000/04/07 10:07:05 $
 </table>
 
 <P><ADDRESS>Eric S. Raymond <A HREF="mailto:esr@thyrsus.com">&lt;esr@snark.thyrsus.com&gt;</A></ADDRESS>
diff --git a/imap.c b/imap.c
index 7f039f51..3e534cf3 100644
--- a/imap.c
+++ b/imap.c
@@ -62,31 +62,17 @@ extern char *strstr();	/* needed on sysV68 R3V7.1. */
 #define IMAP4		0	/* IMAP4 rev 0, RFC1730 */
 #define IMAP4rev1	1	/* IMAP4 rev 1, RFC2060 */
 
-static int count, seen, unseen, deletions, imap_version, preauth; 
-#ifdef USE_SEARCH
-static int unseen_count;
-#endif /* USE_SEARCH */
+static int count, unseen, deletions, imap_version, preauth; 
 static int expunged, expunge_period, saved_timeout;
 static flag do_idle;
 static char capabilities[MSGBUFSIZE+1];
-
-#ifdef USE_SEARCH
-/*
- * unseen_count is a dynamic array that holds the sequence number
- * of unseen_messages. It is created whenever a SEARCH UNSEEN response
- * is returned and deleted at logout time.
- * Data are terminated by 0 in unseen_sequence since sequence numbers
- * are between 1 and 2^32.
- */
-static unsigned int* unseen_sequence;
-#endif /* USE_SEARCH */
+static unsigned int *unseen_messages;
 
 int imap_ok(int sock, char *argbuf)
 /* parse command response */
 {
-    char buf [MSGBUFSIZE+1];
+    char buf[MSGBUFSIZE+1];
 
-    seen = 0;
     do {
 	int	ok;
 	char	*cp;
@@ -102,7 +88,7 @@ int imap_ok(int sock, char *argbuf)
 	/* interpret untagged status responses */
 	if (strstr(buf, "* CAPABILITY"))
 	    strncpy(capabilities, buf + 12, sizeof(capabilities));
-	if (strstr(buf, "EXISTS"))
+	else if (strstr(buf, "EXISTS"))
 	{
 	    count = atoi(buf+2);
 	    /*
@@ -123,78 +109,8 @@ int imap_ok(int sock, char *argbuf)
 		stage = STAGE_FETCH;
 	    }
 	}
-	if (strstr(buf, "UNSEEN"))
-	{
-	    char	*cp;
-
-	    /*
-	     * Handle both "* 42 UNSEEN" (if that ever happens) and 
-	     * "* OK [UNSEEN 42] 42". Note that what this gets us is
-	     * a minimum index, not a count.
-	     */
-	    unseen = 0;
-	    for (cp = buf; *cp && !isdigit(*cp); cp++)
-		continue;
-	    unseen = atoi(cp);
-	}
-	if (strstr(buf, "FLAGS"))
-	    seen = (strstr(buf, "SEEN") != (char *)NULL);
-	if (strstr(buf, "PREAUTH"))
+	else if (strstr(buf, "PREAUTH"))
 	    preauth = TRUE;
-#ifdef USE_SEARCH
-	/*
-	 * The response to SEARCH UNSEEN looks like:
-	 * * SEARCH list of sequence numbers
-         * It means stuffing unseen_sequence and eating white space in between.
-	 */
-	if (strstr(buf, "* SEARCH"))
-	{
-	    char	*cp = strstr(buf, "SEARCH") + 6; /* Start after SEARCH */
-	    char	*endofnumber = NULL;
-
-	    /* unseen_sequence already exists. Discard it for a new use */
-	    if (unseen_sequence)
-		free(unseen_sequence);
-
-	    /* 
-	     * We do unseen_sequence memory allocation here.  Chances
-	     * are it will be grossly oversized, but there is no way
-	     * to size it correctly in one pass without relying on
-	     * the flaky RECENT response.  Which is exactly why
-	     * we're using SEARCH UNSEEN. Fill unseen_sequence with
-	     * zero because 0 means end of data.
-             */
-	    unseen_sequence = xmalloc(count * sizeof(unsigned int));
-	    memset(unseen_sequence, 0, count * sizeof(unsigned int));
-	    unseen_count = 0;
-
-	    while (*cp)
-	    {
-		/* White space */
-		while (*cp && isspace(*cp)) cp++;
-		/* Number is between 1 and 2^32 included so unsigned int is enough. */
-		if (*cp) 
-		{
-		    /*
-		     * To avoid a buffer overflow.  That count would
-		     * be less than unseen_count is not supposed to
-		     * happen, but you never know.
-		     */
-		    if (unseen_count >= count) 
-			break;
-		    unseen_sequence[unseen_count++] = (unsigned int)strtol(cp, &endofnumber, 10);
-
-		    if (outlevel >= O_DEBUG)
-			report(stdout, _("%u is unseen\n"), unseen_sequence[unseen_count - 1]);
-		
-		    cp = endofnumber;
-		}
-	    }
-
-	    if (unseen_sequence[0] > 0 && unseen_count > 0)
-		unseen = unseen_sequence[0];
-	}
-#endif /* USE_SEARCH */
     } while
 	(tag[0] != '\0' && strncmp(buf, tag, strlen(tag)));
 
@@ -1118,6 +1034,7 @@ static int internal_expunge(int sock)
 	return(ok);
 
     expunged += deletions;
+    unseen -= deletions;
     deletions = 0;
 
 #ifdef IMAP_UID	/* not used */
@@ -1134,9 +1051,10 @@ static int imap_getrange(int sock,
 /* get range of messages to be fetched */
 {
     int ok;
+    char buf[MSGBUFSIZE+1], *cp;
 
     /* find out how many messages are waiting */
-    *bytes = unseen = -1;
+    *bytes = -1;
 
     if (pass > 1)
     {
@@ -1164,28 +1082,13 @@ static int imap_getrange(int sock,
 	else if (count == -1)	/* no EXISTS response to NOOP/IDLE */
 	{
 	    count = 0;
-	    unseen = -1;
 	}
     }
     else
     {
-#ifdef USE_SEARCH
 	ok = gen_transact(sock, 
 			  check_only ? "EXAMINE \"%s\"" : "SELECT \"%s\"",
 			  folder ? folder : "INBOX");
-#else
-	if (check_only) {
-	    ok = gen_transact(sock, "EXAMINE \"%s\"", folder? folder: "INBOX");
-	} else {
-            ok = gen_transact(sock, "SELECT \"%s\"", folder? folder: "INBOX");
-	    /*
-	     * SEARCH UNSEEN will modify the static unseen variable.
-	     * It is a much better way than RECENT to determine UNSEEN
-	     * messages.
-             */
-	    ok = gen_transact(sock, "SEARCH UNSEEN");
-	}
-#endif /* USE_SEARCH */
 	if (ok != 0)
 	{
 	    report(stderr, _("mailbox selection failed\n"));
@@ -1195,30 +1098,55 @@ static int imap_getrange(int sock,
 
     *countp = count;
 
-    /*
-     * Note: because IMAP has an is_old method, this number is used
-     * only for the "X messages (Y unseen)" notification.  Accordingly
-     * it doesn't matter much that it can be wrong (e.g. if we see an
-     * UNSEEN response but not all messages above the first UNSEEN one
-     * are likewise).
-     *
-     * We used to fall back to using the value from the last RECENT
-     * response here.  But RECENT is not really a good indication of
-     * recent messages, because it is ill-defined, e.g. if several
-     * connections are made to the folder, messages might appear as
-     * recent to some and not to others.  Using SEARCH UNSEEN provides
-     * us with the exact number of unseen messages.
-     */
-#ifndef USE_SEARCH
-    if (unseen >= 0)
-	*newp = count - unseen + 1;
-#else
-    if (unseen >= 0 && unseen_count > 0)
-	*newp = unseen_count;
-#endif /* USE_SEARCH */
-    else
-	*newp = -1;		/* should never happen */ 
+    /* OK, now get a count of unseen messages and their indices */
 
+    if (unseen_messages)
+	free(unseen_messages);
+    unseen_messages = xmalloc(count * sizeof(unsigned int));
+    memset(unseen_messages, 0, count * sizeof(unsigned int));
+    unseen = 0;
+
+    gen_send(sock, "SEARCH UNSEEN");
+    do {
+	ok = gen_recv(sock, buf, sizeof(buf));
+	if (ok != 0)
+	{
+	    report(stderr, _("search for unseen messages failed\n"));
+	    return(PS_PROTOCOL);
+	}
+	else if ((cp = strstr(buf, "* SEARCH")))
+	{
+	    char	*ep;
+
+	    cp += 8;	/* skip "* SEARCH" */
+
+	    while (*cp && unseen < count)
+	    {
+		/* skip whitespace */
+		while (*cp && isspace(*cp))
+		    cp++;
+		if (*cp) 
+		{
+		    /*
+		     * Message numbers are between 1 and 2^32 inclusive,
+		     * so unsigned int is large enough.
+		     */
+		    unseen_messages[unseen]=(unsigned int)strtol(cp,&ep,10);
+
+		    if (outlevel >= O_DEBUG)
+			report(stdout, 
+			       _("%u is unseen\n"), 
+			       unseen_messages[unseen]);
+		
+		    unseen++;
+		    cp = ep;
+		}
+	    }
+	}
+    } while
+	(tag[0] != '\0' && strncmp(buf, tag, strlen(tag)));
+
+    *newp = unseen;
     expunged = 0;
 
     return(PS_SUCCESS);
@@ -1285,35 +1213,22 @@ static int imap_getsizes(int sock, int count, int *sizes)
 static int imap_is_old(int sock, struct query *ctl, int number)
 /* is the given message old? */
 {
-#ifdef USE_SEARCH
-    int ok;
-#else
-    int ok, i;
-#endif /* USE_SEARCH */
+    flag seen = TRUE;
+    int i;
 
-    /* expunges change the fetch numbers */
-    number -= expunged;
+    /* 
+     * Expunges change the fetch numbers, but unseen_messages contains
+     * indices from before any expungees were done.  So neither the
+     * argument nor the values in message_sequence need to be decremented.
+     */
 
-#ifdef USE_SEARCH
-    if (unseen_count > 0) 
-    {
-        /* !unseen_count[i] means end of data - redundant with unseen_count */
-    	for (i = 0; i < unseen_count && 0 < unseen_sequence[i]; i++)
+    seen = TRUE;
+    for (i = 0; i < unseen; i++)
+	if (unseen_messages[i] == number)
 	{
-	    unseen_sequence[i] -= expunged;	/* expunge adjustment */
-	    if (unseen_sequence[i] == number) {
-		seen = FALSE;			/* message in unseen_sequence */
-		break;
-	    } 
-	    else
-		seen = TRUE;
+	    seen = FALSE;
+	    break;
 	}
-    }
-    else
-	/* Fall back to FETCH FLAGS */
-#endif /* USE_SEARCH */
-	if ((ok = gen_transact(sock, "FETCH %d FLAGS", number)) != 0)
-	    return(PS_ERROR);
 
     return(seen);
 }
@@ -1518,8 +1433,8 @@ static int imap_logout(int sock, struct query *ctl)
 
 #ifdef USE_SEARCH
     /* Memory clean-up */
-    if (unseen_sequence)
-	free(unseen_sequence);
+    if (unseen_messages)
+	free(unseen_messages);
 #endif /* USE_SEARCH */
 
     return(gen_transact(sock, "LOGOUT"));
-- 
cgit v1.2.3