Do not search for UNSEEN messages in ranges.

Add gen_recv_split() to split long protocol messages when the message prefix matches

Order of search commands:

IMAP> A010 SEARCH UNSEEN UNDELETED     (IMAP4 or higher)
IMAP> A011 SEARCH UNSEEN
IMAP> A012 FETCH 1:n FLAGS

4 files changed, 133 insertions, 48 deletions

diff --git a/NEWS b/NEWS
index 107c5f82..86657c13 100644
--- a/NEWS
+++ b/NEWS
@@ -70,6 +70,10 @@ fetchmail-6.3.20 (not yet released):
 
 # BUG FIXES
 * Call strlen() only once when removing CRLF from a line. (Sunil Shetye)
+* Do not search for UNSEEN messages in ranges. Usually, there are very few new
+  messages and most of the range searches result in nothing. Instead, split the
+  long response to make the IMAP driver think that there are multiple lines of
+  response. (Sunil Shetye)
 
 # TRANSLATION UPDATES
   [ja]    Japanese (Takeshi Hamasaki)
diff --git a/fetchmail.h b/fetchmail.h
index f6c6a4ec..fcab0e59 100644
--- a/fetchmail.h
+++ b/fetchmail.h
@@ -517,6 +517,13 @@ void resetidletimeout(void);
 int do_protocol(struct query *, const struct method *);
 
 /* transact.c: transaction support */
+struct RecvSplit
+{
+    char prefix[100];
+    int cached;
+    char buf[MSGBUFSIZE];
+};
+
 void init_transact(const struct method *);
 int readheaders(int sock,
 		       long fetchlen,
@@ -530,12 +537,16 @@ void gen_send(int sock, const char *, ... )
     __attribute__ ((format (printf, 2, 3)))
     ;
 int gen_recv(int sock, char *buf, int size);
+void gen_recv_split_init(const char *prefix, struct RecvSplit *rs);
+int gen_recv_split(int sock, char *buf, int size, struct RecvSplit *rs);
 int gen_transact(int sock, const char *, ... )
     __attribute__ ((format (printf, 2, 3)))
     ;
 #else
 void gen_send();
 int gen_recv();
+void gen_recv_split_init();
+int gen_recv_split();
 int gen_transact();
 #endif
 extern struct msgblk msgblk;
diff --git a/imap.c b/imap.c
index 86d1bfea..6145564b 100644
--- a/imap.c
+++ b/imap.c
@@ -179,7 +179,7 @@ static int imap_untagged_response(int sock, const char *buf)
     return(PS_SUCCESS);
 }
 
-static int imap_response(int sock, char *argbuf)
+static int imap_response(int sock, char *argbuf, struct RecvSplit *rs)
 /* parse command response */
 {
     char buf[MSGBUFSIZE+1];
@@ -188,7 +188,11 @@ static int imap_response(int sock, char *argbuf)
 	int	ok;
 	char	*cp;
 
-	if ((ok = gen_recv(sock, buf, sizeof(buf))))
+	if (rs)
+	    ok = gen_recv_split(sock, buf, sizeof(buf), rs);
+	else
+	    ok = gen_recv(sock, buf, sizeof(buf));
+	if (ok != PS_SUCCESS)
 	    return(ok);
 
 	/* all tokens in responses are caseblind */
@@ -269,7 +273,7 @@ static int imap_ok(int sock, char *argbuf)
 {
     int ok;
 
-    while ((ok = imap_response(sock, argbuf)) == PS_UNTAGGED)
+    while ((ok = imap_response(sock, argbuf, NULL)) == PS_UNTAGGED)
 	; /* wait for the tagged response */
     return(ok);
 }
@@ -754,9 +758,6 @@ static int imap_idle(int sock)
     return(ok);
 }
 
-/* maximum number of numbers we can process in "SEARCH" response */
-# define IMAP_SEARCH_MAX 1000
-
 static int imap_search(int sock, struct query *ctl, int count)
 /* search for unseen messages */
 {
@@ -770,29 +771,20 @@ static int imap_search(int sock, struct query *ctl, int count)
     flag skipdeleted = (imap_version >= IMAP4) && ctl->keep;
     const char *undeleted;
 
-    /* Skip range search if there are less than or equal to
-     * IMAP_SEARCH_MAX mails. */
-    flag skiprangesearch = (count <= IMAP_SEARCH_MAX);
+    /* structure to keep the end portion of the incomplete response */
+    struct RecvSplit rs;
 
     /* startcount is higher than count so that if there are no
      * unseen messages, imap_getsizes() will not need to do
      * anything! */
     startcount = count + 1;
 
-    for (first = 1, last = IMAP_SEARCH_MAX; first <= count; first += IMAP_SEARCH_MAX, last += IMAP_SEARCH_MAX)
+    for (;;)
     {
-	if (last > count)
-	    last = count;
-
-restartsearch:
 	undeleted = (skipdeleted ? " UNDELETED" : "");
-	if (skiprangesearch)
-	    gen_send(sock, "SEARCH UNSEEN%s", undeleted);
-	else if (last == first)
-	    gen_send(sock, "SEARCH %d UNSEEN%s", last, undeleted);
-	else
-	    gen_send(sock, "SEARCH %d:%d UNSEEN%s", first, last, undeleted);
-	while ((ok = imap_response(sock, buf)) == PS_UNTAGGED)
+	gen_send(sock, "SEARCH UNSEEN%s", undeleted);
+	gen_recv_split_init("* SEARCH", &rs);
+	while ((ok = imap_response(sock, buf, &rs)) == PS_UNTAGGED)
 	{
 	    if ((cp = strstr(buf, "* SEARCH")))
 	    {
@@ -824,39 +816,25 @@ restartsearch:
 		}
 	    }
 	}
-	/* if there is a protocol error on the first loop, try a
-	 * different search command */
-	if (ok == PS_ERROR && first == 1)
+	if (ok != PS_ERROR) /* success or non-protocol error */
+	    return(ok);
+
+	/* there is a protocol error. try a different search command. */
+	if (skipdeleted)
 	{
-	    if (skipdeleted)
-	    {
-		/* retry with "SEARCH 1:1000 UNSEEN" */
-		skipdeleted = FALSE;
-		goto restartsearch;
-	    }
-	    if (!skiprangesearch)
-	    {
-		/* retry with "SEARCH UNSEEN" */
-		skiprangesearch = TRUE;
-		goto restartsearch;
-	    }
-	    /* try with "FETCH 1:n FLAGS" */
-	    goto fetchflags;
+	    /* retry with "SEARCH UNSEEN" */
+	    skipdeleted = FALSE;
+	    continue;
 	}
-	if (ok != PS_SUCCESS)
-	    return(ok);
-	/* loop back only when searching in range */
-	if (skiprangesearch)
-	    break;
+	/* try with "FETCH 1:n FLAGS" */
+	break;
     }
-    return(PS_SUCCESS);
 
-fetchflags:
     if (count == 1)
 	gen_send(sock, "FETCH %d FLAGS", count);
     else
 	gen_send(sock, "FETCH %d:%d FLAGS", 1, count);
-    while ((ok = imap_response(sock, buf)) == PS_UNTAGGED)
+    while ((ok = imap_response(sock, buf, NULL)) == PS_UNTAGGED)
     {
 	unsigned int num;
 	int consumed;
@@ -1067,7 +1045,7 @@ static int imap_getpartialsizes(int sock, int first, int last, int *sizes)
 	gen_send(sock, "FETCH %d:%d RFC822.SIZE", first, last);
     else /* no unseen messages! */
 	return(PS_SUCCESS);
-    while ((ok = imap_response(sock, buf)) == PS_UNTAGGED)
+    while ((ok = imap_response(sock, buf, NULL)) == PS_UNTAGGED)
     {
 	unsigned int size;
 	int num;
@@ -1155,7 +1133,7 @@ static int imap_fetch_headers(int sock, struct query *ctl,int number,int *lenp)
     gen_send(sock, "FETCH %d RFC822.HEADER", number);
 
     /* looking for FETCH response */
-    if ((ok = imap_response(sock, buf)) == PS_UNTAGGED)
+    if ((ok = imap_response(sock, buf, NULL)) == PS_UNTAGGED)
     {
 		int consumed;
 	/* expected response formats:
diff --git a/transact.c b/transact.c
index 758e5aaf..ff3a12ec 100644
--- a/transact.c
+++ b/transact.c
@@ -1586,6 +1586,98 @@ int gen_recv(int sock  /** socket to which server is connected */,
     }
 }
 
+/*
+ * gen_recv_split() splits the response from a server which is too
+ * long to fit into the buffer into multiple lines. If the prefix is
+ * set as "MY FEATURES" and the response from the server is too long
+ * to fit in the buffer, as in:
+ *
+ * "MY FEATURES ABC DEF GHI JKLMNOPQRS TU VWX YZ"
+ *
+ * Repeated calls to gen_recv_split() may return:
+ *
+ * "MY FEATURES ABC DEF GHI"
+ * "MY FEATURES JKLMNOPQRS"
+ * "MY FEATURES TU VWX YZ"
+ *
+ * A response not beginning with the prefix "MY FEATURES" will not be
+ * split.
+ *
+ * To use:
+ * - Declare a variable of type struct RecvSplit
+ * - Call gen_recv_split_init() once
+ * - Call gen_recv_split() in a loop
+ */
+
+void gen_recv_split_init (const char *prefix, struct RecvSplit *rs)
+{
+    snprintf(rs->prefix, sizeof(rs->prefix), "%s", prefix);
+    rs->cached = 0;
+    rs->buf[0] = '\0';
+}
+
+int gen_recv_split(int sock  /** socket to which server is connected */,
+	     char *buf /* buffer to receive input */,
+	     int size  /* length of buffer */,
+	     struct RecvSplit *rs /* cached information across calls */)
+{
+    size_t n = 0;
+    int foundnewline = 0;
+    char *p;
+    int oldphase = phase;	/* we don't have to be re-entrant */
+
+    /* if this is not our first call, prepare the buffer */
+    if (rs->cached)
+    {
+	snprintf (buf, size, "%s%s", rs->prefix, rs->buf);
+	n = strlen(buf);
+	/* clear the cache for the next call */
+	rs->cached = 0;
+	rs->buf[0] = '\0';
+    }
+
+    phase = SERVER_WAIT;
+    set_timeout(mytimeout);
+    if (SockRead(sock, buf + n, size - n) == -1)
+    {
+	set_timeout(0);
+	phase = oldphase;
+	return(PS_SOCKET);
+    }
+    set_timeout(0);
+
+    n = strlen(buf);
+    if (n > 0 && buf[n-1] == '\n')
+    {
+	buf[--n] = '\0';
+	foundnewline = 1;
+    }
+    if (n > 0 && buf[n-1] == '\r')
+	buf[--n] = '\0';
+
+    if (foundnewline				/* we have found a complete line */
+	|| strncmp(buf, rs->prefix, strlen(rs->prefix))	/* mismatch in prefix */
+	|| !(p = strrchr(buf, ' '))		/* no space found in response */
+	|| p < buf + strlen(rs->prefix))	/* space is at the wrong location */
+    {
+	if (outlevel >= O_MONITOR)
+	    report(stdout, "%s< %s\n", protocol->name, buf);
+	phase = oldphase;
+	return(PS_SUCCESS);
+    }
+
+    /* we are ready to cache some information now. */
+    rs->cached = 1;
+    snprintf (rs->buf, sizeof(rs->buf), "%s", p);
+    *p = '\0';
+    if (outlevel >= O_MONITOR)
+	report(stdout, "%s< %s\n", protocol->name, buf);
+    if (outlevel >= O_DEBUG)
+	report(stdout, "%s< %s%s...\n", protocol->name, rs->prefix, rs->buf);
+    phase = oldphase;
+    return(PS_SUCCESS);
+}
+
 #if defined(HAVE_STDARG_H)
 int gen_transact(int sock, const char *fmt, ... )
 #else