Better behavior on line hits.

svn path=/trunk/; revision=1485

7 files changed, 88 insertions, 27 deletions

diff --git a/NEWS b/NEWS
index 53968598..5e9a0fd5 100644
--- a/NEWS
+++ b/NEWS
@@ -23,6 +23,7 @@ fetchmail-4.3.0-alpha (Sun Oct  5 11:07:53 EDT 1997)
 * It's now possible to explicitly configure out POP3, IMAP, or ETRN.
 * We no longer get the hostname for address rewrites and log messages from the
   server greeting line, instead it's the server's canonical DNS name.
+* Improved UID handling for RFC1725 POP3 servers coping with a line hit.
 * Created fetchmail-announce list.
 
 There are 285 people on fetchmail-friends and 6 on fetchmail-announce.
diff --git a/driver.c b/driver.c
index 0e4d0997..8eb7494f 100644
--- a/driver.c
+++ b/driver.c
@@ -1862,7 +1862,9 @@ const struct method *proto;	/* protocol method table */
 			    if (ok != 0)
 				goto cleanUp;
 			    set_timeout(ctl->server.timeout);
+#ifdef POP3_ENABLE
 			    delete_str(&ctl->newsaved, num);
+#endif /* POP3_ENABLE */
 			}
 			else if (outlevel > O_SILENT) 
 			    error_complete(0, 0, " not flushed");
diff --git a/fetchmail-FAQ.html b/fetchmail-FAQ.html
index cd3b5f19..0168e0ce 100644
--- a/fetchmail-FAQ.html
+++ b/fetchmail-FAQ.html
@@ -10,7 +10,7 @@
 <table width="100%" cellpadding=0><tr>
 <td width="30%">Back to <a href="index.html">Fetchmail Home Page</a>
 <td width="30%" align=center>To <a href="/~esr/sitemap.html">Site Map</a>
-<td width="30%" align=right>$Date: 1997/10/06 16:14:18 $
+<td width="30%" align=right>$Date: 1997/10/06 20:34:27 $
 </table>
 <HR>
 <H1>Frequently Asked Questions About Fetchmail</H1>
@@ -1164,8 +1164,8 @@ more often.<P>
 
 Qualcomm's qpopper, used at many BSD Unix sites, is better behaved.
 If its connection is dropped, it will first execute all DELE commands (as
-though you had issued an QUIT -- this is a technical violation of
-the RFCs, but a good idea in a world of flaky phone lines). Then it
+though you had issued a QUIT -- this is a technical violation of
+the POP3 RFCs, but a good idea in a world of flaky phone lines). Then it
 will re-queue any message that was being downloaded at hangup time.
 Still, qpopper may require a noticeable amount of time to do deletions
 and clean up its queue.  (Fetchmail waits a bit before retrying in
@@ -1542,10 +1542,13 @@ force an rc file reread, do <code>fetchmail -q; fetchmail</code>.<P>
 <h2><a name="O4">O4. Why do deleted messages show up again when I take
 a line hit while downloading?</a></h2>
 
-Because you're using a POP2 or POP3 server.  According to the POP3 RFCs,
-deletes aren't actually performed until you issue the end-of-session
-QUIT command.  Fetchmail cannot fix this, it takes cooperation from the.
-server. There are two possible remedies:<P>
+Because you're using a POP3 other than Qualcomm qpopper, or an IMAP
+with a long expunge interval.<P>
+
+According to the POP3 RFCs, deletes aren't actually performed until
+you issue the end-of-session QUIT command.  Fetchmail cannot fix this,
+it takes cooperation from the.  server. There are two possible
+remedies:<P>
 
 One is to switch to qpopper (the freeware POP3 server from Qualcomm,
 the Eudora people).  The qpopper software violates the POP3 RFCs by
@@ -1554,8 +1557,13 @@ as on processing a QUIT command.<P>
 
 The other (which we recommend) is to switch to <a
 href="http://www.imap.org">IMAP</a>.  IMAP has an explicit expunge
-command and fetchmail uses it to delete messages quickly after they
-are downloaded.<P>
+command and fetchmail normally uses it to delete messages immediately
+after they are downloaded.<P>
+
+If you get very unlucky, you might take a line hit in the window
+between the delete and the expunge.  If you've set a longer expunge
+interval, the window gets wider.  This problem should correct itself
+the next time you complete a successful query.<P>
 
 <hr>
 <h2><a name="O5">O5. Why is fetched mail being logged with my name, not the real From address?</a></h2>
@@ -1581,7 +1589,7 @@ will look right.<p>
 <table width="100%" cellpadding=0><tr>
 <td width="30%">Back to <a href="index.html">Fetchmail Home Page</a>
 <td width="30%" align=center>To <a href="/~esr/sitemap.html">Site Map</a>
-<td width="30%" align=right>$Date: 1997/10/06 16:14:18 $
+<td width="30%" align=right>$Date: 1997/10/06 20:34:27 $
 </table>
 
 <P><ADDRESS>Eric S. Raymond <A HREF="mailto:esr@thyrsus.com">&lt;esr@snark.thyrsus.com&gt;</A></ADDRESS>
diff --git a/fetchmail.h b/fetchmail.h
index 1b033a21..89891d54 100644
--- a/fetchmail.h
+++ b/fetchmail.h
@@ -142,6 +142,9 @@ struct query
     int	expunge;		/* max # msgs to pass between expunges */
 
     /* unseen, previous state of mailbox (initially from .fetchids) */
+#define UID_KEPT	0	/* this was remembered from a previous run */
+#define UID_DELETED	-1	/* this message has been deleted */
+#define UID_EXPUNGED	-2	/* this message has been expunged */ 
     struct idlist *oldsaved, *newsaved;
 
     /* internal use */
@@ -265,6 +268,7 @@ char *str_from_nr_list( struct idlist **idl, int number );
 char *str_find(struct idlist **, int);
 char *idpair_find(struct idlist **, const char *);
 void append_str_list(struct idlist **, struct idlist **);
+void expunge_uids(struct query *);
 void update_str_lists(struct query *);
 void write_saved_lists(struct query *, const char *);
 
diff --git a/imap.c b/imap.c
index 1d8dbeb1..317a3394 100644
--- a/imap.c
+++ b/imap.c
@@ -415,6 +415,10 @@ static int imap_getrange(int sock,
 	ok = 0;
 	if (deletions && ctl->expunge > 1)
 	    ok = gen_transact(sock, "EXPUNGE");
+#ifdef IMAP_UID	/* not used */
+	if (!ok)
+	    expunge_uids(ctl);
+#endif /* IMAP_UID */
 	if (ok || gen_transact(sock, "NOOP"))
 	{
 	    error(0, 0, "re-poll failed");
@@ -479,7 +483,7 @@ static int imap_is_old(int sock, struct query *ctl, int number)
 {
     int ok;
 
-    /* expunged change the fetch numbers */
+    /* expunges change the fetch numbers */
     number -= expunged;
 
     if ((ok = gen_transact(sock, "FETCH %d FLAGS", number)) != 0)
@@ -494,7 +498,7 @@ static int imap_fetch_headers(int sock, struct query *ctl,int number,int *lenp)
     char buf [POPBUFSIZE+1];
     int	num;
 
-    /* expunged change the fetch numbers */
+    /* expunges change the fetch numbers */
     number -= expunged;
 
     /*
@@ -524,7 +528,7 @@ static int imap_fetch_body(int sock, struct query *ctl, int number, int *lenp)
     char buf [POPBUFSIZE+1], *cp;
     int	num;
 
-    /* expunged change the fetch numbers */
+    /* expunges change the fetch numbers */
     number -= expunged;
 
     /*
@@ -582,7 +586,7 @@ static int imap_fetch_body(int sock, struct query *ctl, int number, int *lenp)
 static int imap_trail(int sock, struct query *ctl, int number)
 /* discard tail of FETCH response after reading message text */
 {
-    /* expunged change the fetch numbers */
+    /* expunges change the fetch numbers */
     /* number -= expunged; */
 
     for (;;)
@@ -606,7 +610,7 @@ static int imap_delete(int sock, struct query *ctl, int number)
 {
     int	ok;
 
-    /* expunged change the fetch numbers */
+    /* expunges change the fetch numbers */
     number -= expunged;
 
     /*
@@ -631,6 +635,10 @@ static int imap_delete(int sock, struct query *ctl, int number)
 	if ((ok = gen_transact(sock, "EXPUNGE")))
 	    return(ok);
 
+#ifdef IMAP_UID	/* not used */
+	expunge_uids(ctl);
+#endif /* IMAP_UID */
+
 	expunged = deletions;
     }
 
@@ -647,6 +655,10 @@ static int imap_logout(int sock, struct query *ctl)
 
 	if ((ok = gen_transact(sock, "EXPUNGE")))
 	    return(ok);
+
+#ifdef IMAP_UID	/* not used */
+	expunge_uids(ctl);
+#endif /* IMAP_UID */
     }
 
     return(gen_transact(sock, "LOGOUT"));
diff --git a/pop3.c b/pop3.c
index 090b292c..d3b95742 100644
--- a/pop3.c
+++ b/pop3.c
@@ -483,7 +483,12 @@ static int pop3_delete(int sock, struct query *ctl, int number)
 static int pop3_logout(int sock, struct query *ctl)
 /* send logout command */
 {
-    return(gen_transact(sock, "QUIT"));
+    int ok = gen_transact(sock, "QUIT");
+
+    if (!ok)
+	expunge_uids(ctl);
+
+    return(ok);
 }
 
 const static struct method pop3 =
diff --git a/uid.c b/uid.c
index 01ace720..000c0233 100644
--- a/uid.c
+++ b/uid.c
@@ -46,16 +46,22 @@
  * (and possibly deleted).  It should be downloaded anyway if --all
  * is on.  It should not be deleted if --keep is on.
  *
- * Each time a message is deleted, we remove its id from the `newsaved'
- * member.
+ * Each time a message is deleted, we mark its id UID_DELETED from the
+ * `newsaved' member.  When we want to assert that an expunge has been
+ * done on the server, we call expunge_uid() to register that all
+ * deleted messages are gone by marking them UID_EXPUNGED.
  *
- * At the end of the query, whatever remains in the `newsaved' member
- * (because it was not deleted) becomes the `oldsaved' list.  The old
- * `oldsaved' list is freed.
+ * At the end of the query, the `newsaved' member becomes the
+ * `oldsaved' list.  The old `oldsaved' list is freed.
  *
- * At the end of the fetchmail run, all current `oldsaved' lists are
- * flushed out to the .fetchids file to be picked up by the next run.
- * If there are no such messages, the file is deleted.
+ * At the end of the fetchmail run, non-EXPUNGED members of all
+ * current `oldsaved' lists are flushed out to the .fetchids file to
+ * be picked up by the next run.  (The UID_EXPUNGED test means that a
+ * message marked UID_DELETED may still have its ID go to disk if
+ * there has been no intervening expunge operation.  This typically
+ * comes up if the query was aborted by a line hit before a quit or
+ * expunge was sent to the server.)  If there are no un-expunged
+ * messages, the file is deleted.
  *
  * Note: all comparisons are caseblind!
  */
@@ -90,14 +96,14 @@ void initialize_saved_lists(struct query *hostlist, const char *idfile)
 		        strcasecmp(host, ctl->server.truename) == 0
 				&& strcasecmp(user, ctl->remotename) == 0)
 		    {
-			save_str(&ctl->oldsaved, -1, id);
+			save_str(&ctl->oldsaved, UID_KEPT, id);
 			break;
 		    }
 		}
 
 		/* if it's not in a host we're querying, save it anyway */
 		if (ctl == (struct query *)NULL)
-		    save_str(&scratchlist, -1, buf);
+		    save_str(&scratchlist, UID_KEPT, buf);
 	    }
 	}
 	fclose(tmpfp);
@@ -235,6 +241,7 @@ char *idpair_find(struct idlist **idl, const char *id)
 int delete_str(struct idlist **idl, int num)
 /* delete given message from given list */
 {
+#ifdef HARD_DELETE	/* not used */
     if (*idl == (struct idlist *)NULL)
 	return(0);
     else if ((*idl)->val.num == num)
@@ -248,6 +255,17 @@ int delete_str(struct idlist **idl, int num)
     }
     else
 	return(delete_str(&(*idl)->next, num));
+#else
+    struct idlist	*idp;
+
+    for (idp = *idl; idp; idp = idp->next)
+	if (idp->val.num == num)
+	{
+	    idp->val.num = UID_DELETED;
+	    return(1);
+	}
+    return(0);
+#endif /* HARD_DELETE */
 }
 
 void append_str_list(struct idlist **idl, struct idlist **nidl)
@@ -262,6 +280,16 @@ void append_str_list(struct idlist **idl, struct idlist **nidl)
 }
 
 #ifdef POP3_ENABLE
+void expunge_uids(struct query *ctl)
+/* assert that all UIDs marked deleted have actually been expunged */
+{
+    struct idlist *idl;
+
+    for (idl = ctl->newsaved; idl; idl = idl->next)
+	if (idl->val.num == UID_DELETED)
+	    idl->val.num = UID_EXPUNGED;
+}
+
 void update_str_lists(struct query *ctl)
 /* perform end-of-query actions on UID lists */
 {
@@ -292,7 +320,8 @@ void write_saved_lists(struct query *hostlist, const char *idfile)
 	if ((tmpfp = fopen(idfile, "w")) != (FILE *)NULL) {
 	    for (ctl = hostlist; ctl; ctl = ctl->next) {
 		for (idp = ctl->oldsaved; idp; idp = idp->next)
-		    fprintf(tmpfp, "%s@%s %s\n", 
+		    if (idp->val.num != UID_EXPUNGED)
+			fprintf(tmpfp, "%s@%s %s\n", 
 			    ctl->remotename, ctl->server.truename, idp->id);
 	    }
 	    for (idp = scratchlist; idp; idp = idp->next)