Enable expunge to controil POP2 and POP3 checkpointing.

svn path=/trunk/; revision=2439

11 files changed, 141 insertions, 79 deletions

diff --git a/NEWS b/NEWS
index a7393693..6ca21291 100644
--- a/NEWS
+++ b/NEWS
@@ -12,8 +12,9 @@ fetchmail-5.0.1 ():
 * Try to discover user/home via getpwuid(getuid()) before using LOGNAME/HOME.
 * Mike Pearce's fix for buggy DEFAULT handling in .netrc.
 * Keep validation errors from generating message lines that qmail will reject.
+* Can now use expunge option to chop POP3 retrievals into subsessions.
 
-There are 255 people on fetchmail-friends and 369 on fetchmail-announce.
+There are 253 people on fetchmail-friends and 370 on fetchmail-announce.
 
 ------------------------------------------------------------------------------
 fetchmail-5.0.0 (Mon Apr  5 11:00:24 EDT 1999):
diff --git a/conf.c b/conf.c
index 24f71a37..23826c8e 100644
--- a/conf.c
+++ b/conf.c
@@ -74,7 +74,7 @@ static void numdump(const char *name, const int num)
 /* dump a numeric quantity at current indent */
 {
     indent('\0');
-    fprintf(stdout, "'%s':%d,\n", name, num);
+    fprintf(stdout, "'%s':%d,\n", name, NUM_VALUE_OUT(num));
 }
 
 static void booldump(const char *name, const int onoff)
diff --git a/driver.c b/driver.c
index d1ef6d48..2e123934 100644
--- a/driver.c
+++ b/driver.c
@@ -73,7 +73,6 @@
 extern char *strstr();	/* needed on sysV68 R3V7.1. */
 #endif /* strstr */
 
-int fetchlimit;		/* how often to tear down the server connection */
 int batchcount;		/* count of messages sent in current batch */
 flag peek_capable;	/* can we peek for better error recovery? */
 int pass;		/* how many times have we re-polled? */
@@ -1354,10 +1353,11 @@ static void send_size_warnings(struct query *ctl)
 #undef OVERHD
 }
 
-int do_protocol(ctl, proto)
+static int do_session(ctl, proto, maxfetch)
 /* retrieve messages from server using given protocol method table */
 struct query *ctl;		/* parsed options with merged-in defaults */
 const struct method *proto;	/* protocol method table */
+const int maxfetch;		/* maximum number of messages to fetch */
 {
     int ok, js;
 #ifdef HAVE_VOLATILE
@@ -1372,47 +1372,6 @@ const struct method *proto;	/* protocol method table */
     protocol = proto;
     ctl->server.base_protocol = protocol;
 
-#ifndef KERBEROS_V4
-    if (ctl->server.preauthenticate == A_KERBEROS_V4)
-    {
-	report(stderr, _("Kerberos V4 support not linked.\n"));
-	return(PS_ERROR);
-    }
-#endif /* KERBEROS_V4 */
-
-#ifndef KERBEROS_V5
-    if (ctl->server.preauthenticate == A_KERBEROS_V5)
-    {
-	report(stderr, _("Kerberos V5 support not linked.\n"));
-	return(PS_ERROR);
-    }
-#endif /* KERBEROS_V5 */
-
-    /* lacking methods, there are some options that may fail */
-    if (!proto->is_old)
-    {
-	/* check for unsupported options */
-	if (ctl->flush) {
-	    report(stderr,
-		    _("Option --flush is not supported with %s\n"),
-		    proto->name);
-	    return(PS_SYNTAX);
-	}
-	else if (ctl->fetchall) {
-	    report(stderr,
-		    _("Option --all is not supported with %s\n"),
-		    proto->name);
-	    return(PS_SYNTAX);
-	}
-    }
-    if (!proto->getsizes && NUM_SPECIFIED(ctl->limit))
-    {
-	report(stderr,
-		_("Option --limit is not supported with %s\n"),
-		proto->name);
-	return(PS_SYNTAX);
-    }
-
     pass = 0;
     tagnum = 0;
     tag[0] = '\0';	/* nuke any tag hanging out from previous query */
@@ -2067,12 +2026,12 @@ const struct method *proto;	/* protocol method table */
 			    report_complete(stdout, _(" not flushed\n"));
 
 			/* perhaps this as many as we're ready to handle */
-			if (NUM_NONZERO(ctl->fetchlimit)
-					&& ctl->fetchlimit <= fetches)
+			if (maxfetch && maxfetch <= fetches && fetches < count)
 			{
-			    report(stdout, _("fetchlimit reached; %d messages left on server\n"),
-				  count - fetches);
-			    goto no_error;
+			    report(stdout, _("fetchlimit %d reached; %d messages left on server\n"),
+				  maxfetch, count - fetches);
+			    ok = PS_MAXFETCH;
+			    goto cleanUp;
 			}
 		    }
 
@@ -2145,6 +2104,7 @@ const struct method *proto;	/* protocol method table */
 	report(stderr, _("undefined error\n"));
 	break;
     }
+    /* no report on PS_MAXFETCH or PS_UNDEFINED */
     if (ok==PS_SOCKET || ok==PS_AUTHFAIL || ok==PS_SYNTAX 
 		|| ok==PS_IOERR || ok==PS_ERROR || ok==PS_PROTOCOL 
 		|| ok==PS_LOCKBUSY || ok==PS_SMTP || ok==PS_DNS)
@@ -2163,6 +2123,84 @@ closeUp:
     return(ok);
 }
 
+int do_protocol(ctl, proto)
+/* retrieve messages from server using given protocol method table */
+struct query *ctl;		/* parsed options with merged-in defaults */
+const struct method *proto;	/* protocol method table */
+{
+    int	ok;
+
+#ifndef KERBEROS_V4
+    if (ctl->server.preauthenticate == A_KERBEROS_V4)
+    {
+	report(stderr, _("Kerberos V4 support not linked.\n"));
+	return(PS_ERROR);
+    }
+#endif /* KERBEROS_V4 */
+
+#ifndef KERBEROS_V5
+    if (ctl->server.preauthenticate == A_KERBEROS_V5)
+    {
+	report(stderr, _("Kerberos V5 support not linked.\n"));
+	return(PS_ERROR);
+    }
+#endif /* KERBEROS_V5 */
+
+    /* lacking methods, there are some options that may fail */
+    if (!proto->is_old)
+    {
+	/* check for unsupported options */
+	if (ctl->flush) {
+	    report(stderr,
+		    _("Option --flush is not supported with %s\n"),
+		    proto->name);
+	    return(PS_SYNTAX);
+	}
+	else if (ctl->fetchall) {
+	    report(stderr,
+		    _("Option --all is not supported with %s\n"),
+		    proto->name);
+	    return(PS_SYNTAX);
+	}
+    }
+    if (!proto->getsizes && NUM_SPECIFIED(ctl->limit))
+    {
+	report(stderr,
+		_("Option --limit is not supported with %s\n"),
+		proto->name);
+	return(PS_SYNTAX);
+    }
+
+    /*
+     * If no expunge limit or we do expunges within the driver,
+     * then just do one session, passing in any fetchlimit.
+     */
+    if (proto->retry || !NUM_SPECIFIED(ctl->expunge))
+	return(do_session(ctl, proto, NUM_VALUE_OUT(ctl->fetchlimit)));
+    /*
+     * There's an expunge limit, and it isn't handled in the driver itself.
+     * OK; do multiple sessions, each fetching a limited # of messages.
+     * Stop if the total count of retrieved messages exceeds ctl->fetchlimit
+     * (if it was nonzero).
+     */
+    else
+    {
+	int totalcount = 0; 
+	int expunge    = NUM_VALUE_OUT(ctl->expunge);
+	int fetchlimit = NUM_VALUE_OUT(ctl->fetchlimit);
+
+	do {
+	    ok = do_session(ctl, proto, expunge);
+	    totalcount += expunge;
+	    if (NUM_SPECIFIED(ctl->fetchlimit) && totalcount >= fetchlimit)
+		break;
+	} while
+	    (ok == PS_MAXFETCH);
+
+	return(ok);
+    }
+}
+
 #if defined(HAVE_STDARG_H)
 void gen_send(int sock, const char *fmt, ... )
 #else
diff --git a/fetchmail-features.html b/fetchmail-features.html
index a243bfea..ab778a90 100644
--- a/fetchmail-features.html
+++ b/fetchmail-features.html
@@ -10,12 +10,17 @@
 <table width="100%" cellpadding=0><tr>
 <td width="30%">Back to <a href="index.html">Fetchmail Home Page</a>
 <td width="30%" align=center>To <a href="/~esr/sitemap.html">Site Map</a>
-<td width="30%" align=right>$Date: 1999/02/07 17:03:04 $
+<td width="30%" align=right>$Date: 1999/04/18 18:32:01 $
 </table>
 <HR>
 
 <H1 ALIGN=CENTER>Fetchmail Feature List</H1>
 
+<H2>Since 5.0:</H2>
+<UL>
+<LI>Expunge option can now be used to break POP3 retrieval into subsessions. 
+</UL>
+
 <H2>Since 4.0:</H2>
 <UL>
 <LI>The interface and monitor options now work with freeBSD.
@@ -168,7 +173,7 @@ get-mail, gwpop, pimp-1.0, pop-perl5-1.2, popc, popmail-1.6 and upop.<P>
 <table width="100%" cellpadding=0><tr>
 <td width="30%">Back to <a href="index.html">Fetchmail Home Page</a>
 <td width="30%" align=center>To <a href="/~esr/sitemap.html">Site Map</a>
-<td width="30%" align=right>$Date: 1999/02/07 17:03:04 $
+<td width="30%" align=right>$Date: 1999/04/18 18:32:01 $
 </table>
 
 <P><ADDRESS>Eric S. Raymond <A HREF="mailto:esr@thyrsus.com">&lt;esr@snark.thyrsus.com&gt;</A></ADDRESS>
diff --git a/fetchmail.c b/fetchmail.c
index 54e54405..5ace5344 100644
--- a/fetchmail.c
+++ b/fetchmail.c
@@ -885,7 +885,6 @@ static int load_params(int argc, char **argv, int optind)
     def_opts.server.timeout = CLIENT_TIMEOUT;
     def_opts.warnings = WARNING_INTERVAL;
     def_opts.remotename = user;
-    def_opts.expunge = 1;
     def_opts.listener = SMTP_MODE;
 
     /* this builds the host list */
@@ -1493,9 +1492,9 @@ static void dump_params (struct runctl *runp,
 		    printf(_("  No SMTP message batch limit (--batchlimit 0).\n"));
 		if (ctl->server.protocol == P_IMAP)
 		    if (NUM_NONZERO(ctl->expunge))
-			printf(_("  Deletion interval between expunges is %d (--expunge %d).\n"), ctl->expunge, ctl->expunge);
+			printf(_("  Deletion interval between expunges forced to %d (--expunge %d).\n"), ctl->expunge, ctl->expunge);
 		    else if (outlevel >= O_VERBOSE)
-			printf(_("  No expunges (--expunge 0).\n"));
+			printf(_("  No forced expunges (--expunge 0).\n"));
 	}
 	if (ctl->bsmtp)
 	    printf(_("  Messages will be appended to %s as BSMTP\n"), visbuf(ctl->bsmtp));
diff --git a/fetchmail.h b/fetchmail.h
index c02bb768..0c0118e7 100644
--- a/fetchmail.h
+++ b/fetchmail.h
@@ -74,6 +74,7 @@
 #define		PS_REFUSED	25	/* mail refused (internal use) */
 #define		PS_RETAINED	26	/* message retained (internal use) */
 #define		PS_TRUNCATED	27	/* headers incomplete (internal use) */
+#define		PS_MAXFETCH	28	/* poll ended by fetch limit */
 
 /* output noise level */
 #define         O_SILENT	0	/* mute, max squelch, etc. */
@@ -285,9 +286,12 @@ struct msgblk			/* message header parsed for open_sink() */
 
 /*
  * Numeric option handling.  Numeric option value of zero actually means
- * it's unspecified.  Value less than zero is zero.
+ * it's unspecified.  Value less than zero is zero.  The reason for this
+ * screwy encoding is so we can zero out an option block in order to set the
+ * numeric flags in it to unspecified.
  */
-#define NUM_VALUE(n)		(((n) == 0) ? -1 : (n))
+#define NUM_VALUE_IN(n)		(((n) == 0) ? -1 : (n))
+#define NUM_VALUE_OUT(n)	(((n) < 0) ? 0 : (n))
 #define NUM_NONZERO(n)		((n) > 0)
 #define NUM_ZERO(n)		((n) < 0)
 #define NUM_SPECIFIED(n)	((n) != 0)
diff --git a/fetchmail.man b/fetchmail.man
index 50e215c5..601e9511 100644
--- a/fetchmail.man
+++ b/fetchmail.man
@@ -365,8 +365,14 @@ overrides any limits set in your run control file.
 This option does not work with ETRN.
 .TP
 .B -e, --expunge
-(keyword: expunge)
-When talking to an IMAP server, 
+(keyword: expunge) 
+Arrange for deletions to be made final after a given number of
+messages.  Under POP2 or POP3, fetchmail cannot make deletions final
+without sending QUIT and ending the session -- with this option on,
+fetchmail will break a long mail retrieval session into multiple
+subsessions, sending QUIT after each sub-session. This is a good
+defense against line drops on POP3 servers that do not do the
+equivalent of a QUIT on hangup.  Under IMAP,
 .I fetchmail
 normally issues an EXPUNGE command after each deletion in order to
 force the deletion to be done immediately.  This is safest when your
@@ -377,10 +383,9 @@ server pretty hard, so if your connection is reliable it is good to do
 expunges less frequently.  If you specify this option to an integer N,
 it tells
 .I fetchmail
-to only issue expunges on every Nth delete.  An argument
-of zero suppresses expunges entirely (so no expunges at all will be
-done until the end of run). 
-This option does not work with ETRN, POP2, or POP3.
+to only issue expunges on every Nth delete.  An argument of zero
+suppresses expunges entirely (so no expunges at all will be done until
+the end of run).  This option does not work with ETRN.
 .SS Authentication Options
 .TP
 .B \-u name, --username name
diff --git a/fetchmailconf b/fetchmailconf
index 2482d456..e904662e 100755
--- a/fetchmailconf
+++ b/fetchmailconf
@@ -220,7 +220,7 @@ class User:
         self.warnings = 0		# Size warning interval
 	self.fetchlimit = 0		# Max messages fetched per batch
 	self.batchlimit = 0		# Max message forwarded per batch
-	self.expunge = 1		# Interval between expunges (IMAP)
+	self.expunge = 0		# Interval between expunges (IMAP)
         self.properties = None		# Extension properties
 	User.typemap = (
 	    ('remote',      'String'),
diff --git a/imap.c b/imap.c
index ad654320..09561ea5 100644
--- a/imap.c
+++ b/imap.c
@@ -50,7 +50,8 @@ extern char *strstr();	/* needed on sysV68 R3V7.1. */
 #define IMAP4		0	/* IMAP4 rev 0, RFC1730 */
 #define IMAP4rev1	1	/* IMAP4 rev 1, RFC2060 */
 
-static int count, seen, recent, unseen, deletions, expunged, imap_version;
+static int count, seen, recent, unseen, deletions, imap_version; 
+static int expunged, expunge_period;
 static char capabilities[MSGBUFSIZE+1];
 
 int imap_ok(int sock, char *argbuf)
@@ -708,6 +709,15 @@ int imap_getauth(int sock, struct query *ctl, char *greeting)
     if (ok)
 	return(ok);
     
+    /* 
+     * Assumption: expunges are cheap, so we want to do them
+     * after every message unless user said otherwise.
+     */
+    if (NUM_SPECIFIED(ctl->expunge))
+	expunge_period = NUM_VALUE_OUT(ctl->expunge);
+    else
+	expunge_period = 1;
+
     return(PS_SUCCESS);
 }
 
@@ -747,7 +757,7 @@ static int imap_getrange(int sock,
 	 * infinite-loop picking up un-expunged message.
 	 */
 	ok = 0;
-	if (deletions && ctl->expunge > 1)
+	if (deletions && expunge_period > 1)
 	    internal_expunge(sock);
 	count = -1;
 	if (ok || gen_transact(sock, "NOOP"))
@@ -973,12 +983,12 @@ static int imap_delete(int sock, struct query *ctl, int number)
 	deletions++;
 
     /*
-     * We do an expunge after ctl->expunge messages, rather than
+     * We do an expunge after expunge_period messages, rather than
      * just before quit, so that a line hit during a long session
      * won't result in lots of messages being fetched again during
      * the next session.
      */
-    if (NUM_NONZERO(ctl->expunge) && (deletions % ctl->expunge) == 0)
+    if (NUM_NONZERO(expunge_period) && (deletions % expunge_period) == 0)
 	internal_expunge(sock);
 
     return(PS_SUCCESS);
@@ -988,7 +998,7 @@ static int imap_logout(int sock, struct query *ctl)
 /* send logout command */
 {
     /* if expunges after deletion have been suppressed, ship one now */
-    if (NUM_SPECIFIED(ctl->expunge) && NUM_ZERO(ctl->expunge) && deletions)
+    if (NUM_SPECIFIED(expunge_period) && NUM_ZERO(expunge_period) && deletions)
 	internal_expunge(sock);
 
     return(gen_transact(sock, "LOGOUT"));
diff --git a/options.c b/options.c
index 9d8cc486..8152bea3 100644
--- a/options.c
+++ b/options.c
@@ -426,7 +426,7 @@ struct query *ctl;	/* option record to be initialized */
 	case 'l':
 	case LA_LIMIT:
 	    c = xatoi(optarg, &errflag);
-	    ctl->limit = NUM_VALUE(c);
+	    ctl->limit = NUM_VALUE_IN(c);
 	    break;
 	case 'r':
 	case LA_FOLDER:
@@ -468,17 +468,17 @@ struct query *ctl;	/* option record to be initialized */
 	case 'b':
 	case LA_BATCHLIMIT:
 	    c = xatoi(optarg, &errflag);
-	    ctl->batchlimit = NUM_VALUE(c);
+	    ctl->batchlimit = NUM_VALUE_IN(c);
 	    break;
 	case 'B':
 	case LA_FETCHLIMIT:
 	    c = xatoi(optarg, &errflag);
-	    ctl->fetchlimit = NUM_VALUE(c);
+	    ctl->fetchlimit = NUM_VALUE_IN(c);
 	    break;
 	case 'e':
 	case LA_EXPUNGE:
 	    c = xatoi(optarg, &errflag);
-	    ctl->expunge = NUM_VALUE(c);
+	    ctl->expunge = NUM_VALUE_IN(c);
 	    break;
 	case 'm':
 	case LA_MDA:
@@ -528,7 +528,7 @@ struct query *ctl;	/* option record to be initialized */
 	case 'w':
 	case LA_WARNINGS:
 	    c = xatoi(optarg, &errflag);
-	    ctl->warnings = NUM_VALUE(c);
+	    ctl->warnings = NUM_VALUE_IN(c);
 	    break;
 
 	case LA_CONFIGDUMP:
diff --git a/rcfile_y.y b/rcfile_y.y
index 793426fc..848ebbb7 100644
--- a/rcfile_y.y
+++ b/rcfile_y.y
@@ -327,11 +327,11 @@ user_option	: TO localnames HERE
 		| NO DROPSTATUS		{current.dropstatus  = FLAG_FALSE;}
 		| NO MIMEDECODE		{current.mimedecode  = FLAG_FALSE;}
 
-		| LIMIT NUMBER		{current.limit       = NUM_VALUE($2);}
-		| WARNINGS NUMBER	{current.warnings    = NUM_VALUE($2);}
-		| FETCHLIMIT NUMBER	{current.fetchlimit  = NUM_VALUE($2);}
-		| BATCHLIMIT NUMBER	{current.batchlimit  = NUM_VALUE($2);}
-		| EXPUNGE NUMBER	{current.expunge     = NUM_VALUE($2);}
+		| LIMIT NUMBER		{current.limit       = NUM_VALUE_IN($2);}
+		| WARNINGS NUMBER	{current.warnings    = NUM_VALUE_IN($2);}
+		| FETCHLIMIT NUMBER	{current.fetchlimit  = NUM_VALUE_IN($2);}
+		| BATCHLIMIT NUMBER	{current.batchlimit  = NUM_VALUE_IN($2);}
+		| EXPUNGE NUMBER	{current.expunge     = NUM_VALUE_IN($2);}
 
 		| PROPERTIES STRING	{current.properties  = xstrdup($2);}
 		;