Treat permanent delivery errors as temporary (configurable).

Fetchmail no longer drops permanently undelivered messages by default,
to match historic documentation.  It does this by adding a new
"softbounce" option, see below.
Fixes Debian Bug#471283, demotes Debian Bug#494418 to wishlist.

There is a new "softbounce" global option that prevents the deletion of
messages that have not been forwarded.  It defaults to "true" for
fetchmail 6.3.X in order to match historic documentation.  This may
change its default in the next major release.

NOTE: untested.

svn path=/branches/BRANCH_6-3/; revision=5315

11 files changed, 95 insertions, 15 deletions

diff --git a/NEWS b/NEWS
index 3f7be799..9fb8cbd8 100644
--- a/NEWS
+++ b/NEWS
@@ -43,11 +43,22 @@ removed from a 6.4.0 or newer release.)
   based systems.
 * The maintainer may migrate fetchmail to C++ with STL or C#, and impose further
   requirements (dependencies), such as Boost or other class libraries.
+* The softbounce option default will change to "false" in the next release.
 
 --------------------------------------------------------------------------------
 
 fetchmail 6.3.10 (not yet released):
 
+# INCOMPATIBLE BUGFIXES AND CHANGES
+* Fetchmail no longer drops permanently undelivered messages by default, to
+  match historic documentation.  It does this by adding a new "softbounce"
+  option, see below.
+  Fixes Debian Bug#471283, demotes Debian Bug#494418 to wishlist.
+* There is a new "softbounce" global option that prevents the deletion of
+  messages that have not been forwarded. It defaults to "true" for fetchmail
+  6.3.X in order to match historic documentation. This may change its default
+  in the next major release.
+
 # BUGFIXES
 * Fix misuse of canonical autoconf target as _TARGET when it should have been
   _HOST. Report and patch courtesy of Diego E. "Flameeyes" Pettenò.
diff --git a/TODO.txt b/TODO.txt
index aaf1852f..47a780ed 100644
--- a/TODO.txt
+++ b/TODO.txt
@@ -14,9 +14,8 @@ soon - MUST:
   https://bugs.launchpad.net/ubuntu/+source/fetchmail/+bug/151333
 
 soon - SHOULD:
-- https://bugzilla.novell.com/show_bug.cgi?id=246829 - fetchmail lost some mail
-  (5XX error code in contradiction with manual?)
-  also http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=471283
+- https://bugzilla.novell.com/show_bug.cgi?id=246829 - recycle part of
+  patch (STARTTLS required; 5XX error code in contradiction with manual?)
 - allow \Deleted without \Seen, rf. 
   http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=466299
 - document IMAP4 ACL requirements
@@ -31,7 +30,7 @@ soon - SHOULD:
 - Debian Bug#449179, smbutil.c:90: unicodeToString: Assertion `len+1 < sizeof
   buf' failed, from Stepan Golosunov <stepan@golosunov.pp.ru>
 - Fix TOCTOU race around prc_filecheck*
-- Get rid of peeking in socket.c? MSG_PEEK seems non-portable and racey.
+- Get rid of peeking in socket.c? MSG_PEEK seems non-portable.
 
 soon - MAY:
 - http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=471176
diff --git a/conf.c b/conf.c
index 75fbd34a..60eb3ebb 100644
--- a/conf.c
+++ b/conf.c
@@ -200,6 +200,7 @@ void dump_config(struct runctl *runp, struct query *querylist)
     stringdump("postmaster", runp->postmaster);
     booldump("bouncemail", runp->bouncemail);
     booldump("spambounce", runp->spambounce);
+    booldump("softbounce", runp->softbounce);
     stringdump("properties", runp->properties);
     booldump("invisible", runp->invisible);
     booldump("showdots", runp->showdots);
diff --git a/driver.c b/driver.c
index 9f332f80..4a2e3233 100644
--- a/driver.c
+++ b/driver.c
@@ -765,16 +765,17 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 
 flagthemail:
 	/*
-	 * At this point in flow of control, either
-	 * we've bombed on a protocol error or had
-	 * delivery refused by the SMTP server
-	 * (unlikely -- I've never seen it) or we've
-	 * seen `accepted for delivery' and the
-	 * message is shipped.  It's safe to mark the
-	 * message seen and delete it on the server
-	 * now.
+	 * At this point in flow of control,
+	 * either we've bombed on a protocol error
+	 * or had delivery refused by the SMTP server
+	 * or we've seen `accepted for delivery' and the message is shipped.
+	 * It's safe to mark the message seen and delete it on the server now.
 	 */
 
+	/* in softbounce mode, suppress deletion and marking as seen */
+	if (suppress_forward)
+	    suppress_delete = suppress_delete || run.softbounce;
+
 	/* maybe we delete this message now? */
 	if (retained)
 	{
diff --git a/fetchmail.c b/fetchmail.c
index 2832397a..264b7e39 100644
--- a/fetchmail.c
+++ b/fetchmail.c
@@ -987,6 +987,7 @@ static int load_params(int argc, char **argv, int optind)
     char *p;
 
     run.bouncemail = TRUE;
+    run.softbounce = TRUE;	/* treat permanent errors as temporary */
     run.spambounce = FALSE;	/* don't bounce back to innocent bystanders */
 
     memset(&def_opts, '\0', sizeof(struct query));
@@ -1112,6 +1113,8 @@ static int load_params(int argc, char **argv, int optind)
 	run.postmaster = cmd_run.postmaster;
     if (cmd_run.bouncemail)
 	run.bouncemail = cmd_run.bouncemail;
+    if (cmd_run.softbounce)
+	run.softbounce = cmd_run.softbounce;
 
     /* check and daemon options are not compatible */
     if (check_only && run.poll_interval)
@@ -1536,6 +1539,11 @@ static void dump_params (struct runctl *runp,
     else if (outlevel >= O_VERBOSE)
 	printf(GT_("Fetchmail will direct error mail to the sender.\n"));
 
+    if (!runp->softbounce)
+	printf(GT_("Fetchmail will treat permanent errors as permanent (drop messsages).\n"));
+    else if (outlevel >= O_VERBOSE)
+	printf(GT_("Fetchmail will treat permanent errors as temporary (keep messages).\n"));
+
     for (ctl = querylist; ctl; ctl = ctl->next)
     {
 	if (!ctl->active || (implicit && ctl->server.skip))
diff --git a/fetchmail.h b/fetchmail.h
index bb3b09bf..afbf0d72 100644
--- a/fetchmail.h
+++ b/fetchmail.h
@@ -180,6 +180,7 @@ struct runctl
     int		poll_interval;	/** poll interval in seconds (daemon mode, 0 == off) */
     flag	bouncemail;
     flag	spambounce;
+    flag	softbounce;
     flag	use_syslog;
     flag	invisible;
     flag	showdots;
diff --git a/fetchmail.man b/fetchmail.man
index 1c4e6375..86a541be 100644
--- a/fetchmail.man
+++ b/fetchmail.man
@@ -159,6 +159,20 @@ Verbose mode.  All control messages passed between \fBfetchmail\fP
 and the mailserver are echoed to stdout.  Overrides \-\-silent.
 Doubling this option (\-v \-v) causes extra diagnostic information
 to be printed.
+.TP
+.B \-\-nosoftbounce
+(since v6.3.10, Keyword: set no softbounce, since v6.3.10)
+.br
+Hard bounce mode. All permanent delivery errors cause messages to be
+deleted from the upstream server, see "no softbounce" below.
+.TP
+.B \-\-softbounce
+(since v6.3.10, Keyword: set softbounce, since v6.3.10)
+.br
+Soft bounce mode. All permanent delivery errors cause messages to be
+left on the upstream server if the protocol supports that. Default to
+match historic fetchmail documentation, to be changed to hard bounce
+mode in the next fetchmail release.
 .SS Disposal Options
 .TP
 .B \-a | \-\-all | (since v6.3.3) \-\-fetchall
@@ -1573,6 +1587,14 @@ Warning: Do not use this to bounce spam back to the sender - most spam
 is sent with false sender address and thus this option hurts innocent
 bystanders.
 T}
+set no softbounce	\&	\&	T{
+Delete permanently undeliverable mail. It is recommended to use this
+option if the configuration has been thoroughly tested.
+T}
+set    spambounce	\&	\&	T{
+Keep permanently undeliverable mail as though a temporary error had
+occurred (default).
+T}
 set logfile  	\-L	\&	T{
 Name of a file to append error and status messages to.
 T}
diff --git a/fetchmailconf.py b/fetchmailconf.py
index 8c6decab..baeb1a0c 100755
--- a/fetchmailconf.py
+++ b/fetchmailconf.py
@@ -5,7 +5,7 @@
 # Matthias Andree <matthias.andree@gmx.de>
 # Requires Python with Tkinter, and the following OS-dependent services:
 #	posix, posixpath, socket
-version = "1.54 $Revision$"
+version = "1.55 $Revision$"
 
 from Tkinter import *
 from Dialog import *
@@ -22,6 +22,7 @@ class Configuration:
 	self.postmaster = None		# No last-resort address, initially
 	self.bouncemail = TRUE		# Bounce errors to users
 	self.spambounce = FALSE		# Bounce spam errors
+	self.softbounce = TRUE		# Treat permanent error as temporary
 	self.properties = None		# No exiguous properties
 	self.invisible = FALSE		# Suppress Received line & spoof?
 	self.syslog = FALSE		# Use syslogd for logging?
@@ -33,6 +34,7 @@ class Configuration:
 	    ('postmaster',	'String'),
 	    ('bouncemail',	'Boolean'),
 	    ('spambounce',	'Boolean'),
+	    ('softbounce',	'Boolean'),
 	    ('properties',	'String'),
 	    ('syslog',	  'Boolean'),
 	    ('invisible',	'Boolean'))
@@ -55,6 +57,10 @@ class Configuration:
 	    str = str + ("set spambounce\n")
 	else:
 	    str = str + ("set no spambounce\n")
+	if self.softbounce:
+	    str = str + ("set softbounce\n")
+	else:
+	    str = str + ("set no softbounce\n")
 	if self.properties != ConfigurationDefaults.properties:
 	    str = str + ("set properties \"%s\"\n" % (self.properties,));
 	if self.poll_interval > 0:
@@ -723,6 +729,14 @@ Send spam bounces?
 	postmaster (depending on the "Bounces to sender?" option.  Otherwise,
 	spam bounces are not sent (the default).
 
+Use soft bounces?
+	If this option is on, permanent delivery errors are treated as
+	temporary, i. e. mail is kept on the upstream server. Useful
+	during testing and after configuration changes, and on by
+	default.
+	  If this option is off, permanent delivery errors delete
+	undeliverable mail from the upstream.
+
 Invisible
 	If false (the default) fetchmail generates a Received line into
 	each message and generates a HELO from the machine it is running on.
@@ -818,6 +832,13 @@ class ConfigurationEdit(Frame, MyWidget):
 		'relief':GROOVE}).pack(side=LEFT, anchor=W)
 	    sb.pack(fill=X)
 
+	    sb = Frame(gf)
+	    Checkbutton(sb,
+		{'text':'treat permanent errors as temporary?',
+		'variable':self.softbounce,
+		'relief':GROOVE}).pack(side=LEFT, anchor=W)
+	    sb.pack(fill=X)
+
 	    sf = Frame(gf)
 	    Checkbutton(sf,
 		{'text':'Log to syslog?',
diff --git a/options.c b/options.c
index 56569bcb..e85682d5 100644
--- a/options.c
+++ b/options.c
@@ -51,7 +51,9 @@ enum {
     LA_FETCHSIZELIMIT,
     LA_FASTUIDL,
     LA_LIMITFLUSH,
-    LA_IDLE
+    LA_IDLE,
+    LA_NOSOFTBOUNCE,
+    LA_SOFTBOUNCE
 };
 
 /* options still left: CgGhHjJoORTWxXYz */
@@ -78,6 +80,8 @@ static const struct option longoptions[] = {
   {"pidfile",	required_argument, (int *) 0, LA_PIDFILE },
   {"postmaster",required_argument, (int *) 0, LA_POSTMASTER },
   {"nobounce",	no_argument,	   (int *) 0, LA_NOBOUNCE },
+  {"nosoftbounce", no_argument,	   (int *) 0, LA_NOSOFTBOUNCE },
+  {"softbounce", no_argument,	   (int *) 0, LA_SOFTBOUNCE },
 
   {"protocol",	required_argument, (int *) 0, 'p' },
   {"proto",	required_argument, (int *) 0, 'p' },
@@ -300,6 +304,12 @@ int parsecmdline (int argc /** argument count */,
 	case LA_NOBOUNCE:
 	    run.bouncemail = FALSE;
 	    break;
+	case LA_NOSOFTBOUNCE:
+	    run.softbounce = FALSE;
+	    break;
+	case LA_SOFTBOUNCE:
+	    run.softbounce = TRUE;
+	    break;
 	case 'p':
 	    /* XXX -- should probably use a table lookup here */
 	    if (strcasecmp(optarg,"auto") == 0)
@@ -607,6 +617,8 @@ int parsecmdline (int argc /** argument count */,
 	P(GT_("      --pidfile     specify alternate PID (lock) file\n"));
 	P(GT_("      --postmaster  specify recipient of last resort\n"));
 	P(GT_("      --nobounce    redirect bounces from user to postmaster.\n"));
+	P(GT_("      --nosoftbounce fetchmail deletes permanently undeliverable messages.\n"));
+	P(GT_("      --softbounce  keep permanently undeliverable messages on server (default).\n"));
 #ifdef CAN_MONITOR
 	P(GT_("  -I, --interface   interface required specification\n"));
 	P(GT_("  -M, --monitor     monitor interface for activity\n"));
diff --git a/rcfile_l.l b/rcfile_l.l
index f19360d1..67317a6b 100644
--- a/rcfile_l.l
+++ b/rcfile_l.l
@@ -18,6 +18,7 @@ int prc_lineno = 1;
 #else
 #define SETSTATE(n)	BEGIN(n)
 #endif /* LEXDEBUG */
+
 %}
 
 /* this doesn't work with Linux lex, see the INSTALL file */
@@ -77,6 +78,7 @@ showdots	{ return SHOWDOTS; }
 postmaster	{ return POSTMASTER; }
 bouncemail	{ return BOUNCEMAIL; }
 spambounce	{ return SPAMBOUNCE; }
+softbounce	{ return SOFTBOUNCE; }
 warnings	{ return WARNINGS; }
 tracepolls	{ return TRACEPOLLS; }
 
diff --git a/rcfile_y.y b/rcfile_y.y
index cb9a262f..c28c6b42 100644
--- a/rcfile_y.y
+++ b/rcfile_y.y
@@ -69,7 +69,7 @@ extern char * yytext;
 %token IS HERE THERE TO MAP WILDCARD
 %token BATCHLIMIT FETCHLIMIT FETCHSIZELIMIT FASTUIDL EXPUNGE PROPERTIES
 %token SET LOGFILE DAEMON SYSLOG IDFILE PIDFILE INVISIBLE POSTMASTER BOUNCEMAIL
-%token SPAMBOUNCE SHOWDOTS
+%token SPAMBOUNCE SOFTBOUNCE SHOWDOTS
 %token <proto> PROTO AUTHTYPE
 %token <sval>  STRING
 %token <number> NUMBER
@@ -102,6 +102,8 @@ statement	: SET LOGFILE optmap STRING	{run.logfile = prependdir ($4, rcfiledir);
 		| SET NO BOUNCEMAIL		{run.bouncemail = FALSE;}
 		| SET SPAMBOUNCE		{run.spambounce = TRUE;}
 		| SET NO SPAMBOUNCE		{run.spambounce = FALSE;}
+		| SET SOFTBOUNCE		{run.softbounce = TRUE;}
+		| SET NO SOFTBOUNCE		{run.softbounce = FALSE;}
 		| SET PROPERTIES optmap STRING	{run.properties =xstrdup($4);}
 		| SET SYSLOG			{run.use_syslog = TRUE;}
 		| SET NO SYSLOG			{run.use_syslog = FALSE;}