From 0d8a8b913f6a15088523bc69cbcf6240de945cda Mon Sep 17 00:00:00 2001 From: "Eric S. Raymond" Date: Wed, 15 Oct 2003 19:26:58 +0000 Subject: Sunil's transaction patches. svn path=/trunk/; revision=3858 --- driver.c | 215 +++++++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 134 insertions(+), 81 deletions(-) (limited to 'driver.c') diff --git a/driver.c b/driver.c index 6de0ffbd..efe74a9c 100644 --- a/driver.c +++ b/driver.c @@ -415,11 +415,55 @@ static void mark_oversized(struct query *ctl, int num, int size) } static int fetch_messages(int mailserver_socket, struct query *ctl, - int count, int *msgsizes, int *msgcodes, int maxfetch, + int count, int *msgsizes, int maxfetch, int *fetches, int *dispatches, int *deletions) /* fetch messages in lockstep mode */ { - int num, err, len; + flag force_retrieval; + int num, firstnum = 1, lastnum = 0, err, len; + int fetchsizelimit = ctl->fetchsizelimit; + int msgsize; + + if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit)) + { + /* for POP3, we can get the size of one mail only! Unfortunately, this + * protocol specific test cannot be done elsewhere as the protocol + * could be "auto". */ + if (ctl->server.protocol == P_POP3) + fetchsizelimit = 1; + + /* Time to allocate memory to store the sizes */ + xalloca(msgsizes, int *, sizeof(int) * fetchsizelimit); + } + + /* + * What forces this code is that in POP2 and + * IMAP2bis you can't fetch a message without + * having it marked `seen'. In POP3 and IMAP4, on the + * other hand, you can (peek_capable is set by + * each driver module to convey this; it's not a + * method constant because of the difference between + * IMAP2bis and IMAP4, and because POP3 doesn't peek + * if fetchall is on). + * + * The result of being unable to peek is that if there's + * any kind of transient error (DNS lookup failure, or + * sendmail refusing delivery due to process-table limits) + * the message will be marked "seen" on the server without + * having been delivered. This is not a big problem if + * fetchmail is running in foreground, because the user + * will see a "skipped" message when it next runs and get + * clued in. + * + * But in daemon mode this leads to the message + * being silently ignored forever. This is not + * acceptable. + * + * We compensate for this by checking the error + * count from the previous pass and forcing all + * messages to be considered new if it's nonzero. + */ + force_retrieval = !peek_capable && (ctl->errcount > 0); for (num = 1; num <= count; num++) { @@ -427,30 +471,82 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, flag suppress_forward = FALSE; flag suppress_readbody = FALSE; flag retained = FALSE; + int msgcode = MSGLEN_UNKNOWN; - if (msgcodes[num-1] < 0) + /* check if the message is old + * Note: the size of the message may not be known here */ + if (ctl->fetchall || force_retrieval) + ; + else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num)) + msgcode = MSGLEN_OLD; + if (msgcode == MSGLEN_OLD) { - if ((msgcodes[num-1] == MSGLEN_TOOLARGE) && !check_only) - { - mark_oversized(ctl, num, msgsizes[num-1]); - suppress_delete = TRUE; - } /* To avoid flooding the syslog when using --keep, * report "Skipped message" only when: * 1) --verbose is on, or - * 2) fetchmail does not use syslog, or - * 3) the message was skipped for some other - * reason than being old. + * 2) fetchmail does not use syslog */ if ( (outlevel >= O_VERBOSE) || - (outlevel > O_SILENT && (!run.use_syslog || msgcodes[num-1] != MSGLEN_OLD)) + (outlevel > O_SILENT && !run.use_syslog) ) { + report_build(stdout, + GT_("skipping message %s@%s:%d"), + ctl->remotename, ctl->server.truename, num); + } + + goto flagthemail; + } + + if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit) && + lastnum < num) + { + /* Instead of getting the sizes of all mails at the start, we get + * the sizes in blocks of fetchsizelimit. This leads to better + * performance when there are too many mails (say, 10000) in + * the mailbox and either we are not getting all the mails at + * one go (--fetchlimit 100) or there is a frequent socket + * error while just getting the sizes of all mails! */ + + int i; + int oldstage = stage; + firstnum = num; + lastnum = num + fetchsizelimit - 1; + if (lastnum > count) + lastnum = count; + for (i = 0; i < fetchsizelimit; i++) + msgsizes[i] = 0; + + stage = STAGE_GETSIZES; + err = (ctl->server.base_protocol->getpartialsizes)(mailserver_socket, num, lastnum, msgsizes); + if (err != 0) + return err; + stage = oldstage; + } + + msgsize = msgsizes ? msgsizes[num-firstnum] : 0; + + /* check if the message is oversized */ + if (NUM_NONZERO(ctl->limit) && (msgsize > ctl->limit)) + msgcode = MSGLEN_TOOLARGE; +/* else if (msgsize == 512) + msgcode = MSGLEN_OLD; (hmh) sample code to skip message */ + + if (msgcode < 0) + { + if ((msgcode == MSGLEN_TOOLARGE) && !check_only) + { + mark_oversized(ctl, num, msgsize); + suppress_delete = TRUE; + } + if (outlevel > O_SILENT) + { + /* old messages are already handled above */ report_build(stdout, GT_("skipping message %s@%s:%d (%d octets)"), ctl->remotename, ctl->server.truename, num, - msgsizes[num-1]); - switch (msgcodes[num-1]) + msgsize); + switch (msgcode) { case MSGLEN_INVALID: /* @@ -483,7 +579,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, report(stdout, GT_("couldn't fetch headers, message %s@%s:%d (%d octets)\n"), ctl->remotename, ctl->server.truename, num, - msgsizes[num-1]); + msgsize); continue; } else if (err != 0) @@ -492,7 +588,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, /* -1 means we didn't see a size in the response */ if (len == -1) { - len = msgsizes[num - 1]; + len = msgsize; wholesize = TRUE; } @@ -515,7 +611,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, * Read the message headers and ship them to the * output sink. */ - err = readheaders(mailserver_socket, len, msgsizes[num-1], + err = readheaders(mailserver_socket, len, msgsize, ctl, num, /* pass the suppress_readbody flag only if the underlying * protocol does not fetch the body separately */ @@ -573,7 +669,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, * string. This violates RFC2060. */ if (len == -1) - len = msgsizes[num-1] - msgblk.msglen; + len = msgsize - msgblk.msglen; if (outlevel > O_SILENT && !wholesize) report_complete(stdout, GT_(" (%d body octets) "), len); @@ -630,13 +726,13 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, * QUALCOMM server (at least) seems to be * reporting the on-disk size correctly. */ - if (msgblk.msglen != msgsizes[num-1]) + if (msgblk.msglen != msgsize) { if (outlevel >= O_DEBUG) report(stdout, GT_("message %s@%s:%d was not the expected length (%d actual != %d expected)\n"), ctl->remotename, ctl->server.truename, num, - msgblk.msglen, msgsizes[num-1]); + msgblk.msglen, msgsize); } /* end-of-message processing starts here */ @@ -649,6 +745,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, (*fetches)++; } +flagthemail: /* * At this point in flow of control, either * we've bombed on a protocol error or had @@ -668,9 +765,8 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, } else if (ctl->server.base_protocol->delete && !suppress_delete - && ((msgcodes[num-1] >= 0 && !ctl->keep) - || (msgcodes[num-1] == MSGLEN_OLD|| - msgcodes[num-1] == MSGLEN_TOOLARGE) && ctl->flush)) + && ((msgcode >= 0 && !ctl->keep) + || (msgcode == MSGLEN_OLD && ctl->flush))) { (*deletions)++; if (outlevel > O_SILENT) @@ -689,14 +785,14 @@ static int fetch_messages(int mailserver_socket, struct query *ctl, * 3) the message was skipped for some other * reason than just being old. */ - (outlevel > O_SILENT && (!run.use_syslog || msgcodes[num-1] != MSGLEN_OLD)) + (outlevel > O_SILENT && (!run.use_syslog || msgcode != MSGLEN_OLD)) ) report_complete(stdout, GT_(" not flushed\n")); /* maybe we mark this message as seen now? */ if (ctl->server.base_protocol->mark_seen && !suppress_delete - && (msgcodes[num-1] >= 0 && ctl->keep)) + && (msgcode >= 0 && ctl->keep)) { err = (ctl->server.base_protocol->mark_seen)(mailserver_socket, ctl, num); if (err != 0) @@ -827,7 +923,6 @@ const int maxfetch; /* maximum number of messages to fetch */ char buf[MSGBUFSIZE+1], *realhost; int count, new, bytes, deletions = 0; int *msgsizes = (int *)NULL; - int *msgcodes = (int *)NULL; #if INET6_ENABLE int fetches, dispatches, oldphase; #else /* INET6_ENABLE */ @@ -1302,37 +1397,7 @@ is restored.")); } else if (count > 0) { - flag force_retrieval; - int i, num; - - /* - * What forces this code is that in POP2 and - * IMAP2bis you can't fetch a message without - * having it marked `seen'. In POP3 and IMAP4, on the - * other hand, you can (peek_capable is set by - * each driver module to convey this; it's not a - * method constant because of the difference between - * IMAP2bis and IMAP4, and because POP3 doesn't peek - * if fetchall is on). - * - * The result of being unable to peek is that if there's - * any kind of transient error (DNS lookup failure, or - * sendmail refusing delivery due to process-table limits) - * the message will be marked "seen" on the server without - * having been delivered. This is not a big problem if - * fetchmail is running in foreground, because the user - * will see a "skipped" message when it next runs and get - * clued in. - * - * But in daemon mode this leads to the message - * being silently ignored forever. This is not - * acceptable. - * - * We compensate for this by checking the error - * count from the previous pass and forcing all - * messages to be considered new if it's nonzero. - */ - force_retrieval = !peek_capable && (ctl->errcount > 0); + int i; /* * Don't trust the message count passed by the server. @@ -1347,23 +1412,23 @@ is restored.")); return(PS_PROTOCOL); } - /* OK, we're going to gather size info next */ - xalloca(msgsizes, int *, sizeof(int) * count); - xalloca(msgcodes, int *, sizeof(int) * count); - for (i = 0; i < count; i++) { - msgsizes[i] = 0; - msgcodes[i] = MSGLEN_UNKNOWN; - } - /* * We need the size of each message before it's * loaded in order to pass it to the ESMTP SIZE * option. If the protocol has a getsizes method, * we presume this means it doesn't get reliable * sizes from message fetch responses. + * + * If the protocol supports getting sizes of subset of + * messages, we skip this step now. */ - if (proto->getsizes) + if (proto->getsizes && + !(proto->getpartialsizes && NUM_NONZERO(ctl->fetchsizelimit))) { + xalloca(msgsizes, int *, sizeof(int) * count); + for (i = 0; i < count; i++) + msgsizes[i] = 0; + stage = STAGE_GETSIZES; err = (proto->getsizes)(mailserver_socket, count, msgsizes); if (err != 0) @@ -1377,25 +1442,12 @@ is restored.")); } } - /* mark some messages not to be retrieved */ - for (num = 1; num <= count; num++) - { - if (NUM_NONZERO(ctl->limit) && (msgsizes[num-1] > ctl->limit)) - msgcodes[num-1] = MSGLEN_TOOLARGE; - else if (ctl->fetchall || force_retrieval) - continue; - else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num)) - msgcodes[num-1] = MSGLEN_OLD; -/* else if (msgsizes[num-1] == 512) - msgcodes[num-1] = MSGLEN_OLD; (hmh) sample code to skip message */ - } - /* read, forward, and delete messages */ stage = STAGE_FETCH; /* fetch in lockstep mode */ err = fetch_messages(mailserver_socket, ctl, - count, msgsizes, msgcodes, + count, msgsizes, maxfetch, &fetches, &dispatches, &deletions); if (err) @@ -1565,7 +1617,8 @@ const struct method *proto; /* protocol method table */ * If no expunge limit or we do expunges within the driver, * then just do one session, passing in any fetchlimit. */ - if (proto->retry || !NUM_SPECIFIED(ctl->expunge)) + if ((ctl->keep && !ctl->flush) || + proto->retry || !NUM_SPECIFIED(ctl->expunge)) return(do_session(ctl, proto, NUM_VALUE_OUT(ctl->fetchlimit))); /* * There's an expunge limit, and it isn't handled in the driver itself. -- cgit v1.2.3