/* texpire -- expire old articles Written by Arnt Gulbrandsen and copyright 1995 Troll Tech AS, Postboks 6133 Etterstad, 0602 Oslo, Norway, fax +47 22646949. Modified by Cornelius Krasel and Randolf Skerka . Copyright of the modifications 1997. Modified by Kent Robotti . Copyright of the modifications 1998. Modified by Markus Enzenberger . Copyright of the modifications 1998. Modified by Cornelius Krasel . Copyright of the modifications 1998, 1999. Modified by Kazushi (Jam) Marukawa . Copyright of the modifications 1998, 1999. Modified by Joerg Dietrich . Copyright of the modifications 1999-2001. See README for restrictions on the use of this software. */ #include "leafnode.h" #ifdef SOCKS #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define ARTNO (0) #define SUBJECT (1) #define FROM (2) #define DATE (3) #define MESSAGEID (4) #define REFERENCES (5) #define BYTES (6) #define LINES (7) #define XREF (8) int debug = 0; int use_atime = 1; /* look for atime on articles to expire */ int repair_spool = 0; /* Try to relink articles in groups with message.id tree. Useful for converting from a different tradspool server to leafnode. Or to repair other damage. */ int default_expire; struct exp { unsigned long artno; time_t atime; time_t mtime; char *xover; }; struct thread { unsigned long members; char **artnos; char **msgid; /* Message-ID of a single article */ char *mids; /* Message-ID and References of all articles in this thread */ time_t atime_max; time_t mtime_max; struct thread *next_thread; }; static int _compare_artno( const void *a, const void *b ); static char *mirror_string( const char *string ); void free_expire( void ); static time_t lookup_expire( char* group ); static void expiregroup( struct newsgroup* g ); static char *readxover( const char *gdir ); static void dogroup( struct newsgroup *g ); void free_threads( struct thread *thrd); static struct thread *sort_into_threads( struct exp *arts, unsigned long acount ); static char *newrefs( const char *mids, const char *refs ); static void new_thread( struct thread *thrd, const char *artno, const char *mid, const char * refs, time_t atime, time_t mtime ); static int findinrefs( const char *mids, const char *refs ); static void combine_arts_xover( struct exp *arts, char *xover, unsigned long acount ); static char *xoverextract( char *xover, unsigned int field ); static void expiremsgid( void ); static void usage( void ); static int _compare_artno( const void *a, const void *b ) { if ( *(const unsigned long **)a < *(const unsigned long **)b ) { return -1; } else { if ( *(const unsigned long **)a > *(const unsigned long **)b ){ return 1; } else { fprintf( stderr, "This is impossible!\n" ); exit( EXIT_FAILURE ); } } } /* * Return a pointer to a mirrored string * E.g. if string is "lewd did I live & evil I did dwel" the function * returns "lewd did I live & evil I did dwel". */ static char *mirror_string(const char *string ) { static char *mstring; char *p; size_t len; size_t i; if ( !string ) return NULL; len = strlen( string ); mstring = critmalloc( sizeof(char)*(len+1), "Allocating space for mirror string." ); p = strchr( string, '\0' ); p--; for ( i = len; i > 0; i-- ) { *(mstring+len-i) = *(p-len+i); } *(mstring+len)='\0'; return mstring; } void free_expire( void ) { struct expire_entry *a, *b; b = expire_base; while ((a = b) != NULL) { b = a->next; free(a); } } /* 05/27/97 - T. Sweeney - Find a group in the expireinfo linked list and return its expire time. Otherwise, return zero. */ static time_t lookup_expire(char* group) { struct expire_entry *a; a = expire_base; while (a != NULL) { if (ngmatch(a->group, group) == 0) return a->xtime; a = a->next; } return 0; } static void expiregroup(struct newsgroup* g) { struct newsgroup * ng; ng = g; while ( ng && ng->name ) { if (!(expire = lookup_expire(ng->name))) expire = default_expire; dogroup( ng ); ng++; } } /* Read overview into memory and return a pointer to it */ static char *readxover( const char *gdir ) { struct stat st; static char *overview; int fd; if (stat(".overview", &st) == 0) { overview = critmalloc(st.st_size + 1, "Reading article overview info"); if ((fd = open(".overview", O_RDONLY)) < 0 || (read(fd, overview, st.st_size) < st.st_size)) { syslog(LOG_ERR, "can't open/read %s/.overview: %m", gdir); *overview = '\0'; if (fd > -1) close(fd); } else { close(fd); overview[st.st_size] = '\0'; /* 0-terminate string */ } return overview; } return NULL; } /* * Expire articles in a group */ static void dogroup(struct newsgroup* g) { char gdir[PATH_MAX]; DIR *d; struct dirent *de; struct stat st; unsigned long first, last; /* low-/high-water mark */ unsigned long current, acount; /* current artno, total number of articles in group */ unsigned long artno; unsigned long deleted, kept; struct exp *articles = NULL; char *overview; char *p; struct thread *threads, *threads_first; acount = current = 0; deleted = kept = 0; clearidtree(); if ( !chdirgroup( g->name, FALSE ) ) return; getcwd(gdir, PATH_MAX); if ( verbose > 1 ) printf("%s: expire: %lu\n", g->name, expire); if ( debugmode ) syslog( LOG_DEBUG, "%s: expire %lu", g->name, expire ); d = opendir("."); if (!d) { syslog(LOG_ERR, "opendir in %s: %m", gdir); return; } getxover(); /* Make sure .overview is up to date */ overview = readxover( gdir ); first = ULONG_MAX; last = 0; while ( ( de = readdir(d) ) != 0 ) { if ( !isdigit((unsigned char)de->d_name[0]) || stat( de->d_name, &st ) || !S_ISREG( st.st_mode ) ) continue; current = strtoul(de->d_name, &p, 10); if (p && !*p) { acount++; articles = (struct exp *) critrealloc( (char *)articles, (acount) * sizeof(struct exp), "Reading articles to expire"); articles[acount-1].artno = current; articles[acount-1].atime = st.st_atime; articles[acount-1].mtime = st.st_mtime; articles[acount-1].xover = '\0'; } } /* Sort articles array by article number */ qsort( articles, acount, sizeof( struct exp ), &_compare_artno ); combine_arts_xover( articles, overview, acount ); threads_first = threads = sort_into_threads( articles, acount ); while( threads ) { if ( threads->mtime_max < expire || ( use_atime && threads->atime_max < expire ) ) { for( current = 0;current <= threads->members; current++ ) { if ( !unlink(threads->artnos[current]) ) { if ( debugmode ) syslog( LOG_DEBUG, "deleted article %s/%s", gdir, threads->artnos[current] ); deleted++; } else { syslog( LOG_ERR, "failed to unlinks %s/%s: %m", gdir, threads->artnos[current] ); kept++; } } } else { for ( current = 0; current <= threads->members; current++ ) { artno = strtoul(threads->artnos[current], NULL, 10); p = threads->msgid[current]; if (findmsgid( p )) { /* another file with same msgid? */ unlink( threads->artnos[current] ); deleted++; } else { insertmsgid(p, artno); if ( repair_spool ) { if ( link(threads->artnos[current], lookup( p )) ) { if (errno != EEXIST) syslog(LOG_ERR, "relink of %s failed: %m (%s)", p, lookup( p )); } else syslog(LOG_INFO, "relinked message %s", p ); } kept++; } if (artno < first) first = artno; if (artno > last) last = artno; } } threads = threads->next_thread; } if ( overview ) free( overview ); if ( threads_first ) free_threads( threads_first ); if ( articles ) free( articles ); if ( deleted || kept ) { printf("%s: %lu articles deleted, %lu kept\n", g->name, deleted, kept); syslog( LOG_INFO, "%s: %lu articles deleted, %lu kept", g->name, deleted, kept); } if ( !kept ) { if ( unlink( ".overview" ) < 0 && errno != ENOENT ) syslog( LOG_ERR, "unlink %s/.overview: %m", gdir ); if ( !chdir("..") ) { /* delete directory and empty parent directories */ while ( !rmdir( gdir ) ) { chdir(".."); getcwd( gdir, PATH_MAX ); } } } if ( deleted ) getxover(); if ( last >= first ) { g->first = first; g->last = last; if ( verbose > 1 ) printf("%s: new low water mark %lu, new high water mark %lu\n", g->name, first, last ); if ( debugmode ) syslog( LOG_DEBUG, "%s: new low water mark %lu, new high water mark %lu", g->name, first, last ); } } /* * Free() list of threads */ void free_threads( struct thread *thrd ) { struct thread *next_thread; unsigned long i; while( thrd ) { for ( i = 0; i <= thrd->members; i++ ) { free( thrd->artnos[i] ); free( thrd->msgid[i] ); } free( thrd->mids ); next_thread = thrd->next_thread; free( thrd ); thrd = next_thread; } } static struct thread *sort_into_threads( struct exp *arts, unsigned long acount ) { static struct thread *thrd, *thrd_first; unsigned long current; unsigned long base; char *artno = NULL; char *msgid = NULL; char *refs = NULL; char *nrefs; if ( !arts ) return NULL; thrd_first = (struct thread *)critmalloc( sizeof( struct thread ), "Allocating space for thread" ); thrd = thrd_first; current = 0; while ( (!artno || !msgid) && current < acount ) { artno = xoverextract( arts[current].xover, ARTNO ); msgid = xoverextract( arts[current].xover, MESSAGEID ); refs = xoverextract( arts[current].xover, REFERENCES ); current++; } if ( !artno || !msgid ) return NULL; new_thread( thrd, artno, msgid, refs, arts[current-1].atime, arts[current-1].mtime ); base = current; for ( current = acount; current > base; current-- ) { artno = xoverextract( arts[current-1].xover, ARTNO ); msgid = xoverextract( arts[current-1].xover, MESSAGEID ); refs = xoverextract( arts[current-1].xover, REFERENCES ); if ( artno && msgid ) { thrd = thrd_first; do { if ( strstr( thrd->mids, msgid ) || findinrefs( thrd->mids, refs ) ) { thrd->members++; thrd->artnos = (char **) critrealloc( (char *)thrd->artnos, sizeof(char*)*(thrd->members+1), "Allocating space for new artno."); thrd->artnos[thrd->members] = strdup( artno ); thrd->msgid = (char **) critrealloc( (char *)thrd->msgid, sizeof(char*)*(thrd->members+1), "Allocating space for new msgid."); thrd->msgid[thrd->members] = strdup( msgid ); thrd->mids = critrealloc( thrd->mids, sizeof(char)* (strlen(msgid)+strlen(thrd->mids)+1), "Allocating space for new mids."); strcat( thrd->mids, msgid ); if ( refs ) { nrefs = newrefs( thrd->mids, refs ); if (nrefs) { thrd->mids = critrealloc( thrd->mids, sizeof(char)* (strlen(nrefs) +strlen(thrd->mids)+1), "Allocating space for new mids."); strcat( thrd->mids, nrefs ); free( nrefs ); } } if ( thrd->atime_max < arts[current-1].atime ) thrd->atime_max = arts[current-1].atime; if ( thrd->mtime_max < arts[current-1].mtime ) thrd->mtime_max = arts[current-1].mtime; goto next_article; } } while( thrd->next_thread && (thrd = thrd->next_thread) ); thrd->next_thread = (struct thread *)critmalloc( sizeof(struct thread), "Allocating space for new" " thread." ); thrd = thrd->next_thread; new_thread( thrd, artno, msgid, refs, arts[current-1].atime, arts[current-1].mtime ); next_article: free( artno ); free( msgid ); if ( refs ) free( refs ); artno = msgid = refs = NULL; } } return thrd_first; } /* * Return the Message-IDs present in refs, but not in mids, until the * the first match is found beginning from the end of refs. * Q: Why? * A: References are added to the end of the References: line. As soon * as we encounter one we already have in mids, we have all the previous * ones as well. */ static char *newrefs( const char *mids, const char *refs ) { static char *nrefs; char *mrefs; char *tmp, *p; if ( !refs || !*refs ) return NULL; tmp = mrefs = mirror_string( refs ); p = strtok( mrefs, " " ); p = mirror_string( p ); if ( !strstr( mids, p ) ) { nrefs = critmalloc( sizeof(char)*(strlen(p)+1), "Allocating space for new ref." ); strcpy( nrefs, p ); free( p ); } else { free( tmp ); free( p ); return NULL; } while ( 1 ) { p = strtok( NULL, " " ); p = mirror_string( p ); if ( p && !strstr( mids, p ) ) { nrefs = critrealloc( nrefs, sizeof(char)*(strlen(nrefs)+strlen(p)+1), "Allocating space for new ref." ); strcat( nrefs, p ); free( p ); } else { free( tmp ); if ( p ) free( p ); return nrefs; } } } /* * Compare Message-IDs from a References: line with a char containing msgids. * Return 1 if found, 0 if not. * Note: It turns out that we only need to compare the first entry in the * References: line because all readers either conform to grandson of RFC 1036 * or only put their immediate parent's Message-ID into References:. * If you think that's wrong uncomment the code below. */ static int findinrefs( const char *mids, const char *refs ) { char *p; char *tmp; int found = 0; if ( !refs || !*refs ) return 0; tmp = p = strdup( refs ); if ( !tmp ) return 0; p = strtok( p, " " ); if ( strstr( mids, p ) ) found = 1; /* See note above! while ( p && !found ) { p = strtok( NULL, " " ); if ( p && strstr( mids, p ) ) found = 1; } */ free( tmp ); return found; } /* * Initialize a new thread */ static void new_thread( struct thread *thrd, const char *artno, const char *mid, const char * refs, time_t atime, time_t mtime ) { thrd->members = 0; thrd->artnos = (char **)critmalloc( sizeof(char*), "Allocating space for artno."); thrd->artnos[0] = strdup( artno ); thrd->msgid = (char **)critmalloc( sizeof(char*), "Allocating space for msgid."); thrd->msgid[0] = strdup( mid ); thrd->mids = critmalloc( sizeof(char)*(strlen(mid)+strlen(refs)+1), "Allocating space for mids."); strcpy( thrd->mids, mid ); if ( refs ) strcat( thrd->mids, refs ); thrd->atime_max = atime; thrd->mtime_max = mtime; thrd->next_thread = NULL; } static char *xoverextract( char *xover, unsigned int field ) { unsigned int n; char *line; char *p, *q, *tmp; if ( !xover ) return NULL; tmp = p = strdup( xover ); for (n = 0; n < field; n++) { if (p && (p = strchr(p + 1, '\t'))) { p++; } } q = p ? strchr(p, '\t') : NULL; if ( p && q ) { *q = '\0'; line = strdup( p ); } else line = NULL; free( tmp ); return line; } /* * Add overview information to sorted articles array */ static void combine_arts_xover( struct exp *arts, char *xover, unsigned long acount ) { unsigned long current, artno; char *p; if ( !arts ) return; current = 0; p = xover; while (p && *p && current < acount) { while (p && isspace((unsigned char)*p)) p++; artno = strtoul(p, NULL, 10); if ( artno == arts[current].artno ) { arts[current].xover = p; current++; } else { /* If we are here something something is probably wrong with the locking mechanism*/ if ( artno < arts[current].artno ) { fprintf( stderr, "Missing article %lu in expire array." "\nDeleted after texpire started?", artno ); } else { fprintf( stderr, "Missing overview information for article " "%lu." "\nAdded after texpire started?", arts[current].artno ); current++; continue; /* Continue the while loop. Do not advance p! */ } } p = strchr(p, '\n'); if (p) { *p = '\0'; if (p[-1] == '\r') p[-1] = '\0'; p++; } } } static void expiremsgid(void) { int n; DIR * d; struct dirent * de; struct stat st; int deleted, kept; deleted = kept = 0; for ( n=0; n<1000; n++ ) { sprintf( s, "%s/message.id/%03d", spooldir, n ); if ( chdir( s ) ) { if ( errno == ENOENT ) mkdir( s, 0755 ); /* file system damage? */ if ( chdir( s ) ) { syslog( LOG_ERR, "chdir %s: %m", s ); continue; } } d = opendir( "." ); if ( !d ) continue; while ((de = readdir(d)) != 0) { if (stat(de->d_name, &st) == 0) { if (st.st_nlink < 2 && !unlink(de->d_name)) deleted++; else if (S_ISREG(st.st_mode)) kept++; } } closedir( d ); } if ( kept || deleted ) { printf("total: %d articles deleted, %d kept\n", deleted, kept); syslog( LOG_INFO, "%d articles deleted, %d kept", deleted, kept ); } } static void usage( void ) { fprintf( stderr, "Usage:\n" "texpire -V\n" " print version on stderr and exit\n" "texpire [-Dfv] [-F configfile]\n" " -D: switch on debugmode\n" " -r: relink articles with message.id tree\n" " -f: force expire irrespective of access time\n" " -v: more verbose (may be repeated)\n" " -F: use \"configfile\" instead of %s/config\n" "See also the leafnode homepage at http://www.leafnode.org/\n", libdir ); } int main(int argc, char** argv) { int option, reply; char * conffile; conffile = critmalloc( strlen(libdir) + 10, "Allocating space for configuration file name" ); sprintf( conffile, "%s/config", libdir ); if ( !initvars( argv[0] ) ) exit(EXIT_FAILURE); #ifdef HAVE_OLD_SYSLOG openlog( "texpire", LOG_PID ); #else openlog( "texpire", LOG_PID|LOG_CONS, LOG_NEWS ); #endif while ( (option=getopt( argc, argv, "F:VDvfr" )) != -1 ) { if ( parseopt( "texpire", option, optarg, conffile ) ) { ; } else if ( option == 'f' ) { use_atime = 0; } else if ( option == 'r' ) { repair_spool = 1; } else { usage(); exit(EXIT_FAILURE); } } debug = debugmode; expire = 0; expire_base = NULL; if ( ( reply = readconfig( conffile ) ) != 0 ) { printf( "Reading configuration from %s failed (%s).\n", conffile, strerror(reply) ); exit( 2 ); } if ( lockfile_exists( FALSE, FALSE ) ) exit(EXIT_FAILURE); readactive(); if ( !active ) { fprintf( stderr, "Reading active file failed, exiting " "(see syslog for more information).\n" "Has fetchnews been run?\n" ); unlink( lockfile ); exit( 2 ); } if ( verbose ) { printf( "texpire %s: ", version ); if ( use_atime ) printf( "check mtime and atime\n" ); else printf( "check mtime only\n" ); } if ( debugmode ) syslog( LOG_DEBUG, "texpire %s: use_atime is %d", version, use_atime ); if ( expire == 0 ) { fprintf( stderr, "%s: no expire time\n", argv[0] ); exit( 2 ); } default_expire = expire; expiregroup( active ); writeactive(); unlink( lockfile ); expiremsgid(); free_expire(); return 0; }