jcs.org
mutt stable branch with some hacks
1/*
2 * Copyright (C) 2004 Thomas Glanzmann <sithglan@stud.uni-erlangen.de>
3 * Copyright (C) 2004 Tobias Werth <sitowert@stud.uni-erlangen.de>
4 * Copyright (C) 2004 Brian Fundakowski Feldman <green@FreeBSD.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
20
21#if HAVE_CONFIG_H
22#include "config.h"
23#endif /* HAVE_CONFIG_H */
24
25#if HAVE_QDBM
26#include <depot.h>
27#include <cabin.h>
28#include <villa.h>
29#elif HAVE_TC
30#include <tcbdb.h>
31#elif HAVE_KC
32#include <kclangc.h>
33#elif HAVE_GDBM
34#include <gdbm.h>
35#elif HAVE_DB4
36#include <db.h>
37#elif HAVE_LMDB
38/* This is the maximum size of the database file (2GiB) which is
39 * mmap(2)'ed into memory. This limit should be good for ~800,000
40 * emails. */
41#define LMDB_DB_SIZE 2147483648
42#include <lmdb.h>
43#endif
44
45#include <errno.h>
46#include <fcntl.h>
47#if HAVE_SYS_TIME_H
48#include <sys/time.h>
49#endif
50#include "mutt.h"
51#include "hcache.h"
52#include "hcversion.h"
53#include "mx.h"
54#include "lib.h"
55#include "md5.h"
56#include "rfc822.h"
57
58unsigned int hcachever = 0x0;
59
60#if HAVE_QDBM
61struct header_cache
62{
63 VILLA *db;
64 char *folder;
65 unsigned int crc;
66};
67#elif HAVE_TC
68struct header_cache
69{
70 TCBDB *db;
71 char *folder;
72 unsigned int crc;
73};
74#elif HAVE_KC
75struct header_cache
76{
77 KCDB *db;
78 char *folder;
79 unsigned int crc;
80};
81#elif HAVE_GDBM
82struct header_cache
83{
84 GDBM_FILE db;
85 char *folder;
86 unsigned int crc;
87};
88#elif HAVE_DB4
89struct header_cache
90{
91 DB_ENV *env;
92 DB *db;
93 char *folder;
94 unsigned int crc;
95 int fd;
96 BUFFER *lockfile;
97};
98
99static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len);
100static void mutt_hcache_dbt_empty_init(DBT * dbt);
101#elif HAVE_LMDB
102enum mdb_txn_mode
103{
104 txn_uninitialized = 0,
105 txn_read,
106 txn_write
107};
108struct header_cache
109{
110 MDB_env *env;
111 MDB_txn *txn;
112 MDB_dbi db;
113 char *folder;
114 unsigned int crc;
115 enum mdb_txn_mode txn_mode;
116};
117
118static int mdb_get_r_txn(header_cache_t *h)
119{
120 int rc;
121
122 if (h->txn)
123 {
124 if (h->txn_mode == txn_read || h->txn_mode == txn_write)
125 return MDB_SUCCESS;
126
127 if ((rc = mdb_txn_renew (h->txn)) != MDB_SUCCESS)
128 {
129 h->txn = NULL;
130 dprint (2, (debugfile, "mdb_get_r_txn: mdb_txn_renew: %s\n",
131 mdb_strerror (rc)));
132 return rc;
133 }
134 h->txn_mode = txn_read;
135 return rc;
136 }
137
138 if ((rc = mdb_txn_begin (h->env, NULL, MDB_RDONLY, &h->txn)) != MDB_SUCCESS)
139 {
140 h->txn = NULL;
141 dprint (2, (debugfile, "mdb_get_r_txn: mdb_txn_begin: %s\n",
142 mdb_strerror (rc)));
143 return rc;
144 }
145 h->txn_mode = txn_read;
146 return rc;
147}
148
149static int mdb_get_w_txn(header_cache_t *h)
150{
151 int rc;
152
153 if (h->txn)
154 {
155 if (h->txn_mode == txn_write)
156 return MDB_SUCCESS;
157
158 /* Free up the memory for readonly or reset transactions */
159 mdb_txn_abort (h->txn);
160 }
161
162 if ((rc = mdb_txn_begin (h->env, NULL, 0, &h->txn)) != MDB_SUCCESS)
163 {
164 h->txn = NULL;
165 dprint (2, (debugfile, "mdb_get_w_txn: mdb_txn_begin %s\n",
166 mdb_strerror (rc)));
167 return rc;
168 }
169
170 h->txn_mode = txn_write;
171 return rc;
172}
173#endif
174
175typedef union
176{
177 struct timeval timeval;
178 unsigned int uidvalidity;
179} validate;
180
181static void *
182lazy_malloc(size_t siz)
183{
184 if (0 < siz && siz < 4096)
185 siz = 4096;
186
187 return safe_malloc(siz);
188}
189
190static void
191lazy_realloc(void *ptr, size_t siz)
192{
193 void **p = (void **) ptr;
194
195 if (p != NULL && 0 < siz && siz < 4096)
196 return;
197
198 safe_realloc(ptr, siz);
199}
200
201static unsigned char *
202dump_int(unsigned int i, unsigned char *d, int *off)
203{
204 lazy_realloc(&d, *off + sizeof (int));
205 memcpy(d + *off, &i, sizeof (int));
206 (*off) += sizeof (int);
207
208 return d;
209}
210
211static void
212restore_int(unsigned int *i, const unsigned char *d, int *off)
213{
214 memcpy(i, d + *off, sizeof (int));
215 (*off) += sizeof (int);
216}
217
218static inline int is_ascii (const char *p, size_t len)
219{
220 register const char *s = p;
221 while (s && (unsigned) (s - p) < len)
222 {
223 if ((*s & 0x80) != 0)
224 return 0;
225 s++;
226 }
227 return 1;
228}
229
230static unsigned char *
231dump_char_size(char *c, unsigned char *d, int *off, ssize_t size, int convert)
232{
233 char *p = c;
234
235 if (c == NULL)
236 {
237 size = 0;
238 d = dump_int(size, d, off);
239 return d;
240 }
241
242 if (convert && !is_ascii (c, size))
243 {
244 p = mutt_substrdup (c, c + size);
245 if (mutt_convert_string (&p, Charset, "utf-8", 0) == 0)
246 {
247 c = p;
248 size = mutt_strlen (c) + 1;
249 }
250 }
251
252 d = dump_int(size, d, off);
253 lazy_realloc(&d, *off + size);
254 memcpy(d + *off, p, size);
255 *off += size;
256
257 if (p != c)
258 FREE(&p);
259
260 return d;
261}
262
263static unsigned char *
264dump_char(char *c, unsigned char *d, int *off, int convert)
265{
266 return dump_char_size (c, d, off, mutt_strlen (c) + 1, convert);
267}
268
269static void
270restore_char(char **c, const unsigned char *d, int *off, int convert)
271{
272 unsigned int size;
273 restore_int(&size, d, off);
274
275 if (size == 0)
276 {
277 *c = NULL;
278 return;
279 }
280
281 *c = safe_malloc(size);
282 memcpy(*c, d + *off, size);
283 if (convert && !is_ascii (*c, size))
284 {
285 char *tmp = safe_strdup (*c);
286 if (mutt_convert_string (&tmp, "utf-8", Charset, 0) == 0)
287 {
288 mutt_str_replace (c, tmp);
289 }
290 else
291 {
292 FREE(&tmp);
293 }
294 }
295 *off += size;
296}
297
298static unsigned char *
299dump_address(ADDRESS * a, unsigned char *d, int *off, int convert)
300{
301 unsigned int counter = 0;
302 unsigned int start_off = *off;
303
304 d = dump_int(0xdeadbeef, d, off);
305
306 while (a)
307 {
308#ifdef EXACT_ADDRESS
309 d = dump_char(a->val, d, off, convert);
310#endif
311 d = dump_char(a->personal, d, off, convert);
312 d = dump_char(a->mailbox, d, off, 0);
313 d = dump_int(a->group, d, off);
314 a = a->next;
315 counter++;
316 }
317
318 memcpy(d + start_off, &counter, sizeof (int));
319
320 return d;
321}
322
323static void
324restore_address(ADDRESS ** a, const unsigned char *d, int *off, int convert)
325{
326 unsigned int counter;
327
328 restore_int(&counter, d, off);
329
330 while (counter)
331 {
332 *a = rfc822_new_address();
333#ifdef EXACT_ADDRESS
334 restore_char(&(*a)->val, d, off, convert);
335#endif
336 restore_char(&(*a)->personal, d, off, convert);
337 restore_char(&(*a)->mailbox, d, off, 0);
338 restore_int((unsigned int *) &(*a)->group, d, off);
339 a = &(*a)->next;
340 counter--;
341 }
342
343 *a = NULL;
344}
345
346static unsigned char *
347dump_list(LIST * l, unsigned char *d, int *off, int convert)
348{
349 unsigned int counter = 0;
350 unsigned int start_off = *off;
351
352 d = dump_int(0xdeadbeef, d, off);
353
354 while (l)
355 {
356 d = dump_char(l->data, d, off, convert);
357 l = l->next;
358 counter++;
359 }
360
361 memcpy(d + start_off, &counter, sizeof (int));
362
363 return d;
364}
365
366static void
367restore_list(LIST ** l, const unsigned char *d, int *off, int convert)
368{
369 unsigned int counter;
370
371 restore_int(&counter, d, off);
372
373 while (counter)
374 {
375 *l = safe_malloc(sizeof (LIST));
376 restore_char(&(*l)->data, d, off, convert);
377 l = &(*l)->next;
378 counter--;
379 }
380
381 *l = NULL;
382}
383
384static unsigned char *
385dump_buffer(BUFFER * b, unsigned char *d, int *off, int convert)
386{
387 if (!b)
388 {
389 d = dump_int(0, d, off);
390 return d;
391 }
392 else
393 d = dump_int(1, d, off);
394
395 d = dump_char_size(b->data, d, off, b->dsize + 1, convert);
396 d = dump_int(b->dptr - b->data, d, off);
397 d = dump_int(b->dsize, d, off);
398 d = dump_int(b->destroy, d, off);
399
400 return d;
401}
402
403static void
404restore_buffer(BUFFER ** b, const unsigned char *d, int *off, int convert)
405{
406 unsigned int used;
407 unsigned int offset;
408 restore_int(&used, d, off);
409 if (!used)
410 {
411 return;
412 }
413
414 *b = safe_malloc(sizeof (BUFFER));
415
416 restore_char(&(*b)->data, d, off, convert);
417 restore_int(&offset, d, off);
418 (*b)->dptr = (*b)->data + offset;
419 restore_int (&used, d, off);
420 (*b)->dsize = used;
421 restore_int (&used, d, off);
422 (*b)->destroy = used;
423}
424
425static unsigned char *
426dump_parameter(PARAMETER * p, unsigned char *d, int *off, int convert)
427{
428 unsigned int counter = 0;
429 unsigned int start_off = *off;
430
431 d = dump_int(0xdeadbeef, d, off);
432
433 while (p)
434 {
435 d = dump_char(p->attribute, d, off, 0);
436 d = dump_char(p->value, d, off, convert);
437 p = p->next;
438 counter++;
439 }
440
441 memcpy(d + start_off, &counter, sizeof (int));
442
443 return d;
444}
445
446static void
447restore_parameter(PARAMETER ** p, const unsigned char *d, int *off, int convert)
448{
449 unsigned int counter;
450
451 restore_int(&counter, d, off);
452
453 while (counter)
454 {
455 *p = safe_malloc(sizeof (PARAMETER));
456 restore_char(&(*p)->attribute, d, off, 0);
457 restore_char(&(*p)->value, d, off, convert);
458 p = &(*p)->next;
459 counter--;
460 }
461
462 *p = NULL;
463}
464
465static unsigned char *
466dump_body(BODY * c, unsigned char *d, int *off, int convert)
467{
468 BODY nb;
469
470 memcpy (&nb, c, sizeof (BODY));
471
472 /* some fields are not safe to cache */
473 nb.content = NULL;
474 nb.charset = NULL;
475 nb.next = NULL;
476 nb.parts = NULL;
477 nb.hdr = NULL;
478 nb.aptr = NULL;
479 nb.mime_headers = NULL;
480
481 lazy_realloc(&d, *off + sizeof (BODY));
482 memcpy(d + *off, &nb, sizeof (BODY));
483 *off += sizeof (BODY);
484
485 d = dump_char(nb.xtype, d, off, 0);
486 d = dump_char(nb.subtype, d, off, 0);
487
488 d = dump_parameter(nb.parameter, d, off, convert);
489
490 d = dump_char(nb.description, d, off, convert);
491 d = dump_char(nb.form_name, d, off, convert);
492 d = dump_char(nb.filename, d, off, convert);
493 d = dump_char(nb.d_filename, d, off, convert);
494
495 return d;
496}
497
498static void
499restore_body(BODY * c, const unsigned char *d, int *off, int convert)
500{
501 memcpy(c, d + *off, sizeof (BODY));
502 *off += sizeof (BODY);
503
504 restore_char(&c->xtype, d, off, 0);
505 restore_char(&c->subtype, d, off, 0);
506
507 restore_parameter(&c->parameter, d, off, convert);
508
509 restore_char(&c->description, d, off, convert);
510 restore_char(&c->form_name, d, off, convert);
511 restore_char(&c->filename, d, off, convert);
512 restore_char(&c->d_filename, d, off, convert);
513}
514
515static unsigned char *
516dump_envelope(ENVELOPE * e, unsigned char *d, int *off, int convert)
517{
518 d = dump_address(e->return_path, d, off, convert);
519 d = dump_address(e->from, d, off, convert);
520 d = dump_address(e->to, d, off, convert);
521 d = dump_address(e->cc, d, off, convert);
522 d = dump_address(e->bcc, d, off, convert);
523 d = dump_address(e->sender, d, off, convert);
524 d = dump_address(e->reply_to, d, off, convert);
525 d = dump_address(e->mail_followup_to, d, off, convert);
526
527 d = dump_char(e->list_post, d, off, convert);
528 d = dump_char(e->subject, d, off, convert);
529
530 if (e->real_subj)
531 d = dump_int(e->real_subj - e->subject, d, off);
532 else
533 d = dump_int(-1, d, off);
534
535 d = dump_char(e->message_id, d, off, 0);
536 d = dump_char(e->supersedes, d, off, 0);
537 d = dump_char(e->date, d, off, 0);
538 d = dump_char(e->x_label, d, off, convert);
539
540 d = dump_buffer(e->spam, d, off, convert);
541
542 d = dump_list(e->references, d, off, 0);
543 d = dump_list(e->in_reply_to, d, off, 0);
544 d = dump_list(e->userhdrs, d, off, convert);
545
546 return d;
547}
548
549static void
550restore_envelope(ENVELOPE * e, const unsigned char *d, int *off, int convert)
551{
552 int real_subj_off;
553
554 restore_address(&e->return_path, d, off, convert);
555 restore_address(&e->from, d, off, convert);
556 restore_address(&e->to, d, off, convert);
557 restore_address(&e->cc, d, off, convert);
558 restore_address(&e->bcc, d, off, convert);
559 restore_address(&e->sender, d, off, convert);
560 restore_address(&e->reply_to, d, off, convert);
561 restore_address(&e->mail_followup_to, d, off, convert);
562
563 restore_char(&e->list_post, d, off, convert);
564 if (option (OPTAUTOSUBSCRIBE))
565 mutt_auto_subscribe (e->list_post);
566
567 restore_char(&e->subject, d, off, convert);
568 restore_int((unsigned int *) (&real_subj_off), d, off);
569
570 if (0 <= real_subj_off)
571 e->real_subj = e->subject + real_subj_off;
572 else
573 e->real_subj = NULL;
574
575 restore_char(&e->message_id, d, off, 0);
576 restore_char(&e->supersedes, d, off, 0);
577 restore_char(&e->date, d, off, 0);
578 restore_char(&e->x_label, d, off, convert);
579
580 restore_buffer(&e->spam, d, off, convert);
581
582 restore_list(&e->references, d, off, 0);
583 restore_list(&e->in_reply_to, d, off, 0);
584 restore_list(&e->userhdrs, d, off, convert);
585}
586
587static int
588crc_matches(const char *d, unsigned int crc)
589{
590 int off = sizeof (validate);
591 unsigned int mycrc = 0;
592
593 if (!d)
594 return 0;
595
596 restore_int(&mycrc, (unsigned char *) d, &off);
597
598 return (crc == mycrc);
599}
600
601/* Append md5sumed folder to path if path is a directory. */
602void
603mutt_hcache_per_folder(BUFFER *hcpath, const char *path, const char *folder,
604 hcache_namer_t namer)
605{
606 BUFFER *hcfile = NULL;
607 struct stat sb;
608 unsigned char md5sum[16];
609 char* s;
610 int ret, plen;
611#ifndef HAVE_ICONV
612 const char *chs = Charset ? Charset : mutt_get_default_charset ();
613#endif
614
615 plen = mutt_strlen (path);
616
617 ret = stat(path, &sb);
618 if (ret < 0 && path[plen-1] != '/')
619 {
620#ifdef HAVE_ICONV
621 mutt_buffer_strcpy (hcpath, path);
622#else
623 mutt_buffer_printf (hcpath, "%s-%s", path, chs);
624#endif
625 return;
626 }
627
628 if (ret >= 0 && !S_ISDIR(sb.st_mode))
629 {
630#ifdef HAVE_ICONV
631 mutt_buffer_strcpy (hcpath, path);
632#else
633 mutt_buffer_printf (hcpath, "%s-%s", path, chs);
634#endif
635 return;
636 }
637
638 hcfile = mutt_buffer_pool_get ();
639
640 if (namer)
641 {
642 namer (folder, hcfile);
643 }
644 else
645 {
646 md5_buffer (folder, strlen (folder), &md5sum);
647 mutt_buffer_printf(hcfile,
648 "%02x%02x%02x%02x%02x%02x%02x%02x"
649 "%02x%02x%02x%02x%02x%02x%02x%02x",
650 md5sum[0], md5sum[1], md5sum[2], md5sum[3],
651 md5sum[4], md5sum[5], md5sum[6], md5sum[7],
652 md5sum[8], md5sum[9], md5sum[10], md5sum[11],
653 md5sum[12], md5sum[13], md5sum[14], md5sum[15]);
654#ifndef HAVE_ICONV
655 mutt_buffer_addch (hcfile, '-');
656 mutt_buffer_addstr (hcfile, chs);
657#endif
658 }
659
660 mutt_buffer_concat_path (hcpath, path, mutt_b2s (hcfile));
661 mutt_buffer_pool_release (&hcfile);
662
663 if (stat (mutt_b2s (hcpath), &sb) >= 0)
664 return;
665
666 s = strchr (hcpath->data + 1, '/');
667 while (s)
668 {
669 /* create missing path components */
670 *s = '\0';
671 if (stat (mutt_b2s (hcpath), &sb) < 0 &&
672 (errno != ENOENT || mkdir (mutt_b2s (hcpath), 0777) < 0))
673 {
674 mutt_buffer_strcpy (hcpath, path);
675 break;
676 }
677 *s = '/';
678 s = strchr (s + 1, '/');
679 }
680}
681
682/* This function transforms a header into a char so that it is usable by
683 * db_store.
684 */
685static void *
686mutt_hcache_dump(header_cache_t *h, HEADER * header, int *off,
687 unsigned int uidvalidity, mutt_hcache_store_flags_t flags)
688{
689 unsigned char *d = NULL;
690 HEADER nh;
691 int convert = !Charset_is_utf8;
692
693 *off = 0;
694 d = lazy_malloc(sizeof (validate));
695
696 if (flags & MUTT_GENERATE_UIDVALIDITY)
697 {
698 struct timeval now;
699 gettimeofday(&now, NULL);
700 memcpy(d, &now, sizeof (struct timeval));
701 }
702 else
703 memcpy(d, &uidvalidity, sizeof (uidvalidity));
704 *off += sizeof (validate);
705
706 d = dump_int(h->crc, d, off);
707
708 lazy_realloc(&d, *off + sizeof (HEADER));
709 memcpy(&nh, header, sizeof (HEADER));
710
711 /* some fields are not safe to cache */
712 nh.tagged = 0;
713 nh.changed = 0;
714 nh.threaded = 0;
715 nh.recip_valid = 0;
716 nh.searched = 0;
717 nh.matched = 0;
718 nh.collapsed = 0;
719 nh.limited = 0;
720 nh.num_hidden = 0;
721 nh.recipient = 0;
722 nh.pair = 0;
723 nh.attach_valid = 0;
724 nh.path = NULL;
725 nh.tree = NULL;
726 nh.thread = NULL;
727#ifdef MIXMASTER
728 nh.chain = NULL;
729#endif
730#if defined USE_POP || defined USE_IMAP
731 nh.data = NULL;
732#endif
733
734 memcpy(d + *off, &nh, sizeof (HEADER));
735 *off += sizeof (HEADER);
736
737 d = dump_envelope(nh.env, d, off, convert);
738 d = dump_body(nh.content, d, off, convert);
739 d = dump_char(nh.maildir_flags, d, off, convert);
740
741 return d;
742}
743
744HEADER *
745mutt_hcache_restore(const unsigned char *d, HEADER ** oh)
746{
747 int off = 0;
748 HEADER *h = mutt_new_header();
749 int convert = !Charset_is_utf8;
750
751 /* skip validate */
752 off += sizeof (validate);
753
754 /* skip crc */
755 off += sizeof (unsigned int);
756
757 memcpy(h, d + off, sizeof (HEADER));
758 off += sizeof (HEADER);
759
760 h->env = mutt_new_envelope();
761 restore_envelope(h->env, d, &off, convert);
762
763 h->content = mutt_new_body();
764 restore_body(h->content, d, &off, convert);
765
766 restore_char(&h->maildir_flags, d, &off, convert);
767
768 /* this is needed for maildir style mailboxes */
769 if (oh)
770 {
771 h->old = (*oh)->old;
772 h->path = safe_strdup((*oh)->path);
773 mutt_free_header(oh);
774 }
775
776 return h;
777}
778
779void *
780mutt_hcache_fetch(header_cache_t *h, const char *filename,
781 size_t(*keylen) (const char *fn))
782{
783 void* data;
784
785 data = mutt_hcache_fetch_raw (h, filename, keylen);
786
787 if (!data || !crc_matches(data, h->crc))
788 {
789 mutt_hcache_free (&data);
790 return NULL;
791 }
792
793 return data;
794}
795
796void *
797mutt_hcache_fetch_raw (header_cache_t *h, const char *filename,
798 size_t(*keylen) (const char *fn))
799{
800#ifndef HAVE_DB4
801 BUFFER *path = NULL;
802 int ksize;
803 void *rv = NULL;
804#endif
805#if HAVE_TC
806 int sp;
807#elif HAVE_KC
808 size_t sp;
809#elif HAVE_GDBM
810 datum key;
811 datum data;
812#elif HAVE_DB4
813 DBT key;
814 DBT data;
815#elif HAVE_LMDB
816 MDB_val key;
817 MDB_val data;
818#endif
819
820 if (!h)
821 return NULL;
822
823#ifdef HAVE_DB4
824 if (filename[0] == '/')
825 filename++;
826
827 mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename));
828 mutt_hcache_dbt_empty_init(&data);
829 data.flags = DB_DBT_MALLOC;
830
831 h->db->get(h->db, NULL, &key, &data, 0);
832
833 return data.data;
834
835#else
836 path = mutt_buffer_pool_get ();
837 mutt_buffer_strcpy (path, h->folder);
838 mutt_buffer_addstr (path, filename);
839
840 ksize = strlen (h->folder) + keylen (filename);
841
842#ifdef HAVE_QDBM
843 rv = vlget(h->db, mutt_b2s (path), ksize, NULL);
844#elif HAVE_TC
845 rv = tcbdbget(h->db, mutt_b2s (path), ksize, &sp);
846#elif HAVE_KC
847 rv = kcdbget(h->db, mutt_b2s (path), ksize, &sp);
848#elif HAVE_GDBM
849 key.dptr = path->data;
850 key.dsize = ksize;
851
852 data = gdbm_fetch(h->db, key);
853
854 rv = data.dptr;
855#elif HAVE_LMDB
856 key.mv_data = path->data;
857 key.mv_size = ksize;
858 /* LMDB claims ownership of the returned data, so this will not be
859 * freed in mutt_hcache_free(). */
860 if ((mdb_get_r_txn (h) == MDB_SUCCESS) &&
861 (mdb_get (h->txn, h->db, &key, &data) == MDB_SUCCESS))
862 rv = data.mv_data;
863#endif
864
865 mutt_buffer_pool_release (&path);
866 return rv;
867#endif
868}
869
870/*
871 * flags
872 *
873 * MUTT_GENERATE_UIDVALIDITY
874 * ignore uidvalidity param and store gettimeofday() as the value
875 */
876int
877mutt_hcache_store(header_cache_t *h, const char *filename, HEADER * header,
878 unsigned int uidvalidity,
879 size_t(*keylen) (const char *fn),
880 mutt_hcache_store_flags_t flags)
881{
882 char* data;
883 int dlen;
884 int ret;
885
886 if (!h)
887 return -1;
888
889 data = mutt_hcache_dump(h, header, &dlen, uidvalidity, flags);
890 ret = mutt_hcache_store_raw (h, filename, data, dlen, keylen);
891
892 FREE(&data);
893
894 return ret;
895}
896
897int
898mutt_hcache_store_raw (header_cache_t* h, const char* filename, void* data,
899 size_t dlen, size_t(*keylen) (const char* fn))
900{
901#ifndef HAVE_DB4
902 BUFFER *path = NULL;
903 int ksize;
904 int rv = 0;
905#endif
906#if HAVE_GDBM
907 datum key;
908 datum databuf;
909#elif HAVE_DB4
910 DBT key;
911 DBT databuf;
912#elif HAVE_LMDB
913 MDB_val key;
914 MDB_val databuf;
915#endif
916
917 if (!h)
918 return -1;
919
920#if HAVE_DB4
921 if (filename[0] == '/')
922 filename++;
923
924 mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename));
925
926 mutt_hcache_dbt_empty_init(&databuf);
927 databuf.flags = DB_DBT_USERMEM;
928 databuf.data = data;
929 databuf.size = dlen;
930 databuf.ulen = dlen;
931
932 return h->db->put(h->db, NULL, &key, &databuf, 0);
933
934#else
935 path = mutt_buffer_pool_get ();
936 mutt_buffer_strcpy (path, h->folder);
937 mutt_buffer_addstr (path, filename);
938
939 ksize = strlen(h->folder) + keylen(filename);
940
941#if HAVE_QDBM
942 rv = vlput(h->db, mutt_b2s (path), ksize, data, dlen, VL_DOVER);
943#elif HAVE_TC
944 rv = tcbdbput(h->db, mutt_b2s (path), ksize, data, dlen);
945#elif HAVE_KC
946 rv = kcdbset(h->db, mutt_b2s (path), ksize, data, dlen);
947#elif HAVE_GDBM
948 key.dptr = path->data;
949 key.dsize = ksize;
950
951 databuf.dsize = dlen;
952 databuf.dptr = data;
953
954 rv = gdbm_store(h->db, key, databuf, GDBM_REPLACE);
955#elif HAVE_LMDB
956 key.mv_data = path->data;
957 key.mv_size = ksize;
958 databuf.mv_data = data;
959 databuf.mv_size = dlen;
960 if ((rv = mdb_get_w_txn (h)) == MDB_SUCCESS)
961 {
962 if ((rv = mdb_put (h->txn, h->db, &key, &databuf, 0)) != MDB_SUCCESS)
963 {
964 dprint (2, (debugfile, "mutt_hcache_store_raw: mdb_put: %s\n",
965 mdb_strerror(rv)));
966 mdb_txn_abort (h->txn);
967 h->txn_mode = txn_uninitialized;
968 h->txn = NULL;
969 }
970 }
971#endif
972
973 mutt_buffer_pool_release (&path);
974 return rv;
975#endif
976}
977
978static char* get_foldername (const char *folder)
979{
980 char *p = NULL;
981 BUFFER *path;
982 struct stat st;
983
984 path = mutt_buffer_pool_get ();
985 mutt_encode_path (path, folder);
986
987 /* if the folder is local, canonify the path to avoid
988 * to ensure equivalent paths share the hcache */
989 if (stat (mutt_b2s (path), &st) == 0)
990 {
991 p = safe_malloc (PATH_MAX+1);
992 if (!realpath (mutt_b2s (path), p))
993 mutt_str_replace (&p, mutt_b2s (path));
994 }
995 else
996 p = safe_strdup (mutt_b2s (path));
997
998 mutt_buffer_pool_release (&path);
999 return p;
1000}
1001
1002#if HAVE_QDBM
1003static int
1004hcache_open_qdbm (struct header_cache* h, const char* path)
1005{
1006 int flags = VL_OWRITER | VL_OCREAT;
1007
1008 if (option(OPTHCACHECOMPRESS))
1009 flags |= VL_OZCOMP;
1010
1011 h->db = vlopen (path, flags, VL_CMPLEX);
1012 if (h->db)
1013 return 0;
1014 else
1015 return -1;
1016}
1017
1018void
1019mutt_hcache_close(header_cache_t *h)
1020{
1021 if (!h)
1022 return;
1023
1024 vlclose(h->db);
1025 FREE(&h->folder);
1026 FREE(&h);
1027}
1028
1029int
1030mutt_hcache_delete(header_cache_t *h, const char *filename,
1031 size_t(*keylen) (const char *fn))
1032{
1033 BUFFER *path = NULL;
1034 int ksize, rc;
1035
1036 if (!h)
1037 return -1;
1038
1039 path = mutt_buffer_pool_get ();
1040 mutt_buffer_strcpy (path, h->folder);
1041 mutt_buffer_addstr (path, filename);
1042
1043 ksize = strlen(h->folder) + keylen(filename);
1044
1045 rc = vlout(h->db, mutt_b2s (path), ksize);
1046
1047 mutt_buffer_pool_release (&path);
1048 return rc;
1049}
1050
1051#elif HAVE_TC
1052static int
1053hcache_open_tc (struct header_cache* h, const char* path)
1054{
1055 h->db = tcbdbnew();
1056 if (!h->db)
1057 return -1;
1058 if (option(OPTHCACHECOMPRESS))
1059 tcbdbtune(h->db, 0, 0, 0, -1, -1, BDBTDEFLATE);
1060 if (tcbdbopen(h->db, path, BDBOWRITER | BDBOCREAT))
1061 return 0;
1062 else
1063 {
1064#ifdef DEBUG
1065 int ecode = tcbdbecode (h->db);
1066 dprint (2, (debugfile, "tcbdbopen failed for %s: %s (ecode %d)\n", path, tcbdberrmsg (ecode), ecode));
1067#endif
1068 tcbdbdel(h->db);
1069 return -1;
1070 }
1071}
1072
1073void
1074mutt_hcache_close(header_cache_t *h)
1075{
1076 if (!h)
1077 return;
1078
1079 if (!tcbdbclose(h->db))
1080 {
1081#ifdef DEBUG
1082 int ecode = tcbdbecode (h->db);
1083 dprint (2, (debugfile, "tcbdbclose failed for %s: %s (ecode %d)\n", h->folder, tcbdberrmsg (ecode), ecode));
1084#endif
1085 }
1086 tcbdbdel(h->db);
1087 FREE(&h->folder);
1088 FREE(&h);
1089}
1090
1091int
1092mutt_hcache_delete(header_cache_t *h, const char *filename,
1093 size_t(*keylen) (const char *fn))
1094{
1095 BUFFER *path = NULL;
1096 int ksize, rc;
1097
1098 if (!h)
1099 return -1;
1100
1101 path = mutt_buffer_pool_get ();
1102 mutt_buffer_strcpy (path, h->folder);
1103 mutt_buffer_addstr (path, filename);
1104
1105 ksize = strlen(h->folder) + keylen(filename);
1106
1107 rc = tcbdbout(h->db, mutt_b2s (path), ksize);
1108
1109 mutt_buffer_pool_release (&path);
1110 return rc;
1111}
1112
1113#elif HAVE_KC
1114static int
1115hcache_open_kc (struct header_cache* h, const char* path)
1116{
1117 BUFFER *fullpath = NULL;
1118 int rc = -1;
1119
1120 fullpath = mutt_buffer_pool_get ();
1121
1122 /* Kyoto cabinet options are discussed at
1123 * http://fallabs.com/kyotocabinet/spex.html
1124 * - rcomp is by default lex, so there is no need to specify it.
1125 * - opts=l enables linear collision chaining as opposed to using a binary tree.
1126 * this isn't suggested unless you are tuning the number of buckets.
1127 * - opts=c enables compression
1128 */
1129 mutt_buffer_printf (fullpath, "%s#type=kct%s", path,
1130 option(OPTHCACHECOMPRESS) ? "#opts=c" : "");
1131 h->db = kcdbnew();
1132 if (!h->db)
1133 goto cleanup;
1134 if (!kcdbopen(h->db, mutt_b2s (fullpath), KCOWRITER | KCOCREATE))
1135 {
1136 dprint (2, (debugfile, "kcdbopen failed for %s: %s (ecode %d)\n",
1137 mutt_b2s (fullpath),
1138 kcdbemsg (h->db), kcdbecode (h->db)));
1139 kcdbdel(h->db);
1140 goto cleanup;
1141 }
1142
1143 rc = 0;
1144
1145cleanup:
1146 mutt_buffer_pool_release (&fullpath);
1147 return rc;
1148}
1149
1150void
1151mutt_hcache_close(header_cache_t *h)
1152{
1153 if (!h)
1154 return;
1155
1156 if (!kcdbclose(h->db))
1157 dprint (2, (debugfile, "kcdbclose failed for %s: %s (ecode %d)\n", h->folder,
1158 kcdbemsg (h->db), kcdbecode (h->db)));
1159 kcdbdel(h->db);
1160 FREE(&h->folder);
1161 FREE(&h);
1162}
1163
1164int
1165mutt_hcache_delete(header_cache_t *h, const char *filename,
1166 size_t(*keylen) (const char *fn))
1167{
1168 BUFFER *path = NULL;
1169 int ksize, rc;
1170
1171 if (!h)
1172 return -1;
1173
1174 path = mutt_buffer_pool_get ();
1175 mutt_buffer_strcpy (path, h->folder);
1176 mutt_buffer_addstr (path, filename);
1177
1178 ksize = strlen(h->folder) + keylen(filename);
1179
1180 rc = kcdbremove(h->db, mutt_b2s (path), ksize);
1181
1182 mutt_buffer_pool_release (&path);
1183 return rc;
1184}
1185
1186#elif HAVE_GDBM
1187static int
1188hcache_open_gdbm (struct header_cache* h, const char* path)
1189{
1190 int pagesize;
1191
1192 pagesize = HeaderCachePageSize;
1193 if (pagesize <= 0)
1194 pagesize = 16384;
1195
1196 h->db = gdbm_open((char *) path, pagesize, GDBM_WRCREAT, 00600, NULL);
1197 if (h->db)
1198 return 0;
1199
1200 /* if rw failed try ro */
1201 h->db = gdbm_open((char *) path, pagesize, GDBM_READER, 00600, NULL);
1202 if (h->db)
1203 return 0;
1204
1205 return -1;
1206}
1207
1208void
1209mutt_hcache_close(header_cache_t *h)
1210{
1211 if (!h)
1212 return;
1213
1214 gdbm_close(h->db);
1215 FREE(&h->folder);
1216 FREE(&h);
1217}
1218
1219int
1220mutt_hcache_delete(header_cache_t *h, const char *filename,
1221 size_t(*keylen) (const char *fn))
1222{
1223 datum key;
1224 BUFFER *path = NULL;
1225 int rc;
1226
1227 if (!h)
1228 return -1;
1229
1230 path = mutt_buffer_pool_get ();
1231 mutt_buffer_strcpy (path, h->folder);
1232 mutt_buffer_addstr (path, filename);
1233
1234 key.dptr = path->data;
1235 key.dsize = strlen(h->folder) + keylen(filename);
1236
1237 rc = gdbm_delete(h->db, key);
1238
1239 mutt_buffer_pool_release (&path);
1240 return rc;
1241}
1242#elif HAVE_DB4
1243
1244static void
1245mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len)
1246{
1247 dbt->data = data;
1248 dbt->size = dbt->ulen = len;
1249 dbt->dlen = dbt->doff = 0;
1250 dbt->flags = DB_DBT_USERMEM;
1251}
1252
1253static void
1254mutt_hcache_dbt_empty_init(DBT * dbt)
1255{
1256 dbt->data = NULL;
1257 dbt->size = dbt->ulen = dbt->dlen = dbt->doff = 0;
1258 dbt->flags = 0;
1259}
1260
1261static int
1262hcache_open_db4 (struct header_cache* h, const char* path)
1263{
1264 struct stat sb;
1265 int ret;
1266 u_int32_t createflags = DB_CREATE;
1267 int pagesize;
1268
1269 pagesize = HeaderCachePageSize;
1270 if (pagesize <= 0)
1271 pagesize = 16384;
1272
1273 h->lockfile = mutt_buffer_new ();
1274 mutt_buffer_printf (h->lockfile, "%s-lock-hack", path);
1275
1276 h->fd = open (mutt_b2s (h->lockfile), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
1277 if (h->fd < 0)
1278 return -1;
1279
1280 if (mx_lock_file (mutt_b2s (h->lockfile), h->fd, 1, 0, 5))
1281 goto fail_close;
1282
1283 ret = db_env_create (&h->env, 0);
1284 if (ret)
1285 goto fail_unlock;
1286
1287 ret = (*h->env->open)(h->env, NULL, DB_INIT_MPOOL | DB_CREATE | DB_PRIVATE,
1288 0600);
1289 if (ret)
1290 goto fail_env;
1291
1292 ret = db_create (&h->db, h->env, 0);
1293 if (ret)
1294 goto fail_env;
1295
1296 if (stat(path, &sb) != 0 && errno == ENOENT)
1297 {
1298 createflags |= DB_EXCL;
1299 h->db->set_pagesize(h->db, pagesize);
1300 }
1301
1302 ret = (*h->db->open)(h->db, NULL, path, h->folder, DB_BTREE, createflags,
1303 0600);
1304 if (ret)
1305 goto fail_db;
1306
1307 return 0;
1308
1309fail_db:
1310 h->db->close (h->db, 0);
1311fail_env:
1312 h->env->close (h->env, 0);
1313fail_unlock:
1314 mx_unlock_file (mutt_b2s (h->lockfile), h->fd, 0);
1315fail_close:
1316 close (h->fd);
1317 unlink (mutt_b2s (h->lockfile));
1318 mutt_buffer_free (&h->lockfile);
1319
1320 return -1;
1321}
1322
1323void
1324mutt_hcache_close(header_cache_t *h)
1325{
1326 if (!h)
1327 return;
1328
1329 h->db->close (h->db, 0);
1330 h->env->close (h->env, 0);
1331 mx_unlock_file (mutt_b2s (h->lockfile), h->fd, 0);
1332 close (h->fd);
1333 unlink (mutt_b2s (h->lockfile));
1334 mutt_buffer_free (&h->lockfile);
1335 FREE (&h->folder);
1336 FREE (&h);
1337}
1338
1339int
1340mutt_hcache_delete(header_cache_t *h, const char *filename,
1341 size_t(*keylen) (const char *fn))
1342{
1343 DBT key;
1344
1345 if (!h)
1346 return -1;
1347
1348 if (filename[0] == '/')
1349 filename++;
1350
1351 mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename));
1352 return h->db->del(h->db, NULL, &key, 0);
1353}
1354#elif HAVE_LMDB
1355
1356static int
1357hcache_open_lmdb (struct header_cache* h, const char* path)
1358{
1359 int rc;
1360
1361 h->txn = NULL;
1362
1363 if ((rc = mdb_env_create(&h->env)) != MDB_SUCCESS)
1364 {
1365 dprint (2, (debugfile, "hcache_open_lmdb: mdb_env_create: %s\n",
1366 mdb_strerror(rc)));
1367 return -1;
1368 }
1369
1370 mdb_env_set_mapsize(h->env, LMDB_DB_SIZE);
1371
1372 if ((rc = mdb_env_open(h->env, path, MDB_NOSUBDIR, 0644)) != MDB_SUCCESS)
1373 {
1374 dprint (2, (debugfile, "hcache_open_lmdb: mdb_env_open: %s\n",
1375 mdb_strerror(rc)));
1376 goto fail_env;
1377 }
1378
1379 if ((rc = mdb_get_r_txn(h)) != MDB_SUCCESS)
1380 goto fail_env;
1381
1382 if ((rc = mdb_dbi_open(h->txn, NULL, MDB_CREATE, &h->db)) != MDB_SUCCESS)
1383 {
1384 dprint (2, (debugfile, "hcache_open_lmdb: mdb_dbi_open: %s\n",
1385 mdb_strerror(rc)));
1386 goto fail_dbi;
1387 }
1388
1389 mdb_txn_reset(h->txn);
1390 h->txn_mode = txn_uninitialized;
1391 return 0;
1392
1393fail_dbi:
1394 mdb_txn_abort(h->txn);
1395 h->txn_mode = txn_uninitialized;
1396 h->txn = NULL;
1397
1398fail_env:
1399 mdb_env_close(h->env);
1400 return -1;
1401}
1402
1403void
1404mutt_hcache_close(header_cache_t *h)
1405{
1406 int rc;
1407
1408 if (!h)
1409 return;
1410
1411 if (h->txn)
1412 {
1413 if (h->txn_mode == txn_write)
1414 {
1415 if ((rc = mdb_txn_commit (h->txn)) != MDB_SUCCESS)
1416 {
1417 dprint (2, (debugfile, "mutt_hcache_close: mdb_txn_commit: %s\n",
1418 mdb_strerror (rc)));
1419 }
1420 }
1421 else
1422 mdb_txn_abort (h->txn);
1423 h->txn_mode = txn_uninitialized;
1424 h->txn = NULL;
1425 }
1426
1427 mdb_env_close(h->env);
1428 FREE (&h->folder);
1429 FREE (&h);
1430}
1431
1432int
1433mutt_hcache_delete(header_cache_t *h, const char *filename,
1434 size_t(*keylen) (const char *fn))
1435{
1436 BUFFER *path = NULL;
1437 int ksize;
1438 MDB_val key;
1439 int rc = -1;
1440
1441 if (!h)
1442 return -1;
1443
1444 path = mutt_buffer_pool_get ();
1445 mutt_buffer_strcpy (path, h->folder);
1446 mutt_buffer_addstr (path, filename);
1447 ksize = strlen (h->folder) + keylen (filename);
1448
1449 key.mv_data = path->data;
1450 key.mv_size = ksize;
1451 if (mdb_get_w_txn(h) != MDB_SUCCESS)
1452 goto cleanup;
1453 rc = mdb_del(h->txn, h->db, &key, NULL);
1454 if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND)
1455 {
1456 dprint (2, (debugfile, "mutt_hcache_delete: mdb_del: %s\n",
1457 mdb_strerror (rc)));
1458 mdb_txn_abort(h->txn);
1459 h->txn_mode = txn_uninitialized;
1460 h->txn = NULL;
1461 goto cleanup;
1462 }
1463
1464 rc = 0;
1465
1466cleanup:
1467 mutt_buffer_pool_release (&path);
1468 return rc;
1469}
1470#endif
1471
1472header_cache_t *
1473mutt_hcache_open(const char *path, const char *folder, hcache_namer_t namer)
1474{
1475 struct header_cache *h = safe_calloc(1, sizeof (struct header_cache));
1476 int (*hcache_open) (struct header_cache* h, const char* path);
1477 struct stat sb;
1478 BUFFER *hcpath = NULL;
1479
1480#if HAVE_QDBM
1481 hcache_open = hcache_open_qdbm;
1482#elif HAVE_TC
1483 hcache_open= hcache_open_tc;
1484#elif HAVE_KC
1485 hcache_open= hcache_open_kc;
1486#elif HAVE_GDBM
1487 hcache_open = hcache_open_gdbm;
1488#elif HAVE_DB4
1489 hcache_open = hcache_open_db4;
1490#elif HAVE_LMDB
1491 hcache_open = hcache_open_lmdb;
1492#endif
1493
1494 /* Calculate the current hcache version from dynamic configuration */
1495 if (hcachever == 0x0)
1496 {
1497 union {
1498 unsigned char charval[16];
1499 unsigned int intval;
1500 } digest;
1501 struct md5_ctx ctx;
1502 REPLACE_LIST *spam;
1503 RX_LIST *nospam;
1504
1505 hcachever = HCACHEVER;
1506
1507 md5_init_ctx(&ctx);
1508
1509 /* Seed with the compiled-in header structure hash */
1510 md5_process_bytes(&hcachever, sizeof(hcachever), &ctx);
1511
1512 /* Mix in user's spam list */
1513 for (spam = SpamList; spam; spam = spam->next)
1514 {
1515 md5_process_bytes(spam->rx->pattern, strlen(spam->rx->pattern), &ctx);
1516 md5_process_bytes(spam->template, strlen(spam->template), &ctx);
1517 }
1518
1519 /* Mix in user's nospam list */
1520 for (nospam = NoSpamList; nospam; nospam = nospam->next)
1521 {
1522 md5_process_bytes(nospam->rx->pattern, strlen(nospam->rx->pattern), &ctx);
1523 }
1524
1525 /* Get a hash and take its bytes as an (unsigned int) hash version */
1526 md5_finish_ctx(&ctx, digest.charval);
1527 hcachever = digest.intval;
1528 }
1529
1530#if HAVE_LMDB
1531 h->db = 0;
1532#else
1533 h->db = NULL;
1534#endif
1535 h->folder = get_foldername(folder);
1536 h->crc = hcachever;
1537
1538 if (!path || path[0] == '\0')
1539 {
1540 FREE(&h->folder);
1541 FREE(&h);
1542 return NULL;
1543 }
1544
1545 hcpath = mutt_buffer_pool_get ();
1546 mutt_hcache_per_folder(hcpath, path, h->folder, namer);
1547
1548 if (hcache_open (h, mutt_b2s (hcpath)))
1549 {
1550 /* remove a possibly incompatible version */
1551 if (stat (mutt_b2s (hcpath), &sb) ||
1552 unlink (mutt_b2s (hcpath)) ||
1553 hcache_open (h, mutt_b2s (hcpath)))
1554 {
1555 FREE(&h->folder);
1556 FREE(&h);
1557 }
1558 }
1559
1560 mutt_buffer_pool_release (&hcpath);
1561 return h;
1562}
1563
1564void mutt_hcache_free (void **data)
1565{
1566 if (!data || !*data)
1567 return;
1568
1569#if HAVE_KC
1570 kcfree (*data);
1571 *data = NULL;
1572#elif HAVE_LMDB
1573 /* LMDB owns the data returned. It should not be freed */
1574#else
1575 FREE (data); /* __FREE_CHECKED__ */
1576#endif
1577}
1578
1579#if HAVE_DB4
1580const char *mutt_hcache_backend (void)
1581{
1582 return DB_VERSION_STRING;
1583}
1584#elif HAVE_LMDB
1585const char *mutt_hcache_backend (void)
1586{
1587 return "lmdb " MDB_VERSION_STRING;
1588}
1589#elif HAVE_GDBM
1590const char *mutt_hcache_backend (void)
1591{
1592 return gdbm_version;
1593}
1594#elif HAVE_QDBM
1595const char *mutt_hcache_backend (void)
1596{
1597 return "qdbm " _QDBM_VERSION;
1598}
1599#elif HAVE_TC
1600const char *mutt_hcache_backend (void)
1601{
1602 return "tokyocabinet " _TC_VERSION;
1603}
1604#elif HAVE_KC
1605const char *mutt_hcache_backend (void)
1606{
1607 static char backend[SHORT_STRING];
1608 snprintf(backend, sizeof(backend), "kyotocabinet %s", KCVERSION);
1609 return backend;
1610}
1611#endif