Benchmarking IMAP servers

There will be 4 tests against mailboxes of varying sizes; 100, 1000, and 10000 messages. Each of the tests will be repeated in full 5 times, and an average calculated.

Test #1: A stat of a fresh mailbox - at this point the mailbox will never have been accessed Test #2: A stat of the mailbox again Test #3: Accessing message 85 - a ‘random’ message Test #4: Flagging all messages as deleted, and then expunging them

The tests currently will be run against cyrus and courier imap servers. Configs are left as-is except what was required to change to get the mailboxes working with plain text login.

The tests were run on a old p3 server, with a single IDE hard disk drive. The system is on a single partition (except / boot) which is formatted as EXT3.

# cat /proc/cpuinfo 
processor       : 0
vendor_id       : GenuineIntel
cpu family      : 6
model           : 8
model name      : Pentium III (Coppermine)
stepping        : 3
cpu MHz         : 802.043
cache size      : 256 KB
fdiv_bug        : no
hlt_bug         : no
f00f_bug        : no
coma_bug        : no
fpu             : yes
fpu_exception   : yes
cpuid level     : 2
wp              : yes
flags           : fpu vme de pse tsc msr pae mce cx8 sep mtrr pge mca cmov pat pse36 mmx fxsr sse
bogomips        : 1581.05

Distro: Gentoo 2005.1
Kernel: Linux p3 2.6.12-gentoo-r6 
net-mail/cyrus-imapd-2.2.12  -afs -drac -idled -kerberos +pam -snmp +ssl +tcpd
mail-mta/exim-4.50-r1  -X -dnsdb -exiscan -exiscan-acl -ipv6 -ldap +lmtp -mailwrapper -mbox -mysql -nis +pam +perl -postgres -sasl +ssl -syslog +tcpd
net-mail/courier-imap-4.0.1  +berkdb -debug -fam +gdbm -ipv6 +nls (-selinux)

Script used to create test data

Main test script: test.sh

#!/bin/sh

NUM_MAILS=10000
USER="cyrus"
PASS="password"

./emails.sh $NUM_MAILS $USER
./virgin.pl localhost $USER $PASS
./virgin.pl localhost $USER $PASS
./fetch.pl localhost $USER $PASS
./delete.pl localhost $USER $PASS $NUM_MAILS

email.sh

#!/bin/sh

n=0
while test $n -lt $1
do
    echo "this is a test mail!!!" | uuencode - | mail -s "Test message $n" $2

    n=`expr $n + 1`
done

./virgin.pl

#!/usr/bin/perl -w

use Expect;
use Time::HiRes qw(gettimeofday tv_interval);

die "Usage: $0 server user password\n" if $#ARGV != 2;

our $server=$ARGV[0];
our $user=$ARGV[1];
our $password=$ARGV[2];
our $ok = '^\d+\s+OK';

print "Testing select on $user\n";
my $exp = Expect->spawn("telnet", "$server", "143") or die "Could not connect"; 

$exp->expect(2, "-re", '^\* OK') or die "didn't see OK";
$exp->send("001 LOGIN $user $password\n");
$exp->expect(20, "-re", "$ok") or die "didn't see OK";

$t0 = [gettimeofday];
$exp->send("002 SELECT INBOX\n");
$exp->expect(30, "-re", "$ok") or die "didn't see OK";
$time = tv_interval($t0 , [gettimeofday]);

$exp->send("002 LOGOUT\n");
$exp->expect(5, "-re", "$ok") or die "didn't see OK";
$exp->hard_close();

open (LOGFILE, ">> output.log");
print LOGFILE "\nSelect INBOX Time: ", $time, " s\n";
close LOGFILE;

fetch.pl

#!/usr/bin/perl -w

# This test does a set of uid fetches

use Expect;
use Time::HiRes qw(gettimeofday tv_interval);

my @commands = (
 "SELECT INBOX",
 "FETCH 85 (FLAGS BODYSTRUCTURE)",
 "FETCH 85 BODY[HEADER]",
 "FETCH 85 BODY[1]"
 );

die "Usage: $0 server user password\n" if $#ARGV != 2;

our $server=$ARGV[0];
our $user=$ARGV[1];
our $password=$ARGV[2];
our $ok = '^\d+\s+OK';
our $num = 2;

my $exp = Expect->spawn("telnet", "$server", "143") or die "Could not connect"; 
$exp->expect(2, "-re", '^\* OK') or die "didn't see OK";
$exp->send("001 LOGIN $user $password\n");
$exp->expect(5, "-re", "$ok") or die "didn't see OK";

$t0 = [gettimeofday];
for $command (@commands) {
   $exp->send(sprintf("%03d %s\n", $num++, $command));
   $exp->expect(20, "-re", "$ok") or die "didn't see OK";
}
$time = tv_interval($t0 , [gettimeofday]);
$exp->send("999 LOGOUT\n");
$exp->expect(5, "-re", "$ok") or die "didn't see OK";
$exp->hard_close();

open (LOGFILE, ">> output.log");
print LOGFILE "\nFetch Mail 85 Time: ", $time, " s\n";
close LOGFILE;

delete.pl

#!/usr/bin/perl -w

# This test does a deletion of 10 messages, including the expunge

use Expect;
use Time::HiRes qw(gettimeofday tv_interval);


die "Usage: $0 server user password\n" if $#ARGV != 3;

our $server=$ARGV[0];
our $user=$ARGV[1];
our $password=$ARGV[2];
our $ok = '^\d+\s+OK';
our $num = 2;

my @commands = (
 'SELECT INBOX',
 'STORE 0:' . $ARGV[3] . ' +FLAGS (\Deleted)',
 'EXPUNGE',
 );

my $exp = Expect->spawn("telnet", "$server", "143") or die "Could not connect"; 
$exp->expect(5, "-re", '^\* OK') or die "didn't see OK";
$exp->send("001 LOGIN $user $password\n");
$exp->expect(5, "-re", "$ok") or die "didn't see OK";

$t0 = [gettimeofday];
for $command (@commands) {
   $exp->send(sprintf("%03d %s\n", $num++, $command));
   $exp->expect(600, "-re", "$ok") or die "didn't see OK";
}
$time = tv_interval($t0 , [gettimeofday]);
$exp->send("999 LOGOUT\n");
$exp->expect(5, "-re", "$ok") or die "didn't see OK";
$exp->hard_close();

open (LOGFILE, ">> output.log");
print LOGFILE "\nDelete All Mail Time: ", $time, " s\n";
close LOGFILE;

begin routers

router_courier:
  driver = accept
  local_parts = courier
  transport = transport_courier
  
route_cyrus:
  driver = accept
  local_parts = cyrus
  transport = transport_cyrus

begin transports

transport_courier:
  driver = appendfile
  directory = /var/spool/mail/$local_part/Maildir
  maildir_format
  delivery_date_add
  envelope_to_add
  return_path_add
  group = mail
  mode = 0660

transport_cyrus:
     driver = pipe
     user = cyrus
     group = mail
     message_prefix = ""
     message_suffix = ""
     log_fail_output
     return_output
     envelope_to_add = true
     command = "/usr/lib/cyrus/deliver -r $return_path $local_part"

# userdb  courier set home=/var/spool/mail/courier/Maildir/ mail=/var/spool/mail/courier/Maildir/ uid=8 gid=12
# userdbpw -md5 | userdb courier set systempw
Password: 
Reenter password: 

# saslpasswd2 cyrus
Password:
# cyradm --user cyrus  localhost
Password: 
localhost> cm user.cyrus
#

Cyrus with 100 emails

Cyrus with 1000 emails

Cyrus with 10000 emails

Courier with 100 emails

Courier with 1000 emails

Courier with 10000 emails

100 emails

1000 emails

10000 emails

—- References:

http://www.courier-mta.org/mbox-vs-maildir/

http://whizzo.uoregon.edu/public/src/mailperf/results.html