SSL enabled slowing down site

Using 0.9.3 (compiled), if I enable the SSL lines then :80 traffic slows
down (as does SSL traffic), remove the SSL lines and everything is
quicker… Looking through my config, am I doing anything wrong or can it
be improved?

#user  nobody;
worker_processes  1;

error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local]
"$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;
    keepalive_timeout  40;

    gzip  on;

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   /var/www;
            index  index.html index.htm index.php;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }


        location ~ \.php$ {
        fastcgi_pass 127.0.0.1:9000;
        fastcgi_index index.php;
        fastcgi_param SCRIPT_FILENAME /var/www$fastcgi_script_name;
        include fastcgi_params;
        }

  }

   server {
        server_name secure.fanboy.co.nz;
        listen       443;
        ssl                  on;
        ssl_ciphers RC4:ALL:-LOW:-EXPORT:!ADH:!MD5;
        ssl_prefer_server_ciphers on;
        keepalive_requests 280;
        keepalive_timeout 0;
        ssl_certificate      /root/server.pem;
        ssl_certificate_key  /root/ssl.key;

           location / {
                root   /var/www;
                index  index.html index.htm index.php;
            }

        }


}

Posted at Nginx Forum:
http://forum.nginx.org/read.php?2,159017,159017#msg-159017