Weird behavior on SSL, and corruption on reload

luislavena · September 25, 2011, 11:14am

Hello,

I had a weird behavior in nginx, in which SSL connections were suddenly
denied. It happened with no apparent reason. A reload of nginx seemed to
have solved it, but on reload, error.log had this stack trace:

*** glibc detected *** nginx: worker process is shutting down: double
free or corruption (out): 0x080ca518 ***
======= Backtrace: =========
/lib/i686/nosegneg/libc.so.6[0xb7d0ccfd]
/lib/i686/nosegneg/libc.so.6(cfree+0x90)[0xb7d103b0]
/lib/libcrypto.so.6(CRYPTO_free+0x3a)[0xb7e3046a]
/lib/libcrypto.so.6(BN_free+0x68)[0xb7e5a1d8]
/lib/libcrypto.so.6(BN_MONT_CTX_free+0x29)[0xb7e63d99]
/lib/libcrypto.so.6[0xb7e683e9]
/lib/libcrypto.so.6(RSA_free+0x61)[0xb7e6afc1]
/lib/libcrypto.so.6[0xb7e8cacd]
/lib/libcrypto.so.6(EVP_PKEY_free+0x67)[0xb7e8cb47]
/lib/libssl.so.6(ssl_cert_free+0xa0)[0xb7f5e5c0]
/lib/libssl.so.6(SSL_CTX_free+0xe3)[0xb7f5bb73]
nginx: worker process is shutting down[0x8066bbb]
nginx: worker process is shutting down[0x804d1d0]
nginx: worker process is shutting down[0x80645fe]
nginx: worker process is shutting down[0x80646eb]
nginx: worker process is shutting down[0x8062ff1]
nginx: worker process is shutting down[0x8064f2c]
nginx: worker process is shutting down[0x804c891]
/lib/i686/nosegneg/libc.so.6(__libc_start_main+0xdc)[0xb7cbbdec]
nginx: worker process is shutting down[0x804b141]
======= Memory map: ========
08048000-080ab000 r-xp 00000000 08:01 298520
/usr/local/nginx/sbin/nginx
080ab000-080b3000 rw-p 00062000 08:01 298520
/usr/local/nginx/sbin/nginx
080b3000-0829e000 rw-p 080b3000 00:00 0 [heap]
b4600000-b4621000 rw-p b4600000 00:00 0
b4621000-b4700000 —p b4621000 00:00 0
b479b000-b47a6000 r-xp 00000000 08:01 262401
/lib/libgcc_s-4.1.1-20070105.so.1
b47a6000-b47a7000 rw-p 0000a000 08:01 262401
/lib/libgcc_s-4.1.1-20070105.so.1
b47a7000-b47ab000 r-xp 00000000 08:01 262411 /lib/libnss_dns-2.5.so
b47ab000-b47ac000 r–p 00003000 08:01 262411 /lib/libnss_dns-2.5.so
b47ac000-b47ad000 rw-p 00004000 08:01 262411 /lib/libnss_dns-2.5.so
b47b3000-b49a0000 rw-p b47b3000 00:00 0
b49a0000-b49a1000 rw-s 00000000 00:08 1620412881 /dev/zero (deleted)
b49a1000-b7ba1000 rw-s 00000000 00:08 1620412878 /dev/zero (deleted)
b7ba1000-b7baa000 r-xp 00000000 08:01 262412
/lib/libnss_files-2.5.so
b7baa000-b7bab000 r–p 00008000 08:01 262412
/lib/libnss_files-2.5.so
b7bab000-b7bac000 rw-p 00009000 08:01 262412
/lib/libnss_files-2.5.so
b7bac000-b7bae000 rw-p b7bac000 00:00 0
b7bae000-b7bb5000 r-xp 00000000 08:01 65580
/usr/lib/libkrb5support.so.0.1
b7bb5000-b7bb6000 rw-p 00006000 08:01 65580
/usr/lib/libkrb5support.so.0.1
b7bb6000-b7bc5000 r-xp 00000000 08:01 262423 /lib/libresolv-2.5.so
b7bc5000-b7bc6000 r–p 0000e000 08:01 262423 /lib/libresolv-2.5.so
b7bc6000-b7bc7000 rw-p 0000f000 08:01 262423 /lib/libresolv-2.5.so
b7bc7000-b7bc9000 rw-p b7bc7000 00:00 0
b7bc9000-b7bee000 r-xp 00000000 08:01 65574
/usr/lib/libk5crypto.so.3.0
b7bee000-b7bef000 rw-p 00025000 08:01 65574
/usr/lib/libk5crypto.so.3.0
b7bef000-b7bf1000 r-xp 00000000 08:01 262387 /lib/libcom_err.so.2.1
b7bf1000-b7bf2000 rw-p 00001000 08:01 262387 /lib/libcom_err.so.2.1
b7bf2000-b7bf3000 rw-p b7bf2000 00:00 0
b7bf3000-b7c79000 r-xp 00000000 08:01 65579
/usr/lib/libkrb5.so.3.2
b7c79000-b7c7b000 rw-p 00086000 08:01 65579
/usr/lib/libkrb5.so.3.2
b7c7b000-b7ca5000 r-xp 00000000 08:01 65561
/usr/lib/libgssapi_krb5.so.2.2
b7ca5000-b7ca6000 rw-p 00029000 08:01 65561
/usr/lib/libgssapi_krb5.so.2.2
b7ca6000-b7de1000 r-xp 00000000 08:01 262438
/lib/i686/nosegneg/libc-2.5.so
b7de1000-b7de3000 r–p 0013a000 08:01 262438
/lib/i686/nosegneg/libc-2.5.so
b7de3000-b7de4000 rw-p 0013c000 08:01 262438
/lib/i686/nosegneg/libc-2.5.so
b7de4000-b7de7000 rw-p b7de4000 00:00 0
b7de7000-b7df9000 r-xp 00000000 08:01 65693 /usr/lib/libz.so.1.2.3
b7df9000-b7dfa000 rw-p 00011000 08:01 65693 /usr/lib/libz.so.1.2.3
b7dfa000-b7dfc000 r-xp 00000000 08:01 262397 /lib/libdl-2.5.so
b7dfc000-b7dfd000 r–p 00001000 08:01 262397 /lib/libdl-2.5.so
b7dfd000-b7dfe000 rw-p 00002000 08:01 262397 /lib/libdl-2.5.so
b7dfe000-b7f1b000 r-xp 00000000 08:01 262389
/lib/libcrypto.so.0.9.8b
b7f1b000-b7f2e000 rw-p 0011c000 08:01 262389
/lib/libcrypto.so.0.9.8b
b7f2e000-b7f32000 rw-p b7f2e000 00:00 0
b7f32000-b7f73000 r-xp 00000000 08:01 262428 /lib/libssl.so.0.9.8b
b7f73000-b7f77000 rw-p 00040000 08:01 262428 /lib/libssl.so.0.9.8b
b7f77000-b7f93000 r-xp 00000000 08:01 262420 /lib/libpcre.so.0.0.1
b7f93000-b7f94000 rw-p 0001b000 08:01 262420 /lib/libpcre.so.0.0.1
b7f94000-b7f99000 r-xp 00000000 08:01 262388 /lib/libcrypt-2.5.so
b7f99000-b7f9a000 r–p 00004000 08:01 262388 /lib/libcrypt-2.5.so
b7f9a000-b7f9b000 rw-p 00005000 08:01 262388 /lib/libcrypt-2.5.so
b7f9b000-b7fc2000 rw-p b7f9b000 00:00 0
b7fc8000-b7fc9000 rw-p b7fc8000 00:00 0
b7fc9000-b7fca000 r-xp b7fc9000 00:00 0 [vdso]
b7fca000-b7fe3000 r-xp 00000000 08:01 262375 /lib/ld-2.5.so
b7fe3000-b7fe4000 r–p 00018000 08:01 262375 /lib/ld-2.5.so
b7fe4000-b7fe5000 rw-p 00019000 08:01 262375 /lib/ld-2.5.so
bfdae000-bfdd2000 rw-p bfdae000 00:00 0 [stack]
2011/09/25 05:01:43 [alert] 21233#0: worker process 1870 exited on
signal 6

Any ideas on what happened, and what can be done to prevent it in the
future?

Thanks,
Oren

Posted at Nginx Forum:

orensol · September 25, 2011, 1:01pm

Hello!

On Sun, Sep 25, 2011 at 05:13:54AM -0400, orensol wrote:

/lib/i686/nosegneg/libc.so.6(cfree+0x90)[0xb7d103b0]
nginx: worker process is shutting down[0x804d1d0]
nginx: worker process is shutting down[0x80645fe]
nginx: worker process is shutting down[0x80646eb]
nginx: worker process is shutting down[0x8062ff1]
nginx: worker process is shutting down[0x8064f2c]
nginx: worker process is shutting down[0x804c891]
/lib/i686/nosegneg/libc.so.6(__libc_start_main+0xdc)[0xb7cbbdec]
nginx: worker process is shutting down[0x804b141]

[…]

2011/09/25 05:01:43 [alert] 21233#0: worker process 1870 exited on
signal 6

Any ideas on what happened, and what can be done to prevent it in the
future?

Are you able to reproduce the problem? If yes, please follow
instructions here:

http://wiki.nginx.org/Debugging

At least proper backtrace is needed to debug this further. And
you may need to make sure your nginx binary isn’t stripped.

If not, please at least provide “nginx -V” output, config and
OpenSSL version details (output of “openssl version -a”).

Maxim D.

orensol · September 25, 2011, 1:15pm

Hello Maxim,

For now I can’t reproduce the problem, if it happens again i’ll try to
catch a real backtrace.

Here are the other details. Thanks!

nginx:
nginx version: nginx/0.8.35
built by gcc 4.1.1 20070105 (Red Hat 4.1.1-52)
TLS SNI support disabled
configure arguments: --with-http_ssl_module
–add-module=/root/ngx_cache_purge-1.0

openssl:
OpenSSL 0.9.8b 04 May 2006
built on: Wed Oct 17 18:15:17 EDT 2007
platform: linux-elf
options: bn(64,32) md2(int) rc4(idx,int) des(ptr,risc1,16,long)
blowfish(idx)
compiler: gcc -fPIC -DOPENSSL_PIC -DZLIB -DOPENSSL_THREADS -D_REENTRANT
-DDSO_DLFCN -DHAVE_DLFCN_H -DKRB5_MIT -I/usr/kerberos/include -DL_ENDIAN
-DTERMIO -Wall -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions
-fstack-protector --param=ssp-buffer-size=4 -m32 -march=i686
-mtune=generic -fasynchronous-unwind-tables -Wa,–noexecstack
-DOPENSSL_BN_ASM_PART_WORDS -DOPENSSL_IA32_SSE2 -DSHA1_ASM -DMD5_ASM
-DRMD160_ASM -DAES_ASM
OPENSSLDIR: “/etc/pki/tls”
engines: dynamic padlock

nginx conf:

user nginx;
worker_processes 1;
worker_rlimit_nofile 10240;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;

events {
worker_connections 10240;
}

http {
include /usr/local/nginx/conf/mime.types;
default_type application/octet-stream;

log_format  main  '$remote_addr - $remote_user [$time_local]

$request ’
'“$status” $body_bytes_sent “$http_referer” ’
‘“$http_user_agent” “$http_x_forwarded_for”’;

access_log  /var/log/nginx/access.log  main;

log_format cache '***$time_local '
                 '$upstream_cache_status '
                 'Cache-Control: $upstream_http_cache_control '
                 'Expires: $upstream_http_expires '
                 '"$host" "$request" ($status) '
                 '"$http_user_agent" ';
access_log  /var/log/nginx/cache.log cache;

sendfile        on;

keepalive_timeout  65;

gzip  on;
gzip_types  text/css text/javascript application/x-javascript;


    proxy_cache_path /mnt/nginx/cache levels=1:2

keys_zone=myzone:50m inactive=2h max_size=3000m;
proxy_temp_path /mnt/nginx/temp;

upstream backend_ssl {
server 10.255.10.255:443;
}

server {
listen 443;
server_name _;

  ssl                  on;
  ssl_certificate      /usr/local/nginx/conf/cert;
  ssl_certificate_key  /usr/local/nginx/conf/key;

  ssl_session_timeout  5m;

  ssl_protocols  SSLv2 SSLv3 TLSv1;
  ssl_ciphers

ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;
ssl_prefer_server_ciphers on;

location / {

        proxy_pass http://backend_ssl;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $remote_addr;
}

# static files served locally
location ~* \.(css|js)$ {
    root   /home/static;
}

# static images served locally and get Expires header
location ~* \.(jpg|jpeg|gif|png|ico|bmp)$ {
    root   /home/static;
    expires 30d;
}

}

Posted at Nginx Forum:

orensol · September 25, 2011, 2:14pm

Hello!

On Sun, Sep 25, 2011 at 07:14:40AM -0400, orensol wrote:

nginx version: nginx/0.8.35
platform: linux-elf
engines: dynamic padlock
Both nginx 0.8.35 and openssl 0.9.8b are rather old and have known
memory corruption issues. You may want to upgrade before doing
anything else.

Maxim D.