Weird behavior on SSL, and corruption on reload

Hello,

I had a weird behavior in nginx, in which SSL connections were suddenly
denied. It happened with no apparent reason. A reload of nginx seemed to
have solved it, but on reload, error.log had this stack trace:

*** glibc detected *** nginx: worker process is shutting down: double
free or corruption (out): 0x080ca518 ***
======= Backtrace: =========
/lib/i686/nosegneg/libc.so.6[0xb7d0ccfd]
/lib/i686/nosegneg/libc.so.6(cfree+0x90)[0xb7d103b0]
/lib/libcrypto.so.6(CRYPTO_free+0x3a)[0xb7e3046a]
/lib/libcrypto.so.6(BN_free+0x68)[0xb7e5a1d8]
/lib/libcrypto.so.6(BN_MONT_CTX_free+0x29)[0xb7e63d99]
/lib/libcrypto.so.6[0xb7e683e9]
/lib/libcrypto.so.6(RSA_free+0x61)[0xb7e6afc1]
/lib/libcrypto.so.6[0xb7e8cacd]
/lib/libcrypto.so.6(EVP_PKEY_free+0x67)[0xb7e8cb47]
/lib/libssl.so.6(ssl_cert_free+0xa0)[0xb7f5e5c0]
/lib/libssl.so.6(SSL_CTX_free+0xe3)[0xb7f5bb73]
nginx: worker process is shutting down[0x8066bbb]
nginx: worker process is shutting down[0x804d1d0]
nginx: worker process is shutting down[0x80645fe]
nginx: worker process is shutting down[0x80646eb]
nginx: worker process is shutting down[0x8062ff1]
nginx: worker process is shutting down[0x8064f2c]
nginx: worker process is shutting down[0x804c891]
/lib/i686/nosegneg/libc.so.6(__libc_start_main+0xdc)[0xb7cbbdec]
nginx: worker process is shutting down[0x804b141]
======= Memory map: ========
08048000-080ab000 r-xp 00000000 08:01 298520
/usr/local/nginx/sbin/nginx
080ab000-080b3000 rw-p 00062000 08:01 298520
/usr/local/nginx/sbin/nginx
080b3000-0829e000 rw-p 080b3000 00:00 0 [heap]
b4600000-b4621000 rw-p b4600000 00:00 0
b4621000-b4700000 —p b4621000 00:00 0
b479b000-b47a6000 r-xp 00000000 08:01 262401
/lib/libgcc_s-4.1.1-20070105.so.1
b47a6000-b47a7000 rw-p 0000a000 08:01 262401
/lib/libgcc_s-4.1.1-20070105.so.1
b47a7000-b47ab000 r-xp 00000000 08:01 262411 /lib/libnss_dns-2.5.so
b47ab000-b47ac000 r–p 00003000 08:01 262411 /lib/libnss_dns-2.5.so
b47ac000-b47ad000 rw-p 00004000 08:01 262411 /lib/libnss_dns-2.5.so
b47b3000-b49a0000 rw-p b47b3000 00:00 0
b49a0000-b49a1000 rw-s 00000000 00:08 1620412881 /dev/zero (deleted)
b49a1000-b7ba1000 rw-s 00000000 00:08 1620412878 /dev/zero (deleted)
b7ba1000-b7baa000 r-xp 00000000 08:01 262412
/lib/libnss_files-2.5.so
b7baa000-b7bab000 r–p 00008000 08:01 262412
/lib/libnss_files-2.5.so
b7bab000-b7bac000 rw-p 00009000 08:01 262412
/lib/libnss_files-2.5.so
b7bac000-b7bae000 rw-p b7bac000 00:00 0
b7bae000-b7bb5000 r-xp 00000000 08:01 65580
/usr/lib/libkrb5support.so.0.1
b7bb5000-b7bb6000 rw-p 00006000 08:01 65580
/usr/lib/libkrb5support.so.0.1
b7bb6000-b7bc5000 r-xp 00000000 08:01 262423 /lib/libresolv-2.5.so
b7bc5000-b7bc6000 r–p 0000e000 08:01 262423 /lib/libresolv-2.5.so
b7bc6000-b7bc7000 rw-p 0000f000 08:01 262423 /lib/libresolv-2.5.so
b7bc7000-b7bc9000 rw-p b7bc7000 00:00 0
b7bc9000-b7bee000 r-xp 00000000 08:01 65574
/usr/lib/libk5crypto.so.3.0
b7bee000-b7bef000 rw-p 00025000 08:01 65574
/usr/lib/libk5crypto.so.3.0
b7bef000-b7bf1000 r-xp 00000000 08:01 262387 /lib/libcom_err.so.2.1
b7bf1000-b7bf2000 rw-p 00001000 08:01 262387 /lib/libcom_err.so.2.1
b7bf2000-b7bf3000 rw-p b7bf2000 00:00 0
b7bf3000-b7c79000 r-xp 00000000 08:01 65579
/usr/lib/libkrb5.so.3.2
b7c79000-b7c7b000 rw-p 00086000 08:01 65579
/usr/lib/libkrb5.so.3.2
b7c7b000-b7ca5000 r-xp 00000000 08:01 65561
/usr/lib/libgssapi_krb5.so.2.2
b7ca5000-b7ca6000 rw-p 00029000 08:01 65561
/usr/lib/libgssapi_krb5.so.2.2
b7ca6000-b7de1000 r-xp 00000000 08:01 262438
/lib/i686/nosegneg/libc-2.5.so
b7de1000-b7de3000 r–p 0013a000 08:01 262438
/lib/i686/nosegneg/libc-2.5.so
b7de3000-b7de4000 rw-p 0013c000 08:01 262438
/lib/i686/nosegneg/libc-2.5.so
b7de4000-b7de7000 rw-p b7de4000 00:00 0
b7de7000-b7df9000 r-xp 00000000 08:01 65693 /usr/lib/libz.so.1.2.3
b7df9000-b7dfa000 rw-p 00011000 08:01 65693 /usr/lib/libz.so.1.2.3
b7dfa000-b7dfc000 r-xp 00000000 08:01 262397 /lib/libdl-2.5.so
b7dfc000-b7dfd000 r–p 00001000 08:01 262397 /lib/libdl-2.5.so
b7dfd000-b7dfe000 rw-p 00002000 08:01 262397 /lib/libdl-2.5.so
b7dfe000-b7f1b000 r-xp 00000000 08:01 262389
/lib/libcrypto.so.0.9.8b
b7f1b000-b7f2e000 rw-p 0011c000 08:01 262389
/lib/libcrypto.so.0.9.8b
b7f2e000-b7f32000 rw-p b7f2e000 00:00 0
b7f32000-b7f73000 r-xp 00000000 08:01 262428 /lib/libssl.so.0.9.8b
b7f73000-b7f77000 rw-p 00040000 08:01 262428 /lib/libssl.so.0.9.8b
b7f77000-b7f93000 r-xp 00000000 08:01 262420 /lib/libpcre.so.0.0.1
b7f93000-b7f94000 rw-p 0001b000 08:01 262420 /lib/libpcre.so.0.0.1
b7f94000-b7f99000 r-xp 00000000 08:01 262388 /lib/libcrypt-2.5.so
b7f99000-b7f9a000 r–p 00004000 08:01 262388 /lib/libcrypt-2.5.so
b7f9a000-b7f9b000 rw-p 00005000 08:01 262388 /lib/libcrypt-2.5.so
b7f9b000-b7fc2000 rw-p b7f9b000 00:00 0
b7fc8000-b7fc9000 rw-p b7fc8000 00:00 0
b7fc9000-b7fca000 r-xp b7fc9000 00:00 0 [vdso]
b7fca000-b7fe3000 r-xp 00000000 08:01 262375 /lib/ld-2.5.so
b7fe3000-b7fe4000 r–p 00018000 08:01 262375 /lib/ld-2.5.so
b7fe4000-b7fe5000 rw-p 00019000 08:01 262375 /lib/ld-2.5.so
bfdae000-bfdd2000 rw-p bfdae000 00:00 0 [stack]
2011/09/25 05:01:43 [alert] 21233#0: worker process 1870 exited on
signal 6

Any ideas on what happened, and what can be done to prevent it in the
future?

Thanks,
Oren

Posted at Nginx Forum:
http://forum.nginx.org/read.php?2,215785,215785#msg-215785

Hello!

On Sun, Sep 25, 2011 at 05:13:54AM -0400, orensol wrote:

/lib/i686/nosegneg/libc.so.6(cfree+0x90)[0xb7d103b0]
nginx: worker process is shutting down[0x804d1d0]
nginx: worker process is shutting down[0x80645fe]
nginx: worker process is shutting down[0x80646eb]
nginx: worker process is shutting down[0x8062ff1]
nginx: worker process is shutting down[0x8064f2c]
nginx: worker process is shutting down[0x804c891]
/lib/i686/nosegneg/libc.so.6(__libc_start_main+0xdc)[0xb7cbbdec]
nginx: worker process is shutting down[0x804b141]

[…]

2011/09/25 05:01:43 [alert] 21233#0: worker process 1870 exited on
signal 6

Any ideas on what happened, and what can be done to prevent it in the
future?

Are you able to reproduce the problem? If yes, please follow
instructions here:

http://wiki.nginx.org/Debugging

At least proper backtrace is needed to debug this further. And
you may need to make sure your nginx binary isn’t stripped.

If not, please at least provide “nginx -V” output, config and
OpenSSL version details (output of “openssl version -a”).

Maxim D.

Hello Maxim,

For now I can’t reproduce the problem, if it happens again i’ll try to
catch a real backtrace.

Here are the other details. Thanks!

nginx:
nginx version: nginx/0.8.35
built by gcc 4.1.1 20070105 (Red Hat 4.1.1-52)
TLS SNI support disabled
configure arguments: --with-http_ssl_module
–add-module=/root/ngx_cache_purge-1.0

openssl:
OpenSSL 0.9.8b 04 May 2006
built on: Wed Oct 17 18:15:17 EDT 2007
platform: linux-elf
options: bn(64,32) md2(int) rc4(idx,int) des(ptr,risc1,16,long)
blowfish(idx)
compiler: gcc -fPIC -DOPENSSL_PIC -DZLIB -DOPENSSL_THREADS -D_REENTRANT
-DDSO_DLFCN -DHAVE_DLFCN_H -DKRB5_MIT -I/usr/kerberos/include -DL_ENDIAN
-DTERMIO -Wall -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions
-fstack-protector --param=ssp-buffer-size=4 -m32 -march=i686
-mtune=generic -fasynchronous-unwind-tables -Wa,–noexecstack
-DOPENSSL_BN_ASM_PART_WORDS -DOPENSSL_IA32_SSE2 -DSHA1_ASM -DMD5_ASM
-DRMD160_ASM -DAES_ASM
OPENSSLDIR: “/etc/pki/tls”
engines: dynamic padlock

nginx conf:

user nginx;
worker_processes 1;
worker_rlimit_nofile 10240;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;

events {
worker_connections 10240;
}

http {
include /usr/local/nginx/conf/mime.types;
default_type application/octet-stream;

log_format  main  '$remote_addr - $remote_user [$time_local]

$request ’
'"$status" $body_bytes_sent “$http_referer” ’
‘"$http_user_agent" “$http_x_forwarded_for”’;

access_log  /var/log/nginx/access.log  main;

log_format cache '***$time_local '
                 '$upstream_cache_status '
                 'Cache-Control: $upstream_http_cache_control '
                 'Expires: $upstream_http_expires '
                 '"$host" "$request" ($status) '
                 '"$http_user_agent" ';
access_log  /var/log/nginx/cache.log cache;

sendfile        on;

keepalive_timeout  65;

gzip  on;
gzip_types  text/css text/javascript application/x-javascript;


    proxy_cache_path /mnt/nginx/cache levels=1:2

keys_zone=myzone:50m inactive=2h max_size=3000m;
proxy_temp_path /mnt/nginx/temp;

upstream backend_ssl {
server 10.255.10.255:443;
}

server {
listen 443;
server_name _;

  ssl                  on;
  ssl_certificate      /usr/local/nginx/conf/cert;
  ssl_certificate_key  /usr/local/nginx/conf/key;

  ssl_session_timeout  5m;

  ssl_protocols  SSLv2 SSLv3 TLSv1;
  ssl_ciphers

ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;
ssl_prefer_server_ciphers on;

location / {

        proxy_pass http://backend_ssl;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $remote_addr;
}

# static files served locally
location ~* \.(css|js)$ {
    root   /home/static;
}

# static images served locally and get Expires header
location ~* \.(jpg|jpeg|gif|png|ico|bmp)$ {
    root   /home/static;
    expires 30d;
}

}

}

Posted at Nginx Forum:
http://forum.nginx.org/read.php?2,215785,215795#msg-215795

Hello!

On Sun, Sep 25, 2011 at 07:14:40AM -0400, orensol wrote:

nginx version: nginx/0.8.35
platform: linux-elf
engines: dynamic padlock
Both nginx 0.8.35 and openssl 0.9.8b are rather old and have known
memory corruption issues. You may want to upgrade before doing
anything else.

Maxim D.

This forum is not affiliated to the Ruby language, Ruby on Rails framework, nor any Ruby applications discussed here.

| Privacy Policy | Terms of Service | Remote Ruby Jobs