We’re running nginx 0.7.68 w/ the s3-proxy patch applied, and we’re
seeing
reproducible segfaults when we hit (or miss) certain files:
Nicholas-Tangs-MacBook-Air:~ ntang$ curl
http://[server]/path/to/test.txt
curl: (52) Empty reply from server
And here’s what happens on the server:
[root@server nginx]# strace -p 9742
Process 9742 attached - interrupt to quit
write(26, “2010/12/20 11:23:52 [info] 9742#”…, 84) = 84
epoll_wait(5, {{EPOLLIN, {u32=857063440, u64=139771977777168}}}, 512,
4294967295) = 1
accept(6, {sa_family=AF_INET, sin_port=htons(44473),
sin_addr=inet_addr(“myip”)}, [29640703181062160]) = 3
ioctl(3, FIONBIO, [1]) = 0
epoll_ctl(5, EPOLL_CTL_ADD, 3, {EPOLLIN|EPOLLET, {u32=857064145,
u64=139771977777873}}) = 0
epoll_wait(5, {{EPOLLIN, {u32=857064145, u64=139771977777873}}}, 512,
5000)
= 1
recvfrom(3, “GET /path/to/…”…, 1024, 0, NULL, NULL) = 210
stat(“/path/to/file/test.txt”, 0x7fff3b23df20) = -1 ENOENT (No such file
or
directory)
epoll_ctl(5, EPOLL_CTL_MOD, 3, {EPOLLIN|EPOLLOUT|EPOLLET,
{u32=857064145,
u64=139771977777873}}) = 0
— SIGSEGV (Segmentation fault) @ 0 (0) —
Process 9742 detached
[root@server nginx]#
Has anyone seen any similar behavior? We’ve got a series of location/
rewrite/ try_files blocks to look for the files. Basically, the way it
works is like this:
- look on the main filesystem
- look on the backup filesystem
- look on the backup backup filesystem
- look on S3
In the case of the request above, the process segfaulted and died after
step
1.
Normally, this is what should happen:
epoll_wait(5, {{EPOLLIN, {u32=857063440, u64=139771977777168}}}, 512,
4294967295) = 1
accept(6, {sa_family=AF_INET, sin_port=htons(46329),
sin_addr=inet_addr(“my_ip”)}, [29640703181062160]) = 3
ioctl(3, FIONBIO, [1]) = 0
epoll_ctl(5, EPOLL_CTL_ADD, 3, {EPOLLIN|EPOLLET, {u32=857063792,
u64=139771977777520}}) = 0
epoll_wait(5, {{EPOLLIN, {u32=857063792, u64=139771977777520}}}, 512,
5000)
= 1
recvfrom(3, “GET /path/to/request”…, 1024, 0, NULL, NULL) = 210
stat(“/path/1/test.txt”, 0x7fff3b23df20) = -1 ENOENT (No such file or
directory)
stat(“/path/2/test.txt”, 0x7fff3b23dd90) = -1 ENOENT (No such file or
directory)
stat(“/path/3/test.txt”, 0x7fff3b23dc00) = -1 ENOENT (No such file or
directory)
epoll_ctl(5, EPOLL_CTL_MOD, 3, {EPOLLIN|EPOLLOUT|EPOLLET,
{u32=857063792,
u64=139771977777520}}) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 7
ioctl(7, FIONBIO, [1]) = 0
epoll_ctl(5, EPOLL_CTL_ADD, 7, {EPOLLIN|EPOLLOUT|EPOLLET,
{u32=857063969,
u64=139771977777697}}) = 0
connect(7, {sa_family=AF_INET, sin_port=htons(8976),
sin_addr=inet_addr(“127.0.0.1”)}, 16) = -1 EINPROGRESS (Operation now in
progress)
epoll_wait(5, {{EPOLLOUT, {u32=857063792, u64=139771977777520}},
{EPOLLOUT,
{u32=857063969, u64=139771977777697}}, {EPOLLIN, {u32=857063440,
u64=139771977777168}}}, 512, 60000) = 3
recvfrom(3, 0x7fff3b23e1f7, 1, 2, 0, 0) = -1 EAGAIN (Resource
temporarily
unavailable)
getsockopt(7, SOL_SOCKET, SO_ERROR, [3681060957225746432], [4]) = 0
writev(7, [{“GET /path/to/request”…, 203}], 1) = 203
accept(6, {sa_family=AF_INET, sin_port=htons(40347),
sin_addr=inet_addr(“127.0.0.1”)}, [51539607568]) = 8
ioctl(8, FIONBIO, [1]) = 0
epoll_ctl(5, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLET, {u32=857064145,
u64=139771977777873}}) = 0
epoll_wait(5, {{EPOLLIN, {u32=857064145, u64=139771977777873}}}, 512,
5000)
= 1
recvfrom(8, “GET /path/to/request”…, 1024, 0, NULL, NULL) = 203
epoll_ctl(5, EPOLL_CTL_MOD, 8, {EPOLLIN|EPOLLOUT|EPOLLET,
{u32=857064145,
u64=139771977777873}}) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 9
ioctl(9, FIONBIO, [1]) = 0
epoll_ctl(5, EPOLL_CTL_ADD, 9, {EPOLLIN|EPOLLOUT|EPOLLET,
{u32=857064321,
u64=139771977778049}}) = 0
connect(9, {sa_family=AF_INET, sin_port=htons(80),
sin_addr=inet_addr(“s3_ip”)}, 16) = -1 EINPROGRESS (Operation now in
progress)
epoll_wait(5, {{EPOLLOUT, {u32=857064145, u64=139771977777873}}}, 512,
60000) = 1
recvfrom(8, 0x7fff3b23e1f7, 1, 2, 0, 0) = -1 EAGAIN (Resource
temporarily
unavailable)
epoll_wait(5, {{EPOLLOUT, {u32=857064321, u64=139771977778049}}}, 512,
59999) = 1
getsockopt(9, SOL_SOCKET, SO_ERROR, [3681062469054234624], [4]) = 0
writev(9, [{“GET /path/to/request”…, 332}], 1) = 332
epoll_wait(5, {{EPOLLIN|EPOLLOUT, {u32=857064321,
u64=139771977778049}}},
512, 59998) = 1
recvfrom(9, “HTTP/1.1 200 OK\r\nx-amz-id-2: [snip]”…, 4096, 0, NULL,
NULL)
= 447
readv(9,
[{“\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0”…,
3649}], 1) = 0
close(9) = 0
writev(8, [{“HTTP/1.1 200 OK\r\nServer: nginx/0”…, 442}, {“bla\n”,
4}], 2)
= 446
write(27, “127.0.0.1 - - [20/Dec/2010:11:27”…, 190) = 190
close(8) = 0
epoll_wait(5, {{EPOLLIN|EPOLLOUT, {u32=857063969,
u64=139771977777697}}},
512, 59987) = 1
recvfrom(7, “HTTP/1.1 200 OK\r\nServer: nginx/0”…, 4096, 0, NULL,
NULL) =
446
readv(7,
[{“\0\0\300\220k\0\0\0\0\0\310\0\0\0\0\0\0\0\6\0\0\0\0\0\0\0\270\240k\0\0\0”…,
3650}], 1) = 0
close(7) = 0
writev(3, [{“HTTP/1.1 200 OK\r\nServer: nginx/0”…, 470}, {“bla\n”,
4}], 2)
= 474
write(27, “my_ip - - [20/Dec/2010:”…, 219) = 219
setsockopt(3, SOL_TCP, TCP_NODELAY, [1], 4) = 0
recvfrom(3, 0x69e5c0, 1024, 0, 0, 0) = -1 EAGAIN (Resource
temporarily
unavailable)
epoll_wait(5, {{EPOLLIN|EPOLLOUT, {u32=857063792,
u64=139771977777520}}},
512, 5000) = 1
recvfrom(3, “”, 1024, 0, NULL, NULL) = 0
write(26, “2010/12/20 11:27:56 [info] 10313”…, 92) = 92
close(3) = 0
Here’s how the config lines work:
Files served from Set A
location ~ ^/dv0([\d])/(.*)$ {
alias /path1/$1b/path/$2;
try_files “” /dv$1b/$2;
}
Files served from Set B
location ~ ^/dv0([\d])b/(.*)$ {
alias /path2/$1/path/$2;
try_files “” /union/$2;
}
Files served from the Union of A and B
location ~ ^/union/path3/(.*)$ {
alias /union/path3/$1;
try_files "" @failUnion;
}
location @failUnion {
rewrite ^/union/path/(.*)$ /$1 break;
proxy_pass http://127.0.0.1:8976;
}
location ~ ^/path/(.)$ {
rewrite ^/path/(.)$ /union/path3/$1 last;
}
Proxy the request back to Amazon S3
location / {
proxy_pass http://bucket.s3.amazonaws.com;
proxy_s3_auth on;
proxy_s3_secure_download off; # optional if you’re also using
secdownload
proxy_s3_bucket bucket;
proxy_s3_user user;
proxy_s3_pass pass;
}
Hopefully that gives you some idea of what we’re doing without revealing
anything confidential… 
Thanks,
Nicholas
Nicholas Tang*:*
VP, Dev Ops
[email protected]
|
t: +1 (646) 495 9707
|
m: +1 (347) 410 6066
|
111 8th Avenue, Floor 15, New York, NY 10011
[image: www.livestream.com] http://www.livestream.com/
