Failover for down host slow - taking full proxy_connect_timeout

I’ve set up nginx as a load balancer (proxy). When one of the
upstream hosts goes down such that connections to the host fail
immediately, it is taking nginx the full proxy_connect_timeout to fail
each request that is sent to that proxy. Is there a way to configure
nginx such that when a connection to an upstream server fails, it gets
treated as a failure without waiting for the full
proxy_connect_timeout before treating it as a failed connection?

Here’s my config file:

user nginx;
worker_processes 2;
worker_rlimit_nofile 10000;

error_log /var/log/nginx/error.log;
pid /var/run/;

events {
worker_connections 4000;
use epoll;

http {
include mime.types;
keepalive_timeout 65;
gzip on;

upstream backend {
server max_fails=2 fail_timeout=120s;
server max_fails=2 fail_timeout=120s;

server {
listen 80 default;
proxy_buffers 64 4k;

proxy_set_header Host $host;
proxy_set_header X-Real_IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-SSL-Request "";

location / {
  proxy_connect_timeout  2;
  proxy_read_timeout    10;

  proxy_pass http://backend;