diff --git a/src/context.c b/src/context.c
index 726a596b..1191fb28 100755
--- a/src/context.c
+++ b/src/context.c
@@ -240,10 +240,11 @@ upstreams_create(getdns_context *context, size_t size)
 {
 	getdns_upstreams *r = (void *) GETDNS_XMALLOC(context->mf, char,
 	    sizeof(getdns_upstreams) +
-	    sizeof(struct getdns_upstream) * size);
+	    sizeof(getdns_upstream) * size);
 	r->mf = context->mf;
 	r->referenced = 1;
 	r->count = 0;
+	r->current = 0;
 	return r;
 }
 
@@ -253,7 +254,7 @@ upstreams_resize(getdns_upstreams *upstreams, size_t size)
 	getdns_upstreams *r = (void *) GETDNS_XREALLOC(
 	    upstreams->mf, upstreams, char,
 	    sizeof(getdns_upstreams) +
-	    sizeof(struct getdns_upstream) * size);
+	    sizeof(getdns_upstream) * size);
 	return r;
 }
 
@@ -265,13 +266,13 @@ upstreams_dereference(getdns_upstreams *upstreams)
 }
 
 static size_t
-upstream_addr_len(struct getdns_upstream *upstream)
+upstream_addr_len(getdns_upstream *upstream)
 {
 	return upstream->addr.ss_family == AF_INET ? 4 : 16;
 }
 
 static uint8_t*
-upstream_addr(struct getdns_upstream *upstream)
+upstream_addr(getdns_upstream *upstream)
 {
 	return upstream->addr.ss_family == AF_INET
 	    ? (void *)&((struct sockaddr_in*)&upstream->addr)->sin_addr
@@ -279,7 +280,7 @@ upstream_addr(struct getdns_upstream *upstream)
 }
 
 static in_port_t
-upstream_port(struct getdns_upstream *upstream)
+upstream_port(getdns_upstream *upstream)
 {
 	return ntohs(upstream->addr.ss_family == AF_INET
 	    ? ((struct sockaddr_in *)&upstream->addr)->sin_port
@@ -287,7 +288,7 @@ upstream_port(struct getdns_upstream *upstream)
 }
 
 static uint32_t *
-upstream_scope_id(struct getdns_upstream *upstream)
+upstream_scope_id(getdns_upstream *upstream)
 {
 	return upstream->addr.ss_family == AF_INET ? NULL
 	    : (upstream_addr(upstream)[0] == 0xFE &&
@@ -296,7 +297,7 @@ upstream_scope_id(struct getdns_upstream *upstream)
 }
 
 static void
-upstream_ntop_buf(struct getdns_upstream *upstream, char *buf, size_t len)
+upstream_ntop_buf(getdns_upstream *upstream, char *buf, size_t len)
 {
 	/* Also possible but prints scope_id by name (nor parsed by unbound)
 	 *
@@ -314,7 +315,7 @@ upstream_ntop_buf(struct getdns_upstream *upstream, char *buf, size_t len)
 }
 
 static getdns_dict *
-upstream_dict(getdns_context *context, struct getdns_upstream *upstream)
+upstream_dict(getdns_context *context, getdns_upstream *upstream)
 {
 	getdns_dict *r = getdns_dict_create_with_context(context);
 	char addrstr[1024], *b;
@@ -346,12 +347,13 @@ net_req_query_id_cmp(const void *id1, const void *id2)
 }
 
 static void
-upstream_init(struct getdns_upstream *upstream, struct addrinfo *ai)
+upstream_init(getdns_upstream *upstream, struct addrinfo *ai)
 {
 	assert(upstream && ai);
 	upstream->addr_len = ai->ai_addrlen;
 	(void) memcpy(&upstream->addr, ai->ai_addr, ai->ai_addrlen);
 	upstream->to_retry =  2;
+	upstream->back_off =  1;
 	upstream->tcp_fd   = -1;
 	(void) memset(&upstream->tcp_event, 0, sizeof(upstream->tcp_event));
 	getdns_rbtree_init(&upstream->netreq_by_query_id,
@@ -372,7 +374,7 @@ set_os_defaults(struct getdns_context *context)
 	struct getdns_bindata bindata;
 	struct addrinfo hints;
 	struct addrinfo *result;
-	struct getdns_upstream *upstream;
+	getdns_upstream *upstream;
 	int s;
 
 	if(context->fchg_resolvconf == NULL) {
@@ -1275,7 +1277,7 @@ getdns_context_set_upstream_recursive_servers(struct getdns_context *context,
 		uint32_t port;
 		getdns_bindata *scope_id;
 		struct addrinfo *ai;
-		struct getdns_upstream *upstream;
+		getdns_upstream *upstream;
 
 		upstream = &upstreams->upstreams[upstreams->count];
 		if ((r = getdns_list_get_dict(upstream_list, i, &dict)))
@@ -1552,7 +1554,7 @@ ub_setup_stub(struct ub_ctx *ctx, getdns_upstreams *upstreams)
 {
 	getdns_return_t r = GETDNS_RETURN_GOOD;
 	size_t i;
-	struct getdns_upstream *upstream;
+	getdns_upstream *upstream;
 	char addr[1024];
 
 	(void) ub_ctx_set_fwd(ctx, NULL);
@@ -1627,7 +1629,7 @@ set_ldns_nameservers(struct getdns_context *context,
 {
 	getdns_return_t r = GETDNS_RETURN_GOOD;
 	size_t i;
-	struct getdns_upstream *upstream;
+	getdns_upstream *upstream;
 	ldns_rdf *pop, *ns_rdf;
 	uint16_t port = 53;
 
@@ -1992,7 +1994,7 @@ priv_get_context_settings(getdns_context* context) {
     r |= priv_dict_set_list_if_not_null(result, "suffix", context->suffix);
 	if (context->upstreams->count > 0) {
 		size_t i;
-		struct getdns_upstream *upstream;
+		getdns_upstream *upstream;
 		getdns_list *upstreams =
 		    getdns_list_create_with_context(context);
 
diff --git a/src/context.h b/src/context.h
index 31cd3244..1b993787 100755
--- a/src/context.h
+++ b/src/context.h
@@ -69,20 +69,22 @@ struct filechg {
 	struct stat *prevstat;
 };
 
-struct getdns_upstream {
+typedef struct getdns_upstream {
 	socklen_t               addr_len;
 	struct sockaddr_storage addr;
 	int                     to_retry;
+	int                     back_off;
 	int                     tcp_fd;
 	getdns_eventloop_event  tcp_event;
 	getdns_rbtree_t         netreq_by_query_id;
-};
+} getdns_upstream;
 
 typedef struct getdns_upstreams {
 	struct mem_funcs mf;
 	size_t referenced;
 	size_t count;
-	struct getdns_upstream upstreams[];
+	size_t current;
+	getdns_upstream upstreams[];
 } getdns_upstreams;
 
 struct getdns_context {
@@ -96,7 +98,7 @@ struct getdns_context {
 	getdns_append_name_t append_name;
 	struct getdns_list   *suffix;
 	struct getdns_list   *dnssec_trust_anchors;
-	getdns_upstreams *upstreams;
+	getdns_upstreams     *upstreams;
 	getdns_transport_t   dns_transport;
 	uint16_t             limit_outstanding_queries;
 	uint32_t             dnssec_allowed_skew;
diff --git a/src/request-internal.c b/src/request-internal.c
index ea63bb67..2f6e0542 100644
--- a/src/request-internal.c
+++ b/src/request-internal.c
@@ -149,7 +149,6 @@ dns_req_new(struct getdns_context *context, getdns_eventloop *loop,
 	result->upstreams = context->upstreams;
 	if (result->upstreams)
 		result->upstreams->referenced++;
-	result->ns_index = 0;
 
 	/* create the requests */
 	req = network_req_new(result, request_type, klass, extensions);
diff --git a/src/stub.c b/src/stub.c
index 2cfdeeac..15e0e55c 100644
--- a/src/stub.c
+++ b/src/stub.c
@@ -269,6 +269,12 @@ stub_resolve_timeout_cb(void *userarg)
 	getdns_network_req *netreq = (getdns_network_req *)userarg;
 	getdns_dns_req *dns_req = netreq->owner;
 
+	if (! --netreq->upstream->to_retry) 
+		netreq->upstream->to_retry = -(netreq->upstream->back_off *= 2);
+
+	if (++dns_req->upstreams->current > dns_req->upstreams->count)
+		dns_req->upstreams->current = 0;
+
 	(void) getdns_context_request_timed_out(dns_req);
 }
 
@@ -300,6 +306,7 @@ stub_resolve_read_cb(void *userarg)
 	close(netreq->udp_fd);
 	netreq->state = NET_REQ_FINISHED;
 	ldns_wire2pkt(&(netreq->result), pkt, read);
+	dns_req->upstreams->current = 0;
 
 	/* Do the dnssec here */
 	netreq->secure = 0;
@@ -361,12 +368,47 @@ done:
 	return;
 }
 
+static getdns_upstream *
+pick_upstream(getdns_dns_req *dns_req)
+{
+	getdns_upstream *upstream;
+	size_t i;
+	
+	if (!dns_req->upstreams->count)
+		return NULL;
+
+	for (i = 0; i < dns_req->upstreams->count; i++)
+		if (dns_req->upstreams->upstreams[i].to_retry <= 0)
+			dns_req->upstreams->upstreams[i].to_retry++;
+
+	i = dns_req->upstreams->current;
+	do {
+		if (dns_req->upstreams->upstreams[i].to_retry > 0) {
+			dns_req->upstreams->current = i;
+			return &dns_req->upstreams->upstreams[i];
+		}
+		if (++i > dns_req->upstreams->count)
+			i = 0;
+	} while (i != dns_req->upstreams->current);
+
+	upstream = dns_req->upstreams->upstreams;
+	for (i = 1; i < dns_req->upstreams->count; i++)
+		if (dns_req->upstreams->upstreams[i].back_off <
+		    upstream->back_off)
+			upstream = &dns_req->upstreams->upstreams[i];
+
+	upstream->back_off++;
+	upstream->to_retry = 1;
+	dns_req->upstreams->current = upstream - dns_req->upstreams->upstreams;
+	return upstream;
+}
+
 getdns_return_t
 priv_getdns_submit_stub_request(getdns_network_req *netreq)
 {
 	getdns_dns_req *dns_req = netreq->owner;
 
-	struct getdns_upstream *upstream;
+	getdns_upstream *upstream;
 
 	/* TODO: TCP */
 	if (dns_req->context->dns_transport != GETDNS_TRANSPORT_UDP_ONLY &&
@@ -374,7 +416,9 @@ priv_getdns_submit_stub_request(getdns_network_req *netreq)
 	    GETDNS_TRANSPORT_UDP_FIRST_AND_FALL_BACK_TO_TCP)
 	    	return GETDNS_RETURN_GENERIC_ERROR;
 
-	upstream = &dns_req->upstreams->upstreams[dns_req->ns_index];
+	if (!(upstream = pick_upstream(dns_req)))
+		return GETDNS_RETURN_GENERIC_ERROR;
+
 	if ((netreq->udp_fd = socket(
 	    upstream->addr.ss_family, SOCK_DGRAM, IPPROTO_UDP)) == -1)
 		return GETDNS_RETURN_GENERIC_ERROR;
diff --git a/src/types-internal.h b/src/types-internal.h
index 285eb29c..2b1c789d 100644
--- a/src/types-internal.h
+++ b/src/types-internal.h
@@ -231,7 +231,6 @@ typedef struct getdns_dns_req
 
 	/* Stuff for stub resolving */
 	struct getdns_upstreams *upstreams;
-	size_t                   ns_index;
 
 } getdns_dns_req;