Merge branch 'release/1.4.1-max_udp_backoff_value' into release/1.4.1

configure.ac

@@ -379,6 +379,10 @@ no)
 	;;
 esac
 
+AC_ARG_WITH(max-udp-backoff, AS_HELP_STRING([--with-max-udp-backoff=<number of queries>],
+    [Set the maximum number of messages that can be sent to other upstreams before the upstream which has previously timed out will be tried again. (defaults to 1000)]),, [withval="1000"])
+AC_DEFINE_UNQUOTED([UDP_MAX_BACKOFF], [$withval], [Maximum number of queries an failed UDP upstream passes before it will retry])
+
 #---- check for pthreads library
 AC_ARG_WITH(libpthread, AS_HELP_STRING([--without-libpthread],
 	[Disable libpthread (default is autodetect)]),

src/const-info.c

@@ -93,6 +93,7 @@ static struct const_info consts_info[] = {
 	{     632, "GETDNS_CONTEXT_CODE_TLS_CA_FILE", GETDNS_CONTEXT_CODE_TLS_CA_FILE_TEXT },
 	{     633, "GETDNS_CONTEXT_CODE_TLS_CIPHER_LIST", GETDNS_CONTEXT_CODE_TLS_CIPHER_LIST_TEXT },
 	{     634, "GETDNS_CONTEXT_CODE_TLS_CURVES_LIST", GETDNS_CONTEXT_CODE_TLS_CURVES_LIST_TEXT },
+	{     699, "GETDNS_CONTEXT_CODE_MAX_BACKOFF_VALUE", GETDNS_CONTEXT_CODE_MAX_BACKOFF_VALUE_TEXT },
 	{     700, "GETDNS_CALLBACK_COMPLETE", GETDNS_CALLBACK_COMPLETE_TEXT },
 	{     701, "GETDNS_CALLBACK_CANCEL", GETDNS_CALLBACK_CANCEL_TEXT },
 	{     702, "GETDNS_CALLBACK_TIMEOUT", GETDNS_CALLBACK_TIMEOUT_TEXT },
@@ -176,6 +177,7 @@ static struct const_name_info consts_name_info[] = {
 	{ "GETDNS_CONTEXT_CODE_HOSTS", 630 },
 	{ "GETDNS_CONTEXT_CODE_IDLE_TIMEOUT", 617 },
 	{ "GETDNS_CONTEXT_CODE_LIMIT_OUTSTANDING_QUERIES", 606 },
+	{ "GETDNS_CONTEXT_CODE_MAX_BACKOFF_VALUE", 699 },
 	{ "GETDNS_CONTEXT_CODE_MEMORY_FUNCTIONS", 615 },
 	{ "GETDNS_CONTEXT_CODE_NAMESPACES", 600 },
 	{ "GETDNS_CONTEXT_CODE_PUBKEY_PINSET", 621 },

src/const-info.h

@@ -39,6 +39,14 @@
 #ifndef CONST_INFO_H_
 #define CONST_INFO_H_
 
+#include "getdns/getdns.h"
+#include "getdns/getdns_extra.h"
+
+#ifndef GETDNS_CONTEXT_CODE_MAX_BACKOFF_VALUE
+#define GETDNS_CONTEXT_CODE_MAX_BACKOFF_VALUE 699
+#define GETDNS_CONTEXT_CODE_MAX_BACKOFF_VALUE_TEXT "Change related to getdns_context_set_max_backoff_value"
+#endif
+
 struct const_info {
 	int code;
 	const char *name;

src/context.c

@@ -93,6 +93,7 @@ typedef unsigned short in_port_t;
 #ifdef USE_DANESSL
 # include "ssl_dane/danessl.h"
 #endif
+#include "const-info.h"
 
 #define GETDNS_PORT_ZERO 0
 #define GETDNS_PORT_DNS 53
@@ -679,6 +680,7 @@ upstreams_create(getdns_context *context, size_t size)
 	r->count = 0;
 	r->current_udp = 0;
 	r->current_stateful = 0;
+	r->max_backoff_value = context->max_backoff_value;
 	r->tls_backoff_time = context->tls_backoff_time;
 	r->tls_connection_retries = context->tls_connection_retries;
 	r->log = context->log;
@@ -1664,6 +1666,7 @@ getdns_context_create_with_extended_memory_functions(
 	result->tls_backoff_time = 3600;
 	result->tls_connection_retries = 2;
 	result->limit_outstanding_queries = 0;
+	result->max_backoff_value = UDP_MAX_BACKOFF;
 
 	/* unbound context is initialized here */
 	/* Unbound needs SSL to be init'ed this early when TLS is used. However we
@@ -2362,6 +2365,28 @@ getdns_context_set_round_robin_upstreams(getdns_context *context, uint8_t value)
     return GETDNS_RETURN_GOOD;
 }               /* getdns_context_set_round_robin_upstreams */
 
+/**
+ * Set the maximum number of messages that can be sent to other upstreams
+ * before the upstream which has previously timed out will be tried again.
+ * @see getdns_context_get_max_backoff_value
+ * @param[in] context  The context to configure
+ * @param[in[ value    Number of messages sent to other upstreams before 
+ *                     retrying the upstream which had timed out.
+ * @return GETDNS_RETURN_GOOD on success
+ * @return GETDNS_RETURN_INVALID_PARAMETER if context is null.
+ */
+getdns_return_t
+getdns_context_set_max_backoff_value(getdns_context *context, uint16_t value)
+{
+    RETURN_IF_NULL(context, GETDNS_RETURN_INVALID_PARAMETER);
+
+    context->max_backoff_value = value;
+
+    dispatch_updated(context, GETDNS_CONTEXT_CODE_MAX_BACKOFF_VALUE);
+
+    return GETDNS_RETURN_GOOD;
+}               /* getdns_context_set_max_backoff_value */
+
 /*
  * getdns_context_set_tls_backoff_time
  *
@@ -3931,6 +3956,8 @@ _get_context_settings(getdns_context* context)
 	                           context->tls_auth)
 	    || getdns_dict_set_int(result, "round_robin_upstreams",
 	                           context->round_robin_upstreams)
+	    || getdns_dict_set_int(result, "max_backoff_value",
+	                           context->max_backoff_value)
 	    || getdns_dict_set_int(result, "tls_backoff_time",
 	                           context->tls_backoff_time)
 	    || getdns_dict_set_int(result, "tls_connection_retries",
@@ -4378,6 +4405,25 @@ getdns_context_get_round_robin_upstreams(getdns_context *context,
     return GETDNS_RETURN_GOOD;
 }
 
+/**
+ * Get the maximum number of messages that can be sent to other upstreams
+ * before the upstream which has previously timed out will be tried again.
+ * @see getdns_context_set_max_backoff_value
+ * @param[in]  context  The context from which to get the setting
+ * @param[out] value    Number of messages sent to other upstreams before 
+ *                      retrying the upstream which had timed out.
+ * @return GETDNS_RETURN_GOOD on success
+ * @return GETDNS_RETURN_INVALID_PARAMETER if context is null.
+ */
+getdns_return_t
+getdns_context_get_max_backoff_value(getdns_context *context,
+    uint16_t* value) {
+    RETURN_IF_NULL(context, GETDNS_RETURN_INVALID_PARAMETER);
+    RETURN_IF_NULL(value, GETDNS_RETURN_INVALID_PARAMETER);
+    *value = context->max_backoff_value;
+    return GETDNS_RETURN_GOOD;
+}
+
 getdns_return_t
 getdns_context_get_tls_backoff_time(getdns_context *context,
     uint16_t* value) {

src/context.h

@@ -263,6 +263,7 @@ typedef struct getdns_upstreams {
 	size_t count;
 	size_t current_udp;
 	size_t current_stateful;
+    uint16_t max_backoff_value;
 	uint16_t tls_backoff_time;
 	uint16_t tls_connection_retries;
 	getdns_log_config log;
@@ -357,6 +358,7 @@ struct getdns_context {
 	getdns_tls_authentication_t  tls_auth;  /* What user requested for TLS*/
 	getdns_tls_authentication_t  tls_auth_min; /* Derived minimum auth allowed*/
 	uint8_t              round_robin_upstreams;
+    uint16_t             max_backoff_value;
 	uint16_t             tls_backoff_time;
 	uint16_t             tls_connection_retries;
 

src/getdns/getdns_extra.h.in

@@ -768,7 +768,6 @@ getdns_return_t
 getdns_context_set_tls_curves_list(
     getdns_context *context, const char *curves_list);
 
-
 /**
  * Get the current resolution type setting from this context.
  * @see getdns_context_set_resolution_type
@@ -1076,7 +1075,7 @@ getdns_context_get_tls_authentication(getdns_context *context,
 /**
  * Get whether the context is configured to round robin queries over the available
  * upstreams.
- * @see getdns_context_get_round_robin_upstreams
+ * @see getdns_context_set_round_robin_upstreams
  * @param[in]  context The context from which to get the setting
  * @param[out] value 1 if the setting is on, 0 otherwise
  * @return GETDNS_RETURN_GOOD when successful
@@ -1303,7 +1302,6 @@ getdns_return_t
 getdns_context_get_tls_curves_list(
     getdns_context *context, const char **curves_list);
 
-
 /** @}
  */
 

src/mk-const-info.c.sh

@@ -14,7 +14,7 @@ cat > const-info.c << END_OF_HEAD
 static struct const_info consts_info[] = {
 	{   -1, NULL, "/* <unknown getdns value> */" },
 END_OF_HEAD
-gawk '/^[ 	]+GETDNS_[A-Z_]+[ 	]+=[ 	]+[0-9]+/{ key = sprintf("%7d", $3); consts[key] = $1; }/^#define GETDNS_[A-Z_]+[ 	]+[0-9]+/ && !/^#define GETDNS_RRTYPE/ && !/^#define GETDNS_RRCLASS/ && !/^#define GETDNS_OPCODE/  && !/^#define GETDNS_RCODE/ && !/_TEXT/{ key = sprintf("%7d", $3); consts[key] = $2; }/^#define GETDNS_[A-Z_]+[ 	]+\(\(getdns_(return|append_name)_t) [0-9]+ \)/{ key = sprintf("%7d", $4); consts[key] = $2; }END{ n = asorti(consts, const_vals); for ( i = 1; i <= n; i++) { val = const_vals[i]; name = consts[val]; print "\t{ "val", \""name"\", "name"_TEXT },"}}' getdns/getdns_extra.h.in getdns/getdns.h.in | sed 's/,,/,/g' >> const-info.c
+gawk '/^[ 	]+GETDNS_[A-Z_]+[ 	]+=[ 	]+[0-9]+/{ key = sprintf("%7d", $3); consts[key] = $1; }/^#define GETDNS_[A-Z_]+[ 	]+[0-9]+/ && !/^#define GETDNS_RRTYPE/ && !/^#define GETDNS_RRCLASS/ && !/^#define GETDNS_OPCODE/  && !/^#define GETDNS_RCODE/ && !/_TEXT/{ key = sprintf("%7d", $3); consts[key] = $2; }/^#define GETDNS_[A-Z_]+[ 	]+\(\(getdns_(return|append_name)_t) [0-9]+ \)/{ key = sprintf("%7d", $4); consts[key] = $2; }END{ n = asorti(consts, const_vals); for ( i = 1; i <= n; i++) { val = const_vals[i]; name = consts[val]; print "\t{ "val", \""name"\", "name"_TEXT },"}}' getdns/getdns_extra.h.in getdns/getdns.h.in const-info.h| sed 's/,,/,/g' >> const-info.c
 cat >> const-info.c << END_OF_TAIL
 };
 
@@ -49,7 +49,7 @@ getdns_get_errorstr_by_id(uint16_t err)
 
 static struct const_name_info consts_name_info[] = {
 END_OF_TAIL
-gawk '/^[ 	]+GETDNS_[A-Z_]+[ 	]+=[ 	]+[0-9]+/{ key = sprintf("%d", $3); consts[$1] = key; }/^#define GETDNS_[A-Z_]+[ 	]+[0-9]+/ && !/_TEXT/{ key = sprintf("%d", $3); consts[$2] = key; }/^#define GETDNS_[A-Z_]+[ 	]+\(\(getdns_(return|append_name)_t) [0-9]+ \)/{ key = sprintf("%d", $4); consts[$2] = key; }END{ n = asorti(consts, const_vals); for ( i = 1; i <= n; i++) { val = const_vals[i]; name = consts[val]; print "\t{ \""val"\", "name" },"}}' getdns/getdns.h.in getdns/getdns_extra.h.in | sed 's/,,/,/g' >> const-info.c
+gawk '/^[ 	]+GETDNS_[A-Z_]+[ 	]+=[ 	]+[0-9]+/{ key = sprintf("%d", $3); consts[$1] = key; }/^#define GETDNS_[A-Z_]+[ 	]+[0-9]+/ && !/_TEXT/{ key = sprintf("%d", $3); consts[$2] = key; }/^#define GETDNS_[A-Z_]+[ 	]+\(\(getdns_(return|append_name)_t) [0-9]+ \)/{ key = sprintf("%d", $4); consts[$2] = key; }END{ n = asorti(consts, const_vals); for ( i = 1; i <= n; i++) { val = const_vals[i]; name = consts[val]; print "\t{ \""val"\", "name" },"}}' getdns/getdns.h.in getdns/getdns_extra.h.in const-info.h| sed 's/,,/,/g' >> const-info.c
 cat >> const-info.c << END_OF_TAIL
 };
 

src/stub.c

@@ -460,8 +460,13 @@ stub_next_upstream(getdns_network_req *netreq)
 {
 	getdns_dns_req *dnsreq = netreq->owner;
 
-	if (! --netreq->upstream->to_retry) 
+	if (! --netreq->upstream->to_retry) {
-		netreq->upstream->to_retry = -(netreq->upstream->back_off *= 2);
+        /* Limit back_off value to configured maximum */
+        if (netreq->upstream->back_off * 2 > dnsreq->context->max_backoff_value)
+            netreq->upstream->to_retry = -(dnsreq->context->max_backoff_value);
+        else
+            netreq->upstream->to_retry = -(netreq->upstream->back_off *= 2);
+    }
 
 	dnsreq->upstreams->current_udp+=GETDNS_UPSTREAM_TRANSPORTS;
 	if (dnsreq->upstreams->current_udp >= dnsreq->upstreams->count)
@@ -1590,6 +1595,7 @@ stub_udp_read_cb(void *userarg)
 	netreq->debug_end_time = _getdns_get_time_as_uintt64();
 	_getdns_netreq_change_state(netreq, NET_REQ_FINISHED);
 	upstream->udp_responses++;
+    upstream->back_off = 1;
 	if (upstream->udp_responses == 1 || 
 	    upstream->udp_responses % 100 == 0)
 		_getdns_upstream_log(upstream, GETDNS_LOG_UPSTREAM_STATS, GETDNS_LOG_INFO,
@@ -2152,6 +2158,7 @@ upstream_select_stateful(getdns_network_req *netreq, getdns_transport_list_t tra
 	return upstream;
 }
 
+/* Used for UDP only */
 static getdns_upstream *
 upstream_select(getdns_network_req *netreq)
 {
@@ -2161,6 +2168,7 @@ upstream_select(getdns_network_req *netreq)
 
 	if (!upstreams->count)
 		return NULL;
+
 	/* First UPD/TCP upstream is always at i=0 and then start of each upstream block*/
 	/* TODO: Have direct access to sets of upstreams for different transports*/
 	for (i = 0; i < upstreams->count; i+=GETDNS_UPSTREAM_TRANSPORTS)
@@ -2178,14 +2186,18 @@ upstream_select(getdns_network_req *netreq)
 			i = 0;
 	} while (i != upstreams->current_udp);
 
+	/* Select upstream with the lowest back_off value */
 	upstream = upstreams->upstreams;
 	for (i = 0; i < upstreams->count; i+=GETDNS_UPSTREAM_TRANSPORTS)
-		if (upstreams->upstreams[i].back_off <
+		if (upstreams->upstreams[i].back_off < upstream->back_off)
-		    upstream->back_off)
 			upstream = &upstreams->upstreams[i];
 
-	if (upstream->back_off > 1)
+	/* Restrict back_off in case no upstream is available to achieve
-		upstream->back_off--;
+	   (more or less) round-robin retry on all upstreams. */
+	if (upstream->back_off > 4) {
+		for (i = 0; i < upstreams->count; i+=GETDNS_UPSTREAM_TRANSPORTS)
+			upstreams->upstreams[i].back_off = 2;
+	}
 	upstream->to_retry = 1;
 	upstreams->current_udp = upstream - upstreams->upstreams;
 	return upstream;