From fdbefa17ecc10aa735776a87b02d1e6ac9013865 Mon Sep 17 00:00:00 2001 From: Sara Dickinson Date: Fri, 5 Aug 2016 17:25:27 +0100 Subject: [PATCH] Add timer for back off on upstream (use 1 hr). Reset as new upstream when re-instated. --- src/context.c | 38 ++++++++++++++++++++++++++++---------- src/context.h | 2 ++ src/stub.c | 14 +++++++++++--- src/types-internal.h | 4 ++-- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/context.c b/src/context.c index 1c851e64..5e120317 100644 --- a/src/context.c +++ b/src/context.c @@ -84,6 +84,9 @@ typedef unsigned short in_port_t; #define GETDNS_STR_PORT_ZERO "0" #define GETDNS_STR_PORT_DNS "53" #define GETDNS_STR_PORT_DNS_OVER_TLS "853" +/* How long to wait in seconds before re-trying a connection based backed-off + upstream. Using 1 hour for all transports - based on RFC7858 value for for TLS.*/ +#define BACKOFF_RETRY 3600 void *plain_mem_funcs_user_arg = MF_PLAIN; @@ -692,15 +695,16 @@ _getdns_upstream_shutdown(getdns_upstream *upstream) upstream->past_tls_auth_state = upstream->tls_auth_state; #if defined(DAEMON_DEBUG) && DAEMON_DEBUG - DEBUG_DAEMON("%s Upstream %s : Connection closed: Connection stats - Resp=%d,Timeouts=%d,Keepalive(ms)=%d,Auth=%s\n", + DEBUG_DAEMON("%s %s : Conn closed: Conn stats - Resp=%d,Timeouts=%d,Auth=%s,Keepalive(ms)=%d\n", STUB_DEBUG_DAEMON, upstream->addr_str, (int)upstream->responses_received, (int)upstream->responses_timeouts, - (int)upstream->keepalive_timeout, getdns_auth_str_array[upstream->tls_auth_state]); - DEBUG_DAEMON("%s Upstream %s : Connection closed: Upstream stats - Resp=%d,Timeouts=%d,Conns=%d,Conn_fails=%d,Conn_shutdowns=%d,Auth=%s\n", + getdns_auth_str_array[upstream->tls_auth_state], (int)upstream->keepalive_timeout); + DEBUG_DAEMON("%s %s : Upstream stats - Resp=%d,Timeouts=%d,Auth=%s,Conns=%d,Conn_fails=%d,Conn_shutdowns=%d,Backoffs=%d\n", STUB_DEBUG_DAEMON, upstream->addr_str, (int)upstream->total_responses, (int)upstream->total_timeouts, - (int)upstream->conn_completed, (int)upstream->conn_setup_failed, - (int)upstream->conn_shutdowns, getdns_auth_str_array[upstream->tls_auth_state]); + getdns_auth_str_array[upstream->tls_auth_state], + (int)upstream->conn_completed, (int)upstream->conn_setup_failed, + (int)upstream->conn_shutdowns, (int)upstream->conn_backoffs); #endif /* Back off connections that never got up service at all (probably no @@ -716,10 +720,18 @@ _getdns_upstream_shutdown(getdns_upstream *upstream) (upstream->conn_completed >= GETDNS_CONN_ATTEMPTS && upstream->total_responses == 0 && upstream->total_timeouts > GETDNS_TRANSPORT_FAIL_MULT)) { - DEBUG_STUB("%s %-35s: FD: %d BACKING OFF THIS UPSTREAM! \n", - STUB_DEBUG_CLEANUP, __FUNCTION__, upstream->fd); upstream->conn_state = GETDNS_CONN_BACKOFF; - } + upstream->conn_retry_time = time(NULL) + BACKOFF_RETRY; + upstream->total_responses = 0; + upstream->total_timeouts = 0; + upstream->conn_completed = 0; + upstream->conn_setup_failed = 0; + upstream->conn_shutdowns = 0; + upstream->conn_backoffs++; + DEBUG_DAEMON("%s %s : !Backing off this upstream - will retry as new upstream at %s\n", + STUB_DEBUG_DAEMON, upstream->addr_str, + asctime(gmtime(&upstream->conn_retry_time))); + } // Reset per connection counters upstream->queries_sent = 0; upstream->responses_received = 0; @@ -848,15 +860,21 @@ upstream_init(getdns_upstream *upstream, upstream->addr_str, INET6_ADDRSTRLEN); #endif - /* How is this upstream doing? */ - upstream->conn_setup_failed = 0; + /* How is this upstream doing on connections? */ + upstream->conn_completed = 0; upstream->conn_shutdowns = 0; + upstream->conn_setup_failed = 0; + upstream->conn_retry_time = 0; + upstream->conn_backoffs = 0; + upstream->total_responses = 0; + upstream->total_timeouts = 0; upstream->conn_state = GETDNS_CONN_CLOSED; upstream->queries_sent = 0; upstream->responses_received = 0; upstream->responses_timeouts = 0; upstream->keepalive_shutdown = 0; upstream->keepalive_timeout = 0; + /* How is this upstream doing on UDP? */ upstream->to_retry = 2; upstream->back_off = 1; diff --git a/src/context.h b/src/context.h index 9caf458d..8d192d39 100644 --- a/src/context.h +++ b/src/context.h @@ -143,6 +143,8 @@ typedef struct getdns_upstream { size_t conn_completed; size_t conn_shutdowns; size_t conn_setup_failed; + time_t conn_retry_time; + size_t conn_backoffs; size_t total_responses; size_t total_timeouts; getdns_auth_state_t past_tls_auth_state; diff --git a/src/stub.c b/src/stub.c index 74f25d39..5fd5e9d4 100644 --- a/src/stub.c +++ b/src/stub.c @@ -1626,12 +1626,20 @@ upstream_select_stateful(getdns_network_req *netreq, getdns_transport_list_t tra getdns_upstream *upstream = NULL; getdns_upstreams *upstreams = netreq->owner->upstreams; size_t i; + time_t now = time(NULL); if (!upstreams->count) return NULL; - /* [TLS1]TODO: Add check to re-instate backed-off upstreams after X amount - of time*/ + /* A check to re-instate backed-off upstreams after X amount of time*/ + for (i = 0; i < upstreams->count; i++) { + if (upstreams->upstreams[i].conn_state == GETDNS_CONN_BACKOFF && + upstreams->upstreams[i].conn_retry_time < now) { + upstreams->upstreams[i].conn_state = GETDNS_CONN_CLOSED; + DEBUG_DAEMON("%s %s : Re-instating upstream\n", + STUB_DEBUG_DAEMON, upstreams->upstreams[i].addr_str); + } + } /* First find if an open upstream has the correct properties and use that*/ for (i = 0; i < upstreams->count; i++) { @@ -1745,7 +1753,7 @@ upstream_connect(getdns_upstream *upstream, getdns_transport_list_t transport, /* Nothing to do*/ } #if defined(DAEMON_DEBUG) && DAEMON_DEBUG - DEBUG_DAEMON("%s Upstream %s : Connection initialised\n", + DEBUG_DAEMON("%s %s : Conn init\n", STUB_DEBUG_DAEMON, upstream->addr_str); #endif return fd; diff --git a/src/types-internal.h b/src/types-internal.h index 5f67f7cd..bd1f993c 100644 --- a/src/types-internal.h +++ b/src/types-internal.h @@ -63,8 +63,8 @@ typedef enum getdns_auth_state { GETDNS_AUTH_OK, /* Tried and worked (Strict) */ } getdns_auth_state_t; -#define GETDNS_STR_AUTH_NONE "N/A" -#define GETDNS_STR_AUTH_FAILED "Failed or not tried" +#define GETDNS_STR_AUTH_NONE "None" +#define GETDNS_STR_AUTH_FAILED "Failed" #define GETDNS_STR_AUTH_OK "Success" static char*