Merge pull request #234 from saradickinson/stubby_enhancements

Stubby enhancements

README.md

@@ -19,6 +19,8 @@ Traditional access to DNS data from applications has several limitations:
 
 * Sophisticated uses of the DNS (things like IDNA and DNSSEC validation) require considerable application work, possibly by application developers with little experience with the vagaries of DNS.
 
+getdns also provides a experimental DNS Privacy enabled client called 'stubby' - see below for more details.
+
 ## Motivation for providing the API
 
 The developers are of the opinion that DNSSEC offers a unique global infrastructure for establishing and enhancing cryptographic trust relations.  With the development of this API we intend to offer application developers a modern and flexible interface that enables end-to-end trust in the DNS architecture, and which will inspire application developers to implement innovative security solutions in their applications.
@@ -73,9 +75,13 @@ If you want to make use of the configuration files that utilise a JSON-like form
 
 before building. 
 
-If you want to use the getdns_query command line wrapper script for testing or to enable getdns as a daemon then you must build it using
+As well as building the getdns library 2 other tools are installed by default by the above process:
+
+* getdns_query: a command line test script wrapper for getdns
+* stubby: a experimental DNS Privacy enabled client
+
+Note: If you only want to build stubby, then use the `--enable-stub-only` and `--without-libidn` options when running 'configure'.
 
-    # make getdns_query
 
 ## Minimizing dependencies
 
@@ -91,6 +97,18 @@ The implementation works with a variety of event loops, each built as a separate
 * [libuv](https://github.com/joyent/libuv)
 * [libev](http://software.schmorp.de/pkg/libev.html)
 
+## Stubby
+
+* Stubby is an experimental implementation of a DNS Privacy enabled stub resolver. It is currently suitable for advanced/technical users - all feedback is welcome! Also see [dnsprivacy.org](https://dnsprivacy.org) for more information on DNS Privacy and stubby.
+* By default stubby will attempt to use 'Opportunistic' Privacy for DNS queries.
+* A sample configuration file is available in the source code (src/tools/stubby.conf) which uses 'Strict' Privacy and some of the available test DNS Privacy servers to resolve queries. Note these servers are test servers that offer no service guarantees. The location of a configuration file can be specified with the '-C' flag
+* RECOMMENDED: Minimal logging output from Stubby is available  (e.g. which servers are used and connection level statistics) by also using the '--enable-debug-daemon' flag when running 'configure'.
+
+To use stubby
+* Start stubby from the command line
+* Test it by doing, for example, 'dig @127.0.0.1 www.example.com'
+* Alter the default DNS resolvers on your system to point at localhost (127.0.0.1, ::1)
+
 ## Regression Tests
 
 A suite of regression tests are included with the library, if you make changes or just

src/context.c

@@ -716,20 +716,21 @@ _getdns_upstream_shutdown(getdns_upstream *upstream)
 	if (upstream->tls_auth_state > upstream->best_tls_auth_state)
 		upstream->best_tls_auth_state = upstream->tls_auth_state;
 #if defined(DAEMON_DEBUG) && DAEMON_DEBUG
-	DEBUG_DAEMON("%s %s : Conn closed: Conn stats     - Resp=%d,Timeouts=%d,Auth=%s,Keepalive(ms)=%d\n",
+	DEBUG_DAEMON("%s %s : Conn closed   : Transport=%s - Resp=%d,Timeouts=%d,Auth=%s,Keepalive(ms)=%d\n",
 	             STUB_DEBUG_DAEMON, upstream->addr_str,
+	             (upstream->transport == GETDNS_TRANSPORT_TLS ? "TLS" : "TCP"),
 	             (int)upstream->responses_received, (int)upstream->responses_timeouts,
 	             getdns_auth_str_array[upstream->tls_auth_state], (int)upstream->keepalive_timeout);
-	DEBUG_DAEMON("%s %s :              Upstream stats - Resp=%d,Timeouts=%d,Best_auth=%s,Conns=%d\n",
+	DEBUG_DAEMON("%s %s : Upstream stats: Transport=%s - Resp=%d,Timeouts=%d,Best_auth=%s\n",
 	             STUB_DEBUG_DAEMON, upstream->addr_str,
+	             (upstream->transport == GETDNS_TRANSPORT_TLS ? "TLS" : "TCP"),
 	             (int)upstream->total_responses, (int)upstream->total_timeouts,
-	             getdns_auth_str_array[upstream->best_tls_auth_state], 
+	             getdns_auth_str_array[upstream->best_tls_auth_state]);
-	             (int)upstream->conn_completed);
+	DEBUG_DAEMON("%s %s : Upstream stats: Transport=%s - Conns=%d,Conn_fails=%d,Conn_shutdowns=%d,Backoffs=%d\n",
-	DEBUG_DAEMON("%s %s :              Upstream stats - Conn_fails=%d,Conn_shutdowns=%d,Backoffs=%d\n",
 	             STUB_DEBUG_DAEMON, upstream->addr_str,
-	             (int)upstream->conn_setup_failed,
+	             (upstream->transport == GETDNS_TRANSPORT_TLS ? "TLS" : "TCP"),
+	             (int)upstream->conn_completed, (int)upstream->conn_setup_failed,
 	             (int)upstream->conn_shutdowns, (int)upstream->conn_backoffs);
-
 #endif
 
 	/* Back off connections that never got up service at all (probably no
@@ -754,7 +755,7 @@ _getdns_upstream_shutdown(getdns_upstream *upstream)
 		upstream->conn_shutdowns = 0;
 		upstream->conn_backoffs++;
 #if defined(DAEMON_DEBUG) && DAEMON_DEBUG
-		DEBUG_DAEMON("%s %s : !Backing off this upstream  - will retry as new upstream at %s\n",
+		DEBUG_DAEMON("%s %s : !Backing off this upstream    - Will retry as new upstream at %s",
 		            STUB_DEBUG_DAEMON, upstream->addr_str,
 		            asctime(gmtime(&upstream->conn_retry_time)));
 #endif
@@ -904,6 +905,8 @@ upstream_init(getdns_upstream *upstream,
 	/* How is this upstream doing on UDP? */
 	upstream->to_retry =  2;
 	upstream->back_off =  1;
+	upstream->udp_responses = 0;
+	upstream->udp_timeouts = 0;
 
 	/* For sharing a socket to this upstream with TCP  */
 	upstream->fd       = -1;

src/context.h

@@ -131,6 +131,8 @@ typedef struct getdns_upstream {
 	/* How is this upstream doing over UDP? */
 	int                      to_retry;
 	int                      back_off;
+	size_t                   udp_responses;
+	size_t                   udp_timeouts;
 
 	/* For stateful upstreams, need to share the connection and track the
 	   activity on the connection */

src/stub.c

@@ -571,6 +571,13 @@ stub_timeout_cb(void *userarg)
 	/* Handle upstream*/
 	if (netreq->fd >= 0) {
 		close(netreq->fd);
+		netreq->upstream->udp_timeouts++;
+#if defined(DAEMON_DEBUG) && DAEMON_DEBUG
+	if (netreq->upstream->udp_timeouts % 100 == 0)
+		DEBUG_DAEMON("%s %s : Upstream stats: Transport=UDP - Resp=%d,Timeouts=%d\n",
+		             STUB_DEBUG_DAEMON, netreq->upstream->addr_str,
+		             (int)netreq->upstream->udp_responses, (int)netreq->upstream->udp_timeouts);
+#endif
 		stub_next_upstream(netreq);
 	} else {
 		netreq->upstream->responses_timeouts++;
@@ -1305,6 +1312,14 @@ stub_udp_read_cb(void *userarg)
 	dnsreq->upstreams->current_udp = 0;
 	netreq->debug_end_time = _getdns_get_time_as_uintt64();
 	netreq->state = NET_REQ_FINISHED;
+	upstream->udp_responses++;
+#if defined(DAEMON_DEBUG) && DAEMON_DEBUG
+	if (upstream->udp_responses == 1 || 
+	    upstream->udp_responses % 100 == 0)
+		DEBUG_DAEMON("%s %s : Upstream stats: Transport=UDP - Resp=%d,Timeouts=%d\n",
+		             STUB_DEBUG_DAEMON, upstream->addr_str,
+		             (int)upstream->udp_responses, (int)upstream->udp_timeouts);
+#endif
 	_getdns_check_dns_req_complete(dnsreq);
 }
 
@@ -1531,6 +1546,11 @@ upstream_write_cb(void *userarg)
 	case STUB_NO_AUTH:
 		/* Cleaning up after connection or auth check failure. Need to fallback. */
 		stub_cleanup(netreq);
+#if defined(DAEMON_DEBUG) && DAEMON_DEBUG
+		DEBUG_DAEMON("%s %s : Conn closed   : Transport=%s - *Failure*\n",
+		             STUB_DEBUG_DAEMON, upstream->addr_str,
+		             (upstream->transport == GETDNS_TRANSPORT_TLS ? "TLS" : "TCP"));
+#endif
 		if (fallback_on_write(netreq) == STUB_TCP_ERROR) {
 			/* TODO: Need new state to report transport unavailable*/
 			netreq->state = NET_REQ_FINISHED;
@@ -1767,15 +1787,16 @@ upstream_connect(getdns_upstream *upstream, getdns_transport_list_t transport,
 			upstream->tls_hs_state = GETDNS_HS_WRITE;
 		}
 		upstream->conn_state = GETDNS_CONN_SETUP;
+#if defined(DAEMON_DEBUG) && DAEMON_DEBUG
+	DEBUG_DAEMON("%s %s : Conn init     : Transport=%s - Profile=%s\n", STUB_DEBUG_DAEMON, 
+	             upstream->addr_str, transport == GETDNS_TRANSPORT_TLS ? "TLS":"TCP",
+	             dnsreq->context->tls_auth_min == GETDNS_AUTHENTICATION_NONE ? "Opportunistic":"Strict");
+#endif
 		break;
 	default:
 		return -1;
 		/* Nothing to do*/
 	}
-#if defined(DAEMON_DEBUG) && DAEMON_DEBUG
-	DEBUG_DAEMON("%s %s : Conn init\n",
-                  STUB_DEBUG_DAEMON, upstream->addr_str);
-#endif
 	return fd;
 }
 
@@ -1828,6 +1849,10 @@ upstream_find_for_netreq(getdns_network_req *netreq)
 	}
 	/* Handle better, will give generic error*/
 	DEBUG_STUB("%s %-35s: MSG: %p No valid upstream! \n", STUB_DEBUG_SCHEDULE, __FUNCTION__, netreq);
+#if defined(DAEMON_DEBUG) && DAEMON_DEBUG
+	DEBUG_DAEMON("%s *FAILURE* no valid transports or upstreams available!\n",
+	              STUB_DEBUG_DAEMON);
+#endif
 	return -1;
 }
 

src/tools/getdns_query.c

@@ -51,8 +51,11 @@ typedef unsigned short in_port_t;
 static int i_am_stubby = 0;
 static const char *default_stubby_config =
 "{ resolution_type: GETDNS_RESOLUTION_STUB"
+", dns_transport_list: [ GETDNS_TRANSPORT_TLS, GETDNS_TRANSPORT_UDP, GETDNS_TRANSPORT_TCP ]"
 ", idle_timeout: 10000"
 ", listen_addresses: [ 127.0.0.1@53, 0::1@53 ]"
+", tls_query_padding_blocksize: 256"
+", edns_client_subnet_private : 1"
 "}";
 static int clear_listen_list_on_arg = 0;
 #ifndef GETDNS_ON_WINDOWS
@@ -161,13 +164,19 @@ print_usage(FILE *out, const char *progname)
 {
 	fprintf(out, "usage: %s [<option> ...] \\\n"
 	    "\t[@<upstream> ...] [+<extension> ...] [\'{ <settings> }\'] [<name>] [<type>]\n", progname);
-	fprintf(out, "\ndefault mode: "
+	if (!i_am_stubby) {
+		fprintf(out, "\ndefault mode: "
 #ifdef HAVE_LIBUNBOUND
-	    "recursive"
+	            "recursive"
 #else
-	    "stub"
+	            "stub"
 #endif
-	    ", synchronous resolution of NS record\n\t\tusing UDP with TCP fallback\n");
+	            ", synchronous resolution of NS record\n\t\tusing UDP with TCP fallback\n");
+	}
+	else {
+		fprintf(out, "\ndefault mode: "
+			    "stub, asynchronous resolution \n\t\tusing TLS with UDP then TCP fallback\n");
+	}
 	fprintf(out, "\nupstreams: @<ip>[%%<scope_id>][@<port>][#<tls port>][~<tls name>][^<tsig spec>]");
 	fprintf(out, "\n            <ip>@<port> may be given as <IPv4>:<port>");
 	fprintf(out, "\n                  or \'[\'<IPv6>[%%<scope_id>]\']\':<port> too\n");
@@ -192,10 +201,12 @@ print_usage(FILE *out, const char *progname)
 	fprintf(out, "\t+0\t\t\tClear all extensions\n");
 	fprintf(out, "\nsettings in json dict format (like outputted by -i option).\n");
 	fprintf(out, "\noptions:\n");
-	fprintf(out, "\t-a\tPerform asynchronous resolution "
+	if (!i_am_stubby) {
-	    "(default = synchronous)\n");
+		fprintf(out, "\t-a\tPerform asynchronous resolution "
-	fprintf(out, "\t-A\taddress lookup (<type> is ignored)\n");
+		    "(default = synchronous)\n");
-	fprintf(out, "\t-B\tBatch mode. Schedule all messages before processing responses.\n");
+		fprintf(out, "\t-A\taddress lookup (<type> is ignored)\n");
+		fprintf(out, "\t-B\tBatch mode. Schedule all messages before processing responses.\n");
+	}
 	fprintf(out, "\t-b <bufsize>\tSet edns0 max_udp_payload size\n");
 	fprintf(out, "\t-c\tSend Client Subnet privacy request\n");
 	fprintf(out, "\t-C\t<filename>\n");
@@ -209,17 +220,21 @@ print_usage(FILE *out, const char *progname)
 	fprintf(out, "\t-D\tSet edns0 do bit\n");
 	fprintf(out, "\t-d\tclear edns0 do bit\n");
 	fprintf(out, "\t-e <idle_timeout>\tSet idle timeout in miliseconds\n");
-	fprintf(out, "\t-F <filename>\tread the queries from the specified file\n");
+	if (!i_am_stubby)
+		fprintf(out, "\t-F <filename>\tread the queries from the specified file\n");
 	fprintf(out, "\t-f <filename>\tRead DNSSEC trust anchors from <filename>\n");
 #ifndef GETDNS_ON_WINDOWS
 	if (i_am_stubby)
 		fprintf(out, "\t-g\tRun stubby in background (default is foreground)\n");
 #endif
-	fprintf(out, "\t-G\tgeneral lookup\n");
+	if (!i_am_stubby) {
-	fprintf(out, "\t-H\thostname lookup. (<name> must be an IP address; <type> is ignored)\n");
+		fprintf(out, "\t-G\tgeneral lookup\n");
+		fprintf(out, "\t-H\thostname lookup. (<name> must be an IP address; <type> is ignored)\n");
+	}
 	fprintf(out, "\t-h\tPrint this help\n");
 	fprintf(out, "\t-i\tPrint api information\n");
-	fprintf(out, "\t-I\tInteractive mode (> 1 queries on same context)\n");
+	if (!i_am_stubby)
+		fprintf(out, "\t-I\tInteractive mode (> 1 queries on same context)\n");
 	fprintf(out, "\t-j\tOutput json response dict\n");
 	fprintf(out, "\t-J\tPretty print json response dict\n");
 	fprintf(out, "\t-k\tPrint root trust anchors\n");
@@ -235,8 +250,10 @@ print_usage(FILE *out, const char *progname)
 	fprintf(out, "\t-R <filename>\tRead root hints from <filename>\n");
 	fprintf(out, "\t-s\tSet stub resolution type%s\n"
 	       , i_am_stubby ? "" : "(default = recursing)" );
-	fprintf(out, "\t-S\tservice lookup (<type> is ignored)\n");
+	if (!i_am_stubby)
+		fprintf(out, "\t-S\tservice lookup (<type> is ignored)\n");
 	fprintf(out, "\t-t <timeout>\tSet timeout in miliseconds\n");
+	fprintf(out, "\t-v\tPrint getdns release version\n");
 	fprintf(out, "\t-x\tDo not follow redirects\n");
 	fprintf(out, "\t-X\tFollow redirects (default)\n");
 
@@ -840,6 +857,9 @@ getdns_return_t parse_args(int argc, char **argv)
 				getdns_context_set_timeout(
 					context, timeout);
 				goto next;
+			case 'v':
+				fprintf(stdout, "Version %s\n", GETDNS_VERSION);
+				return CONTINUE;
 			case 'x': 
 				getdns_context_set_follow_redirects(
 				    context, GETDNS_REDIRECTS_DO_NOT_FOLLOW);

src/tools/stubby.conf

@@ -0,0 +1,52 @@
+{ resolution_type: GETDNS_RESOLUTION_STUB
+, dns_transport_list: [ GETDNS_TRANSPORT_TLS ]
+, upstream_recursive_servers:
+  [ { address_data: 145.100.185.15
+    , tls_auth_name: "dnsovertls.sinodun.com"
+    , tls_pubkey_pinset:
+      [ { digest: "sha256"
+        , value: 0xA132D34D34C181765337C70B83E3697B9524DDDB05A7118B43C0284033D5A0CC
+      } ] 
+    },
+    { address_data: 145.100.185.16
+    , tls_auth_name: "dnsovertls1.sinodun.com"
+    , tls_pubkey_pinset:
+      [ { digest: "sha256"
+        , value: 0x659B41EB08DCC70EE9D624E6219C76EE31954DA1548B0C8519EAE5228CB24150
+      } ] 
+    },
+    { address_data: 185.49.141.38
+    , tls_auth_name: "getdnsapi.net"
+    , tls_pubkey_pinset:
+      [ { digest: "sha256"
+        , value: 0x7e8c59467221f606695a797ecc488a6b4109dab7421aba0c5a6d3681ac5273d4
+      } ]
+    },
+    { address_data: 2001:610:1:40ba:145:100:185:15
+    , tls_auth_name: "dnsovertls.sinodun.com"
+    , tls_pubkey_pinset:
+      [ { digest: "sha256"
+        , value: 0xA132D34D34C181765337C70B83E3697B9524DDDB05A7118B43C0284033D5A0CC
+      } ] 
+    },
+    { address_data: 2001:610:1:40ba:145:100:185:16
+    , tls_auth_name: "dnsovertls1.sinodun.com"
+    , tls_pubkey_pinset:
+      [ { digest: "sha256"
+        , value: 0x659B41EB08DCC70EE9D624E6219C76EE31954DA1548B0C8519EAE5228CB24150
+      } ] 
+    },
+    { address_data: 2a04:b900:0:100::38
+    , tls_auth_name: "getdnsapi.net"
+    , tls_pubkey_pinset:
+      [ { digest: "sha256"
+        , value: 0x7e8c59467221f606695a797ecc488a6b4109dab7421aba0c5a6d3681ac5273d4
+      } ]
+    }
+  ]
+, tls_authentication: GETDNS_AUTHENTICATION_REQUIRED
+, tls_query_padding_blocksize: 256
+, edns_client_subnet_private : 1
+, listen_addresses: [ 127.0.0.1, 0::1 ]
+, idle_timeout: 10000 
+}