Merge branch 'develop' into release/1.1.0-alpha3

ChangeLog

@@ -1,3 +1,19 @@
+* 2017-01-13: Version 1.0.0
+  * edns0_cookies extension enabled by default (per RFC7873)
+  * dnssec_roadblock_avoidance enabled by default (per RFC8027)
+  * bugfix: DSA support with OpenSSL 1.1.0
+  * Initialize OpenSSL just once in a thread safe way
+  * Thread safety with arc4random function
+  * Improvements that came from Visual Studio static analysis
+    Thanks Christian Huitema
+  * Conventional RFC3986 IPv6 [address]:port parsing from getdns_query
+  * bugfix: OpenSSL 1.1.0 style crypto locking
+    Thanks volkommenheit
+  * configure tells *which* dependency is missing
+  * bugfix: Exclude terminating '\0' from bindata's returned by
+    getdns_get_suffix(). Thanks Jim Hague
+  * Better README.md.  Thanks Andrew Sullivan
+
 * 2016-10-19: Version 1.1.0-a2
   * Improved TLS connection management
   * OpenSSL 1.1 support

Makefile.in

@@ -150,18 +150,22 @@ distclean:
 	rm -f m4/ltoptions.m4
 	rm -f m4/ltsugar.m4
 	rm -f m4/ltversion.m4
-	rm -f $(distdir).tar.gz $(distdir).tar.gz.sha1
+	rm -f $(distdir).tar.gz $(distdir).tar.gz.sha256
 	rm -f $(distdir).tar.gz.md5 $(distdir).tar.gz.asc
 
 megaclean:
-	cd $(srcdir) && rm -fr * .dir-locals.el .gitignore .indent.pro .travis.yml && git reset --hard
+	cd $(srcdir) && rm -fr * .dir-locals.el .gitignore .indent.pro .travis.yml && git reset --hard && git submodule update --init
+
+autoclean: megaclean
+	libtoolize -ci
+	autoreconf -fi
 
 dist: $(distdir).tar.gz
 
-pub: $(distdir).tar.gz.sha1 $(distdir).tar.gz.md5 $(distdir).tar.gz.asc
+pub: $(distdir).tar.gz.sha256 $(distdir).tar.gz.md5 $(distdir).tar.gz.asc
 
-$(distdir).tar.gz.sha1: $(distdir).tar.gz
+$(distdir).tar.gz.sha256: $(distdir).tar.gz
-	openssl sha1 $(distdir).tar.gz >$@
+	openssl sha256 $(distdir).tar.gz >$@
 
 $(distdir).tar.gz.md5: $(distdir).tar.gz
 	openssl md5 $(distdir).tar.gz >$@

README.md

@@ -89,7 +89,7 @@ Note: If you only want to build stubby, then use the `--enable-stub-only` and `-
 * Currently getdns only offers two helper functions to deal with IDN: `getdns_convert_ulabel_to_alabel` and `getdns_convert_alabel_to_ulabel`.  If you do not need these functions, getdns can be configured to compile without them with the `--without-libidn` option to configure.
 * When both `--enable-stub-only` and `--without-libidn` options are used, getdns has only one dependency left, which is OpenSSL.
 
-## Extensions / Event loop dependencies
+## Extensions and Event loop dependencies
 
 The implementation works with a variety of event loops, each built as a separate shared library.  See [the wiki](https://github.com/getdnsapi/getdns/wiki/Asynchronous-Support#wiki-included-event-loop-integrations) for more details.
 
@@ -142,7 +142,7 @@ We have a [getdns users list](https://getdnsapi.net/mailman/listinfo/users) for
 
 The [getdns-api mailing list](https://getdnsapi.net/mailman/listinfo/spec) is a good place to engage in discussions regarding the design of the API.
 
-# Tickets/Bug Reports
+# Tickets and Bug Reports
 
 Tickets and bug reports should be reported via the [GitHub issues list](https://github.com/getdnsapi/getdns/issues).
 
@@ -197,7 +197,18 @@ Stub mode does not support:
 
 # Known Issues
 
-* None
+* The synchronous lookup functions will not work when new file descriptors
+  needed for the lookup will be larger than `FD_SETSIZE`.  This is because
+  the synchronous functions use a "default" event loop under the hood
+  which is based on `select()` and thus inherits the limits that `select()` has.
+
+  If you need only slightly more file descriptors, it is possible to enlarge
+  the `FD_SETSIZE` with the `--with-fd-setsize=`*`size`* flag to `configure`.
+
+  To resolve, use the asynchronous functions with an event loop extension for
+  libevent, libev or libuv.  Note that the asynchronous functions will have
+  the same problem when used in combination with `getdns_context_run()`, which
+  also uses the default event loop.
 
 # Supported Platforms
 
@@ -221,7 +232,7 @@ If you're using [FreeBSD](https://www.freebsd.org/), you may install getdns via
 
 If you are using FreeBSD 10 getdns can be intalled via 'pkg install getdns'.
 
-### CentOS/RHEL 6.5
+### CentOS and RHEL 6.5
 
 We rely on the most excellent package manager fpm to build the linux packages, which
 means that the packaging platform requires ruby 2.1.0.  There are other ways to
@@ -279,7 +290,7 @@ The build has been tested using the following:
 32 bit only Mingw: [Mingw(3.21.0) and Msys 1.0](http://www.mingw.org/) on Windows 8.1
 32 bit build on a 64 bit Mingw [Download latest from: http://mingw-w64.org/doku.php/download/mingw-builds and http://msys2.github.io/]. IMPORTANT: Install tested ONLY on the  "x86_64" for 64-bit installer of msys2.
 
-#### Dependencies: 
+#### Dependencies
 The following dependencies are 
 * openssl-1.0.2j
 * libidn

configure.ac

@@ -263,7 +263,7 @@ fi
 AC_CHECK_HEADERS([openssl/conf.h],,, [AC_INCLUDES_DEFAULT])
 AC_CHECK_HEADERS([openssl/engine.h],,, [AC_INCLUDES_DEFAULT])
 AC_CHECK_HEADERS([openssl/bn.h openssl/rsa.h openssl/dsa.h],,, [AC_INCLUDES_DEFAULT])
-AC_CHECK_FUNCS([OPENSSL_config EVP_md5 EVP_sha1 EVP_sha224 EVP_sha256 EVP_sha384 EVP_sha512 FIPS_mode ENGINE_load_cryptodev EVP_PKEY_keygen ECDSA_SIG_get0 EVP_MD_CTX_new EVP_PKEY_base_id HMAC_CTX_new HMAC_CTX_free TLS_client_method])
+AC_CHECK_FUNCS([OPENSSL_config EVP_md5 EVP_sha1 EVP_sha224 EVP_sha256 EVP_sha384 EVP_sha512 FIPS_mode ENGINE_load_cryptodev EVP_PKEY_keygen ECDSA_SIG_get0 EVP_MD_CTX_new EVP_PKEY_base_id HMAC_CTX_new HMAC_CTX_free TLS_client_method DSA_SIG_set0 EVP_dss1])
 AC_CHECK_DECLS([SSL_COMP_get_compression_methods,sk_SSL_COMP_pop_free,SSL_CTX_set_ecdh_auto], [], [], [
 AC_INCLUDES_DEFAULT
 #ifdef HAVE_OPENSSL_ERR_H
@@ -450,39 +450,38 @@ case "$enable_dsa" in
       ;;
     *) dnl default
       # detect if DSA is supported, and turn it off if not.
-      AC_CHECK_FUNC(EVP_dss1, [
+      AC_CHECK_FUNC(DSA_SIG_new, [
 	AC_DEFINE_UNQUOTED([USE_DSA], [1], [Define this to enable DSA support.])
       ], [if test "x$enable_dsa" = "xyes"; then AC_MSG_ERROR([OpenSSL does not support DSA and you used --enable-dsa.])
 	  fi ])
       ;;
 esac
 
-AC_ARG_ENABLE(draft-dnssec-roadblock-avoidance, AC_HELP_STRING([--enable-draft-dnssec-roadblock-avoidance], [Enable experimental dnssec roadblock avoidance]))
+AC_ARG_ENABLE(all-drafts, AC_HELP_STRING([--enable-all-drafts], [No drafts in this release]))
-AC_ARG_ENABLE(draft-edns-cookies, AC_HELP_STRING([--enable-draft-edns-cookies], [Enable experimental edns cookies]))
-AC_ARG_ENABLE(all-drafts, AC_HELP_STRING([--enable-all-drafts], [Enable cookies and roadblock avoidance]))
 case "$enable_all_drafts" in
 	yes)
-		enable_draft_dnssec_roadblock_avoidance=yes
-		enable_draft_edns_cookies=yes
 		;;
 	no|*)
 		;;
 esac
-case "$enable_draft_dnssec_roadblock_avoidance" in
+AC_ARG_ENABLE(dnssec-roadblock-avoidance, AC_HELP_STRING([--disable-dnssec-roadblock-avoidance], [Disable dnssec roadblock avoidance]))
-	yes)
+case "$enable_dnssec_roadblock_avoidance" in
-		AC_DEFINE_UNQUOTED([DNSSEC_ROADBLOCK_AVOIDANCE], [1], [Define this to enable the experimental draft dnssec roadblock avoidance.])
+	no)
 		;;
-	no|*)
+	yes|*)
+		AC_DEFINE_UNQUOTED([DNSSEC_ROADBLOCK_AVOIDANCE], [1], [Define this to enable the experimental dnssec roadblock avoidance.])
 		;;
 esac
-case "$enable_draft_edns_cookies" in
+
-	yes)
+AC_ARG_ENABLE(edns-cookies, AC_HELP_STRING([--disable-edns-cookies], [Disable edns cookies]))
+case "$enable_edns_cookies" in
+	no)
+		;;
+	yes|*)
 		if test "x_$HAVE_SSL" != "x_yes"; then
-			AC_MSG_ERROR([edns cookies need openssl libcrypto which is not available, please rerun without --enable-draft-edns-cookies])
+			AC_MSG_ERROR([edns cookies need openssl libcrypto which is not available, please rerun with --disable-edns-cookies])
 		fi
-		AC_DEFINE_UNQUOTED([EDNS_COOKIES], [1], [Define this to enable the experimental draft edns cookies.])
+		AC_DEFINE_UNQUOTED([EDNS_COOKIES], [1], [Define this to enable the experimental edns cookies.])
-		;;
-	no|*)
 		;;
 esac
 AC_DEFINE_UNQUOTED([EDNS_COOKIE_OPCODE], [10], [The edns cookie option code.])

src/compat/arc4_lock.c

@@ -48,9 +48,65 @@ void _ARC4_UNLOCK(void)
 {
     pthread_mutex_unlock(&arc_lock);
 }
+#elif defined(GETDNS_ON_WINDOWS)
+ /*
+ * There is no explicit arc4random_init call, and thus
+ * the critical section must be allocated on the first call to
+ * ARC4_LOCK(). The interlocked test is used to verify that
+ * the critical section will be allocated only once.
+ *
+ * The work around is for the main program to call arc4random()
+ * at the beginning of execution, before spinning new threads.
+ *
+ * There is also no explicit arc4random_close call, and thus
+ * the critical section is never deleted. It will remain allocated
+ * as long as the program runs.
+ */
+static CRITICAL_SECTION arc_critical_section;
+static volatile long arc_critical_section_initialized = 0;
+
+void _ARC4_LOCK(void)
+{
+	long r = InterlockedCompareExchange(&arc_critical_section_initialized, 1, 0);
+
+	if (r != 2)
+	{
+		if (r == 0)
+		{
+			InitializeCriticalSection(&arc_critical_section);
+			arc_critical_section_initialized = 2;
+		}
+		else if (r == 1)
+		{
+			/*
+			* If the critical section is initialized, the first test
+			* will return the value 2.
+			*
+			* If several threads try to initialize the arc4random
+			* state "at the same time", the first one will find
+			* the "initialized" variable at 0, the other ones at 1.
+			*
+			* Since this is a fairly rare event, we resolve it with a
+			* simple active wait loop.
+			*/
+
+			while (arc_critical_section_initialized != 2)
+			{
+				Sleep(1);
+			}
+		}
+	}
+
+	EnterCriticalSection(&arc_critical_section);
+}
+
+void _ARC4_UNLOCK(void)
+{
+	LeaveCriticalSection(&arc_critical_section);
+}
 
 #else
-/* XXX - add windows-(or at least non pthread) specific lock routines here */
+ /* XXX - add non pthread specific lock routines here */
 void _ARC4_LOCK(void)
 {
 }

src/context.c

@@ -62,6 +62,11 @@ typedef unsigned short in_port_t;
 #include <assert.h>
 #include <ctype.h>
 
+#ifdef HAVE_PTHREADS
+#include <pthread.h>
+#endif
+#include <stdbool.h>
+
 #include "config.h"
 #ifdef HAVE_LIBUNBOUND
 #include <unbound.h>
@@ -88,6 +93,11 @@ typedef unsigned short in_port_t;
    upstream. Using 1 hour for all transports - based on RFC7858 value for for TLS.*/
 #define BACKOFF_RETRY 3600
 
+#ifdef HAVE_PTHREADS
+static pthread_mutex_t ssl_init_lock = PTHREAD_MUTEX_INITIALIZER;
+#endif
+static bool ssl_init=false;
+
 void *plain_mem_funcs_user_arg = MF_PLAIN;
 
 typedef struct host_name_addrs {
@@ -1426,8 +1436,21 @@ getdns_context_create_with_extended_memory_functions(
 	/* Unbound needs SSL to be init'ed this early when TLS is used. However we
 	 * don't know that till later so we will have to do this every time. */
 
-	if ((set_from_os & 2) == 0)
+#ifdef HAVE_PTHREADS
+	pthread_mutex_lock(&ssl_init_lock);
+#else
+	/* XXX implement Windows-style lock here */
+#endif
+	/* Only initialise SSL once and ideally in a thread-safe manner */
+	if (ssl_init == false) {
 		SSL_library_init();
+		ssl_init = true;
+	}
+#ifdef HAVE_PTHREADS
+	pthread_mutex_unlock(&ssl_init_lock);
+#else
+	/* XXX implement Windows-style unlock here */
+#endif
 
 #ifdef HAVE_LIBUNBOUND
 	result->unbound_ctx = NULL;

src/convert.c

@@ -160,6 +160,7 @@ getdns_convert_ulabel_to_alabel(const char *ulabel)
     free(prepped2);
     return buf;
 #else
+    (void)ulabel;
     return NULL;
 #endif
 }
@@ -189,6 +190,7 @@ getdns_convert_alabel_to_ulabel(const char *alabel)
     }
     return buf;
 #else
+    (void)alabel;
     return NULL;
 #endif
 }

src/extension/default_eventloop.c

@@ -212,7 +212,7 @@ default_eventloop_run_once(getdns_eventloop *loop, int blocking)
 		else if (default_loop->timeout_times[i] < timeout)
 			timeout = default_loop->timeout_times[i];
 	}
-	for (fd = 0; fd < FD_SETSIZE; fd++) {
+	for (fd = 0; fd < (int)FD_SETSIZE; fd++) {
 		if (!default_loop->fd_events[fd])
 			continue;
 		if (default_loop->fd_events[fd]->read_cb)
@@ -240,7 +240,7 @@ default_eventloop_run_once(getdns_eventloop *loop, int blocking)
 		exit(EXIT_FAILURE);
 	}
 	now = get_now_plus(0);
-	for (fd = 0; fd < FD_SETSIZE; fd++) {
+	for (fd = 0; fd < (int)FD_SETSIZE; fd++) {
 		if (default_loop->fd_events[fd] &&
 		    default_loop->fd_events[fd]->read_cb &&
 		    FD_ISSET(fd, &readfds))

src/getdns/getdns.h.in

@@ -485,6 +485,7 @@ typedef enum getdns_callback_type_t {
 #define GETDNS_RCODE_BADNAME  20
 #define GETDNS_RCODE_BADALG   21
 #define GETDNS_RCODE_BADTRUNC 22
+#define GETDNS_RCODE_COOKIE   23
 /** @}
  */
 

src/rr-iter.h

@@ -76,7 +76,7 @@ _getdns_rr_iter *_getdns_single_rr_iter_init(_getdns_rr_iter *i,
     const uint8_t *wire, const size_t wire_len);
 
 static inline _getdns_rr_iter *_getdns_rr_iter_rewind(_getdns_rr_iter *i)
-{ return _getdns_rr_iter_init(i, i->pkt, i->pkt_end - i->pkt); }
+{ return i ? _getdns_rr_iter_init(i, i->pkt, i->pkt_end - i->pkt) : NULL; }
 
 _getdns_rr_iter *_getdns_rr_iter_next(_getdns_rr_iter *i);
 
@@ -181,7 +181,7 @@ static inline _getdns_rrset *_getdns_rrset_iter_value(_getdns_rrset_iter *i)
 { return i && i->rr_i.pos ? &i->rrset : NULL; }
 
 static inline _getdns_rrset_iter *_getdns_rrset_iter_rewind(_getdns_rrset_iter *i)
-{ return _getdns_rrset_iter_init(i, i->rrset.pkt, i->rrset.pkt_len, i->rrset.sections); }
+{ return i ? _getdns_rrset_iter_init(i, i->rrset.pkt, i->rrset.pkt_len, i->rrset.sections) : NULL; }
 
 typedef struct _getdns_rdf_iter {
 	const uint8_t           *pkt;

src/test/check_getdns_address.h

@@ -224,7 +224,7 @@
 
        EVENT_BASE_CREATE;
 
-       ASSERT_RC(getdns_address(context, "hostnamedoesntexist", NULL,
+       ASSERT_RC(getdns_address(context, "hostnamedoesntexist.", NULL,
          &fn_ref, &transaction_id, callbackfn),
          GETDNS_RETURN_GOOD, "Return code from getdns_address()");
 

src/test/check_getdns_service_sync.h

@@ -124,7 +124,7 @@
        */
        struct getdns_context *context = NULL;   
        struct getdns_dict *response = NULL;
-       const char *name = "labelsizeofsixtythreecharacterscom";
+       const char *name = "labelsizeofsixtythreecharacterscom.";
 
        CONTEXT_CREATE(TRUE);
 

src/test/tpkg/run-all-lcov.sh

@@ -15,7 +15,7 @@ LCOV_MERGE=""
 for TEST_PKG in ${SRCDIR}/*.tpkg
 do
     # when we run our test, we need to compile with profiling
-    CFLAGS="-fprofile-arcs -ftest-coverage -O0" "${TPKG}" $* exe "${TEST_PKG}"
+    LDFLAGS="-lgcov --coverage" CFLAGS="-fprofile-arcs -ftest-coverage -O0" "${TPKG}" $* exe "${TEST_PKG}"
     # after the test is complete, we need to collect the coverage data
     INFO_FILE=`echo $TEST_PKG | sed 's/.tpkg$//'`.info
     geninfo $SRCDIR/.. -o $INFO_FILE

src/util/val_secalgo.c

@@ -230,6 +230,7 @@ log_crypto_error(const char* str, unsigned long e)
 	ERR_error_string_n(e, buf, sizeof(buf));
 	/* buf now contains */
 	/* error:[error code]:[library name]:[function name]:[reason string] */
+	(void)str;
 	log_err("%s crypto %s", str, buf);
 }
 
@@ -262,8 +263,12 @@ setup_dsa_sig(unsigned char** sig, unsigned int* len)
 	dsasig = DSA_SIG_new();
 	if(!dsasig) return 0;
 
+#ifdef HAVE_DSA_SIG_SET0
+	if(!DSA_SIG_set0(dsasig, R, S)) return 0;
+#else
 	dsasig->r = R;
 	dsasig->s = S;
+#endif
 	*sig = NULL;
 	newlen = i2d_DSA_SIG(dsasig, sig);
 	if(newlen < 0) {
@@ -404,7 +409,11 @@ setup_key_digest(int algo, EVP_PKEY** evp_key, const EVP_MD** digest_type,
 					"EVP_PKEY_assign_DSA failed");
 				return 0;
 			}
+#ifdef HAVE_EVP_DSS1
 			*digest_type = EVP_dss1();
+#else
+			*digest_type = EVP_sha1();
+#endif
 
 			break;
 #endif /* USE_DSA */