From 639239f45cb44c4fed0853320d9cbd7d0c63af0a Mon Sep 17 00:00:00 2001 From: Willem Toorop Date: Mon, 13 Mar 2017 14:20:47 +0100 Subject: [PATCH 1/8] Schedule dnsreqs with absolute timeout/expiry time --- src/dnssec.c | 3 ++- src/general.c | 19 +++++++++++-------- src/general.h | 2 +- src/request-internal.c | 8 +++++++- src/stub.c | 23 ++++++++++++----------- src/stub.h | 3 ++- src/types-internal.h | 7 ++++++- src/util-internal.h | 20 ++++++++++++++++++++ 8 files changed, 61 insertions(+), 24 deletions(-) diff --git a/src/dnssec.c b/src/dnssec.c index 0396f045..b689aba5 100644 --- a/src/dnssec.c +++ b/src/dnssec.c @@ -3032,6 +3032,7 @@ static void check_chain_complete(chain_head *chain) int r = GETDNS_RETURN_GOOD; getdns_network_req **netreq_p, *netreq; + uint64_t now_ms = 0; dnsreq->avoid_dnssec_roadblocks = 1; @@ -3041,7 +3042,7 @@ static void check_chain_complete(chain_head *chain) netreq->state = NET_REQ_NOT_SENT; netreq->owner = dnsreq; - r = _getdns_submit_netreq(netreq); + r = _getdns_submit_netreq(netreq, &now_ms); } return; } diff --git a/src/general.c b/src/general.c index 69dc7462..c71a4944 100644 --- a/src/general.c +++ b/src/general.c @@ -91,6 +91,7 @@ _getdns_check_dns_req_complete(getdns_dns_req *dns_req) { getdns_network_req **netreq_p, *netreq; int results_found = 0, r; + uint64_t now_ms = 0; for (netreq_p = dns_req->netreqs; (netreq = *netreq_p); netreq_p++) if (!_getdns_netreq_finished(netreq)) @@ -126,7 +127,7 @@ _getdns_check_dns_req_complete(getdns_dns_req *dns_req) ; (netreq = *netreq_p) ; netreq_p++ ) { _getdns_netreq_reinit(netreq); - if ((r = _getdns_submit_netreq(netreq))) { + if ((r = _getdns_submit_netreq(netreq, &now_ms))) { if (r == DNS_REQ_FINISHED) return; netreq->state = NET_REQ_FINISHED; @@ -164,7 +165,7 @@ _getdns_check_dns_req_complete(getdns_dns_req *dns_req) ; (netreq = *netreq_p) ; netreq_p++ ) { _getdns_netreq_reinit(netreq); - if ((r = _getdns_submit_netreq(netreq))) { + if ((r = _getdns_submit_netreq(netreq, &now_ms))) { if (r == DNS_REQ_FINISHED) return; netreq->state = NET_REQ_FINISHED; @@ -248,7 +249,7 @@ ub_resolve_callback(void* arg, int err, struct ub_result* ub_res) int -_getdns_submit_netreq(getdns_network_req *netreq) +_getdns_submit_netreq(getdns_network_req *netreq, uint64_t *now_ms) { getdns_return_t r; getdns_dns_req *dns_req = netreq->owner; @@ -284,7 +285,8 @@ _getdns_submit_netreq(getdns_network_req *netreq) _getdns_context_request_timed_out; dns_req->timeout.ev = NULL; if ((r = dns_req->loop->vmt->schedule(dns_req->loop, -1, - dns_req->context->timeout, &dns_req->timeout))) + _getdns_ms_until_expiry2(dns_req->expires, now_ms), + &dns_req->timeout))) return r; } (void) gldns_wire2str_dname_buf(dns_req->name, @@ -314,7 +316,7 @@ _getdns_submit_netreq(getdns_network_req *netreq) } /* Submit with stub resolver */ dns_req->freed = &dnsreq_freed; - r = _getdns_submit_stub_request(netreq); + r = _getdns_submit_stub_request(netreq, now_ms); if (dnsreq_freed) return DNS_REQ_FINISHED; dns_req->freed = NULL; @@ -413,6 +415,7 @@ getdns_general_ns(getdns_context *context, getdns_eventloop *loop, getdns_dns_req *req; getdns_dict *localnames_response; size_t i; + uint64_t now_ms = 0; if (!context || !name || (!callbackfn && !internal_cb)) return GETDNS_RETURN_INVALID_PARAMETER; @@ -430,7 +433,7 @@ getdns_general_ns(getdns_context *context, getdns_eventloop *loop, /* create the request */ if (!(req = _getdns_dns_req_new( - context, loop, name, request_type, extensions))) + context, loop, name, request_type, extensions, &now_ms))) return GETDNS_RETURN_MEMORY_ERROR; req->user_pointer = userarg; @@ -448,7 +451,7 @@ getdns_general_ns(getdns_context *context, getdns_eventloop *loop, for ( netreq_p = req->netreqs ; !r && (netreq = *netreq_p) ; netreq_p++) { - if ((r = _getdns_submit_netreq(netreq))) { + if ((r = _getdns_submit_netreq(netreq, &now_ms))) { if (r == DNS_REQ_FINISHED) { if (return_netreq_p) *return_netreq_p = NULL; @@ -500,7 +503,7 @@ getdns_general_ns(getdns_context *context, getdns_eventloop *loop, for ( netreq_p = req->netreqs ; !r && (netreq = *netreq_p) ; netreq_p++) { - if ((r = _getdns_submit_netreq(netreq))) { + if ((r = _getdns_submit_netreq(netreq, &now_ms))) { if (r == DNS_REQ_FINISHED) { if (return_netreq_p) *return_netreq_p = NULL; diff --git a/src/general.h b/src/general.h index 29b52f47..dcd9b9be 100644 --- a/src/general.h +++ b/src/general.h @@ -46,7 +46,7 @@ void _getdns_call_user_callback(getdns_dns_req *, getdns_dict *); void _getdns_check_dns_req_complete(getdns_dns_req *dns_req); -int _getdns_submit_netreq(getdns_network_req *netreq); +int _getdns_submit_netreq(getdns_network_req *netreq, uint64_t *now_ms); getdns_return_t diff --git a/src/request-internal.c b/src/request-internal.c index eae467e9..6c9cd9ec 100644 --- a/src/request-internal.c +++ b/src/request-internal.c @@ -658,7 +658,8 @@ static const uint8_t no_suffixes[] = { 1, 0 }; /* create a new dns req to be submitted */ getdns_dns_req * _getdns_dns_req_new(getdns_context *context, getdns_eventloop *loop, - const char *name, uint16_t request_type, getdns_dict *extensions) + const char *name, uint16_t request_type, getdns_dict *extensions, + uint64_t *now_ms) { int dnssec_return_status = is_extension_set( extensions, "dnssec_return_status", @@ -953,5 +954,10 @@ _getdns_dns_req_new(getdns_context *context, getdns_eventloop *loop, netreq_sz - sizeof(getdns_network_req), max_query_sz, extensions); + if (*now_ms == 0 && (*now_ms = _getdns_get_now_ms()) == 0) + result->expires = 0; + else + result->expires = *now_ms + context->timeout; + return result; } diff --git a/src/stub.c b/src/stub.c index e905f600..f7f1fddb 100644 --- a/src/stub.c +++ b/src/stub.c @@ -1358,8 +1358,8 @@ stub_udp_read_cb(void *userarg) dnsreq)) == -1) break; upstream_schedule_netreq(netreq->upstream, netreq); - GETDNS_SCHEDULE_EVENT( - dnsreq->loop, -1, dnsreq->context->timeout, + GETDNS_SCHEDULE_EVENT(dnsreq->loop, -1, + _getdns_ms_until_expiry(dnsreq->expires), getdns_eventloop_event_init(&netreq->event, netreq, NULL, NULL, stub_timeout_cb)); @@ -1421,8 +1421,8 @@ stub_udp_write_cb(void *userarg) #endif return; } - GETDNS_SCHEDULE_EVENT( - dnsreq->loop, netreq->fd, dnsreq->context->timeout, + GETDNS_SCHEDULE_EVENT(dnsreq->loop, netreq->fd, + _getdns_ms_until_expiry(dnsreq->expires), getdns_eventloop_event_init(&netreq->event, netreq, stub_udp_read_cb, NULL, stub_timeout_cb)); } @@ -1928,12 +1928,13 @@ upstream_find_for_netreq(getdns_network_req *netreq) static int fallback_on_write(getdns_network_req *netreq) { + uint64_t now_ms = 0; /* Deal with UDP one day*/ DEBUG_STUB("%s %-35s: MSG: %p FALLING BACK \n", STUB_DEBUG_SCHEDULE, __FUNC__, (void*)netreq); /* Try to find a fallback transport*/ - getdns_return_t result = _getdns_submit_stub_request(netreq); + getdns_return_t result = _getdns_submit_stub_request(netreq, &now_ms); if (result != GETDNS_RETURN_GOOD) return STUB_TCP_ERROR; @@ -1997,8 +1998,8 @@ upstream_schedule_netreq(getdns_upstream *upstream, getdns_network_req *netreq) if (upstream->queries_sent == 0) { /* Set a timeout on the upstream so we can catch failed setup*/ upstream->event.timeout_cb = upstream_setup_timeout_cb; - GETDNS_SCHEDULE_EVENT(upstream->loop, - upstream->fd, netreq->owner->context->timeout / 2, + GETDNS_SCHEDULE_EVENT(upstream->loop, upstream->fd, + _getdns_ms_until_expiry(netreq->owner->expires)/2, &upstream->event); } else { GETDNS_SCHEDULE_EVENT(upstream->loop, @@ -2027,7 +2028,7 @@ upstream_schedule_netreq(getdns_upstream *upstream, getdns_network_req *netreq) } getdns_return_t -_getdns_submit_stub_request(getdns_network_req *netreq) +_getdns_submit_stub_request(getdns_network_req *netreq, uint64_t *now_ms) { DEBUG_STUB("%s %-35s: MSG: %p TYPE: %d\n", STUB_DEBUG_ENTRY, __FUNC__, (void*)netreq, netreq->request_type); @@ -2046,8 +2047,8 @@ _getdns_submit_stub_request(getdns_network_req *netreq) case GETDNS_TRANSPORT_UDP: netreq->fd = fd; GETDNS_CLEAR_EVENT(dnsreq->loop, &netreq->event); - GETDNS_SCHEDULE_EVENT( - dnsreq->loop, netreq->fd, dnsreq->context->timeout, + GETDNS_SCHEDULE_EVENT(dnsreq->loop, netreq->fd, + _getdns_ms_until_expiry2(dnsreq->expires, now_ms), getdns_eventloop_event_init(&netreq->event, netreq, NULL, stub_udp_write_cb, stub_timeout_cb)); return GETDNS_RETURN_GOOD; @@ -2121,7 +2122,7 @@ _getdns_submit_stub_request(getdns_network_req *netreq) */ GETDNS_SCHEDULE_EVENT( dnsreq->loop, -1, - dnsreq->context->timeout, + _getdns_ms_until_expiry2(dnsreq->expires, now_ms), getdns_eventloop_event_init( &netreq->event, netreq, NULL, NULL, stub_timeout_cb)); diff --git a/src/stub.h b/src/stub.h index 41aa629a..da45eded 100644 --- a/src/stub.h +++ b/src/stub.h @@ -37,7 +37,8 @@ #include "getdns/getdns.h" #include "types-internal.h" -getdns_return_t _getdns_submit_stub_request(getdns_network_req *netreq); +getdns_return_t _getdns_submit_stub_request( + getdns_network_req *netreq, uint64_t *now_ms); void _getdns_cancel_stub_request(getdns_network_req *netreq); diff --git a/src/types-internal.h b/src/types-internal.h index 78f1935a..9afb105d 100644 --- a/src/types-internal.h +++ b/src/types-internal.h @@ -338,6 +338,11 @@ typedef struct getdns_dns_req { /* the transaction id */ getdns_transaction_t trans_id; + /* Absolute time (in miliseconds since epoch), + * after which this dns request is expired; i.e. timed out + */ + uint64_t expires; + /* for scheduling timeouts when using libunbound */ getdns_eventloop_event timeout; @@ -408,7 +413,7 @@ extern getdns_dict *dnssec_ok_checking_disabled_avoid_roadblocks; /* dns request utils */ getdns_dns_req *_getdns_dns_req_new(getdns_context *context, getdns_eventloop *loop, - const char *name, uint16_t request_type, getdns_dict *extensions); + const char *name, uint16_t request_type, getdns_dict *extensions, uint64_t *now_ms); void _getdns_dns_req_free(getdns_dns_req * req); diff --git a/src/util-internal.h b/src/util-internal.h index 54c81d9d..6f8de235 100644 --- a/src/util-internal.h +++ b/src/util-internal.h @@ -198,5 +198,25 @@ INLINE void _dname_canonicalize2(uint8_t *dname) _dname_canonicalize(dname, dname); } +INLINE uint64_t _getdns_get_now_ms() +{ + struct timeval tv; + + (void) gettimeofday(&tv, NULL); + return tv.tv_sec * 1000 + tv.tv_usec / 1000; +} + +INLINE uint64_t _getdns_ms_until_expiry(uint64_t expires) +{ + uint64_t now_ms = _getdns_get_now_ms(); + return now_ms >= expires ? 0 : expires - now_ms; +} + +INLINE uint64_t _getdns_ms_until_expiry2(uint64_t expires, uint64_t *now_ms) +{ + if (*now_ms == 0) *now_ms = _getdns_get_now_ms(); + return *now_ms >= expires ? 0 : expires - *now_ms; +} + #endif /* util-internal.h */ From 0655a08fa7a1b050151ba73a2669adf3ec488899 Mon Sep 17 00:00:00 2001 From: Willem Toorop Date: Tue, 14 Mar 2017 15:03:43 +0100 Subject: [PATCH 2/8] Unit test for set_limit_outstanding_queries --- .../280-limit_outstanding_queries.Makefile | 15 + .../280-limit_outstanding_queries.c | 136 +++ .../280-limit_outstanding_queries.dsc | 16 + .../280-limit_outstanding_queries.pre | 14 + .../280-limit_outstanding_queries.queries | 1000 +++++++++++++++++ .../280-limit_outstanding_queries.test | 21 + 6 files changed, 1202 insertions(+) create mode 100644 src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.Makefile create mode 100644 src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.c create mode 100644 src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.dsc create mode 100644 src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.pre create mode 100644 src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.queries create mode 100644 src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.Makefile b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.Makefile new file mode 100644 index 00000000..c297df6d --- /dev/null +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.Makefile @@ -0,0 +1,15 @@ +builddir = @BUILDDIR@ +testname = @TPKG_NAME@ +LIBTOOL = $(builddir)/libtool + +CFLAGS=-I$(builddir)/src +LDLIBS=$(builddir)/src/libgetdns.la + +.SUFFIXES: .c .o .a .lo .h + +.c.lo: + $(LIBTOOL) --quiet --tag=CC --mode=compile $(CC) $(CFLAGS) -c $< -o $@ + +$(testname): $(testname).lo + $(LIBTOOL) --tag=CC --mode=link $(CC) $(LDLIBS) $(LDFLAGS) -o $(testname) $(testname).lo + diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.c b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.c new file mode 100644 index 00000000..78107b57 --- /dev/null +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.c @@ -0,0 +1,136 @@ +/* + * delaydns.c - A DNS proxy that adds delay to replies + * + * Copyright (c) 2016, NLnet Labs. All rights reserved. + * + * This software is open source. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of the NLNET LABS nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + + +static int n_requests = 0; + +typedef struct transaction_t { + getdns_transaction_t request_id; + getdns_dict *request; + + getdns_context *context; + getdns_eventloop *loop; + getdns_eventloop_event ev; +} transaction_t; + + +void delay_cb(void *userarg) +{ + transaction_t *trans = userarg; + + trans->loop->vmt->clear(trans->loop, &trans->ev); + (void) getdns_reply(trans->context, trans->request, trans->request_id); + getdns_dict_destroy(trans->request); + free(trans); + n_requests -= 1; +} + +void handler(getdns_context *context, getdns_callback_type_t callback_type, + getdns_dict *request, void *userarg, getdns_transaction_t request_id) +{ + transaction_t *trans; + getdns_bindata *qname; + char nreq_str[255]; + getdns_bindata nreq_bd = { 0, nreq_str }; + + (void) userarg; + nreq_bd.size = snprintf(nreq_str, sizeof(nreq_str), "n_requests: %d", ++n_requests); + + if (getdns_dict_get_bindata(request, "/question/qname", &qname) || + getdns_dict_set_bindata(request, "/answer/0/name", qname) || + getdns_dict_set_int(request, "/answer/0/type", GETDNS_RRTYPE_TXT) || + getdns_dict_set_bindata(request, "/answer/0/rdata/txt_strings/-", &nreq_bd)) + fprintf(stderr, "Request init error\n"); + + else if (qname->size >= 6 && qname->data[0] == 4 && + qname->data[1] == 'q' && qname->data[2] == 'u' && + qname->data[3] == 'i' && qname->data[4] == 't') { + + (void) getdns_reply(context, request, request_id); + (void) getdns_context_set_listen_addresses(trans->context, NULL, NULL, NULL); + n_requests -= 1; + return; + + } else if (!(trans = malloc(sizeof(transaction_t)))) + perror("memerror"); + else { + (void) memset(trans, 0, sizeof(transaction_t)); + trans->request_id = request_id; + trans->request = request; + trans->context = context; + trans->ev.userarg = trans; + trans->ev.timeout_cb = delay_cb; + + if (getdns_context_get_eventloop(context, &trans->loop) + || trans->loop->vmt->schedule(trans->loop, -1, 300, &trans->ev)) + fprintf(stderr, "Could not schedule delay\n"); + else return; + } + exit(EXIT_FAILURE); +} + +int main() +{ + getdns_context *context = NULL; + getdns_list *listeners = NULL; + getdns_dict *address = NULL; + uint32_t port = 18000; + getdns_return_t r; + + if ((r = getdns_str2list("[ 127.0.0.1:18000 ]", &listeners)) || + (r = getdns_list_get_dict(listeners, 0, &address)) || + (r = getdns_context_create(&context, 0))) + fprintf(stderr, "Error initializing: "); + + else while (++port < 18200 && + !(r = getdns_dict_set_int(address, "port", port)) && + (r = getdns_context_set_listen_addresses( + context, listeners, NULL, handler))) + ; /* pass */ + + if (r) fprintf(stderr, "%s\n", getdns_get_errorstr_by_id(r)); + else { + fprintf(stdout, "%d\n", (int)port); + fflush(stdout); + getdns_context_run(context); + } + getdns_list_destroy(listeners); + getdns_context_destroy(context); + return r; +} diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.dsc b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.dsc new file mode 100644 index 00000000..f1f0aa4b --- /dev/null +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.dsc @@ -0,0 +1,16 @@ +BaseName: 280-limit_outstanding_queries +Version: 1.0 +Description: Test if outstanding queries setting is obeyed +CreationDate: Tue Mar 14 10:43:45 CET 2017 +Maintainer: Willem Toorop +Category: +Component: +CmdDepends: +Depends: 210-stub-only-link.tpkg +Help: +Pre: 280-limit_outstanding_queries.pre +Post: +Test: 280-limit_outstanding_queries.test +AuxFiles: +Passed: +Failure: diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.pre b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.pre new file mode 100644 index 00000000..b4ee91ac --- /dev/null +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.pre @@ -0,0 +1,14 @@ +# #-- 280-limit_outstanding_queries.test --# +# source the master var file when it's there +[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master +# use .tpkg.var.test for in test variable passing +[ -f .tpkg.var.test ] && source .tpkg.var.test + +( + grep '^CC=' "${BUILDDIR}/build-stub-only/src/Makefile" + grep '^LDFLAGS=' "${BUILDDIR}/build-stub-only/src/Makefile" + + BUILDDIR4SED=`echo "${BUILDDIR}/build-stub-only" | sed 's/\//\\\\\//g'` + sed -e "s/@BUILDDIR@/${BUILDDIR4SED}/g" \ + -e "s/@TPKG_NAME@/${TPKG_NAME}/g" "${TPKG_NAME}.Makefile" +) > Makefile diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.queries b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.queries new file mode 100644 index 00000000..0cc2103d --- /dev/null +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.queries @@ -0,0 +1,1000 @@ +q1. +q2. +q3. +q4. +q5. +q6. +q7. +q8. +q9. +q10. +q11. +q12. +q13. +q14. +q15. +q16. +q17. +q18. +q19. +q20. +q21. +q22. +q23. +q24. +q25. +q26. +q27. +q28. +q29. +q30. +q31. +q32. +q33. +q34. +q35. +q36. +q37. +q38. +q39. +q40. +q41. +q42. +q43. +q44. +q45. +q46. +q47. +q48. +q49. +q50. +q51. +q52. +q53. +q54. +q55. +q56. +q57. +q58. +q59. +q60. +q61. +q62. +q63. +q64. +q65. +q66. +q67. +q68. +q69. +q70. +q71. +q72. +q73. +q74. +q75. +q76. +q77. +q78. +q79. +q80. +q81. +q82. +q83. +q84. +q85. +q86. +q87. +q88. +q89. +q90. +q91. +q92. +q93. +q94. +q95. +q96. +q97. +q98. +q99. +q100. +q101. +q102. +q103. +q104. +q105. +q106. +q107. +q108. +q109. +q110. +q111. +q112. +q113. +q114. +q115. +q116. +q117. +q118. +q119. +q120. +q121. +q122. +q123. +q124. +q125. +q126. +q127. +q128. +q129. +q130. +q131. +q132. +q133. +q134. +q135. +q136. +q137. +q138. +q139. +q140. +q141. +q142. +q143. +q144. +q145. +q146. +q147. +q148. +q149. +q150. +q151. +q152. +q153. +q154. +q155. +q156. +q157. +q158. +q159. +q160. +q161. +q162. +q163. +q164. +q165. +q166. +q167. +q168. +q169. +q170. +q171. +q172. +q173. +q174. +q175. +q176. +q177. +q178. +q179. +q180. +q181. +q182. +q183. +q184. +q185. +q186. +q187. +q188. +q189. +q190. +q191. +q192. +q193. +q194. +q195. +q196. +q197. +q198. +q199. +q200. +q201. +q202. +q203. +q204. +q205. +q206. +q207. +q208. +q209. +q210. +q211. +q212. +q213. +q214. +q215. +q216. +q217. +q218. +q219. +q220. +q221. +q222. +q223. +q224. +q225. +q226. +q227. +q228. +q229. +q230. +q231. +q232. +q233. +q234. +q235. +q236. +q237. +q238. +q239. +q240. +q241. +q242. +q243. +q244. +q245. +q246. +q247. +q248. +q249. +q250. +q251. +q252. +q253. +q254. +q255. +q256. +q257. +q258. +q259. +q260. +q261. +q262. +q263. +q264. +q265. +q266. +q267. +q268. +q269. +q270. +q271. +q272. +q273. +q274. +q275. +q276. +q277. +q278. +q279. +q280. +q281. +q282. +q283. +q284. +q285. +q286. +q287. +q288. +q289. +q290. +q291. +q292. +q293. +q294. +q295. +q296. +q297. +q298. +q299. +q300. +q301. +q302. +q303. +q304. +q305. +q306. +q307. +q308. +q309. +q310. +q311. +q312. +q313. +q314. +q315. +q316. +q317. +q318. +q319. +q320. +q321. +q322. +q323. +q324. +q325. +q326. +q327. +q328. +q329. +q330. +q331. +q332. +q333. +q334. +q335. +q336. +q337. +q338. +q339. +q340. +q341. +q342. +q343. +q344. +q345. +q346. +q347. +q348. +q349. +q350. +q351. +q352. +q353. +q354. +q355. +q356. +q357. +q358. +q359. +q360. +q361. +q362. +q363. +q364. +q365. +q366. +q367. +q368. +q369. +q370. +q371. +q372. +q373. +q374. +q375. +q376. +q377. +q378. +q379. +q380. +q381. +q382. +q383. +q384. +q385. +q386. +q387. +q388. +q389. +q390. +q391. +q392. +q393. +q394. +q395. +q396. +q397. +q398. +q399. +q400. +q401. +q402. +q403. +q404. +q405. +q406. +q407. +q408. +q409. +q410. +q411. +q412. +q413. +q414. +q415. +q416. +q417. +q418. +q419. +q420. +q421. +q422. +q423. +q424. +q425. +q426. +q427. +q428. +q429. +q430. +q431. +q432. +q433. +q434. +q435. +q436. +q437. +q438. +q439. +q440. +q441. +q442. +q443. +q444. +q445. +q446. +q447. +q448. +q449. +q450. +q451. +q452. +q453. +q454. +q455. +q456. +q457. +q458. +q459. +q460. +q461. +q462. +q463. +q464. +q465. +q466. +q467. +q468. +q469. +q470. +q471. +q472. +q473. +q474. +q475. +q476. +q477. +q478. +q479. +q480. +q481. +q482. +q483. +q484. +q485. +q486. +q487. +q488. +q489. +q490. +q491. +q492. +q493. +q494. +q495. +q496. +q497. +q498. +q499. +q500. +q501. +q502. +q503. +q504. +q505. +q506. +q507. +q508. +q509. +q510. +q511. +q512. +q513. +q514. +q515. +q516. +q517. +q518. +q519. +q520. +q521. +q522. +q523. +q524. +q525. +q526. +q527. +q528. +q529. +q530. +q531. +q532. +q533. +q534. +q535. +q536. +q537. +q538. +q539. +q540. +q541. +q542. +q543. +q544. +q545. +q546. +q547. +q548. +q549. +q550. +q551. +q552. +q553. +q554. +q555. +q556. +q557. +q558. +q559. +q560. +q561. +q562. +q563. +q564. +q565. +q566. +q567. +q568. +q569. +q570. +q571. +q572. +q573. +q574. +q575. +q576. +q577. +q578. +q579. +q580. +q581. +q582. +q583. +q584. +q585. +q586. +q587. +q588. +q589. +q590. +q591. +q592. +q593. +q594. +q595. +q596. +q597. +q598. +q599. +q600. +q601. +q602. +q603. +q604. +q605. +q606. +q607. +q608. +q609. +q610. +q611. +q612. +q613. +q614. +q615. +q616. +q617. +q618. +q619. +q620. +q621. +q622. +q623. +q624. +q625. +q626. +q627. +q628. +q629. +q630. +q631. +q632. +q633. +q634. +q635. +q636. +q637. +q638. +q639. +q640. +q641. +q642. +q643. +q644. +q645. +q646. +q647. +q648. +q649. +q650. +q651. +q652. +q653. +q654. +q655. +q656. +q657. +q658. +q659. +q660. +q661. +q662. +q663. +q664. +q665. +q666. +q667. +q668. +q669. +q670. +q671. +q672. +q673. +q674. +q675. +q676. +q677. +q678. +q679. +q680. +q681. +q682. +q683. +q684. +q685. +q686. +q687. +q688. +q689. +q690. +q691. +q692. +q693. +q694. +q695. +q696. +q697. +q698. +q699. +q700. +q701. +q702. +q703. +q704. +q705. +q706. +q707. +q708. +q709. +q710. +q711. +q712. +q713. +q714. +q715. +q716. +q717. +q718. +q719. +q720. +q721. +q722. +q723. +q724. +q725. +q726. +q727. +q728. +q729. +q730. +q731. +q732. +q733. +q734. +q735. +q736. +q737. +q738. +q739. +q740. +q741. +q742. +q743. +q744. +q745. +q746. +q747. +q748. +q749. +q750. +q751. +q752. +q753. +q754. +q755. +q756. +q757. +q758. +q759. +q760. +q761. +q762. +q763. +q764. +q765. +q766. +q767. +q768. +q769. +q770. +q771. +q772. +q773. +q774. +q775. +q776. +q777. +q778. +q779. +q780. +q781. +q782. +q783. +q784. +q785. +q786. +q787. +q788. +q789. +q790. +q791. +q792. +q793. +q794. +q795. +q796. +q797. +q798. +q799. +q800. +q801. +q802. +q803. +q804. +q805. +q806. +q807. +q808. +q809. +q810. +q811. +q812. +q813. +q814. +q815. +q816. +q817. +q818. +q819. +q820. +q821. +q822. +q823. +q824. +q825. +q826. +q827. +q828. +q829. +q830. +q831. +q832. +q833. +q834. +q835. +q836. +q837. +q838. +q839. +q840. +q841. +q842. +q843. +q844. +q845. +q846. +q847. +q848. +q849. +q850. +q851. +q852. +q853. +q854. +q855. +q856. +q857. +q858. +q859. +q860. +q861. +q862. +q863. +q864. +q865. +q866. +q867. +q868. +q869. +q870. +q871. +q872. +q873. +q874. +q875. +q876. +q877. +q878. +q879. +q880. +q881. +q882. +q883. +q884. +q885. +q886. +q887. +q888. +q889. +q890. +q891. +q892. +q893. +q894. +q895. +q896. +q897. +q898. +q899. +q900. +q901. +q902. +q903. +q904. +q905. +q906. +q907. +q908. +q909. +q910. +q911. +q912. +q913. +q914. +q915. +q916. +q917. +q918. +q919. +q920. +q921. +q922. +q923. +q924. +q925. +q926. +q927. +q928. +q929. +q930. +q931. +q932. +q933. +q934. +q935. +q936. +q937. +q938. +q939. +q940. +q941. +q942. +q943. +q944. +q945. +q946. +q947. +q948. +q949. +q950. +q951. +q952. +q953. +q954. +q955. +q956. +q957. +q958. +q959. +q960. +q961. +q962. +q963. +q964. +q965. +q966. +q967. +q968. +q969. +q970. +q971. +q972. +q973. +q974. +q975. +q976. +q977. +q978. +q979. +q980. +q981. +q982. +q983. +q984. +q985. +q986. +q987. +q988. +q989. +q990. +q991. +q992. +q993. +q994. +q995. +q996. +q997. +q998. +q999. +q1000. diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test new file mode 100644 index 00000000..828d0a96 --- /dev/null +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test @@ -0,0 +1,21 @@ +# #-- 280-limit_outstanding_queries.test --# +# source the master var file when it's there +[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master +# use .tpkg.var.test for in test variable passing +[ -f .tpkg.var.test ] && source .tpkg.var.test + +QLIMIT=10 +make && "./${TPKG_NAME}" | ( + read PORT + ${GETDNS_STUB_QUERY} @127.0.0.1:$PORT TXT \ + -a -F "./${TPKG_NAME}.queries" \ + "{limit_outstanding_queries:$QLIMIT}" 2>&1 > out + + ${GETDNS_STUB_QUERY} -q @127.0.0.1:$PORT TXT quit. +) && grep '"n_requests: [0-9][0-9]*"' out | sed -e 's/^.*n_requests: //g' -e 's/".*$//g' \ + | awk -vQLIMIT=$QLIMIT '{ + if ($1 > QLIMIT) { + print "ERROR: More than "QLIMIT" outstanding queries!"; + exit(-1); + } +}' && echo "SUCCESS: No more than ${QLIMIT} outstanding queries" From b8f43c8acd81f4b61c4e584a83341740d3a6f2f0 Mon Sep 17 00:00:00 2001 From: Willem Toorop Date: Tue, 14 Mar 2017 15:20:56 +0100 Subject: [PATCH 3/8] Mention the number of simultaneous queries in error (and success) --- .../280-limit_outstanding_queries.test | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test index 828d0a96..d033f6ee 100644 --- a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test @@ -13,9 +13,19 @@ make && "./${TPKG_NAME}" | ( ${GETDNS_STUB_QUERY} -q @127.0.0.1:$PORT TXT quit. ) && grep '"n_requests: [0-9][0-9]*"' out | sed -e 's/^.*n_requests: //g' -e 's/".*$//g' \ - | awk -vQLIMIT=$QLIMIT '{ - if ($1 > QLIMIT) { - print "ERROR: More than "QLIMIT" outstanding queries!"; + | awk -vQLIMIT=$QLIMIT ' + +BEGIN{ + max_outstanding = 0; +} +{ + if ($1 > max_outstanding) + max_outstanding = $1; +} +END{ + if (max_outstanding > QLIMIT) { + print "ERROR: More than "QLIMIT" outstanding queries: "max_outstanding; exit(-1); - } -}' && echo "SUCCESS: No more than ${QLIMIT} outstanding queries" + } else + print "SUCCESS: No more than "QLIMIT" outstanding queries: "max_outstanding; +}' From 14c9f3aafcf9c41778d1224060ae7b9bb0ee9eaf Mon Sep 17 00:00:00 2001 From: Willem Toorop Date: Tue, 14 Mar 2017 17:17:56 +0100 Subject: [PATCH 4/8] Track netreqs "in flight" --- src/context.c | 23 +++++++++++++++++++ src/context.h | 17 +++++++++++++- src/dnssec.c | 2 +- src/general.c | 16 +++++++------ src/mdns.c | 4 ++-- src/request-internal.c | 6 ++++- src/stub.c | 10 ++++---- .../280-limit_outstanding_queries.test | 4 +++- 8 files changed, 64 insertions(+), 18 deletions(-) diff --git a/src/context.c b/src/context.c index 5421fd81..1f739fff 100644 --- a/src/context.c +++ b/src/context.c @@ -1330,6 +1330,8 @@ getdns_context_create_with_extended_memory_functions( _getdns_rbtree_init(&result->outbound_requests, transaction_id_cmp); _getdns_rbtree_init(&result->local_hosts, local_host_cmp); + /* TODO: Initialize pending_netreqs */ + result->netreqs_in_flight = 0; result->server = NULL; @@ -2980,6 +2982,27 @@ _getdns_context_request_timed_out(getdns_dns_req *dnsreq) _getdns_context_cancel_request(dnsreq); } + +void +_getdns_netreq_change_state(getdns_network_req *netreq, network_req_state new_state) +{ + if (!netreq) + return; + + if (netreq->state != NET_REQ_IN_FLIGHT) { + if (new_state == NET_REQ_IN_FLIGHT) + netreq->owner->context->netreqs_in_flight += 1; + netreq->state = new_state; + return; + } + if (new_state == NET_REQ_IN_FLIGHT) /* No change */ + return; + netreq->state = new_state; + netreq->owner->context->netreqs_in_flight -= 1; + /* TODO: Schedule pending netreqs + * when netreqs_in_flight < oustanding_queries */ +} + static void accumulate_outstanding_transactions(_getdns_rbnode_t *node, void* arg) { diff --git a/src/context.h b/src/context.h index e6ed49fb..6fe02d23 100644 --- a/src/context.h +++ b/src/context.h @@ -291,6 +291,11 @@ struct getdns_context { */ _getdns_rbtree_t outbound_requests; + /* network requests + */ + size_t netreqs_in_flight; + _getdns_rbtree_t pending_netreqs; + struct listen_set *server; /* Event loop extension. */ @@ -372,12 +377,22 @@ void _getdns_context_clear_outbound_request(getdns_dns_req *dnsreq); */ void _getdns_context_cancel_request(getdns_dns_req *dnsreq); - /* Calls user callback (with GETDNS_CALLBACK_TIMEOUT + response dict), then * cancels and frees the getdns_dns_req with _getdns_context_cancel_request() */ void _getdns_context_request_timed_out(getdns_dns_req *dnsreq); +/* Change state of the netreq req. + * - Increments context->netreqs_in_flight + * when state changes from NOT_SENT to IN_FLIGHT + * - Decrements context->netreqs_in_flight + * when state changes from IN_FLIGHT to FINISHED, TIMED_OUT or ERRORED + * - Resubmits NOT_SENT netreqs from context->pending_netreqs, + * when # pending_netreqs < limit_outstanding_queries + */ +void _getdns_netreq_change_state( + getdns_network_req *req, network_req_state new_state); + char *_getdns_strdup(const struct mem_funcs *mfs, const char *str); struct getdns_bindata *_getdns_bindata_copy( diff --git a/src/dnssec.c b/src/dnssec.c index b689aba5..b3e86bae 100644 --- a/src/dnssec.c +++ b/src/dnssec.c @@ -3040,7 +3040,7 @@ static void check_chain_complete(chain_head *chain) ; !r && (netreq = *netreq_p) ; netreq_p++) { - netreq->state = NET_REQ_NOT_SENT; + _getdns_netreq_change_state(netreq, NET_REQ_NOT_SENT); netreq->owner = dnsreq; r = _getdns_submit_netreq(netreq, &now_ms); } diff --git a/src/general.c b/src/general.c index c71a4944..3781e1da 100644 --- a/src/general.c +++ b/src/general.c @@ -130,7 +130,7 @@ _getdns_check_dns_req_complete(getdns_dns_req *dns_req) if ((r = _getdns_submit_netreq(netreq, &now_ms))) { if (r == DNS_REQ_FINISHED) return; - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_ERRORED); } } _getdns_check_dns_req_complete(dns_req); @@ -168,7 +168,7 @@ _getdns_check_dns_req_complete(getdns_dns_req *dns_req) if ((r = _getdns_submit_netreq(netreq, &now_ms))) { if (r == DNS_REQ_FINISHED) return; - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_ERRORED); } } _getdns_check_dns_req_complete(dns_req); @@ -209,7 +209,7 @@ ub_resolve_event_callback(void* arg, int rcode, void *pkt, int pkt_len, getdns_network_req *netreq = (getdns_network_req *) arg; getdns_dns_req *dns_req = netreq->owner; - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_FINISHED); /* parse */ if (getdns_apply_network_result( netreq, rcode, pkt, pkt_len, sec, why_bogus)) { @@ -227,7 +227,7 @@ ub_resolve_callback(void* arg, int err, struct ub_result* ub_res) getdns_network_req *netreq = (getdns_network_req *) arg; getdns_dns_req *dns_req = netreq->owner; - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_FINISHED); if (err != 0) { _getdns_call_user_callback(dns_req, NULL); return; @@ -259,6 +259,8 @@ _getdns_submit_netreq(getdns_network_req *netreq, uint64_t *now_ms) int ub_resolve_r; #endif + _getdns_netreq_change_state(netreq, NET_REQ_IN_FLIGHT); + #ifdef STUB_NATIVE_DNSSEC # ifdef DNSSEC_ROADBLOCK_AVOIDANCE @@ -457,7 +459,7 @@ getdns_general_ns(getdns_context *context, getdns_eventloop *loop, *return_netreq_p = NULL; return GETDNS_RETURN_GOOD; } - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_ERRORED); } } @@ -486,7 +488,7 @@ getdns_general_ns(getdns_context *context, getdns_eventloop *loop, *return_netreq_p = NULL; return GETDNS_RETURN_GOOD; } - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_ERRORED); } } /* Stop processing more namespaces, since there was a match */ @@ -509,7 +511,7 @@ getdns_general_ns(getdns_context *context, getdns_eventloop *loop, *return_netreq_p = NULL; return GETDNS_RETURN_GOOD; } - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_ERRORED); } } break; diff --git a/src/mdns.c b/src/mdns.c index 253bf972..441701c0 100644 --- a/src/mdns.c +++ b/src/mdns.c @@ -121,7 +121,7 @@ mdns_timeout_cb(void *userarg) #else close(netreq->fd); #endif - netreq->state = NET_REQ_TIMED_OUT; + _getdns_netreq_change_state(netreq, NET_REQ_TIMED_OUT); if (netreq->owner->user_callback) { netreq->debug_end_time = _getdns_get_time_as_uintt64(); (void)_getdns_context_request_timed_out(netreq->owner); @@ -182,7 +182,7 @@ mdns_udp_read_cb(void *userarg) netreq->response_len = read; netreq->debug_end_time = _getdns_get_time_as_uintt64(); - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_FINISHED); _getdns_check_dns_req_complete(dnsreq); } diff --git a/src/request-internal.c b/src/request-internal.c index 6c9cd9ec..8eec6b7a 100644 --- a/src/request-internal.c +++ b/src/request-internal.c @@ -118,7 +118,7 @@ netreq_reset(getdns_network_req *net_req) /* variables that need to be reset on reinit */ net_req->unbound_id = -1; - net_req->state = NET_REQ_NOT_SENT; + _getdns_netreq_change_state(net_req, NET_REQ_NOT_SENT); net_req->dnssec_status = GETDNS_DNSSEC_INDETERMINATE; net_req->tsig_status = GETDNS_DNSSEC_INDETERMINATE; net_req->query_id = 0; @@ -183,6 +183,10 @@ network_req_init(getdns_network_req *net_req, getdns_dns_req *owner, net_req->debug_tls_auth_status = GETDNS_AUTH_NONE; net_req->debug_udp = 0; + /* Scheduling, touch only via _getdns_netreq_change_state! + */ + net_req->state = NET_REQ_NOT_SENT; + if (max_query_sz == 0) { net_req->query = NULL; net_req->opt = NULL; diff --git a/src/stub.c b/src/stub.c index f7f1fddb..1fefaa8c 100644 --- a/src/stub.c +++ b/src/stub.c @@ -550,7 +550,7 @@ upstream_failed(getdns_upstream *upstream, int during_setup) netreq = (getdns_network_req *) _getdns_rbtree_first(&upstream->netreq_by_query_id); stub_cleanup(netreq); - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_FINISHED); _getdns_check_dns_req_complete(netreq->owner); } } @@ -580,7 +580,7 @@ stub_timeout_cb(void *userarg) DEBUG_STUB("%s %-35s: MSG: %p\n", STUB_DEBUG_CLEANUP, __FUNC__, (void*)netreq); stub_cleanup(netreq); - netreq->state = NET_REQ_TIMED_OUT; + _getdns_netreq_change_state(netreq, NET_REQ_TIMED_OUT); /* Handle upstream*/ if (netreq->fd >= 0) { #ifdef USE_WINSOCK @@ -1368,7 +1368,7 @@ stub_udp_read_cb(void *userarg) netreq->response_len = read; dnsreq->upstreams->current_udp = 0; netreq->debug_end_time = _getdns_get_time_as_uintt64(); - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_FINISHED); upstream->udp_responses++; #if defined(DAEMON_DEBUG) && DAEMON_DEBUG if (upstream->udp_responses == 1 || @@ -1495,7 +1495,7 @@ upstream_read_cb(void *userarg) DEBUG_STUB("%s %-35s: MSG: %p (read)\n", STUB_DEBUG_READ, __FUNC__, (void*)netreq); - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_FINISHED); netreq->response = upstream->tcp.read_buf; netreq->response_len = upstream->tcp.read_pos - upstream->tcp.read_buf; @@ -1614,7 +1614,7 @@ upstream_write_cb(void *userarg) #endif if (fallback_on_write(netreq) == STUB_TCP_ERROR) { /* TODO: Need new state to report transport unavailable*/ - netreq->state = NET_REQ_FINISHED; + _getdns_netreq_change_state(netreq, NET_REQ_FINISHED); _getdns_check_dns_req_complete(netreq->owner); } return; diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test index d033f6ee..37915e43 100644 --- a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test @@ -4,7 +4,9 @@ # use .tpkg.var.test for in test variable passing [ -f .tpkg.var.test ] && source .tpkg.var.test -QLIMIT=10 +# TODO: Change QLIMIT to 10 once implement pending netreq resubmission +# +QLIMIT=1000 make && "./${TPKG_NAME}" | ( read PORT ${GETDNS_STUB_QUERY} @127.0.0.1:$PORT TXT \ From 5ea181172a6325ecb6996295948f7e1b2ff33694 Mon Sep 17 00:00:00 2001 From: Willem Toorop Date: Wed, 15 Mar 2017 15:16:42 +0100 Subject: [PATCH 5/8] Reschedule pending netreqs --- src/context.c | 50 ++++---- src/context.h | 16 +-- src/general.c | 114 ++++++++++++++++-- src/general.h | 12 ++ src/request-internal.c | 1 + .../280-limit_outstanding_queries.test | 15 ++- 6 files changed, 159 insertions(+), 49 deletions(-) diff --git a/src/context.c b/src/context.c index 1f739fff..aabe13f5 100644 --- a/src/context.c +++ b/src/context.c @@ -1267,6 +1267,26 @@ NULL_update_callback( getdns_context *context, getdns_context_code_t code, void *userarg) { (void)context; (void)code; (void)userarg; } +static int +netreq_expiry_cmp(const void *id1, const void *id2) +{ + getdns_network_req *req1 = (getdns_network_req *)id1; + getdns_network_req *req2 = (getdns_network_req *)id2; + + return req1->owner->expires < req2->owner->expires ? -1 : + req1->owner->expires > req2->owner->expires ? 1 : + req1 < req2 ? -1 : + req1 > req2 ? 1 : 0; +} + +void _getdns_check_expired_pending_netreqs( + getdns_context *context, uint64_t *now_ms); +static void _getdns_check_expired_pending_netreqs_cb(void *arg) +{ + uint64_t now_ms = 0; + _getdns_check_expired_pending_netreqs((getdns_context *)arg, &now_ms); +} + /* * getdns_context_create * @@ -1330,8 +1350,15 @@ getdns_context_create_with_extended_memory_functions( _getdns_rbtree_init(&result->outbound_requests, transaction_id_cmp); _getdns_rbtree_init(&result->local_hosts, local_host_cmp); - /* TODO: Initialize pending_netreqs */ + _getdns_rbtree_init(&result->pending_netreqs, netreq_expiry_cmp); + result->first_pending_netreq = NULL; result->netreqs_in_flight = 0; + result->pending_timeout_event.userarg = result; + result->pending_timeout_event.read_cb = NULL; + result->pending_timeout_event.write_cb = NULL; + result->pending_timeout_event.timeout_cb = + _getdns_check_expired_pending_netreqs_cb; + result->pending_timeout_event.ev = NULL; result->server = NULL; @@ -2982,27 +3009,6 @@ _getdns_context_request_timed_out(getdns_dns_req *dnsreq) _getdns_context_cancel_request(dnsreq); } - -void -_getdns_netreq_change_state(getdns_network_req *netreq, network_req_state new_state) -{ - if (!netreq) - return; - - if (netreq->state != NET_REQ_IN_FLIGHT) { - if (new_state == NET_REQ_IN_FLIGHT) - netreq->owner->context->netreqs_in_flight += 1; - netreq->state = new_state; - return; - } - if (new_state == NET_REQ_IN_FLIGHT) /* No change */ - return; - netreq->state = new_state; - netreq->owner->context->netreqs_in_flight -= 1; - /* TODO: Schedule pending netreqs - * when netreqs_in_flight < oustanding_queries */ -} - static void accumulate_outstanding_transactions(_getdns_rbnode_t *node, void* arg) { diff --git a/src/context.h b/src/context.h index 6fe02d23..e1c838f6 100644 --- a/src/context.h +++ b/src/context.h @@ -294,7 +294,10 @@ struct getdns_context { /* network requests */ size_t netreqs_in_flight; - _getdns_rbtree_t pending_netreqs; + + _getdns_rbtree_t pending_netreqs; + getdns_network_req *first_pending_netreq; + getdns_eventloop_event pending_timeout_event; struct listen_set *server; @@ -382,17 +385,6 @@ void _getdns_context_cancel_request(getdns_dns_req *dnsreq); */ void _getdns_context_request_timed_out(getdns_dns_req *dnsreq); -/* Change state of the netreq req. - * - Increments context->netreqs_in_flight - * when state changes from NOT_SENT to IN_FLIGHT - * - Decrements context->netreqs_in_flight - * when state changes from IN_FLIGHT to FINISHED, TIMED_OUT or ERRORED - * - Resubmits NOT_SENT netreqs from context->pending_netreqs, - * when # pending_netreqs < limit_outstanding_queries - */ -void _getdns_netreq_change_state( - getdns_network_req *req, network_req_state new_state); - char *_getdns_strdup(const struct mem_funcs *mfs, const char *str); struct getdns_bindata *_getdns_bindata_copy( diff --git a/src/general.c b/src/general.c index 3781e1da..14b7ee40 100644 --- a/src/general.c +++ b/src/general.c @@ -90,14 +90,23 @@ void _getdns_check_dns_req_complete(getdns_dns_req *dns_req) { getdns_network_req **netreq_p, *netreq; - int results_found = 0, r; + int results_found = 0, timed_out = 1, r; uint64_t now_ms = 0; for (netreq_p = dns_req->netreqs; (netreq = *netreq_p); netreq_p++) if (!_getdns_netreq_finished(netreq)) return; - else if (netreq->response_len > 0) - results_found = 1; + else { + if (netreq->state != NET_REQ_TIMED_OUT) + timed_out = 0; + if (netreq->response_len > 0) + results_found = 1; + } + + if (timed_out) { + _getdns_context_request_timed_out(dns_req); + return; + } /* Do we have to check more suffixes on nxdomain/nodata? */ @@ -248,30 +257,113 @@ ub_resolve_callback(void* arg, int err, struct ub_result* ub_res) #endif +void _getdns_check_expired_pending_netreqs( + getdns_context *context, uint64_t *now_ms) +{ + getdns_network_req *first; + + assert(context); + + while (context->pending_netreqs.count) { + first = (getdns_network_req *) + _getdns_rbtree_first(&context->pending_netreqs); + + if (_getdns_ms_until_expiry2(first->owner->expires, now_ms) > 0) + break; + + (void) _getdns_rbtree_delete(&context->pending_netreqs, first); + _getdns_netreq_change_state(first, NET_REQ_TIMED_OUT); + _getdns_check_dns_req_complete(first->owner); + } + first = context->pending_netreqs.count ? (getdns_network_req *) + _getdns_rbtree_first(&context->pending_netreqs) : NULL; + + if (first == context->first_pending_netreq || + (first && context->first_pending_netreq && + first->owner->expires == context->first_pending_netreq->owner->expires)) + return; /* Nothing changed */ + + if (context->first_pending_netreq) + GETDNS_CLEAR_EVENT( context->extension + , &context->pending_timeout_event); + + if ((context->first_pending_netreq = first)) + GETDNS_SCHEDULE_EVENT( context->extension, -1, + _getdns_ms_until_expiry2(first->owner->expires, now_ms), + &context->pending_timeout_event); +} + +void +_getdns_netreq_change_state( + getdns_network_req *netreq, network_req_state new_state) +{ + getdns_context *context; + uint64_t now_ms; + + if (!netreq) + return; + + context = netreq->owner->context; + + if (netreq->state != NET_REQ_IN_FLIGHT) { + if (new_state == NET_REQ_IN_FLIGHT) + context->netreqs_in_flight += 1; + netreq->state = new_state; + return; + } + if (new_state == NET_REQ_IN_FLIGHT) /* No change */ + return; + netreq->state = new_state; + context->netreqs_in_flight -= 1; + + now_ms = 0; + while (context->limit_outstanding_queries > 0 && + context->pending_netreqs.count > 0 && + context->netreqs_in_flight < context->limit_outstanding_queries) { + + getdns_network_req *first = (getdns_network_req *) + _getdns_rbtree_first(&context->pending_netreqs); + (void) _getdns_rbtree_delete(&context->pending_netreqs, first); + (void) _getdns_submit_netreq(first, &now_ms); + } +} + int _getdns_submit_netreq(getdns_network_req *netreq, uint64_t *now_ms) { getdns_return_t r; getdns_dns_req *dns_req = netreq->owner; + getdns_context *context = dns_req->context; char name[1024]; int dnsreq_freed = 0; #ifdef HAVE_LIBUNBOUND int ub_resolve_r; #endif + if (context->limit_outstanding_queries > 0 && + context->netreqs_in_flight >= context->limit_outstanding_queries) { + + netreq->node.key = netreq; + if (_getdns_rbtree_insert( + &context->pending_netreqs, &netreq->node)) { + + _getdns_check_expired_pending_netreqs(context, now_ms); + return GETDNS_RETURN_GOOD; + } + } _getdns_netreq_change_state(netreq, NET_REQ_IN_FLIGHT); #ifdef STUB_NATIVE_DNSSEC # ifdef DNSSEC_ROADBLOCK_AVOIDANCE - if ((dns_req->context->resolution_type == GETDNS_RESOLUTION_RECURSING + if ((context->resolution_type == GETDNS_RESOLUTION_RECURSING && !dns_req->dnssec_roadblock_avoidance) || dns_req->avoid_dnssec_roadblocks) { # else - if ( dns_req->context->resolution_type == GETDNS_RESOLUTION_RECURSING) { + if ( context->resolution_type == GETDNS_RESOLUTION_RECURSING) { # endif #else - if ( dns_req->context->resolution_type == GETDNS_RESOLUTION_RECURSING + if ( context->resolution_type == GETDNS_RESOLUTION_RECURSING || dns_req->dnssec_return_status || dns_req->dnssec_return_only_secure || dns_req->dnssec_return_all_statuses @@ -297,15 +389,15 @@ _getdns_submit_netreq(getdns_network_req *netreq, uint64_t *now_ms) #ifdef HAVE_LIBUNBOUND dns_req->freed = &dnsreq_freed; #ifdef HAVE_UNBOUND_EVENT_API - if (_getdns_ub_loop_enabled(&dns_req->context->ub_loop)) - ub_resolve_r = ub_resolve_event(dns_req->context->unbound_ctx, - name, netreq->request_type, netreq->owner->request_class, + if (_getdns_ub_loop_enabled(&context->ub_loop)) + ub_resolve_r = ub_resolve_event(context->unbound_ctx, + name, netreq->request_type, dns_req->request_class, netreq, ub_resolve_event_callback, &(netreq->unbound_id)) ? GETDNS_RETURN_GENERIC_ERROR : GETDNS_RETURN_GOOD; else #endif - ub_resolve_r = ub_resolve_async(dns_req->context->unbound_ctx, - name, netreq->request_type, netreq->owner->request_class, + ub_resolve_r = ub_resolve_async(context->unbound_ctx, + name, netreq->request_type, dns_req->request_class, netreq, ub_resolve_callback, &(netreq->unbound_id)) ? GETDNS_RETURN_GENERIC_ERROR : GETDNS_RETURN_GOOD; if (dnsreq_freed) diff --git a/src/general.h b/src/general.h index dcd9b9be..e0860c78 100644 --- a/src/general.h +++ b/src/general.h @@ -45,6 +45,18 @@ #define DNS_REQ_FINISHED -1 void _getdns_call_user_callback(getdns_dns_req *, getdns_dict *); + +/* Change state of the netreq req. + * - Increments context->netreqs_in_flight + * when state changes from NOT_SENT to IN_FLIGHT + * - Decrements context->netreqs_in_flight + * when state changes from IN_FLIGHT to FINISHED, TIMED_OUT or ERRORED + * - Resubmits NOT_SENT netreqs from context->pending_netreqs, + * when # pending_netreqs < limit_outstanding_queries + */ +void _getdns_netreq_change_state( + getdns_network_req *netreq, network_req_state new_state); + void _getdns_check_dns_req_complete(getdns_dns_req *dns_req); int _getdns_submit_netreq(getdns_network_req *netreq, uint64_t *now_ms); diff --git a/src/request-internal.c b/src/request-internal.c index 8eec6b7a..2259286e 100644 --- a/src/request-internal.c +++ b/src/request-internal.c @@ -43,6 +43,7 @@ #include "dict.h" #include "debug.h" #include "convert.h" +#include "general.h" /* MAXIMUM_TSIG_SPACE = TSIG name (dname) : 256 * TSIG type (uint16_t) : 2 diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test index 37915e43..177cc1c7 100644 --- a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.test @@ -4,9 +4,15 @@ # use .tpkg.var.test for in test variable passing [ -f .tpkg.var.test ] && source .tpkg.var.test -# TODO: Change QLIMIT to 10 once implement pending netreq resubmission -# -QLIMIT=1000 + +QLIMIT=64 +NQUERIES=`wc "./${TPKG_NAME}.queries"|sed 's/ .*$//g'` + +# Test will take NQUERIES / QLIMIT * answer delay +# For current parameters this is 1000 / 64 * 0.3 = 4.6875 +# which is smaller than 5 seconds default query timeout value, +# so the test should succeed. + make && "./${TPKG_NAME}" | ( read PORT ${GETDNS_STUB_QUERY} @127.0.0.1:$PORT TXT \ @@ -15,7 +21,7 @@ make && "./${TPKG_NAME}" | ( ${GETDNS_STUB_QUERY} -q @127.0.0.1:$PORT TXT quit. ) && grep '"n_requests: [0-9][0-9]*"' out | sed -e 's/^.*n_requests: //g' -e 's/".*$//g' \ - | awk -vQLIMIT=$QLIMIT ' + | awk -vQLIMIT=$QLIMIT -vNQUERIES=$NQUERIES ' BEGIN{ max_outstanding = 0; @@ -25,6 +31,7 @@ BEGIN{ max_outstanding = $1; } END{ + printf("%d of %d queries answered (%.1f%%)\n", NR, NQUERIES, (NR / NQUERIES * 100)); if (max_outstanding > QLIMIT) { print "ERROR: More than "QLIMIT" outstanding queries: "max_outstanding; exit(-1); From 8b09633c94cc3aabb1c42c5d4cc43b0af8ecf26b Mon Sep 17 00:00:00 2001 From: Willem Toorop Date: Mon, 20 Mar 2017 11:03:15 +0100 Subject: [PATCH 6/8] Bug and mem-leak fix --- .../280-limit_outstanding_queries.Makefile | 2 +- .../280-limit_outstanding_queries.c | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.Makefile b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.Makefile index c297df6d..70d86616 100644 --- a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.Makefile +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.Makefile @@ -2,7 +2,7 @@ builddir = @BUILDDIR@ testname = @TPKG_NAME@ LIBTOOL = $(builddir)/libtool -CFLAGS=-I$(builddir)/src +CFLAGS=-Wall -Wextra -I$(builddir)/src LDLIBS=$(builddir)/src/libgetdns.la .SUFFIXES: .c .o .a .lo .h diff --git a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.c b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.c index 78107b57..1467ec34 100644 --- a/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.c +++ b/src/test/tpkg/280-limit_outstanding_queries.tpkg/280-limit_outstanding_queries.c @@ -64,12 +64,12 @@ void delay_cb(void *userarg) void handler(getdns_context *context, getdns_callback_type_t callback_type, getdns_dict *request, void *userarg, getdns_transaction_t request_id) { - transaction_t *trans; + transaction_t *trans = NULL; getdns_bindata *qname; char nreq_str[255]; - getdns_bindata nreq_bd = { 0, nreq_str }; + getdns_bindata nreq_bd = { 0, (void *)nreq_str }; - (void) userarg; + (void) userarg; (void)callback_type; nreq_bd.size = snprintf(nreq_str, sizeof(nreq_str), "n_requests: %d", ++n_requests); if (getdns_dict_get_bindata(request, "/question/qname", &qname) || @@ -83,8 +83,8 @@ void handler(getdns_context *context, getdns_callback_type_t callback_type, qname->data[3] == 'i' && qname->data[4] == 't') { (void) getdns_reply(context, request, request_id); - (void) getdns_context_set_listen_addresses(trans->context, NULL, NULL, NULL); - n_requests -= 1; + (void) getdns_context_set_listen_addresses(context, NULL, NULL, NULL); + getdns_dict_destroy(request); return; } else if (!(trans = malloc(sizeof(transaction_t)))) @@ -102,6 +102,8 @@ void handler(getdns_context *context, getdns_callback_type_t callback_type, fprintf(stderr, "Could not schedule delay\n"); else return; } + getdns_dict_destroy(trans->request); + if (trans) free(trans); exit(EXIT_FAILURE); } From 0891e16147d77d0bbe1a6c2cd8927776f9500977 Mon Sep 17 00:00:00 2001 From: Willem Toorop Date: Mon, 20 Mar 2017 15:20:17 +0100 Subject: [PATCH 7/8] Pend netreqs when out of filedescriptors --- src/general.c | 17 +- src/stub.c | 32 +- .../285-out_of_filedescriptors.Makefile | 15 + .../285-out_of_filedescriptors.c | 143 +++ .../285-out_of_filedescriptors.dsc | 16 + .../285-out_of_filedescriptors.pre | 14 + .../285-out_of_filedescriptors.queries | 1000 +++++++++++++++++ .../285-out_of_filedescriptors.test | 48 + 8 files changed, 1276 insertions(+), 9 deletions(-) create mode 100644 src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.Makefile create mode 100644 src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.c create mode 100644 src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.dsc create mode 100644 src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.pre create mode 100644 src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.queries create mode 100644 src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.test diff --git a/src/general.c b/src/general.c index 14b7ee40..6ed44004 100644 --- a/src/general.c +++ b/src/general.c @@ -299,6 +299,7 @@ _getdns_netreq_change_state( { getdns_context *context; uint64_t now_ms; + getdns_network_req *prev; if (!netreq) return; @@ -317,12 +318,22 @@ _getdns_netreq_change_state( context->netreqs_in_flight -= 1; now_ms = 0; - while (context->limit_outstanding_queries > 0 && - context->pending_netreqs.count > 0 && - context->netreqs_in_flight < context->limit_outstanding_queries) { + prev = NULL; + while (context->pending_netreqs.count > 0 && + ( context->limit_outstanding_queries > context->netreqs_in_flight + || context->limit_outstanding_queries == 0 )) { getdns_network_req *first = (getdns_network_req *) _getdns_rbtree_first(&context->pending_netreqs); + + /* To prevent loops due to _getdns_submit_netreq re-inserting + * because of errno == EMFILE + */ + if (first == prev) + break; + else + prev = first; + (void) _getdns_rbtree_delete(&context->pending_netreqs, first); (void) _getdns_submit_netreq(first, &now_ms); } diff --git a/src/stub.c b/src/stub.c index 1fefaa8c..20695d25 100644 --- a/src/stub.c +++ b/src/stub.c @@ -54,15 +54,18 @@ typedef u_short sa_family_t; #define _getdns_EWOULDBLOCK (WSAGetLastError() == WSATRY_AGAIN ||\ WSAGetLastError() == WSAEWOULDBLOCK) #define _getdns_EINPROGRESS (WSAGetLastError() == WSAEINPROGRESS) +#define _getdns_EMFILE (WSAGetLastError() == WSAEMFILE) #else #define _getdns_EWOULDBLOCK (errno == EAGAIN || errno == EWOULDBLOCK) #define _getdns_EINPROGRESS (errno == EINPROGRESS) +#define _getdns_EMFILE (errno == EMFILE) #endif /* WSA TODO: * STUB_TCP_WOULDBLOCK added to deal with edge triggered event loops (versus * level triggered). See also lines containing WSA TODO below... */ +#define STUB_TRY_AGAIN_LATER -24 /* EMFILE, i.e. Out of OS resources */ #define STUB_NO_AUTH -8 /* Existing TLS connection is not authenticated */ #define STUB_CONN_GONE -7 /* Connection has failed, clear queue*/ #define STUB_TCP_WOULDBLOCK -6 @@ -1905,8 +1908,14 @@ upstream_find_for_netreq(getdns_network_req *netreq) upstream = upstream_find_for_transport(netreq, netreq->transports[i], &fd); - if (fd == -1 || !upstream) + if (!upstream) continue; + + if (fd == -1) { + if (_getdns_EMFILE) + return STUB_TRY_AGAIN_LATER; + return -1; + } netreq->transport_current = i; netreq->upstream = upstream; netreq->keepalive_sent = 0; @@ -2030,10 +2039,15 @@ upstream_schedule_netreq(getdns_upstream *upstream, getdns_network_req *netreq) getdns_return_t _getdns_submit_stub_request(getdns_network_req *netreq, uint64_t *now_ms) { + int fd = -1; + getdns_dns_req *dnsreq; + getdns_context *context; + DEBUG_STUB("%s %-35s: MSG: %p TYPE: %d\n", STUB_DEBUG_ENTRY, __FUNC__, (void*)netreq, netreq->request_type); - int fd = -1; - getdns_dns_req *dnsreq = netreq->owner; + + dnsreq = netreq->owner; + context = dnsreq->context; /* This does a best effort to get a initial fd. * All other set up is done async*/ @@ -2041,9 +2055,15 @@ _getdns_submit_stub_request(getdns_network_req *netreq, uint64_t *now_ms) if (fd == -1) return GETDNS_RETURN_NO_UPSTREAM_AVAILABLE; - getdns_transport_list_t transport = - netreq->transports[netreq->transport_current]; - switch(transport) { + else if (fd == STUB_TRY_AGAIN_LATER) { + _getdns_netreq_change_state(netreq, NET_REQ_NOT_SENT); + netreq->node.key = netreq; + if (_getdns_rbtree_insert( + &context->pending_netreqs, &netreq->node)) + return GETDNS_RETURN_GOOD; + return GETDNS_RETURN_NO_UPSTREAM_AVAILABLE; + } + switch(netreq->transports[netreq->transport_current]) { case GETDNS_TRANSPORT_UDP: netreq->fd = fd; GETDNS_CLEAR_EVENT(dnsreq->loop, &netreq->event); diff --git a/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.Makefile b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.Makefile new file mode 100644 index 00000000..70d86616 --- /dev/null +++ b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.Makefile @@ -0,0 +1,15 @@ +builddir = @BUILDDIR@ +testname = @TPKG_NAME@ +LIBTOOL = $(builddir)/libtool + +CFLAGS=-Wall -Wextra -I$(builddir)/src +LDLIBS=$(builddir)/src/libgetdns.la + +.SUFFIXES: .c .o .a .lo .h + +.c.lo: + $(LIBTOOL) --quiet --tag=CC --mode=compile $(CC) $(CFLAGS) -c $< -o $@ + +$(testname): $(testname).lo + $(LIBTOOL) --tag=CC --mode=link $(CC) $(LDLIBS) $(LDFLAGS) -o $(testname) $(testname).lo + diff --git a/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.c b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.c new file mode 100644 index 00000000..e495466d --- /dev/null +++ b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.c @@ -0,0 +1,143 @@ +/* + * delaydns.c - A DNS proxy that adds delay to replies + * + * Copyright (c) 2016, NLnet Labs. All rights reserved. + * + * This software is open source. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of the NLNET LABS nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + + +static int n_requests = 0; + +typedef struct transaction_t { + getdns_transaction_t request_id; + getdns_dict *request; + + getdns_context *context; + getdns_eventloop *loop; + getdns_eventloop_event ev; +} transaction_t; + + +void delay_cb(void *userarg) +{ + transaction_t *trans = userarg; + + trans->loop->vmt->clear(trans->loop, &trans->ev); + (void) getdns_reply(trans->context, trans->request, trans->request_id); + getdns_dict_destroy(trans->request); + free(trans); + n_requests -= 1; +} + +void handler(getdns_context *context, getdns_callback_type_t callback_type, + getdns_dict *request, void *userarg, getdns_transaction_t request_id) +{ + transaction_t *trans = NULL; + getdns_bindata *qname; + char nreq_str[255]; + getdns_bindata nreq_bd = { 0, (void *)nreq_str }; + + (void) userarg; (void)callback_type; + nreq_bd.size = snprintf(nreq_str, sizeof(nreq_str), "n_requests: %d", ++n_requests); + + if (getdns_dict_get_bindata(request, "/question/qname", &qname) || + getdns_dict_set_bindata(request, "/answer/0/name", qname) || + getdns_dict_set_int(request, "/answer/0/type", GETDNS_RRTYPE_TXT) || + getdns_dict_set_bindata(request, "/answer/0/rdata/txt_strings/-", &nreq_bd)) + fprintf(stderr, "Request init error\n"); + + else if (qname->size >= 6 && qname->data[0] == 4 && + qname->data[1] == 'q' && qname->data[2] == 'u' && + qname->data[3] == 'i' && qname->data[4] == 't') { + + (void) getdns_reply(context, request, request_id); + (void) getdns_context_set_listen_addresses(context, NULL, NULL, NULL); + getdns_dict_destroy(request); + return; + + } else if (!(trans = malloc(sizeof(transaction_t)))) + perror("memerror"); + else { + char *fqdn; + getdns_convert_dns_name_to_fqdn(qname, &fqdn); + + (void) memset(trans, 0, sizeof(transaction_t)); + trans->request_id = request_id; + trans->request = request; + trans->context = context; + trans->ev.userarg = trans; + trans->ev.timeout_cb = delay_cb; + + fprintf(stderr, "sched delay for query %s, n_request %d\n", fqdn, (int)n_requests); + free(fqdn); + if (getdns_context_get_eventloop(context, &trans->loop) + || trans->loop->vmt->schedule(trans->loop, -1, 300, &trans->ev)) + fprintf(stderr, "Could not schedule delay\n"); + else return; + } + getdns_dict_destroy(trans->request); + if (trans) free(trans); + exit(EXIT_FAILURE); +} + +int main() +{ + getdns_context *context = NULL; + getdns_list *listeners = NULL; + getdns_dict *address = NULL; + uint32_t port = 18000; + getdns_return_t r; + + if ((r = getdns_str2list("[ 127.0.0.1:18000 ]", &listeners)) || + (r = getdns_list_get_dict(listeners, 0, &address)) || + (r = getdns_context_create(&context, 0))) + fprintf(stderr, "Error initializing: "); + + else while (++port < 18200 && + !(r = getdns_dict_set_int(address, "port", port)) && + (r = getdns_context_set_listen_addresses( + context, listeners, NULL, handler))) + ; /* pass */ + + if (r) fprintf(stderr, "%s\n", getdns_get_errorstr_by_id(r)); + else { + fprintf(stdout, "%d\n", (int)port); + fflush(stdout); + getdns_context_run(context); + } + getdns_list_destroy(listeners); + getdns_context_destroy(context); + return r; +} diff --git a/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.dsc b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.dsc new file mode 100644 index 00000000..6b00775a --- /dev/null +++ b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.dsc @@ -0,0 +1,16 @@ +BaseName: 285-out_of_filedescriptors +Version: 1.0 +Description: Test if outstanding queries setting is obeyed +CreationDate: ma 20 mrt 2017 15:17:45 CET +Maintainer: Willem Toorop +Category: +Component: +CmdDepends: +Depends: 210-stub-only-link.tpkg +Help: +Pre: 285-out_of_filedescriptors.pre +Post: +Test: 285-out_of_filedescriptors.test +AuxFiles: +Passed: +Failure: diff --git a/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.pre b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.pre new file mode 100644 index 00000000..6e7ff3ff --- /dev/null +++ b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.pre @@ -0,0 +1,14 @@ +# #-- 285-out_of_filedescriptors.test --# +# source the master var file when it's there +[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master +# use .tpkg.var.test for in test variable passing +[ -f .tpkg.var.test ] && source .tpkg.var.test + +( + grep '^CC=' "${BUILDDIR}/build-stub-only/src/Makefile" + grep '^LDFLAGS=' "${BUILDDIR}/build-stub-only/src/Makefile" + + BUILDDIR4SED=`echo "${BUILDDIR}/build-stub-only" | sed 's/\//\\\\\//g'` + sed -e "s/@BUILDDIR@/${BUILDDIR4SED}/g" \ + -e "s/@TPKG_NAME@/${TPKG_NAME}/g" "${TPKG_NAME}.Makefile" +) > Makefile diff --git a/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.queries b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.queries new file mode 100644 index 00000000..0cc2103d --- /dev/null +++ b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.queries @@ -0,0 +1,1000 @@ +q1. +q2. +q3. +q4. +q5. +q6. +q7. +q8. +q9. +q10. +q11. +q12. +q13. +q14. +q15. +q16. +q17. +q18. +q19. +q20. +q21. +q22. +q23. +q24. +q25. +q26. +q27. +q28. +q29. +q30. +q31. +q32. +q33. +q34. +q35. +q36. +q37. +q38. +q39. +q40. +q41. +q42. +q43. +q44. +q45. +q46. +q47. +q48. +q49. +q50. +q51. +q52. +q53. +q54. +q55. +q56. +q57. +q58. +q59. +q60. +q61. +q62. +q63. +q64. +q65. +q66. +q67. +q68. +q69. +q70. +q71. +q72. +q73. +q74. +q75. +q76. +q77. +q78. +q79. +q80. +q81. +q82. +q83. +q84. +q85. +q86. +q87. +q88. +q89. +q90. +q91. +q92. +q93. +q94. +q95. +q96. +q97. +q98. +q99. +q100. +q101. +q102. +q103. +q104. +q105. +q106. +q107. +q108. +q109. +q110. +q111. +q112. +q113. +q114. +q115. +q116. +q117. +q118. +q119. +q120. +q121. +q122. +q123. +q124. +q125. +q126. +q127. +q128. +q129. +q130. +q131. +q132. +q133. +q134. +q135. +q136. +q137. +q138. +q139. +q140. +q141. +q142. +q143. +q144. +q145. +q146. +q147. +q148. +q149. +q150. +q151. +q152. +q153. +q154. +q155. +q156. +q157. +q158. +q159. +q160. +q161. +q162. +q163. +q164. +q165. +q166. +q167. +q168. +q169. +q170. +q171. +q172. +q173. +q174. +q175. +q176. +q177. +q178. +q179. +q180. +q181. +q182. +q183. +q184. +q185. +q186. +q187. +q188. +q189. +q190. +q191. +q192. +q193. +q194. +q195. +q196. +q197. +q198. +q199. +q200. +q201. +q202. +q203. +q204. +q205. +q206. +q207. +q208. +q209. +q210. +q211. +q212. +q213. +q214. +q215. +q216. +q217. +q218. +q219. +q220. +q221. +q222. +q223. +q224. +q225. +q226. +q227. +q228. +q229. +q230. +q231. +q232. +q233. +q234. +q235. +q236. +q237. +q238. +q239. +q240. +q241. +q242. +q243. +q244. +q245. +q246. +q247. +q248. +q249. +q250. +q251. +q252. +q253. +q254. +q255. +q256. +q257. +q258. +q259. +q260. +q261. +q262. +q263. +q264. +q265. +q266. +q267. +q268. +q269. +q270. +q271. +q272. +q273. +q274. +q275. +q276. +q277. +q278. +q279. +q280. +q281. +q282. +q283. +q284. +q285. +q286. +q287. +q288. +q289. +q290. +q291. +q292. +q293. +q294. +q295. +q296. +q297. +q298. +q299. +q300. +q301. +q302. +q303. +q304. +q305. +q306. +q307. +q308. +q309. +q310. +q311. +q312. +q313. +q314. +q315. +q316. +q317. +q318. +q319. +q320. +q321. +q322. +q323. +q324. +q325. +q326. +q327. +q328. +q329. +q330. +q331. +q332. +q333. +q334. +q335. +q336. +q337. +q338. +q339. +q340. +q341. +q342. +q343. +q344. +q345. +q346. +q347. +q348. +q349. +q350. +q351. +q352. +q353. +q354. +q355. +q356. +q357. +q358. +q359. +q360. +q361. +q362. +q363. +q364. +q365. +q366. +q367. +q368. +q369. +q370. +q371. +q372. +q373. +q374. +q375. +q376. +q377. +q378. +q379. +q380. +q381. +q382. +q383. +q384. +q385. +q386. +q387. +q388. +q389. +q390. +q391. +q392. +q393. +q394. +q395. +q396. +q397. +q398. +q399. +q400. +q401. +q402. +q403. +q404. +q405. +q406. +q407. +q408. +q409. +q410. +q411. +q412. +q413. +q414. +q415. +q416. +q417. +q418. +q419. +q420. +q421. +q422. +q423. +q424. +q425. +q426. +q427. +q428. +q429. +q430. +q431. +q432. +q433. +q434. +q435. +q436. +q437. +q438. +q439. +q440. +q441. +q442. +q443. +q444. +q445. +q446. +q447. +q448. +q449. +q450. +q451. +q452. +q453. +q454. +q455. +q456. +q457. +q458. +q459. +q460. +q461. +q462. +q463. +q464. +q465. +q466. +q467. +q468. +q469. +q470. +q471. +q472. +q473. +q474. +q475. +q476. +q477. +q478. +q479. +q480. +q481. +q482. +q483. +q484. +q485. +q486. +q487. +q488. +q489. +q490. +q491. +q492. +q493. +q494. +q495. +q496. +q497. +q498. +q499. +q500. +q501. +q502. +q503. +q504. +q505. +q506. +q507. +q508. +q509. +q510. +q511. +q512. +q513. +q514. +q515. +q516. +q517. +q518. +q519. +q520. +q521. +q522. +q523. +q524. +q525. +q526. +q527. +q528. +q529. +q530. +q531. +q532. +q533. +q534. +q535. +q536. +q537. +q538. +q539. +q540. +q541. +q542. +q543. +q544. +q545. +q546. +q547. +q548. +q549. +q550. +q551. +q552. +q553. +q554. +q555. +q556. +q557. +q558. +q559. +q560. +q561. +q562. +q563. +q564. +q565. +q566. +q567. +q568. +q569. +q570. +q571. +q572. +q573. +q574. +q575. +q576. +q577. +q578. +q579. +q580. +q581. +q582. +q583. +q584. +q585. +q586. +q587. +q588. +q589. +q590. +q591. +q592. +q593. +q594. +q595. +q596. +q597. +q598. +q599. +q600. +q601. +q602. +q603. +q604. +q605. +q606. +q607. +q608. +q609. +q610. +q611. +q612. +q613. +q614. +q615. +q616. +q617. +q618. +q619. +q620. +q621. +q622. +q623. +q624. +q625. +q626. +q627. +q628. +q629. +q630. +q631. +q632. +q633. +q634. +q635. +q636. +q637. +q638. +q639. +q640. +q641. +q642. +q643. +q644. +q645. +q646. +q647. +q648. +q649. +q650. +q651. +q652. +q653. +q654. +q655. +q656. +q657. +q658. +q659. +q660. +q661. +q662. +q663. +q664. +q665. +q666. +q667. +q668. +q669. +q670. +q671. +q672. +q673. +q674. +q675. +q676. +q677. +q678. +q679. +q680. +q681. +q682. +q683. +q684. +q685. +q686. +q687. +q688. +q689. +q690. +q691. +q692. +q693. +q694. +q695. +q696. +q697. +q698. +q699. +q700. +q701. +q702. +q703. +q704. +q705. +q706. +q707. +q708. +q709. +q710. +q711. +q712. +q713. +q714. +q715. +q716. +q717. +q718. +q719. +q720. +q721. +q722. +q723. +q724. +q725. +q726. +q727. +q728. +q729. +q730. +q731. +q732. +q733. +q734. +q735. +q736. +q737. +q738. +q739. +q740. +q741. +q742. +q743. +q744. +q745. +q746. +q747. +q748. +q749. +q750. +q751. +q752. +q753. +q754. +q755. +q756. +q757. +q758. +q759. +q760. +q761. +q762. +q763. +q764. +q765. +q766. +q767. +q768. +q769. +q770. +q771. +q772. +q773. +q774. +q775. +q776. +q777. +q778. +q779. +q780. +q781. +q782. +q783. +q784. +q785. +q786. +q787. +q788. +q789. +q790. +q791. +q792. +q793. +q794. +q795. +q796. +q797. +q798. +q799. +q800. +q801. +q802. +q803. +q804. +q805. +q806. +q807. +q808. +q809. +q810. +q811. +q812. +q813. +q814. +q815. +q816. +q817. +q818. +q819. +q820. +q821. +q822. +q823. +q824. +q825. +q826. +q827. +q828. +q829. +q830. +q831. +q832. +q833. +q834. +q835. +q836. +q837. +q838. +q839. +q840. +q841. +q842. +q843. +q844. +q845. +q846. +q847. +q848. +q849. +q850. +q851. +q852. +q853. +q854. +q855. +q856. +q857. +q858. +q859. +q860. +q861. +q862. +q863. +q864. +q865. +q866. +q867. +q868. +q869. +q870. +q871. +q872. +q873. +q874. +q875. +q876. +q877. +q878. +q879. +q880. +q881. +q882. +q883. +q884. +q885. +q886. +q887. +q888. +q889. +q890. +q891. +q892. +q893. +q894. +q895. +q896. +q897. +q898. +q899. +q900. +q901. +q902. +q903. +q904. +q905. +q906. +q907. +q908. +q909. +q910. +q911. +q912. +q913. +q914. +q915. +q916. +q917. +q918. +q919. +q920. +q921. +q922. +q923. +q924. +q925. +q926. +q927. +q928. +q929. +q930. +q931. +q932. +q933. +q934. +q935. +q936. +q937. +q938. +q939. +q940. +q941. +q942. +q943. +q944. +q945. +q946. +q947. +q948. +q949. +q950. +q951. +q952. +q953. +q954. +q955. +q956. +q957. +q958. +q959. +q960. +q961. +q962. +q963. +q964. +q965. +q966. +q967. +q968. +q969. +q970. +q971. +q972. +q973. +q974. +q975. +q976. +q977. +q978. +q979. +q980. +q981. +q982. +q983. +q984. +q985. +q986. +q987. +q988. +q989. +q990. +q991. +q992. +q993. +q994. +q995. +q996. +q997. +q998. +q999. +q1000. diff --git a/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.test b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.test new file mode 100644 index 00000000..64a53cfb --- /dev/null +++ b/src/test/tpkg/285-out_of_filedescriptors.tpkg/285-out_of_filedescriptors.test @@ -0,0 +1,48 @@ +# #-- 285-out_of_filedescriptors.test --# +# source the master var file when it's there +[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master +# use .tpkg.var.test for in test variable passing +[ -f .tpkg.var.test ] && source .tpkg.var.test + + +QLIMIT=79 +NQUERIES=`wc "./${TPKG_NAME}.queries"|sed 's/ .*$//g'` + +# This time the query limit is set by setting the maximum open +# filedescriptors. We seem to be needing a higher QLIMIT, than +# with limit_outstanding_queries unit test. +# +# 4 filedescriptors are already needed for overhead (logging etc), +# but experiments showed that to prevent timeouts, we should +# have a higher value than 72 at least. +# +# Test will take NQUERIES / QLIMIT * answer delay +# For current parameters this is 1000 / 75 * 0.3 = 4.0 +# which is smaller than 5 seconds default query timeout value, +# so the test should succeed. + +make && "./${TPKG_NAME}" | ( + read PORT + ulimit -n $QLIMIT + ${GETDNS_STUB_QUERY} @127.0.0.1:$PORT TXT \ + -a -F "./${TPKG_NAME}.queries" 2>&1 > out + + ${GETDNS_STUB_QUERY} -q @127.0.0.1:$PORT TXT quit. +) && grep '"n_requests: [0-9][0-9]*"' out | sed -e 's/^.*n_requests: //g' -e 's/".*$//g' \ + | awk -vQLIMIT=$QLIMIT -vNQUERIES=$NQUERIES ' + +BEGIN{ + max_outstanding = 0; +} +{ + if ($1 > max_outstanding) + max_outstanding = $1; +} +END{ + printf("%d of %d queries answered (%.1f%%)\n", NR, NQUERIES, (NR / NQUERIES * 100)); + if (max_outstanding > QLIMIT) { + print "ERROR: More than "QLIMIT" outstanding queries: "max_outstanding; + exit(-1); + } else + print "SUCCESS: No more than "QLIMIT" outstanding queries: "max_outstanding; +}' From 0048066a2a6c7e99f0767ce00f3293f88f6e3f7e Mon Sep 17 00:00:00 2001 From: Willem Toorop Date: Mon, 20 Mar 2017 16:15:02 +0100 Subject: [PATCH 8/8] Test op coding practices --- .../070-coding-practice.dsc | 16 ++++++++ .../070-coding-practice.pre | 14 +++++++ .../070-coding-practice.test | 40 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.dsc create mode 100644 src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.pre create mode 100644 src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.test diff --git a/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.dsc b/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.dsc new file mode 100644 index 00000000..b2101c47 --- /dev/null +++ b/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.dsc @@ -0,0 +1,16 @@ +BaseName: 070-coding-practice +Version: 1.0 +Description: Check for non-recommended coding practices +CreationDate: ma 20 mrt 2017 15:55:19 CET +Maintainer: Willem Toorop +Category: +Component: +CmdDepends: +Depends: +Help: +Pre: 070-coding-practice.pre +Post: +Test: 070-coding-practice.test +AuxFiles: +Passed: +Failure: diff --git a/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.pre b/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.pre new file mode 100644 index 00000000..150497af --- /dev/null +++ b/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.pre @@ -0,0 +1,14 @@ +# #-- 070-coding-practice.pre--# +# source the master var file when it's there +if [ -f ../.tpkg.var.master ] +then + source ../.tpkg.var.master +else + ( + cd .. + [ -f "${TPKG_SRCDIR}/setup-env.sh" ] \ + && sh "${TPKG_SRCDIR}/setup-env.sh" + ) && source ../.tpkg.var.master +fi +# use .tpkg.var.test for in test variable passing +[ -f .tpkg.var.test ] && source .tpkg.var.test diff --git a/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.test b/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.test new file mode 100644 index 00000000..b0360279 --- /dev/null +++ b/src/test/tpkg/070-coding-practice.tpkg/070-coding-practice.test @@ -0,0 +1,40 @@ +# #-- 070-coding-practice.test --# +# source the master var file when it's there +[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master +# use .tpkg.var.test for in test variable passing +[ -f .tpkg.var.test ] && source .tpkg.var.test + +rm -f report.txt +( + cd ${SRCROOT}/src + if [ `grep '[^!=]=[ ][ ]*NET_REQ_' *.[ch] */*.[ch] | wc -l` -gt 1 ] + then + echo "*** " + echo "*** Setting getdns_network_req->state should be done via" + echo "*** _getdns_netreq_change_state() only, for anticipating" + echo "*** running out of filedescriptors (sockets) and for the" + echo "*** limit_outstanding_queries feature." + echo "*** " + grep '[^!=]=[ ][ ]*NET_REQ_' *.[ch] */*.[ch] + echo "" + fi +) >> report.txt +( + cd ${SRCROOT}/src + if [ `grep '__FUNCTION__' *.[ch] */*.[ch] | wc -l` -gt 0 ] + then + echo "*** " + echo "*** Use __FUNC__ instead of __FUNCTION__ for portability" + echo "*** __FUNC__ is aliases in config.h to name to be used" + echo "*** for the system with a #define" + echo "*** " + grep '__FUNCION__' *.[ch] */*.[ch] + echo "" + fi +) >> report.txt + +if [ -s report.txt ] +then + cat report.txt + false +fi