mirror of https://github.com/getdnsapi/getdns.git
Pend netreqs when out of filedescriptors
This commit is contained in:
Willem Toorop
parent
8b09633c94
commit
0891e16147
|
|
@ -299,6 +299,7 @@ _getdns_netreq_change_state(
|
|||
{
|
||||
getdns_context *context;
|
||||
uint64_t now_ms;
|
||||
getdns_network_req *prev;
|
||||
|
||||
if (!netreq)
|
||||
return;
|
||||
|
|
@ -317,12 +318,22 @@ _getdns_netreq_change_state(
|
|||
context->netreqs_in_flight -= 1;
|
||||
|
||||
now_ms = 0;
|
||||
while (context->limit_outstanding_queries > 0 &&
|
||||
context->pending_netreqs.count > 0 &&
|
||||
context->netreqs_in_flight < context->limit_outstanding_queries) {
|
||||
prev = NULL;
|
||||
while (context->pending_netreqs.count > 0 &&
|
||||
( context->limit_outstanding_queries > context->netreqs_in_flight
|
||||
|| context->limit_outstanding_queries == 0 )) {
|
||||
|
||||
getdns_network_req *first = (getdns_network_req *)
|
||||
_getdns_rbtree_first(&context->pending_netreqs);
|
||||
|
||||
/* To prevent loops due to _getdns_submit_netreq re-inserting
|
||||
* because of errno == EMFILE
|
||||
*/
|
||||
if (first == prev)
|
||||
break;
|
||||
else
|
||||
prev = first;
|
||||
|
||||
(void) _getdns_rbtree_delete(&context->pending_netreqs, first);
|
||||
(void) _getdns_submit_netreq(first, &now_ms);
|
||||
}
|
||||
|
|
|
|||
32
src/stub.c
32
src/stub.c
|
|
@ -54,15 +54,18 @@ typedef u_short sa_family_t;
|
|||
#define _getdns_EWOULDBLOCK (WSAGetLastError() == WSATRY_AGAIN ||\
|
||||
WSAGetLastError() == WSAEWOULDBLOCK)
|
||||
#define _getdns_EINPROGRESS (WSAGetLastError() == WSAEINPROGRESS)
|
||||
#define _getdns_EMFILE (WSAGetLastError() == WSAEMFILE)
|
||||
#else
|
||||
#define _getdns_EWOULDBLOCK (errno == EAGAIN || errno == EWOULDBLOCK)
|
||||
#define _getdns_EINPROGRESS (errno == EINPROGRESS)
|
||||
#define _getdns_EMFILE (errno == EMFILE)
|
||||
#endif
|
||||
|
||||
/* WSA TODO:
|
||||
* STUB_TCP_WOULDBLOCK added to deal with edge triggered event loops (versus
|
||||
* level triggered). See also lines containing WSA TODO below...
|
||||
*/
|
||||
#define STUB_TRY_AGAIN_LATER -24 /* EMFILE, i.e. Out of OS resources */
|
||||
#define STUB_NO_AUTH -8 /* Existing TLS connection is not authenticated */
|
||||
#define STUB_CONN_GONE -7 /* Connection has failed, clear queue*/
|
||||
#define STUB_TCP_WOULDBLOCK -6
|
||||
|
|
@ -1905,8 +1908,14 @@ upstream_find_for_netreq(getdns_network_req *netreq)
|
|||
upstream = upstream_find_for_transport(netreq,
|
||||
netreq->transports[i],
|
||||
&fd);
|
||||
if (fd == -1 || !upstream)
|
||||
if (!upstream)
|
||||
continue;
|
||||
|
||||
if (fd == -1) {
|
||||
if (_getdns_EMFILE)
|
||||
return STUB_TRY_AGAIN_LATER;
|
||||
return -1;
|
||||
}
|
||||
netreq->transport_current = i;
|
||||
netreq->upstream = upstream;
|
||||
netreq->keepalive_sent = 0;
|
||||
|
|
@ -2030,10 +2039,15 @@ upstream_schedule_netreq(getdns_upstream *upstream, getdns_network_req *netreq)
|
|||
getdns_return_t
|
||||
_getdns_submit_stub_request(getdns_network_req *netreq, uint64_t *now_ms)
|
||||
{
|
||||
int fd = -1;
|
||||
getdns_dns_req *dnsreq;
|
||||
getdns_context *context;
|
||||
|
||||
DEBUG_STUB("%s %-35s: MSG: %p TYPE: %d\n", STUB_DEBUG_ENTRY, __FUNC__,
|
||||
(void*)netreq, netreq->request_type);
|
||||
int fd = -1;
|
||||
getdns_dns_req *dnsreq = netreq->owner;
|
||||
|
||||
dnsreq = netreq->owner;
|
||||
context = dnsreq->context;
|
||||
|
||||
/* This does a best effort to get a initial fd.
|
||||
* All other set up is done async*/
|
||||
|
|
@ -2041,9 +2055,15 @@ _getdns_submit_stub_request(getdns_network_req *netreq, uint64_t *now_ms)
|
|||
if (fd == -1)
|
||||
return GETDNS_RETURN_NO_UPSTREAM_AVAILABLE;
|
||||
|
||||
getdns_transport_list_t transport =
|
||||
netreq->transports[netreq->transport_current];
|
||||
switch(transport) {
|
||||
else if (fd == STUB_TRY_AGAIN_LATER) {
|
||||
_getdns_netreq_change_state(netreq, NET_REQ_NOT_SENT);
|
||||
netreq->node.key = netreq;
|
||||
if (_getdns_rbtree_insert(
|
||||
&context->pending_netreqs, &netreq->node))
|
||||
return GETDNS_RETURN_GOOD;
|
||||
return GETDNS_RETURN_NO_UPSTREAM_AVAILABLE;
|
||||
}
|
||||
switch(netreq->transports[netreq->transport_current]) {
|
||||
case GETDNS_TRANSPORT_UDP:
|
||||
netreq->fd = fd;
|
||||
GETDNS_CLEAR_EVENT(dnsreq->loop, &netreq->event);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
builddir = @BUILDDIR@
|
||||
testname = @TPKG_NAME@
|
||||
LIBTOOL = $(builddir)/libtool
|
||||
|
||||
CFLAGS=-Wall -Wextra -I$(builddir)/src
|
||||
LDLIBS=$(builddir)/src/libgetdns.la
|
||||
|
||||
.SUFFIXES: .c .o .a .lo .h
|
||||
|
||||
.c.lo:
|
||||
$(LIBTOOL) --quiet --tag=CC --mode=compile $(CC) $(CFLAGS) -c $< -o $@
|
||||
|
||||
$(testname): $(testname).lo
|
||||
$(LIBTOOL) --tag=CC --mode=link $(CC) $(LDLIBS) $(LDFLAGS) -o $(testname) $(testname).lo
|
||||
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
/*
|
||||
* delaydns.c - A DNS proxy that adds delay to replies
|
||||
*
|
||||
* Copyright (c) 2016, NLnet Labs. All rights reserved.
|
||||
*
|
||||
* This software is open source.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* Neither the name of the NLNET LABS nor the names of its contributors may
|
||||
* be used to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <getdns/getdns_extra.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
|
||||
static int n_requests = 0;
|
||||
|
||||
typedef struct transaction_t {
|
||||
getdns_transaction_t request_id;
|
||||
getdns_dict *request;
|
||||
|
||||
getdns_context *context;
|
||||
getdns_eventloop *loop;
|
||||
getdns_eventloop_event ev;
|
||||
} transaction_t;
|
||||
|
||||
|
||||
void delay_cb(void *userarg)
|
||||
{
|
||||
transaction_t *trans = userarg;
|
||||
|
||||
trans->loop->vmt->clear(trans->loop, &trans->ev);
|
||||
(void) getdns_reply(trans->context, trans->request, trans->request_id);
|
||||
getdns_dict_destroy(trans->request);
|
||||
free(trans);
|
||||
n_requests -= 1;
|
||||
}
|
||||
|
||||
void handler(getdns_context *context, getdns_callback_type_t callback_type,
|
||||
getdns_dict *request, void *userarg, getdns_transaction_t request_id)
|
||||
{
|
||||
transaction_t *trans = NULL;
|
||||
getdns_bindata *qname;
|
||||
char nreq_str[255];
|
||||
getdns_bindata nreq_bd = { 0, (void *)nreq_str };
|
||||
|
||||
(void) userarg; (void)callback_type;
|
||||
nreq_bd.size = snprintf(nreq_str, sizeof(nreq_str), "n_requests: %d", ++n_requests);
|
||||
|
||||
if (getdns_dict_get_bindata(request, "/question/qname", &qname) ||
|
||||
getdns_dict_set_bindata(request, "/answer/0/name", qname) ||
|
||||
getdns_dict_set_int(request, "/answer/0/type", GETDNS_RRTYPE_TXT) ||
|
||||
getdns_dict_set_bindata(request, "/answer/0/rdata/txt_strings/-", &nreq_bd))
|
||||
fprintf(stderr, "Request init error\n");
|
||||
|
||||
else if (qname->size >= 6 && qname->data[0] == 4 &&
|
||||
qname->data[1] == 'q' && qname->data[2] == 'u' &&
|
||||
qname->data[3] == 'i' && qname->data[4] == 't') {
|
||||
|
||||
(void) getdns_reply(context, request, request_id);
|
||||
(void) getdns_context_set_listen_addresses(context, NULL, NULL, NULL);
|
||||
getdns_dict_destroy(request);
|
||||
return;
|
||||
|
||||
} else if (!(trans = malloc(sizeof(transaction_t))))
|
||||
perror("memerror");
|
||||
else {
|
||||
char *fqdn;
|
||||
getdns_convert_dns_name_to_fqdn(qname, &fqdn);
|
||||
|
||||
(void) memset(trans, 0, sizeof(transaction_t));
|
||||
trans->request_id = request_id;
|
||||
trans->request = request;
|
||||
trans->context = context;
|
||||
trans->ev.userarg = trans;
|
||||
trans->ev.timeout_cb = delay_cb;
|
||||
|
||||
fprintf(stderr, "sched delay for query %s, n_request %d\n", fqdn, (int)n_requests);
|
||||
free(fqdn);
|
||||
if (getdns_context_get_eventloop(context, &trans->loop)
|
||||
|| trans->loop->vmt->schedule(trans->loop, -1, 300, &trans->ev))
|
||||
fprintf(stderr, "Could not schedule delay\n");
|
||||
else return;
|
||||
}
|
||||
getdns_dict_destroy(trans->request);
|
||||
if (trans) free(trans);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
getdns_context *context = NULL;
|
||||
getdns_list *listeners = NULL;
|
||||
getdns_dict *address = NULL;
|
||||
uint32_t port = 18000;
|
||||
getdns_return_t r;
|
||||
|
||||
if ((r = getdns_str2list("[ 127.0.0.1:18000 ]", &listeners)) ||
|
||||
(r = getdns_list_get_dict(listeners, 0, &address)) ||
|
||||
(r = getdns_context_create(&context, 0)))
|
||||
fprintf(stderr, "Error initializing: ");
|
||||
|
||||
else while (++port < 18200 &&
|
||||
!(r = getdns_dict_set_int(address, "port", port)) &&
|
||||
(r = getdns_context_set_listen_addresses(
|
||||
context, listeners, NULL, handler)))
|
||||
; /* pass */
|
||||
|
||||
if (r) fprintf(stderr, "%s\n", getdns_get_errorstr_by_id(r));
|
||||
else {
|
||||
fprintf(stdout, "%d\n", (int)port);
|
||||
fflush(stdout);
|
||||
getdns_context_run(context);
|
||||
}
|
||||
getdns_list_destroy(listeners);
|
||||
getdns_context_destroy(context);
|
||||
return r;
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
BaseName: 285-out_of_filedescriptors
|
||||
Version: 1.0
|
||||
Description: Test if outstanding queries setting is obeyed
|
||||
CreationDate: ma 20 mrt 2017 15:17:45 CET
|
||||
Maintainer: Willem Toorop
|
||||
Category:
|
||||
Component:
|
||||
CmdDepends:
|
||||
Depends: 210-stub-only-link.tpkg
|
||||
Help:
|
||||
Pre: 285-out_of_filedescriptors.pre
|
||||
Post:
|
||||
Test: 285-out_of_filedescriptors.test
|
||||
AuxFiles:
|
||||
Passed:
|
||||
Failure:
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
# #-- 285-out_of_filedescriptors.test --#
|
||||
# source the master var file when it's there
|
||||
[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master
|
||||
# use .tpkg.var.test for in test variable passing
|
||||
[ -f .tpkg.var.test ] && source .tpkg.var.test
|
||||
|
||||
(
|
||||
grep '^CC=' "${BUILDDIR}/build-stub-only/src/Makefile"
|
||||
grep '^LDFLAGS=' "${BUILDDIR}/build-stub-only/src/Makefile"
|
||||
|
||||
BUILDDIR4SED=`echo "${BUILDDIR}/build-stub-only" | sed 's/\//\\\\\//g'`
|
||||
sed -e "s/@BUILDDIR@/${BUILDDIR4SED}/g" \
|
||||
-e "s/@TPKG_NAME@/${TPKG_NAME}/g" "${TPKG_NAME}.Makefile"
|
||||
) > Makefile
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,48 @@
|
|||
# #-- 285-out_of_filedescriptors.test --#
|
||||
# source the master var file when it's there
|
||||
[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master
|
||||
# use .tpkg.var.test for in test variable passing
|
||||
[ -f .tpkg.var.test ] && source .tpkg.var.test
|
||||
|
||||
|
||||
QLIMIT=79
|
||||
NQUERIES=`wc "./${TPKG_NAME}.queries"|sed 's/ .*$//g'`
|
||||
|
||||
# This time the query limit is set by setting the maximum open
|
||||
# filedescriptors. We seem to be needing a higher QLIMIT, than
|
||||
# with limit_outstanding_queries unit test.
|
||||
#
|
||||
# 4 filedescriptors are already needed for overhead (logging etc),
|
||||
# but experiments showed that to prevent timeouts, we should
|
||||
# have a higher value than 72 at least.
|
||||
#
|
||||
# Test will take NQUERIES / QLIMIT * answer delay
|
||||
# For current parameters this is 1000 / 75 * 0.3 = 4.0
|
||||
# which is smaller than 5 seconds default query timeout value,
|
||||
# so the test should succeed.
|
||||
|
||||
make && "./${TPKG_NAME}" | (
|
||||
read PORT
|
||||
ulimit -n $QLIMIT
|
||||
${GETDNS_STUB_QUERY} @127.0.0.1:$PORT TXT \
|
||||
-a -F "./${TPKG_NAME}.queries" 2>&1 > out
|
||||
|
||||
${GETDNS_STUB_QUERY} -q @127.0.0.1:$PORT TXT quit.
|
||||
) && grep '"n_requests: [0-9][0-9]*"' out | sed -e 's/^.*n_requests: //g' -e 's/".*$//g' \
|
||||
| awk -vQLIMIT=$QLIMIT -vNQUERIES=$NQUERIES '
|
||||
|
||||
BEGIN{
|
||||
max_outstanding = 0;
|
||||
}
|
||||
{
|
||||
if ($1 > max_outstanding)
|
||||
max_outstanding = $1;
|
||||
}
|
||||
END{
|
||||
printf("%d of %d queries answered (%.1f%%)\n", NR, NQUERIES, (NR / NQUERIES * 100));
|
||||
if (max_outstanding > QLIMIT) {
|
||||
print "ERROR: More than "QLIMIT" outstanding queries: "max_outstanding;
|
||||
exit(-1);
|
||||
} else
|
||||
print "SUCCESS: No more than "QLIMIT" outstanding queries: "max_outstanding;
|
||||
}'
|
||||
Loading…
Reference in New Issue