interesting
/
getdns
mirror of https://github.com/getdnsapi/getdns.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rewrote the continuous query organization to use the LRU cache instead of an RB tree.

This commit is contained in:
Christian Huitema 2017-02-14 11:30:29 -10:00
parent 1587e2f8f5
commit 6d3e0c7ca2
4 changed files with 74 additions and 549 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/context.h
View File

@ -351,7 +351,6 @@ struct getdns_context {
int mdns_connection_nb; /* typically 0 or 2 for IPv4 and IPv6 */
struct mdns_network_connection * mdns_connection;
struct lruhash * mdns_cache;
_getdns_rbtree_t mdns_continuous_queries_by_name_rrtype;
#endif /* HAVE_MDNS_SUPPORT */
}; /* getdns_context */

615
src/mdns.c
View File

@ -263,9 +263,32 @@ static void msdn_cache_delkey(void* vkey, void* vcontext)
GETDNS_FREE(((struct getdns_context *) vcontext)->mf, vkey);
}
/** old data is deleted. This function is called: func(data, userarg). */
/** old data is deleted. This function is called: func(data, userarg).
* Since we use the hash table for both data and requests, need to
* terminate whatever request was ongoing. TODO: we should have some smarts
* in cache management and never drop cached entries with active requests.
*/
static void msdn_cache_deldata(void* vdata, void* vcontext)
{
/* need to terminate the pending queries? */
getdns_mdns_cached_record_header* header = ((getdns_mdns_cached_record_header*)vdata);
while (header->netreq_first)
{
/* Need to unchain the request from that entry */
getdns_network_req* netreq = header->netreq_first;
header->netreq_first = netreq->mdns_netreq_next;
netreq->mdns_netreq_next = NULL;
/* TODO: treating as a timeout for now, may consider treating as error */
netreq->debug_end_time = _getdns_get_time_as_uintt64();
netreq->state = NET_REQ_TIMED_OUT;
if (netreq->owner->user_callback) {
(void)_getdns_context_request_timed_out(netreq->owner);
}
_getdns_check_dns_req_complete(netreq->owner);
}
GETDNS_FREE(((struct getdns_context *) vcontext)->mf, vdata);
}
@ -309,7 +332,7 @@ static uint8_t * mdns_cache_create_data(
{
getdns_mdns_cached_record_header * header;
int current_index;
size_t data_size = sizeof(getdns_mdns_cached_record_header) + name_len + 4;
size_t data_size = sizeof(getdns_mdns_cached_record_header) + 12 + name_len + 4;
size_t alloc_size = mdns_util_suggest_size(data_size + record_data_len + 2 + 2 + 2 + 4 + 2);
uint8_t* data = GETDNS_XMALLOC(context->mf, uint8_t, alloc_size);
@ -320,7 +343,10 @@ static uint8_t * mdns_cache_create_data(
header->insertion_microsec = current_time;
header->content_len = data_size;
header->allocated_length = alloc_size;
header->netreq_first = NULL;
current_index = sizeof(getdns_mdns_cached_record_header);
memset(data + current_index, 0, 12);
current_index += 12;
memcpy(data + current_index, name, name_len);
current_index += name_len;
data[current_index++] = (uint8_t)(record_type >> 8);
@ -553,6 +579,7 @@ mdns_propose_entry_to_cache(
uint8_t * name, int name_len,
int record_type, int record_class, int ttl,
uint8_t * record_data, int record_data_len,
getdns_network_req * netreq,
uint64_t current_time)
{
int ret = 0;
@ -561,6 +588,7 @@ mdns_propose_entry_to_cache(
hashvalue_type hash;
struct lruhash_entry *entry, *new_entry;
uint8_t *key, *data;
getdns_mdns_cached_record_header * header;
msdn_cache_create_key_in_buffer(temp_key, name, name_len, record_type, record_class);
@ -621,10 +649,19 @@ mdns_propose_entry_to_cache(
if (entry != NULL)
{
ret = mdns_update_cache_ttl_and_prune(context,
(uint8_t*)entry->data, &data,
record_type, record_class, ttl, record_data, record_data_len,
current_time);
if (record_data != NULL && record_data_len > 0)
ret = mdns_update_cache_ttl_and_prune(context,
(uint8_t*)entry->data, &data,
record_type, record_class, ttl, record_data, record_data_len,
current_time);
if (netreq != NULL)
{
/* chain the continuous request to the cache line */
header = (getdns_mdns_cached_record_header *) entry->data;
netreq->mdns_netreq_next = header->netreq_first;
header->netreq_first = netreq;
}
/* then, unlock the entry */
lock_rw_unlock(entry->lock);
@ -635,482 +672,6 @@ mdns_propose_entry_to_cache(
return ret;
}
#if 0
/*
* Add record function for the MDNS record cache.
*/
static int
mdns_add_record_to_cache(struct getdns_context *context,
uint8_t * name, int name_len,
int record_type, int record_class, int ttl,
uint8_t * record_data, int record_data_len)
{
int ret = 0;
/* First, format a key */
/* Next, get the record from the LRU cache */
/* If there is no record, need to create one */
/* Else, just do simple update */
return ret;
}
/*
* MDNS cache management.
*
* This is provisional, until we can get a proper cache by reusing the outbound code.
*
* The cache is a collection of getdns_mdns_known_record structures, each holding
* one record: name, type, class, ttl, rdata length and rdata, plus a 64 bit time stamp.
* The collection is organized as an RB tree. The comparison function in mdns_cmp_known_records
* is somewhat arbitrary: compare by numeric fields class, type, name length, and
* record length first, then by name value, and then by record value.
*
* When a message is received, it will be parsed, and each valid record in the
* ANSWER, AUTH or additional section will be added to the cache. If there is already
* a matching record, only the TTL and time stamp will be updated. If the new TTL is zero,
* the matching record will be removed from the cache.
*
* When a request is first presented, we will try to solve it from the cache. If there
* is response present, we will format a reply containing all the matching records, with
* an updated TTL. If the updated TTL is negative, the record will be deleted from the
* cache and will not be added to the query.
*
* For continuing requests, we may need to send queries with the list of known answers.
* These will be extracted from the cache.
*
* We may want to periodically check all the records in the cache and remove all those
* that have expired. This could be treated as a garbage collection task.
*
* The cache is emptied and deleted upon context deletion.
*
* In a multithreaded environment, we will assume that whoever accesses the cache holds a
* on the context. This is not ideal, but fine grain locks can lead to all kinds of
* synchronization issues.
*/
/*
* Compare function for the getdns_mdns_known_record type,
* used in the red-black tree of known records per query.
*/
static int mdns_cmp_known_records(const void * nkr1, const void * nkr2)
{
int ret = 0;
getdns_mdns_known_record * kr1 = (getdns_mdns_known_record *)nkr1;
getdns_mdns_known_record * kr2 = (getdns_mdns_known_record *)nkr2;
if (kr1->record_class != kr2->record_class)
{
ret = (kr1->record_class < kr2->record_class) ? -1 : 1;
}
else if (kr1->record_type != kr2->record_type)
{
ret = (kr1->record_type < kr2->record_type) ? -1 : 1;
}
else if (kr1->name_len != kr2->name_len)
{
ret = (kr1->name_len < kr2->name_len) ? -1 : 1;
}
else if (kr1->record_data_len != kr2->record_data_len)
{
ret = (kr1->record_data_len < kr2->record_data_len) ? -1 : 1;
}
else if ((ret = memcmp((void*)kr1->name, (void*)kr2->name, kr2->name_len)) == 0)
{
ret = memcmp((const void*)kr1->record_data, (const void*)kr2->record_data, kr1->record_data_len);
}
return ret;
}
/*
* Add record function for the MDNS record cache.
*/
static int
mdns_add_known_record_to_cache(
struct getdns_context *context,
uint8_t * name, int name_len,
int record_type, int record_class, int ttl,
uint8_t * record_data, int record_data_len)
{
int ret = 0;
getdns_mdns_known_record temp, *record, *inserted_record;
size_t required_memory = 0;
_getdns_rbnode_t * node_found, *to_delete;
temp.name = name;
temp.name_len = name_len;
temp.record_class = record_class;
temp.record_type = record_type;
temp.record_data = record_data;
temp.record_data_len = record_data_len;
if (ttl == 0)
{
to_delete = _getdns_rbtree_delete(
&context->mdns_known_records_by_value, &temp);
if (to_delete != NULL)
{
GETDNS_FREE(context->mf, to_delete->key);
}
}
else
{
node_found = _getdns_rbtree_search(&context->mdns_known_records_by_value, &temp);
if (node_found != NULL)
{
record = (getdns_mdns_known_record *)node_found->key;
record->ttl = ttl;
record->insertion_microsec = _getdns_get_time_as_uintt64();
}
else
{
required_memory = sizeof(getdns_mdns_known_record) + name_len + record_data_len;
record = (getdns_mdns_known_record *)
GETDNS_XMALLOC(context->mf, uint8_t, required_memory);
if (record == NULL)
{
ret = GETDNS_RETURN_MEMORY_ERROR;
}
else
{
record->node.parent = NULL;
record->node.left = NULL;
record->node.right = NULL;
record->node.key = (void*)record;
record->record_class = temp.record_class;
record->name = ((uint8_t*)record) + sizeof(getdns_mdns_known_record);
record->name_len = name_len;
record->record_class = record_class;
record->record_type = record_type;
record->record_data = record->name + name_len;
record->record_data_len = record_data_len;
record->insertion_microsec = _getdns_get_time_as_uintt64();
memcpy(record->name, name, name_len);
memcpy(record->record_data, record_data, record_data_len);
inserted_record = (getdns_mdns_known_record *)
_getdns_rbtree_insert(&context->mdns_known_records_by_value,
&record->node);
if (inserted_record == NULL)
{
/* Weird. This can only happen in a race condition */
GETDNS_FREE(context->mf, record);
ret = GETDNS_RETURN_GENERIC_ERROR;
}
}
}
}
return ret;
}
/*
* Get the position of the first matching record in the cache
*/
static _getdns_rbnode_t *
mdns_get_first_record_from_cache(
struct getdns_context *context,
uint8_t * name, int name_len,
int record_type, int record_class,
getdns_mdns_known_record* next_key)
{
/* First, get a search key */
getdns_mdns_known_record temp;
_getdns_rbnode_t *next_node;
if (next_key == NULL)
{
temp.name = name;
temp.name_len = name_len;
temp.record_class = record_class;
temp.record_type = record_type;
temp.record_data = name;
temp.record_data_len = 0;
next_key = &temp;
}
/*
* Find the starting point
*/
if (!_getdns_rbtree_find_less_equal(
&context->mdns_known_records_by_value,
next_key,
&next_node))
{
/*
* The key was not an exact match. Need to find the first node larger
* than the key.
*/
if (next_node == NULL)
{
/*
* The key was smallest than the smallest key in the tree, or the tree is empty
*/
next_node = _getdns_rbtree_first(&context->mdns_known_records_by_value);
}
else
{
/*
* Search retrurned a key smaller than target, so we pick the next one.
*/
next_node = _getdns_rbtree_next(next_node);
}
}
/*
* At this point, we do not check that this is the right node
*/
return next_node;
}
/*
* Count the number of records for this name and type, purging those that have expired.
* Return the first valid node in the set, or NULL if there are no such nodes.
*/
static _getdns_rbnode_t *
mdns_count_and_purge_record_from_cache(
struct getdns_context *context,
uint8_t * name, int name_len,
int record_type, int record_class,
_getdns_rbnode_t* next_node,
uint64_t current_time_microsec,
int *nb_records, int *total_content)
{
int current_record = 0;
int record_length_sum = 0;
int valid_ttl = 0;
getdns_mdns_known_record *next_key;
_getdns_rbnode_t *first_valid_node = NULL, *old_node = NULL;
while (next_node)
{
next_key = (getdns_mdns_known_record *)next_node->key;
if (next_key->name_len != name_len ||
next_key->record_class != record_class ||
next_key->record_type != record_type ||
memcmp(next_key->name, name, name_len) != 0)
{
next_node = NULL;
next_key = NULL;
break;
}
old_node = next_node;
next_node = _getdns_rbtree_next(next_node);
if (next_key->insertion_microsec + (((uint64_t)next_key->ttl) << 20) >=
current_time_microsec)
{
current_record++;
record_length_sum += next_key->record_data_len;
if (first_valid_node == NULL)
{
first_valid_node = old_node;
}
}
else
{
_getdns_rbnode_t * deleted = _getdns_rbtree_delete(
&context->mdns_known_records_by_value, next_key);
if (deleted != NULL)
{
GETDNS_FREE(context->mf, next_key);
}
}
}
*nb_records = current_record;
*total_content = record_length_sum;
return first_valid_node;
}
/*
* Fill a response buffer with the records present in the set,
* up to a limit
*/
_getdns_rbnode_t *
mdns_fill_response_buffer_from_cache(
struct getdns_context *context,
uint8_t * name, int name_len,
int record_type, int record_class,
uint8_t * response, int* response_len, int response_len_max,
int * nb_records,
_getdns_rbnode_t* next_node,
uint64_t current_time_microsec,
int name_offset
)
{
int current_length = 0;
getdns_mdns_known_record * next_key;
int64_t ttl_64;
int32_t current_ttl;
int current_record = 0;
int coding_length;
int name_coding_length = (name_offset < 0) ? name_len : 2;
while (next_node)
{
next_key = (getdns_mdns_known_record *)next_node->key;
if (next_key->name_len != name_len ||
next_key->record_class != record_class ||
next_key->record_type != record_type ||
memcmp(next_key->name, name, name_len) != 0)
{
next_node = NULL;
next_key = NULL;
break;
}
/*
* Compute the required size.
*/
coding_length = name_len + name_coding_length + 2 + 4 + 2 + next_key->record_data_len;
if (current_length + coding_length > response_len_max)
{
break;
}
/*
* encode the record with the updated TTL
*/
ttl_64 = current_time_microsec - next_key->insertion_microsec;
ttl_64 += (((uint64_t)next_key->ttl) << 20);
if (ttl_64 > 0)
{
current_ttl = max(ttl_64 >> 20, 1);
}
else
{
current_ttl = 1;
}
if (name_offset >= 0)
{
/* Perform name compression, per DNS spec */
response[current_length++] = (uint8_t)((0xC0 | (name_offset >> 8)) & 0xFF);
response[current_length++] = (uint8_t)(name_offset & 0xFF);
}
else
{
memcpy(response + current_length, name, name_len);
current_length += name_len;
}
response[current_length++] = (uint8_t)((record_type >> 8) & 0xFF);
response[current_length++] = (uint8_t)((record_type)& 0xFF);
response[current_length++] = (uint8_t)((record_class >> 8) & 0xFF);
response[current_length++] = (uint8_t)((record_class)& 0xFF);
response[current_length++] = (uint8_t)((current_ttl >> 24) & 0xFF);
response[current_length++] = (uint8_t)((current_ttl >> 16) & 0xFF);
response[current_length++] = (uint8_t)((current_ttl >> 8) & 0xFF);
response[current_length++] = (uint8_t)((current_ttl)& 0xFF);
response[current_length++] = (uint8_t)((next_key->record_data_len >> 8) & 0xFF);
response[current_length++] = (uint8_t)((next_key->record_data_len)& 0xFF);
memcpy(response + current_length, next_key->record_data, next_key->record_data_len);
current_length += next_key->record_data_len;
/*
* continue with the next node
*/
current_record++;
next_node = _getdns_rbtree_next(next_node);
}
*response_len = current_length;
*nb_records = current_record;
return next_node;
}
/*
* Compose a response from the MDNS record cache.
*/
static int
mdns_compose_response_from_cache(
struct getdns_context *context,
uint8_t * name, int name_len,
int record_type, int record_class,
uint8_t ** response, int* response_len, int response_len_max,
int query_id,
getdns_mdns_known_record** next_key)
{
int ret = 0;
/* First, get a search key */
int nb_records = 0;
int total_content = 0;
int total_length = 0;
uint64_t current_time = _getdns_get_time_as_uintt64();
_getdns_rbnode_t * first_node = mdns_get_first_record_from_cache(
context, name, name_len, record_type, record_class, *next_key);
/* Purge the expired records and compute the desired length */
first_node = mdns_count_and_purge_record_from_cache(
context, name, name_len, record_type, record_class, first_node, current_time,
&nb_records, &total_content);
/* todo: check whether encoding an empty message is OK */
/* todo: check whether something special is needed for continuation records */
/* Allocate the required memory */
total_length = 12 /* DNS header */
+ name_len + 4 /* Query */
+ total_content + (2 + 2 + 2 + 4 + 2)*nb_records /* answers */;
/* TODO: do we need EDNS encoding? */
if (response_len_max == 0)
{
/* setting this parameter to zero indicates that a full buffer allocation is desired */
{
if (*response == NULL)
{
*response = GETDNS_XMALLOC(context->mf, uint8_t, total_length);
}
else
{
*response = GETDNS_XREALLOC(context->mf, *response, uint8_t, total_length);
}
if (*response == NULL)
{
ret = GETDNS_RETURN_MEMORY_ERROR;
}
else
{
response_len_max = total_length;
}
}
}
if (ret == 0)
{
/*
* Now, proceed with the encoding
*/
}
}
#endif /* if 0, remove the RB based cache code */
/*
* Compare function for the mdns_continuous_query_by_name_rrtype,
* used in the red-black tree of all ongoing queries.
@ -1344,7 +905,7 @@ static getdns_return_t mdns_delayed_network_init(struct getdns_context *context)
if (context->mdns_extended_support == 2)
{
context->mdns_cache = lruhash_create(100, 1000,
context->mdns_cache = lruhash_create(128, 10000000,
mdns_cache_entry_size, mdns_cache_key_comp,
msdn_cache_delkey, msdn_cache_deldata,
context);
@ -1435,64 +996,12 @@ static getdns_return_t mdns_delayed_network_init(struct getdns_context *context)
static getdns_return_t mdns_initialize_continuous_request(getdns_network_req *netreq)
{
int ret = 0;
getdns_mdns_continuous_query temp_query, *continuous_query, *inserted_query;
getdns_dns_req *dnsreq = netreq->owner;
struct getdns_context *context = dnsreq->context;
_getdns_rbnode_t * node_found;
/*
* Fill the target request, but only initialize name and request_type
*/
temp_query.request_class = dnsreq->request_class;
temp_query.request_type = netreq->request_type;
temp_query.name_len = dnsreq->name_len;
/* TODO: check that dnsreq is in canonical form */
memcpy(temp_query.name, dnsreq->name, dnsreq->name_len);
/*
* Check whether the continuous query is already in the RB tree.
* if there is not, create one.
* TODO: should lock the context object when doing that.
*/
node_found = _getdns_rbtree_search(&context->mdns_continuous_queries_by_name_rrtype, &temp_query);
if (node_found != NULL)
{
continuous_query = (getdns_mdns_continuous_query *)node_found->key;
}
else
{
continuous_query = (getdns_mdns_continuous_query *)
GETDNS_MALLOC(context->mf, getdns_mdns_continuous_query);
if (continuous_query != NULL)
{
continuous_query->node.parent = NULL;
continuous_query->node.left = NULL;
continuous_query->node.right = NULL;
continuous_query->node.key = (void*)continuous_query;
continuous_query->request_class = temp_query.request_class;
continuous_query->request_type = temp_query.request_type;
continuous_query->name_len = temp_query.name_len;
memcpy(continuous_query->name, temp_query.name, temp_query.name_len);
continuous_query->netreq_first = NULL;
/* Add the new continuous query to the context */
inserted_query = (getdns_mdns_continuous_query *)
_getdns_rbtree_insert(&context->mdns_continuous_queries_by_name_rrtype,
&continuous_query->node);
if (inserted_query == NULL)
{
/* Weird. This can only happen in a race condition */
GETDNS_FREE(context->mf, &continuous_query);
ret = GETDNS_RETURN_GENERIC_ERROR;
}
}
else
{
ret = GETDNS_RETURN_MEMORY_ERROR;
}
}
/* insert netreq into query list */
netreq->mdns_netreq_next = continuous_query->netreq_first;
continuous_query->netreq_first = netreq;
ret = mdns_propose_entry_to_cache(context, dnsreq->name, dnsreq->name_len,
netreq->request_type, dnsreq->request_class, 1, NULL, 0,
netreq, _getdns_get_time_as_uintt64());
/* to do: queue message request to socket */
@ -1508,8 +1017,6 @@ void _getdns_mdns_context_init(struct getdns_context *context)
context->mdns_connection = NULL;
context->mdns_connection_nb = 0;
context->mdns_cache = NULL;
_getdns_rbtree_init(&context->mdns_continuous_queries_by_name_rrtype
, mdns_cmp_continuous_queries_by_name_rrtype);
}
/*
@ -1517,11 +1024,31 @@ void _getdns_mdns_context_init(struct getdns_context *context)
*/
void _getdns_mdns_context_destroy(struct getdns_context *context)
{
/* Close the sockets */
/* Clear all the cached records. This will terminate all pending network requests */
if (context->mdns_cache != NULL)
{
lruhash_delete(context->mdns_cache);
context->mdns_cache = NULL;
}
/* Close the connections */
if (context->mdns_connection != NULL)
{
for (int i = 0; i < context->mdns_connection_nb; i++)
{
/* suppress the receive event */
GETDNS_CLEAR_EVENT(context->extension, &context->mdns_connection[i].event);
/* close the socket */
#ifdef USE_WINSOCK
closesocket(context->mdns_connection[i].fd);
#else
close(context->mdns_connection[i].fd);
#endif
}
/* Clear all the continuous queries */
/* Clear all the cached records */
GETDNS_FREE(context->mf, context->mdns_connection);
context->mdns_connection = NULL;
context->mdns_connection_nb = 0;
}
}
/* TODO: actualy delete what is required.. */

2
src/mdns.h
View File

@ -72,6 +72,8 @@ typedef struct getdns_mdns_cached_record_header
uint64_t insertion_microsec;
uint32_t content_len;
uint32_t allocated_length;
/* list of user queries */
getdns_network_req *netreq_first;
} getdns_mdns_cached_record_header;
typedef struct getdns_mdns_continuous_query

5
src/types-internal.h
View File

@ -192,12 +192,9 @@ typedef struct getdns_network_req
_getdns_rbnode_t node;
#ifdef HAVE_MDNS_SUPPORT
/*
* for storage in continuous query context. We never
* expect much more than one query per msdn context,
* so no need for RB Tree.
* for storage of continuous query context in hash table of cached results.
*/
struct getdns_network_req * mdns_netreq_next;
struct getdns_mdns_continuous_query * mdns_continuous_query;
#endif /* HAVE_MDNS_SUPPORT */
/* the async_id from unbound */
int unbound_id;