ipsets lookup working first time

src/dict.c

+/* dict.c is Copyright (c) 2015 Chen Wei <weichen302@gmail.com>
+
+   Use a dictionary like structure to store config options for fast lookup
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 dated June, 1991, or
+   (at your option) version 3 dated 29 June, 2007.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "dnsmasq.h"
+
+#define MAXLABELS 128
+#define OPEN_ADDRESSING_MAXJUMP 7                /* no reason, just like 7 */
+#define OPEN_ADDRESSING_DEFAULT_SLOT 4
+#define FNV1_32A_INIT ((uint32_t)0x811c9dc5)
+#define FNV_32_PRIME  ((uint32_t)0x01000193)
+#define max(A, B) ((A) > (B) ? (A) : (B))
+
+/* hash function 1 for double hashing
+ * 32 bit Fowler/Noll/Vo hash */
+static inline uint32_t fnv_32_hash (char *str)
+{
+  uint32_t hval = FNV1_32A_INIT;
+  unsigned char *s = (unsigned char *) str;
+
+  while (*s)
+    {
+      hval ^= (uint32_t) * s++;
+      hval +=
+        (hval << 1) + (hval << 4) + (hval << 7) + (hval << 8) + (hval << 24);
+    }
+
+  return hval;
+}
+
+/* hash function 2 for double hashing
+ * the modified Bernstein hash, return an odd number */
+static inline unsigned int bernstein_odd (char *key)
+{
+  unsigned char *s = (unsigned char *) key;
+  unsigned int h = 0;
+
+  while (*s)
+    h = 33 * h ^ *s++;
+
+  return h % 2 ? h : h + 1;
+}
+
+/* convert domain to lower cases, remove leading blank, leading and trailing
+ * dot, string end with \0 */
+static inline void memcpy_lower (void *dst, void *src, int len)
+{
+  char *d = (char *) dst;
+  char *s = (char *) src;
+  int i;
+
+  /* skip leading dot and blank */
+  for ( ; *s != '\0' && (*s == '.' || *s == '\t' || *s == ' '); s++ );
+
+  for (i = 0; i < len; i++, d++, s++)
+    {
+      if (*s >= 'A' && *s <= 'Z')
+        *d = *s + 'a' - 'A';
+      else
+        *d = *s;
+    }
+
+  if (*--d == '.')
+      *d = '\0';
+  else
+      *++d = '\0';
+}
+
+
+
+struct dict_node * init_sub_dictnode (struct dict_node *node)
+{
+  unsigned n;
+
+  if (node->sub != NULL)
+    return node;
+
+  node->sub_slots = OPEN_ADDRESSING_DEFAULT_SLOT;
+  node->sub_loadmax = node->sub_slots * 3 / 4;        // loading factor 0.75
+  node->sub = safe_malloc (node->sub_slots * sizeof (struct dict_node *));
+  for (n = 0; n < node->sub_slots; n++)
+    node->sub[n] = NULL;
+
+  return node;
+}
+
+/* allocate and initialize a new node */
+struct dict_node * new_dictnode (char *label, int label_len, int level)
+{
+  struct dict_node *node;
+
+  node = safe_malloc (sizeof (struct dict_node));
+  if (node == NULL)
+    return NULL;
+
+  if (label == NULL || label_len == 0)
+    {
+      node->h1 = 0;
+      node->h2 = 0;
+      node->label = NULL;
+    }
+  else
+    {
+      node->label = strdup (label);
+      node->h1 = fnv_32_hash (label);
+      node->h2 = bernstein_odd (label);
+    }
+
+  node->sub_count = 0;
+  node->sub_slots = 0;
+  node->sub_loadmax = 0;
+  node->sub_maxjump = 0;
+  node->level = level;
+  node->sub = NULL;
+  node->sets = NULL;
+  node->sets_count = 0;
+
+  return node;
+}
+
+
+/* double the slots of dns node, it calls with add_dicttree each other
+ * the table size starts with 2^2, so that the new size remains 2^x, the
+ * double hash used is choosed to work with 2^n slots and perform well */
+void upsize_dicttree (struct dict_node *np)
+{
+  struct dict_node **oldnodes;
+  unsigned i, oldsize;
+
+  oldsize = np->sub_slots;
+  oldnodes = np->sub;
+  np->sub_slots = oldsize * 2;
+  np->sub_loadmax = np->sub_slots * 3 / 4;
+  np->sub_count = 0;
+  np->sub_maxjump = 0;
+  np->sub = safe_malloc (np->sub_slots * sizeof (struct dict_node *));
+  for (i = 0; i < np->sub_slots; i++)
+    np->sub[i] = NULL;
+
+  for (i = 0; i < oldsize; i++)
+    {
+      if (oldnodes[i] != NULL)
+        {
+          add_dicttree (np, oldnodes[i]);
+        }
+    }
+
+  free (oldnodes);
+}
+
+/* add a sub-node, upsize if needed, calls with upsize_dicttree each other */
+void add_dicttree (struct dict_node *node, struct dict_node *sub)
+{
+  int n;
+  uint32_t dh, idx;
+
+  if (node->sub == NULL)
+    init_sub_dictnode (node);
+
+  n = 0;
+  dh = sub->h1;
+  while (1)
+    {
+      idx = dh % node->sub_slots;
+      if (node->sub[idx] == NULL)
+        {
+          node->sub[idx] = sub;
+          node->sub_count += 1;
+          break;
+        }
+      else
+        {
+          dh += sub->h2;
+          n++;
+        }
+    }
+
+  node->sub_maxjump = max (n, node->sub_maxjump);
+  /* If it takes a lots of jumps to find an empty slot, or the used slots
+   * close to loading max, upsize the table */
+  if (node->sub_maxjump > OPEN_ADDRESSING_MAXJUMP ||
+      node->sub_count > node->sub_loadmax)
+    {
+      upsize_dicttree (node);
+    }
+
+  return;
+}
+
+/* add a new subnode to node, or update the attr of the subnode with same
+ * label
+ * return the subnode */
+struct dict_node * add_or_replace_dictnode (struct dict_node *node, char *label)
+{
+  struct dict_node *np;
+
+  if ((np = lookup_dictnode (node, label)) == NULL)
+    {
+      if (node->sub == NULL)
+        {
+          init_sub_dictnode (node);
+        }
+      np = new_dictnode (label, strlen (label), node->level + 1);
+      add_dicttree (node, np);
+    }
+
+
+  return np;
+}
+
+/* lookup the label in node's sub, return pointer if found, NULL if not */
+struct dict_node * lookup_dictnode (struct dict_node *node, char *label)
+{
+  uint32_t h1, h2, dh, idx;
+  struct dict_node *np;
+
+  /* this domain doesn't have sub-domains */
+  if (node->sub == NULL)
+    {
+      return NULL;
+    }
+
+  dh = h1 = fnv_32_hash (label);
+  h2 = bernstein_odd (label);
+  idx = dh % node->sub_slots;
+  while ((np = node->sub[idx]) != NULL)
+    {
+      if (np->h1 == h1 && np->h2 == h2)
+        if (strcmp (np->label, label) == 0)
+          {
+            return np;
+          }
+
+      dh += h2;
+      idx = dh % node->sub_slots;
+    }
+
+  return NULL;
+}
+
+
+/* look up the whole domain pattern by step over DNS name hierarchy top down.
+ * for example, if the pattern is cn.debian.org, the lookup will start with
+ * org, then debian, then cn */
+struct dict_node * match_domain_ipsets (struct dict_node *root, char *domain)
+{
+  char buf[MAXDNAME];
+  char *labels[MAXLABELS];
+  int i, label_num;
+  int len = strlen (domain);
+  struct dict_node *node, *res;
+
+  memset(buf, 0, sizeof(buf));
+  memcpy_lower (buf, domain, len);
+  /*
+  remove the trailing dot, make the last label top domain
+  if (buf[len - 1] == '.')
+    buf[len - 1] = '\0';
+  else
+    buf[len] = '\0';
+  */
+
+  for (i = 0; i < MAXLABELS; i++)
+    labels[i] = NULL;
+
+  label_num = 0;
+  labels[label_num++] = &buf[0];
+
+  /* split domain name into labels */
+  for (i = 0; buf[i] != '\0'; i++)
+    {
+      if (buf[i] == '.')
+        {
+          buf[i] = '\0';
+          labels[label_num++] = &buf[i + 1];
+        }
+    }
+
+  node = root;
+  res = NULL;
+  for (i = label_num - 1; i >= 0; i--)
+    {
+      node = lookup_dictnode (node, labels[i]);
+
+      /* match longest pattern, e.g. for pattern debian.org and cn.debian.org,
+       * domain name ftp.cn.debian.org will match pattern cn.debian.org */
+      if (node == NULL)
+          break;
+
+      if (node->sets != NULL)
+          res = node;
+    }
+
+    if (res == NULL)
+      return NULL;
+
+  return res;
+}
+
+/* look up the whole domain pattern by step over DNS name hierarchy top down.
+ * for example, if the pattern is cache.google.com, the lookup will start with
+ * com, then google, then cache */
+struct dict_node * lookup_domain (struct dict_node *root, char *domain)
+{
+  char buf[MAXDNAME];
+  char *labels[MAXLABELS];
+  int i, label_num;
+  int len = strlen (domain);
+  struct dict_node *node;
+
+  memset(buf, 0, sizeof(buf));
+  memcpy_lower (buf, domain, len);
+
+  for (i = 0; i < MAXLABELS; i++)
+    labels[i] = NULL;
+
+  label_num = 0;
+
+  labels[label_num++] = &buf[0];
+
+  for (i = 0; buf[i] != '\0'; i++)
+    {
+      if (buf[i] == '.')
+        {
+          buf[i] = '\0';
+          labels[label_num++] = &buf[i + 1];
+        }
+    }
+
+  node = root;
+  for (i = label_num - 1; i >= 0 && node != NULL; i--)
+    {
+      node = lookup_dictnode (node, labels[i]);
+    }
+
+  return i == -1 ? node : NULL;
+}
+
+/* add a domain pattern in the form of google.com to root
+ * return number of levels in that pattern, the level is unused now */
+int add_domain (struct dict_node *root, char *domain)
+{
+  char buf[MAXDNAME];
+  char *labels[MAXLABELS];
+  int i, label_num;
+  int len = strlen (domain);
+  struct dict_node *node;
+
+  memset(buf, 0, sizeof(buf));
+  memcpy_lower (buf, domain, len);
+
+  for (i = 0; i < MAXLABELS; i++)
+    labels[i] = NULL;
+
+  label_num = 0;
+  labels[label_num++] = &buf[0];
+
+  for (i = 0; buf[i] != '\0'; i++)
+    {
+      if (buf[i] == '.')
+        {
+          buf[i] = '\0';
+          labels[label_num++] = &buf[i + 1];
+        }
+    }
+
+  node = root;
+  for (i = label_num - 1; i >= 0; i--)
+    {
+      node = add_or_replace_dictnode (node, labels[i]);
+    }
+
+  return node->level;
+}
+
+/* free node and all sub-nodes recursively. Unused. */
+void free_dicttree (struct dict_node *node)
+{
+  struct dict_node *np;
+  unsigned i;
+
+  if (node->sub_count > 0)
+    {
+      for (i = 0; i < node->sub_slots; i++)
+        {
+          np = node->sub[i];
+          if (np != NULL)
+            {
+              if (np->label != NULL)
+                free (np->label);
+
+              if (np->sets != NULL)
+                free (np->sets);
+
+              free_dicttree (np);
+            }
+        }
+      free (node->sub);
+    }
+
+  free (node);
+}

src/dnsmasq.c

@@ -250,7 +250,7 @@ int main (int argc, char **argv)
 #endif
 
 #ifdef HAVE_IPSET
-  if (daemon->ipsets)
+  if (daemon->dh_ipsets)
     ipset_init();
 #endif
 

src/dnsmasq.h

@@ -516,6 +516,25 @@ struct ipsets {
   struct ipsets *next;
 };
 
+/* a dictionary node for open addressing hash table
+ * it has a key, "label", and several values.
+ *
+ * For ipsets match, only INSERT and LOOKUP operation needed
+ */
+struct dict_node {
+  char *label;          /* key */
+  uint32_t h1;          /* from hash function 1, fnv_32_hash */
+  uint32_t h2;          /* from hash function 2, bernstein_odd */
+  int sub_count;        /* items stored in sub */
+  unsigned sub_slots;   /* size of hash table sub */
+  int sub_loadmax;      /* max items stored before upsize sub */
+  int sub_maxjump;      /* max jumps for insertion, upsize when reach */
+  char **sets;          /* ipsets names end with NULL ptr */
+  int    sets_count;
+  int level;            /* unused */
+  struct dict_node **sub;
+};
+
 struct irec {
   union mysockaddr addr;
   struct in_addr netmask; /* only valid for IPv4 */
@@ -942,6 +961,8 @@ extern struct daemon {
   struct bogus_addr *bogus_addr, *ignore_addr;
   struct server *servers;
   struct ipsets *ipsets;
+  struct dict_node *dh_ipsets;
+  struct dict_node *dh_ipsets_names;
   int log_fac; /* log facility */
   char *log_file; /* optional log file */
   int max_logs;  /* queue limit */
@@ -1364,6 +1385,17 @@ void ipset_init(void);
 int add_to_ipset(const char *setname, const struct all_addr *ipaddr, int flags, int remove);
 #endif
 
+/* dict.c */
+void upsize_dicttree (struct dict_node *np);
+void add_dicttree (struct dict_node *node, struct dict_node *sub);
+struct dict_node *new_dictnode (char *label, int len, int level);
+struct dict_node *lookup_dictnode (struct dict_node *node, char *label);
+struct dict_node *lookup_domain(struct dict_node *root, char *domain);
+struct dict_node *match_domain_ipsets (struct dict_node *root, char *domain);
+int add_domain(struct dict_node *root, char *domain);
+struct dict_node *add_or_replace_dictnode (struct dict_node *node, char *label);
+void free_dicttree (struct dict_node *node);
+
 /* helper.c */
 #if defined(HAVE_SCRIPT)
 int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd);

src/forward.c

@@ -534,30 +534,17 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server
   char **sets = 0;
   int munged = 0, is_sign;
   size_t plen; 
+  struct dict_node *np;
 
   (void)ad_reqd;
   (void) do_bit;
 
 #ifdef HAVE_IPSET
-  // TODO add hash lookup
-  if (daemon->ipsets && extract_request(header, n, daemon->namebuff, NULL))
+  if (daemon->dh_ipsets && extract_request(header, n, daemon->namebuff, NULL))
     {
-      /* Similar algorithm to search_servers. */
-      struct ipsets *ipset_pos;
-      unsigned int namelen = strlen(daemon->namebuff);
-      unsigned int matchlen = 0;
-      for (ipset_pos = daemon->ipsets; ipset_pos; ipset_pos = ipset_pos->next) 
-	{
-	  unsigned int domainlen = strlen(ipset_pos->domain);
-	  char *matchstart = daemon->namebuff + namelen - domainlen;
-	  if (namelen >= domainlen && hostname_isequal(matchstart, ipset_pos->domain) &&
-	      (domainlen == 0 || namelen == domainlen || *(matchstart - 1) == '.' ) &&
-	      domainlen >= matchlen) 
-	    {
-	      matchlen = domainlen;
-	      sets = ipset_pos->sets;
-	    }
-	}
+      np = match_domain_ipsets(daemon->dh_ipsets, daemon->namebuff);
+      if (np != NULL)
+        sets = np->sets;
     }
 #endif
   

src/option.c

@@ -2338,65 +2338,70 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
       break;
 #else
       {
-	 struct ipsets ipsets_head;
-	 struct ipsets *ipsets = &ipsets_head;
-	 int size;
-	 char *end;
-	 char **sets, **sets_pos;
-	 memset(ipsets, 0, sizeof(struct ipsets));
-	 unhide_metas(arg);
-	 if (arg && *arg == '/') 
-	   {
-	     arg++;
-	     while ((end = split_chr(arg, '/'))) 
-	       {
-		 char *domain = NULL;
-		 /* elide leading dots - they are implied in the search algorithm */
-		 while (*arg == '.')
-		   arg++;
-		 /* # matches everything and becomes a zero length domain string */
-		 if (strcmp(arg, "#") == 0 || !*arg)
-		   domain = "";
-		 else if (strlen(arg) != 0 && !(domain = canonicalise_opt(arg)))
-		   option = '?';
-		 ipsets->next = opt_malloc(sizeof(struct ipsets));
-		 ipsets = ipsets->next;
-		 memset(ipsets, 0, sizeof(struct ipsets));
-		 ipsets->domain = domain;
-		 arg = end;
-	       }
-	   } 
-	 else 
-	   {
-	     ipsets->next = opt_malloc(sizeof(struct ipsets));
-	     ipsets = ipsets->next;
-	     memset(ipsets, 0, sizeof(struct ipsets));
-	     ipsets->domain = "";
-	   }
-	 if (!arg || !*arg)
-	   {
-	     option = '?';
-	     break;
-	   }
-	 size = 2;
-	 for (end = arg; *end; ++end) 
-	   if (*end == ',')
-	       ++size;
-     
-	 sets = sets_pos = opt_malloc(sizeof(char *) * size);
-	 
-	 do {
-	   end = split(arg);
-	   *sets_pos++ = opt_string_alloc(arg);
-	   arg = end;
-	 } while (end);
-	 *sets_pos = 0;
-	 for (ipsets = &ipsets_head; ipsets->next; ipsets = ipsets->next)
-	   ipsets->next->sets = sets;
-	 ipsets->next = daemon->ipsets;
-	 daemon->ipsets = ipsets_head.next;
-	 
-	 break;
+        int size;
+        char *end;
+        char **sets, **sets_pos;
+        int sets_count = 0;
+        unhide_metas (arg);
+        struct dict_node *np;
+        char *domain = NULL;
+
+        if (daemon->dh_ipsets == NULL)
+          daemon->dh_ipsets = new_dictnode (NULL, 0, 0);
+
+        if (arg && *arg == '/')
+          {
+            arg++;
+            while ((end = split_chr (arg, '/')))
+              {
+                /* elide leading dots - they are implied in the search algorithm */
+                while (*arg == '.')
+                  arg++;
+                /* # matches everything and becomes a zero length domain string */
+                if (strcmp (arg, "#") == 0 || !*arg)
+                  /* ignore match all domain directive # for now */
+                  /* domain = ""; */
+                  ;
+                else if (strlen (arg) != 0 && !(domain = canonicalise_opt (arg)))
+                  option = '?';
+
+                if (domain != NULL)
+                  add_domain (daemon->dh_ipsets, domain);
+
+                arg = end;
+              }
+          }
+
+        if (!arg || !*arg)
+          {
+            option = '?';
+            break;
+          }
+        size = 2;
+        for (end = arg; *end; ++end)
+          if (*end == ',')
+            ++size;
+
+        sets = sets_pos = opt_malloc (sizeof (char *) * size);
+
+        do
+          {
+            end = split (arg);
+            *sets_pos++ = opt_string_alloc (arg);
+            sets_count++;
+            arg = end;
+          }
+        while (end);
+
+        *sets_pos = NULL;
+        if (domain != NULL)
+          {
+            np = lookup_domain (daemon->dh_ipsets, domain);
+            np->sets = sets;
+            np->sets_count = sets_count;
+          }
+
+        break;
       }
 #endif