fix gnunet-gns performance issue for many egos

ChangeLog

@@ -1,3 +1,6 @@
+Sat 15 Jun 2019 11:45:35 PM CEST
+  Improved gnunet-gns performance for many ego scenario. -CG
+
 Fri 14 Jun 2019 07:17:40 PM CEST
   Add option to gnunet-identity to display private keys. -CG
 

src/gns/gns_tld_api.c

@@ -72,7 +72,7 @@ struct GNUNET_GNS_LookupWithTldRequest
   /**
    * Lookup an ego with the identity service.
    */
-  struct GNUNET_IDENTITY_Handle *id_co;
+  struct GNUNET_IDENTITY_EgoSuffixLookup *id_co;
 
   /**
    * Name of the longest matching ego found so far.
@@ -191,68 +191,37 @@ lookup_with_public_key (struct GNUNET_GNS_LookupWithTldRequest *ltr,
  */
 static void
 identity_zone_cb (void *cls,
-                  struct GNUNET_IDENTITY_Ego *ego,
-                  void **ctx,
-                  const char *name)
+		  const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv,
+		  const char *ego_name)
 {
   struct GNUNET_GNS_LookupWithTldRequest *ltr = cls;
   struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
 
-  if (NULL == ego)
+  ltr->id_co = NULL;
+  if (NULL == priv)
   {
-    if (NULL != ltr->longest_match)
-    {
-      /* Final case: TLD matches one of our egos */
-      // FIXME: eat all of the match (not just TLD!)
-      if (0 == strcmp (ltr->name, ltr->longest_match))
-      {
-        /* name matches ego name perfectly, only "@" remains */
-        strcpy (ltr->name, GNUNET_GNS_EMPTY_LABEL_AT);
-      }
-      else
-      {
-        GNUNET_assert (strlen (ltr->longest_match) < strlen (ltr->name));
-        ltr->name[strlen (ltr->name) - strlen (ltr->longest_match) - 1] = '\0';
-      }
-
-      /* if the name is of the form 'label' (and not 'label.SUBDOMAIN'), never go to the DHT */
-      GNUNET_free (ltr->longest_match);
-      ltr->longest_match = NULL;
-      if (NULL == strchr (ltr->name, (unsigned char) '.'))
-        ltr->options = GNUNET_GNS_LO_NO_DHT;
-      else
-        ltr->options = GNUNET_GNS_LO_LOCAL_MASTER;
-
-      GNUNET_IDENTITY_ego_get_public_key (ltr->longest_match_ego, &pkey);
-      GNUNET_IDENTITY_disconnect (ltr->id_co);
-      ltr->id_co = NULL;
-      lookup_with_public_key (ltr, &pkey);
-    }
-    else
-    {
-      /* no matching ego found */
-      GNUNET_IDENTITY_disconnect (ltr->id_co);
-      ltr->id_co = NULL;
-      ltr->lookup_proc (ltr->lookup_proc_cls, GNUNET_NO, 0, NULL);
-      GNUNET_GNS_lookup_with_tld_cancel (ltr);
-    }
+    /* no matching ego found */
+    ltr->lookup_proc (ltr->lookup_proc_cls, GNUNET_NO, 0, NULL);
     return;
   }
-  else if (NULL != name)
+  /* Final case: TLD matches one of our egos */
+  if (0 == strcmp (ltr->name, ego_name))
   {
-    if ((strlen (name) <= strlen (ltr->name)) &&
-        (0 == strcmp (name, &ltr->name[strlen (ltr->name) - strlen (name)])) &&
-        ((strlen (name) == strlen (ltr->name)) ||
-         ('.' == ltr->name[strlen (ltr->name) - strlen (name) - 1])) &&
-        ((NULL == ltr->longest_match) ||
-         (strlen (name) > strlen (ltr->longest_match))))
-    {
-      /* found better match, update! */
-      GNUNET_free_non_null (ltr->longest_match);
-      ltr->longest_match = GNUNET_strdup (name);
-      ltr->longest_match_ego = ego;
-    }
+    /* name matches ego name perfectly, only "@" remains */
+    strcpy (ltr->name, GNUNET_GNS_EMPTY_LABEL_AT);
   }
+  else
+  {
+    GNUNET_assert (strlen (ego_name) < strlen (ltr->name));
+    ltr->name[strlen (ltr->name) - strlen (ego_name) - 1] = '\0';
+  }
+  /* if the name is of the form 'label' (and not 'label.SUBDOMAIN'), never go to the DHT */
+  if (NULL == strchr (ltr->name, (unsigned char) '.'))
+    ltr->options = GNUNET_GNS_LO_NO_DHT;
+  else
+    ltr->options = GNUNET_GNS_LO_LOCAL_MASTER;
+  GNUNET_CRYPTO_ecdsa_key_get_public (priv, &pkey);
+  lookup_with_public_key (ltr, &pkey);
 }
 
 
@@ -336,12 +305,11 @@ GNUNET_GNS_lookup_with_tld (struct GNUNET_GNS_Handle *handle,
     }
     GNUNET_free (dot_tld);
   }
-  /* FIXME: this call is still shitty slow to do the longest
-     suffix if we have thousands of egos. We should modify
-     the IDENTITY API to do the longest suffix matching
-     inside of the identity service and not do an O(n) IPC! */
   ltr->id_co =
-    GNUNET_IDENTITY_connect (ltr->gns_handle->cfg, &identity_zone_cb, ltr);
+    GNUNET_IDENTITY_ego_lookup_by_suffix (ltr->gns_handle->cfg,
+					  ltr->name,
+					  &identity_zone_cb,
+					  ltr);
   if (NULL == ltr->id_co)
   {
     GNUNET_free (ltr->name);
@@ -365,7 +333,7 @@ GNUNET_GNS_lookup_with_tld_cancel (struct GNUNET_GNS_LookupWithTldRequest *ltr)
 
   if (NULL != ltr->id_co)
   {
-    GNUNET_IDENTITY_disconnect (ltr->id_co);
+    GNUNET_IDENTITY_ego_lookup_by_suffix_cancel (ltr->id_co);
     ltr->id_co = NULL;
   }
   if (NULL != ltr->lr)

src/identity/Makefile.am

@@ -43,6 +43,7 @@ libgnunet_plugin_rest_identity_la_LDFLAGS = \
 libgnunetidentity_la_SOURCES = \
   identity_api.c \
   identity_api_lookup.c \
+  identity_api_suffix_lookup.c \
   identity.h
 libgnunetidentity_la_LIBADD = \
   $(top_builddir)/src/util/libgnunetutil.la \

src/identity/gnunet-service-identity.c

@@ -377,6 +377,73 @@ handle_lookup_message (void *cls,
 }
 
 
+/**
+ * Handler for LOOKUP message from client, sends information
+ * about ONE identity to the client immediately.
+ *
+ * @param cls unused
+ * @param message the message received
+ * @return #GNUNET_SYSERR if message was ill-formed
+ */
+static int
+check_lookup_by_suffix_message (void *cls,
+				const struct LookupMessage *message)
+{
+  GNUNET_MQ_check_zero_termination (message);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handler for LOOKUP_BY_SUFFIX message from client, sends information
+ * about ONE identity to the client immediately.
+ *
+ * @param cls a `struct GNUNET_SERVICE_Client *`
+ * @param message the message received
+ */
+static void
+handle_lookup_by_suffix_message (void *cls,
+				 const struct LookupMessage *message)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+  const char *name;
+  struct GNUNET_MQ_Envelope *env;
+  struct Ego *lprefix;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received LOOKUP_BY_SUFFIX message from client\n");
+  name = (const char *) &message[1];
+  lprefix = NULL;
+  for (struct Ego *ego = ego_head; NULL != ego; ego = ego->next)
+  {
+
+    if ((strlen (ego->identifier) <= strlen (name)) &&
+        (0 == strcmp (ego->identifier,
+		      &name[strlen (name) - strlen (ego->identifier)])) &&
+        ((strlen (name) == strlen (ego->identifier)) ||
+         ('.' == name[strlen (name) -
+		      strlen (ego->identifier) - 1])) &&
+        ((NULL == lprefix) ||
+         (strlen (ego->identifier) > strlen (lprefix->identifier))))
+    {
+      /* found better match, update! */
+      lprefix = ego;
+    }
+  }
+  if (NULL != lprefix)
+  {
+    env = create_update_message (lprefix);
+    GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
+    GNUNET_SERVICE_client_continue (client);
+    return;
+  }
+  send_result_code (client,
+                    0,
+                    "ego not found");
+  GNUNET_SERVICE_client_continue (client);
+}
+
+
 /**
  * Checks a #GNUNET_MESSAGE_TYPE_IDENTITY_GET_DEFAULT message
  *
@@ -1118,6 +1185,10 @@ GNUNET_SERVICE_MAIN
                         GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP,
                         struct LookupMessage,
                         NULL),
+ GNUNET_MQ_hd_var_size (lookup_by_suffix_message,
+                        GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP_BY_SUFFIX,
+                        struct LookupMessage,
+                        NULL),
  GNUNET_MQ_hd_var_size (get_default_message,
                         GNUNET_MESSAGE_TYPE_IDENTITY_GET_DEFAULT,
                         struct GetDefaultMessage,

src/identity/identity.h

@@ -62,7 +62,8 @@ struct ResultCodeMessage
 struct LookupMessage
 {
   /**
-   * Type: #GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP
+   * Type: #GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP or
+   * #GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP_BY_SUFFIX
    */
   struct GNUNET_MessageHeader header;
 

src/include/gnunet_identity_service.h

@@ -328,6 +328,24 @@ GNUNET_IDENTITY_ego_lookup (const struct GNUNET_CONFIGURATION_Handle *cfg,
 void
 GNUNET_IDENTITY_ego_lookup_cancel (struct GNUNET_IDENTITY_EgoLookup *el);
 
+/**
+ * Function called with the result.
+ *
+ * @param cls closure
+ * @param ego NULL on error / ego not found
+ * @param ego_name NULL on error, name of the ego otherwise
+ */
+typedef void (*GNUNET_IDENTITY_EgoSuffixCallback) (
+  void *cls,
+  const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv,
+  const char *ego_name);
+
+
+/**
+ * Handle for suffix lookup.
+ */
+struct GNUNET_IDENTITY_EgoSuffixLookup;
+
 
 /**
  * Obtain the ego with the maximum suffix match between the
@@ -335,20 +353,26 @@ GNUNET_IDENTITY_ego_lookup_cancel (struct GNUNET_IDENTITY_EgoLookup *el);
  * a @a suffix "a.b.c" and egos with names "d.a.b.c", "b.c" and "c",
  * we return the ego for "b.c".
  *
- * @param id identity service to query
+ * @param cfg configuration to use
  * @param suffix for which domain name suffix is an identity wanted
  * @param cb function to call with the result (will only be called once)
  * @param cb_cls closure for @a cb
  * @return handle to abort the operation
  */
-struct GNUNET_IDENTITY_EgoLookup *
-GNUNET_IDENTITY_ego_lookup_by_suffix (struct GNUNET_IDENTITY_Handle *id,
+struct GNUNET_IDENTITY_EgoSuffixLookup *
+GNUNET_IDENTITY_ego_lookup_by_suffix (const struct GNUNET_CONFIGURATION_Handle *cfg,
 				      const char *suffix,
-				      GNUNET_IDENTITY_EgoCallback cb,
+				      GNUNET_IDENTITY_EgoSuffixCallback cb,
 				      void *cb_cls);
 
 
-
+/**
+ * Abort ego suffix lookup attempt.
+ *
+ * @param el handle for lookup to abort
+ */
+void
+GNUNET_IDENTITY_ego_lookup_by_suffix_cancel (struct GNUNET_IDENTITY_EgoSuffixLookup *el);
 
 #if 0 /* keep Emacsens' auto-indent happy */
 {

src/include/gnunet_protocols.h

@@ -1948,6 +1948,15 @@ extern "C" {
  */
 #define GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP 632
 
+/**
+ * First message send from identity client to service to lookup a
+ * single ego matching the given suffix (longest match).  The service
+ * will respond with a #GNUNET_MESSAGE_TYPE_IDENTITY_UPDATE message if
+ * the ego exists, or a #GNUNET_MESSAGE_TYPE_IDENTITY_RESULT_CODE if
+ * not.
+ */
+#define GNUNET_MESSAGE_TYPE_IDENTITY_LOOKUP_BY_SUFFIX 633
+
 
 /*******************************************************************************
  * REVOCATION message types