gnunet/src/hostlist/hostlist-client.c

/*
     This file is part of GNUnet.
     (C) 2001, 2002, 2003, 2004, 2005, 2006, 2009, 2010 Christian Grothoff (and other contributing authors)

     GNUnet is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published
     by the Free Software Foundation; either version 3, or (at your
     option) any later version.

     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     General Public License for more details.

     You should have received a copy of the GNU General Public License
     along with GNUnet; see the file COPYING.  If not, write to the
     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
     Boston, MA 02111-1307, USA.
*/

/**
 * @file hostlist/hostlist-client.c
 * @brief hostlist support.  Downloads HELLOs via HTTP.
 * @author Christian Grothoff
 * @author Matthias Wachs
 */

#include "platform.h"
#include "hostlist-client.h"
#include "gnunet_core_service.h"
#include "gnunet_hello_lib.h"
#include "gnunet_statistics_service.h"
#include "gnunet_transport_service.h"
#include "gnunet-daemon-hostlist.h"
#include <curl/curl.h>
#include "gnunet_common.h"
#include "gnunet_bio_lib.h"

#define DEBUG_HOSTLIST_CLIENT GNUNET_NO


/**
 * Number of connections that we must have to NOT download
 * hostlists anymore.
 */
#define MIN_CONNECTIONS 4

/**
 * Interval between two advertised hostlist tests
 */
#define TESTING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)

/**
 * A single hostlist obtained by hostlist advertisements
 */
struct Hostlist
{
  /**
   * previous entry, used to manage entries in a double linked list
   */
  struct Hostlist * prev;

  /**
   * next entry, used to manage entries in a double linked list
   */
  struct Hostlist * next;

  /**
   * URI where hostlist can be obtained
   */
  const char *hostlist_uri;

  /**
   * Value describing the quality of the hostlist, the bigger the better but (should) never < 0
   * used for deciding which hostlist is replaced if MAX_NUMBER_HOSTLISTS in data structure is reached
   * intial value = HOSTLIST_INITIAL
   * increased every successful download by HOSTLIST_SUCCESSFULL_DOWNLOAD
   * increased every successful download by number of obtained HELLO messages
   * decreased every failed download by HOSTLIST_SUCCESSFULL_DOWNLOAD
   */
  uint64_t quality;

  /**
   * Time the hostlist advertisement was recieved and the entry was created
   */
  struct GNUNET_TIME_Absolute time_creation;

  /**
   * Last time the hostlist was obtained
   */
  struct GNUNET_TIME_Absolute time_last_usage;

  /**
   * Number of HELLO messages obtained during last download
   */
  uint32_t hello_count;

  /**
   * Number of times the hostlist was successfully obtained
   */
  uint32_t times_used;

};


/**
 * Our configuration.
 */
static const struct GNUNET_CONFIGURATION_Handle *cfg;

/**
 * Statistics handle.
 */
static struct GNUNET_STATISTICS_Handle *stats; 

/**
 * Transport handle.
 */
static struct GNUNET_TRANSPORT_Handle *transport;
                       
/**
 * Proxy that we are using (can be NULL).
 */
static char *proxy;

/**
 * Number of bytes valid in 'download_buffer'.
 */
static size_t download_pos;

/**
 * Current URL that we are using.
 */
static char *current_url;

/**
 * Current CURL handle.
 */
static CURL *curl;

/**
 * Current multi-CURL handle.
 */
static CURLM *multi;

/**
 * How many bytes did we download from the current hostlist URL?
 */
static uint32_t stat_bytes_downloaded;

/**
 * Amount of time we wait between hostlist downloads.
 */
static struct GNUNET_TIME_Relative hostlist_delay;

/**
 * ID of the task, checking if hostlist download should take plate
 */
static GNUNET_SCHEDULER_TaskIdentifier ti_check_download;

/**
 * ID of the task downloading the hostlist
 */
static GNUNET_SCHEDULER_TaskIdentifier ti_download;

/**
 * ID of the task saving the hostlsit in a regular intervall
 */
static GNUNET_SCHEDULER_TaskIdentifier ti_saving_task;

/**
 * ID of the task called to initiate a download
 */
static GNUNET_SCHEDULER_TaskIdentifier ti_download_dispatcher_task;

/**
 * ID of the task controlling the locking between two hostlist tests
 */
static GNUNET_SCHEDULER_TaskIdentifier ti_testing_intervall_task;

/**
 * At what time MUST the current hostlist request be done?
 */
static struct GNUNET_TIME_Absolute end_time;

/**
 * Head of the linked list used to store hostlists
 */
static struct Hostlist * linked_list_head;

/**
 *  Tail of the linked list used to store hostlists
 */
static struct Hostlist * linked_list_tail;

/**
 *  Current hostlist used for downloading
 */
static struct Hostlist * current_hostlist;

/**
 *  Size of the linke list  used to store hostlists
 */
static unsigned int linked_list_size;

/**
 * Head of the linked list used to store hostlists
 */
static struct Hostlist * hostlist_to_test;

/**
 * Set to GNUNET_YES if the current URL had some problems.
 */
static int stat_bogus_url;

/**
 * Value controlling if a hostlist is tested at the moment
 */
static int stat_testing_hostlist;

/**
 * Value controlling if a hostlist testing is allowed at the moment
 */
static int stat_testing_allowed;

/**
 * Value controlling if a hostlist download is running at the moment
 */
static int stat_download_in_progress;

/**
 * Value saying if a preconfigured bootstrap server is used
 */
static unsigned int stat_use_bootstrap;
/**
 * Set if we are allowed to learn new hostlists and use them
 */
static int stat_learning;

/**
 * Value saying if hostlist download was successful
 */
static unsigned int stat_download_successful;

/**
 * Value saying how many valid HELLO messages were obtained during download
 */
static unsigned int stat_hellos_obtained;

/**
 * Number of active connections (according to core service).
 */
static unsigned int stat_connection_count;


/**
 * Process downloaded bits by calling callback on each HELLO.
 *
 * @param ptr buffer with downloaded data
 * @param size size of a record
 * @param nmemb number of records downloaded
 * @param ctx unused
 * @return number of bytes that were processed (always size*nmemb)
 */
static size_t
callback_download (void *ptr, 
		   size_t size, 
		   size_t nmemb, 
		   void *ctx)
{
  static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
  const char * cbuf = ptr;
  const struct GNUNET_MessageHeader *msg;
  size_t total;
  size_t cpy;
  size_t left;
  uint16_t msize;

  total = size * nmemb;
  stat_bytes_downloaded += total;
  if ( (total == 0) || (stat_bogus_url) )
    {
      return total;  /* ok, no data or bogus data */
    }

  GNUNET_STATISTICS_update (stats, 
			    gettext_noop ("# bytes downloaded from hostlist servers"), 
			    (int64_t) total, 
			    GNUNET_NO);  
  left = total;
  while ( (left > 0) ||
	  (download_pos > 0) )
    {
      cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - download_pos);
      memcpy (&download_buffer[download_pos],
	      cbuf,
	      cpy);      
      cbuf += cpy;
      download_pos += cpy;
      left -= cpy;
      if (download_pos < sizeof(struct GNUNET_MessageHeader))
	{
	  GNUNET_assert (left == 0);
	  break;
	}
      msg = (const struct GNUNET_MessageHeader *) download_buffer;
      msize = ntohs(msg->size);
      if (msize < sizeof(struct GNUNET_MessageHeader))
	{	 
	  GNUNET_STATISTICS_update (stats, 
				    gettext_noop ("# invalid HELLOs downloaded from hostlist servers"), 
				    1, 
				    GNUNET_NO);  
	  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
		      _("Invalid `%s' message received from hostlist at `%s'\n"),
		      "HELLO",
		      current_url);
          stat_hellos_obtained++;
	  stat_bogus_url = 1;
	  return total;
	}
      if (download_pos < msize)
	{
	  GNUNET_assert (left == 0);
	  break;
	}
      if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message*)msg) == msize)
	{
#if DEBUG_HOSTLIST_CLIENT
	  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
		      "Received valid `%s' message from hostlist server.\n",
		      "HELLO");
#endif
	  GNUNET_STATISTICS_update (stats, 
				    gettext_noop ("# valid HELLOs downloaded from hostlist servers"), 
				    1, 
				    GNUNET_NO);
	  stat_hellos_obtained++;
	  GNUNET_TRANSPORT_offer_hello (transport, msg, NULL, NULL);
	}
      else
	{
	  GNUNET_STATISTICS_update (stats, 
				    gettext_noop ("# invalid HELLOs downloaded from hostlist servers"), 
				    1, 
				    GNUNET_NO);  
	  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
		      _("Invalid `%s' message received from hostlist at `%s'\n"),
		      "HELLO",
		      current_url);
	  stat_bogus_url = GNUNET_YES;
          stat_hellos_obtained++;
	  return total;
	}
      memmove (download_buffer,
	       &download_buffer[msize],
	       download_pos - msize);
      download_pos -= msize;
    }
  return total;
}


/**
 * Obtain a hostlist URL that we should use.
 *
 * @return NULL if there is no URL available
 */
static char *
get_bootstrap_server ()
{
  char *servers;
  char *ret;
  size_t urls;
  size_t pos;

  if (GNUNET_OK != 
      GNUNET_CONFIGURATION_get_value_string (cfg,
                                             "HOSTLIST",
                                             "SERVERS",
                                             &servers))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
                  "SERVERS", "HOSTLIST");
      return NULL;
    }

  urls = 0;
  if (strlen (servers) > 0)
    {
      urls++;
      pos = strlen (servers) - 1;
      while (pos > 0)
        {
          if (servers[pos] == ' ')
            urls++;
          pos--;
        }
    }
  if (urls == 0)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
                  "SERVERS", "HOSTLIST");
      GNUNET_free (servers);
      return NULL;
    }

  urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
  pos = strlen (servers) - 1;
  while (pos > 0)
    {
      if (servers[pos] == ' ')
        {
          urls--;
          servers[pos] = '\0';
        }
      if (urls == 0)
        {
          pos++;
          break;
        }
      pos--;    
    }
  ret = GNUNET_strdup (&servers[pos]);
  GNUNET_free (servers);
  return ret;
}

/**
 * Method deciding if a preconfigured or advertisied hostlist is used on a 50:50 ratio
 * @return uri to use, NULL if there is no URL available
 */
static char *
download_get_url ()
{
  uint32_t index;
  unsigned int counter;
  struct Hostlist * pos;

  if ( GNUNET_NO == stat_learning)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Using preconfigured bootstrap server\n");
    current_hostlist = NULL;
    return get_bootstrap_server();
  }

  if ( ( GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test) )
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Testing new advertised hostlist if it is obtainable\n");
    current_hostlist = hostlist_to_test;
    return strdup(hostlist_to_test->hostlist_uri);
  }

  if ( (GNUNET_YES == stat_use_bootstrap) ||
       (linked_list_size == 0) )
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Using preconfigured bootstrap server\n");
    current_hostlist = NULL;
    return get_bootstrap_server();
  }
  index = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 
				    linked_list_size);
  counter = 0;
  pos = linked_list_head;
  while ( counter < index )
    {
      pos = pos->next;
      counter ++;
    }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Using learned hostlist `%s'\n", pos->hostlist_uri);
  current_hostlist = pos;
  return strdup(pos->hostlist_uri);
}


#define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt(c, a, b); if (ret != CURLE_OK) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror(ret)); } while (0)


/**
 * Method to save hostlist to a file during hostlist client shutdown
 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
 */
static void save_hostlist_file ( int shutdown );


/**
 * add val2 to val1 with overflow check
 * @param val1 value 1
 * @param val2 value 2
 * @return result
 */
static uint64_t checked_add (uint64_t val1, uint64_t val2)
{
  static uint64_t temp;
  static uint64_t maxv;

  maxv = 0;
  maxv--;

  temp = val1+val2;
  if ( temp < val1)
    return maxv;
  else
    return temp;
}

/**
 * Subtract val2 from val1 with underflow check
 * @param val1 value 1
 * @param val2 value 2
 * @return result
 */
static uint64_t 
checked_sub (uint64_t val1, 
	     uint64_t val2)
{
  if ( val1 <= val2)
    return 0;
  else
    return (val1-val2);
}

/**
 * Method to check if  a URI is in hostlist linked list
 * @param uri uri to check
 * @return GNUNET_YES if existing in linked list, GNUNET_NO if not
 */
static int
linked_list_contains (const char * uri)
{
  struct Hostlist * pos;

  pos = linked_list_head;
  while (pos != NULL)
    {
      if (0 == strcmp(pos->hostlist_uri, uri) )
        return GNUNET_YES;
      pos = pos->next;
    }
  return GNUNET_NO;
}


/**
 * Method returning the hostlist element with the lowest quality in the datastore
 * @return hostlist with lowest quality
 */
static struct Hostlist *
linked_list_get_lowest_quality ( )
{
  struct Hostlist * pos;
  struct Hostlist * lowest;

  if (linked_list_size == 0)
    return NULL;
  lowest = linked_list_head;
  pos = linked_list_head->next;
  while (pos != NULL)
    {
      if (pos->quality < lowest->quality)
        lowest = pos;
      pos = pos->next;
    }
  return lowest;
}


/**
 * Method to insert a hostlist into the datastore. If datastore
 * contains maximum number of elements, the elements with lowest
 * quality is dismissed
 */
static void
insert_hostlist ()
{
  struct Hostlist * lowest_quality;

  if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
    {
      /* No free entries available, replace existing entry  */
      lowest_quality = linked_list_get_lowest_quality();
      GNUNET_assert (lowest_quality != NULL);
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
		  "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
		  lowest_quality->hostlist_uri,
		  (unsigned long long) lowest_quality->quality);
      GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, lowest_quality);
      linked_list_size--;
      GNUNET_free (lowest_quality);
    }
  GNUNET_CONTAINER_DLL_insert(linked_list_head,
			      linked_list_tail,
			      hostlist_to_test);
  linked_list_size++;
  GNUNET_STATISTICS_set (stats,
                         gettext_noop("# advertised hostlist URIs"),
                         linked_list_size,
                         GNUNET_NO);
  stat_testing_hostlist = GNUNET_NO;
}


/**
 * Method updating hostlist statistics
 */
static void update_hostlist ( )
{
  char *stat;
  if ( ((stat_use_bootstrap == GNUNET_NO) && ( NULL != current_hostlist )) ||
       ((stat_testing_hostlist == GNUNET_YES) && ( NULL != current_hostlist )) )
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Updating hostlist statics for URI `%s'\n",current_hostlist->hostlist_uri );
     current_hostlist->hello_count = stat_hellos_obtained;
     current_hostlist->time_last_usage = GNUNET_TIME_absolute_get();
     current_hostlist->quality = checked_add ( current_hostlist->quality, (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
     if ( GNUNET_YES == stat_download_successful )
     {
       current_hostlist->times_used++;
       current_hostlist->quality = checked_add ( current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
       GNUNET_asprintf (&stat,
                        gettext_noop("# advertised URI `%s' downloaded"),
                        current_hostlist->hostlist_uri);

       GNUNET_STATISTICS_update ( stats,
                                  stat,
                                  1,
                                  GNUNET_YES);
       GNUNET_free (stat);
     }
     else
       current_hostlist->quality = checked_sub ( current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD );
  }
  current_hostlist = NULL;
  /* Alternating the usage of preconfigured and learned hostlists */

  if (stat_testing_hostlist == GNUNET_YES)
    return;

  if ( GNUNET_YES == stat_learning)
    {
    if (stat_use_bootstrap == GNUNET_YES)
      stat_use_bootstrap = GNUNET_NO;
    else
      stat_use_bootstrap = GNUNET_YES;
    }
  else
    stat_use_bootstrap = GNUNET_YES;
}

/**
 * Clean up the state from the task that downloaded the
 * hostlist and schedule the next task.
 */
static void 
clean_up ()
{
  CURLMcode mret;

  if ( (stat_testing_hostlist == GNUNET_YES) && 
       (GNUNET_NO == stat_download_successful) &&
       (NULL != hostlist_to_test))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),
		hostlist_to_test->hostlist_uri);
  }

  if (stat_testing_hostlist == GNUNET_YES)
    {
      stat_testing_hostlist = GNUNET_NO;
    }
  if ( NULL != hostlist_to_test)
  {
    GNUNET_free (hostlist_to_test);
    hostlist_to_test = NULL;
  }

  if (multi != NULL)
    {
      mret = curl_multi_remove_handle (multi, curl);
      if (mret != CURLM_OK)
	{
	  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
		      _("%s failed at %s:%d: `%s'\n"),
		      "curl_multi_remove_handle", __FILE__, __LINE__,
		      curl_multi_strerror (mret));
	}
      mret = curl_multi_cleanup (multi);
      if (mret != CURLM_OK)
	GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
		    _("%s failed at %s:%d: `%s'\n"),
		    "curl_multi_cleanup", __FILE__, __LINE__,
		    curl_multi_strerror (mret));
      multi = NULL;
    }
  if (curl != NULL)
    {
      curl_easy_cleanup (curl);
      curl = NULL;
    }  
  GNUNET_free_non_null (current_url);
  current_url = NULL;
  stat_bytes_downloaded = 0;
  stat_download_in_progress = GNUNET_NO;
}


/**
 * Task that is run when we are ready to receive more data from the hostlist
 * server.
 *
 * @param cls closure, unused
 * @param tc task context, unused
 */
static void
task_download (void *cls,
	       const struct GNUNET_SCHEDULER_TaskContext *tc);


/**
 * Ask CURL for the select set and then schedule the
 * receiving task with the scheduler.
 */
static void
download_prepare ()
{
  CURLMcode mret;
  fd_set rs;
  fd_set ws;
  fd_set es;
  int max;
  struct GNUNET_NETWORK_FDSet *grs;
  struct GNUNET_NETWORK_FDSet *gws;
  long timeout;
  struct GNUNET_TIME_Relative rtime;

  max = -1;
  FD_ZERO (&rs);
  FD_ZERO (&ws);
  FD_ZERO (&es);
  mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
  if (mret != CURLM_OK)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  _("%s failed at %s:%d: `%s'\n"),
                  "curl_multi_fdset", __FILE__, __LINE__,
                  curl_multi_strerror (mret));
      clean_up ();
      return;
    }
  mret = curl_multi_timeout (multi, &timeout);
  if (mret != CURLM_OK)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  _("%s failed at %s:%d: `%s'\n"),
                  "curl_multi_timeout", __FILE__, __LINE__,
                  curl_multi_strerror (mret));
      clean_up ();
      return;
    }
  rtime = GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
                                    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
                                                                   timeout));
  grs = GNUNET_NETWORK_fdset_create ();
  gws = GNUNET_NETWORK_fdset_create ();
  GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
  GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
#if DEBUG_HOSTLIST_CLIENT
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Scheduling task for hostlist download using cURL\n");
#endif
  ti_download
    = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
                                   GNUNET_SCHEDULER_NO_TASK,
                                   rtime,
                                   grs,
                                   gws,
                                   &task_download,
                                   multi);
  GNUNET_NETWORK_fdset_destroy (gws);
  GNUNET_NETWORK_fdset_destroy (grs);
}


/**
 * Task that is run when we are ready to receive more data from the hostlist
 * server. 
 *
 * @param cls closure, unused
 * @param tc task context, unused
 */
static void
task_download (void *cls,
	       const struct GNUNET_SCHEDULER_TaskContext *tc)
{
  int running;
  struct CURLMsg *msg;
  CURLMcode mret;
  
  ti_download = GNUNET_SCHEDULER_NO_TASK;
  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
    {
#if DEBUG_HOSTLIST_CLIENT
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
		  "Shutdown requested while trying to download hostlist from `%s'\n",
		  current_url);
#endif
      update_hostlist();
      clean_up ();
      return;
    }
  if (GNUNET_TIME_absolute_get_remaining (end_time).rel_value == 0)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
		  _("Timeout trying to download hostlist from `%s'\n"),
		  current_url);
      update_hostlist();
      clean_up ();
      return;
    }
#if DEBUG_HOSTLIST_CLIENT
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
	      "Ready for processing hostlist client request\n");
#endif

  do 
    {
      running = 0;
      if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
        {
        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
		    _("Download limit of %u bytes exceeded, stopping download\n"),
		    MAX_BYTES_PER_HOSTLISTS);
        clean_up();
        return;
        }
      mret = curl_multi_perform (multi, &running);
      if (running == 0)
	{
	  do
	    {
	      msg = curl_multi_info_read (multi, &running);
	      GNUNET_break (msg != NULL);
	      if (msg == NULL)
		break;
	      switch (msg->msg)
		{
		case CURLMSG_DONE:
		  if ( (msg->data.result != CURLE_OK) &&
		       (msg->data.result != CURLE_GOT_NOTHING) )		       
		    GNUNET_log(GNUNET_ERROR_TYPE_INFO,
			       _("%s failed for `%s' at %s:%d: `%s'\n"),
			       "curl_multi_perform", 
			       current_url,
			       __FILE__,
			       __LINE__,
			       curl_easy_strerror (msg->data.result));		  
		  else
		    {
		    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
				_("Download of hostlist `%s' completed.\n"),
				current_url);
		    stat_download_successful = GNUNET_YES;
	            update_hostlist();
		    if (GNUNET_YES == stat_testing_hostlist)
		     {
		       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
				   _("Adding successfully tested hostlist `%s' datastore.\n"),
				   current_url);
		      insert_hostlist();
		      hostlist_to_test = NULL;
		      stat_testing_hostlist = GNUNET_NO;
		     }
		    }
		  clean_up ();
		  return;
		default:
		  break;
		}

	    }
	  while ( (running > 0) );
	}
    }
  while (mret == CURLM_CALL_MULTI_PERFORM);

  if (mret != CURLM_OK)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
		  _("%s failed at %s:%d: `%s'\n"),
		  "curl_multi_perform", __FILE__, __LINE__,
		  curl_multi_strerror (mret));
      clean_up ();
    }
  download_prepare ();
}


/**
 * Main function that will download a hostlist and process its
 * data.
 */
static void
download_hostlist () 
{
  CURLcode ret;
  CURLMcode mret;


  current_url = download_get_url ();
  if (current_url == NULL)
    return;
  curl = curl_easy_init ();
  multi = NULL;
  if (curl == NULL)
    {
      GNUNET_break (0);
      clean_up ();
      return;
    }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
	      _("Bootstrapping using hostlist at `%s'.\n"), 
	      current_url);

  stat_download_in_progress = GNUNET_YES;
  stat_download_successful = GNUNET_NO;
  stat_hellos_obtained = 0;
  stat_bytes_downloaded = 0;

  GNUNET_STATISTICS_update (stats, 
			    gettext_noop ("# hostlist downloads initiated"), 
			    1, 
			    GNUNET_NO);  
  if (proxy != NULL)
    CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);    
  download_pos = 0;
  stat_bogus_url = 0;
  CURL_EASY_SETOPT (curl,
		    CURLOPT_WRITEFUNCTION, 
		    &callback_download);
  if (ret != CURLE_OK)
    {
      clean_up ();
      return;
    }
  CURL_EASY_SETOPT (curl,
		    CURLOPT_WRITEDATA, 
		    NULL);
  if (ret != CURLE_OK)
    {
      clean_up ();
      return;
    }
  CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
  CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
  /* no need to abort if the above failed */
  CURL_EASY_SETOPT (curl, 
		    CURLOPT_URL, 
		    current_url);
  if (ret != CURLE_OK)
    {
      clean_up ();
      return;
    }
  CURL_EASY_SETOPT (curl, 
		    CURLOPT_FAILONERROR, 
		    1);
#if 0
  CURL_EASY_SETOPT (curl, 
		    CURLOPT_VERBOSE, 
		    1);
#endif
  CURL_EASY_SETOPT (curl, 
		    CURLOPT_BUFFERSIZE, 
		    GNUNET_SERVER_MAX_MESSAGE_SIZE);
  if (0 == strncmp (current_url, "http", 4))
    CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
  CURL_EASY_SETOPT (curl, 
		    CURLOPT_CONNECTTIMEOUT, 
		    60L);
  CURL_EASY_SETOPT (curl, 
		    CURLOPT_TIMEOUT, 
		    60L);
#if 0
  /* this should no longer be needed; we're now single-threaded! */
  CURL_EASY_SETOPT (curl,
		    CURLOPT_NOSIGNAL, 
		    1);
#endif
  multi = curl_multi_init ();
  if (multi == NULL)
    {
      GNUNET_break (0);
      /* clean_up (); */
      return;
    }
  mret = curl_multi_add_handle (multi, curl);
  if (mret != CURLM_OK)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
		  _("%s failed at %s:%d: `%s'\n"),
		  "curl_multi_add_handle", __FILE__, __LINE__,
		  curl_multi_strerror (mret));
      mret = curl_multi_cleanup (multi);
      if (mret != CURLM_OK)
	GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
		    _("%s failed at %s:%d: `%s'\n"),
		    "curl_multi_cleanup", __FILE__, __LINE__,
		    curl_multi_strerror (mret));
      multi = NULL;
      clean_up ();
      return;
    }
  end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
  download_prepare ();
}  


static void
task_download_dispatcher (void *cls,
            const struct GNUNET_SCHEDULER_TaskContext *tc)
{
  ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
    if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
      return;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
             "Download is initiated...\n");
   if ( GNUNET_NO == stat_download_in_progress )
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Download can start immediately...\n");
     download_hostlist();
   }
   else
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Download in progess, have to wait...\n");
     ti_download_dispatcher_task = GNUNET_SCHEDULER_add_delayed (WAITING_INTERVALL,
                                                              &task_download_dispatcher,
                                                              NULL);
   }
}

/**
 * Task that checks if we should try to download a hostlist.
 * If so, we initiate the download, otherwise we schedule
 * this task again for a later time.
 */
static void
task_check (void *cls,
	    const struct GNUNET_SCHEDULER_TaskContext *tc)
{
  static int once;
  struct GNUNET_TIME_Relative delay;

  ti_check_download = GNUNET_SCHEDULER_NO_TASK;
  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
    return;

  if (stat_connection_count < MIN_CONNECTIONS)
  {
    ti_download_dispatcher_task = GNUNET_SCHEDULER_add_now (&task_download_dispatcher,
                                                          NULL);
  }

  if (stats == NULL)
  {
    curl_global_cleanup ();
    return; /* in shutdown */
  }
  delay = hostlist_delay;
  if (hostlist_delay.rel_value == 0)
    hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
  else
    hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
  if (hostlist_delay.rel_value > GNUNET_TIME_UNIT_HOURS.rel_value * (1 + stat_connection_count))
    hostlist_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
                                                    (1 + stat_connection_count));
  GNUNET_STATISTICS_set (stats,
                         gettext_noop("# milliseconds between hostlist downloads"),
                         hostlist_delay.rel_value,
                         GNUNET_YES);
  if (0 == once)
    {
      delay = GNUNET_TIME_UNIT_ZERO;
      once = 1;
    }  
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Have %u/%u connections.  Will consider downloading hostlist in %llums\n"),
              stat_connection_count,
              MIN_CONNECTIONS,
              (unsigned long long) delay.rel_value);
  ti_check_download = GNUNET_SCHEDULER_add_delayed (delay,
                                               &task_check,
                                               NULL);
}


/**
 * This tasks sets hostlist testing to allowed after intervall between to testings is reached
 *
 * @param cls closure
 * @param tc TaskContext
 */
static void
task_testing_intervall_reset (void *cls,
            const struct GNUNET_SCHEDULER_TaskContext *tc)
{
  ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
    return;
   stat_testing_allowed = GNUNET_OK;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
             "Testing new hostlist advertisements is allowed again\n");
}


/**
 * Task that writes hostlist entries to a file on a regular base
 *
 * @param cls closure
 * @param tc TaskContext
 */
static void
task_hostlist_saving (void *cls,
            const struct GNUNET_SCHEDULER_TaskContext *tc)
{
  ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
    return;
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Scheduled saving of hostlists\n"));
  save_hostlist_file ( GNUNET_NO );

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Hostlists will be saved to file again in %llums\n"),
              (unsigned long long) SAVING_INTERVALL.rel_value);
  ti_saving_task = GNUNET_SCHEDULER_add_delayed (SAVING_INTERVALL,
                                               &task_hostlist_saving,
                                               NULL);
}


/**
 * Method called whenever a given peer connects.
 *
 * @param cls closure
 * @param peer peer identity this notification is about
 * @param atsi performance data
 */
static void
handler_connect (void *cls,
		 const struct
		 GNUNET_PeerIdentity * peer,
		 const struct GNUNET_TRANSPORT_ATS_Information *atsi)
{
  GNUNET_assert (stat_connection_count < UINT_MAX);
  stat_connection_count++;
  GNUNET_STATISTICS_update (stats, 
			    gettext_noop ("# active connections"), 
			    1, 
			    GNUNET_NO);
}


/**
 * Method called whenever a given peer disconnects.
 *
 * @param cls closure
 * @param peer peer identity this notification is about
 */
static void
handler_disconnect (void *cls,
		    const struct
		    GNUNET_PeerIdentity * peer)
{
  GNUNET_assert (stat_connection_count > 0);
  stat_connection_count--;
  GNUNET_STATISTICS_update (stats,
			    gettext_noop ("# active connections"),
			    -1,
			    GNUNET_NO);
}


/**
 * Method called whenever an advertisement message arrives.
 *
 * @param cls closure (always NULL)
 * @param peer the peer sending the message
 * @param message the actual message
 * @param atsi performance data
 * @return GNUNET_OK to keep the connection open,
 *         GNUNET_SYSERR to close it (signal serious error)
 */
static int
handler_advertisement (void *cls,
		       const struct GNUNET_PeerIdentity * peer,
		       const struct GNUNET_MessageHeader * message,
		       const struct GNUNET_TRANSPORT_ATS_Information *atsi)
{
  size_t size;
  size_t uri_size;
  const struct GNUNET_MessageHeader * incoming;
  const char *uri;
  struct Hostlist * hostlist;

  GNUNET_assert (ntohs (message->type) == GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT);
  size = ntohs (message->size);
  if (size <= sizeof(struct GNUNET_MessageHeader))
    {
      GNUNET_break_op (0);
      return GNUNET_SYSERR;
    }
  incoming = (const struct GNUNET_MessageHeader *) message;
  uri = (const char*) &incoming[1];
  uri_size = size - sizeof (struct GNUNET_MessageHeader);
  if (uri [uri_size - 1] != '\0')
    {
      GNUNET_break_op (0);
      return GNUNET_SYSERR;
    }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlist client recieved advertisement from `%s' containing URI `%s'\n", 
	      GNUNET_i2s (peer), 
	      uri);
  if (GNUNET_NO != linked_list_contains (uri))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "URI `%s' is already known\n",
                uri);
      return GNUNET_OK;
    }

  if ( GNUNET_NO == stat_testing_allowed )
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
      return GNUNET_SYSERR;
    }
  if ( GNUNET_YES == stat_testing_hostlist )
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Currently not accepting new advertisements: we are already testing a hostlist\n");
      return GNUNET_SYSERR;
    }

  hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
  hostlist->hostlist_uri = (const char*) &hostlist[1];
  memcpy (&hostlist[1], uri, uri_size);
  hostlist->time_creation = GNUNET_TIME_absolute_get();
  hostlist->time_last_usage = GNUNET_TIME_absolute_get_zero();
  hostlist->quality = HOSTLIST_INITIAL;
  hostlist_to_test = hostlist;

  stat_testing_hostlist = GNUNET_YES;
  stat_testing_allowed = GNUNET_NO;
  ti_testing_intervall_task = GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
                                                         &task_testing_intervall_reset,
                                                         NULL);

  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
            "Testing new hostlist advertisements is locked for the next %u ms\n",
            TESTING_INTERVAL.rel_value);

  ti_download_dispatcher_task = GNUNET_SCHEDULER_add_now (&task_download_dispatcher,
                                                     NULL);

  return GNUNET_OK;
}



/**
 * Continuation called by the statistics code once 
 * we go the stat.  Initiates hostlist download scheduling.
 *
 * @param cls closure
 * @param success GNUNET_OK if statistics were
 *        successfully obtained, GNUNET_SYSERR if not.
 */
static void
primary_task (void *cls, int success)
{
  if (stats == NULL)
    return; /* in shutdown */
#if DEBUG_HOSTLIST_CLIENT
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
	      "Statistics request done, scheduling hostlist download\n");
#endif
  ti_check_download = GNUNET_SCHEDULER_add_now (&task_check,
                                           NULL);
}


static int
process_stat (void *cls,
	      const char *subsystem,
	      const char *name,
	      uint64_t value,
	      int is_persistent)
{
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
	      _("Initial time between hostlist downloads is %llums\n"),
	      (unsigned long long) value);
  hostlist_delay.rel_value = value;
  return GNUNET_OK;
}

/**
 * Method to load persistent hostlist file during hostlist client startup
 */
static void 
load_hostlist_file ()
{
  char *filename;
  char *uri;
  char *emsg;
  struct Hostlist * hostlist;
  uri = NULL;
  uint32_t times_used;
  uint32_t hellos_returned;
  uint64_t quality;
  uint64_t last_used;
  uint64_t created;
  uint32_t counter;

  if (GNUNET_OK !=
		  GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                   "HOSTLIST",
                                                   "HOSTLISTFILE",
                                                   &filename))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  _("No `%s' specified in `%s' configuration, cannot load hostlists from file.\n"),
                  "HOSTLISTFILE", "HOSTLIST");
      return;
    }

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Loading saved hostlist entries from file `%s' \n"), filename);
  if ( GNUNET_NO == GNUNET_DISK_file_test (filename) )
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _("Hostlist file `%s' is not existing\n"), filename);
    GNUNET_free ( filename );
    return;
  }

  struct GNUNET_BIO_ReadHandle * rh = GNUNET_BIO_read_open (filename);
  if (NULL == rh)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  _("Could not open file `%s' for reading to load hostlists: %s\n"), 
		  filename,
		  STRERROR (errno));
      GNUNET_free (filename);
      return;
    }

  counter = 0;
  while ( (GNUNET_OK == GNUNET_BIO_read_string (rh, "url" , &uri, MAX_URL_LEN)) &&
	  (NULL != uri) &&
	  (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &times_used)) &&
	  (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
	  (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
	  (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
	  (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)) )
    {
      hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
      hostlist->hello_count = hellos_returned;
      hostlist->hostlist_uri = (const char *) &hostlist[1];
      memcpy (&hostlist[1], uri, strlen(uri)+1);
      hostlist->quality = quality;
      hostlist->time_creation.abs_value = created;
      hostlist->time_last_usage.abs_value = last_used;
      GNUNET_CONTAINER_DLL_insert(linked_list_head, linked_list_tail, hostlist);
      linked_list_size++;
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Added hostlist entry eith URI `%s' \n", hostlist->hostlist_uri);
      GNUNET_free (uri);
      uri = NULL;
      counter++;
      if ( counter >= MAX_NUMBER_HOSTLISTS ) break;
    }

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("%u hostlist URIs loaded from file\n"), counter);
  GNUNET_STATISTICS_set (stats,
                         gettext_noop("# hostlist URIs read from file"),
                         counter,
                         GNUNET_YES);
  GNUNET_STATISTICS_set (stats,
                         gettext_noop("# advertised hostlist URIs"),
                         linked_list_size,
                         GNUNET_NO);

  GNUNET_free_non_null (uri);
  emsg = NULL;
  GNUNET_BIO_read_close (rh, &emsg);
  if (emsg != NULL)
    GNUNET_free (emsg);
  GNUNET_free (filename);
}


/**
 * Method to save persistent hostlist file during hostlist client shutdown
 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
 */
static void save_hostlist_file ( int shutdown )
{
  char *filename;
  struct Hostlist *pos;
  struct GNUNET_BIO_WriteHandle * wh;
  int ok;
  uint32_t counter;

  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_filename (cfg,
                                             "HOSTLIST",
                                             "HOSTLISTFILE",
                                             &filename))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  _("No `%s' specified in `%s' configuration, cannot save hostlists to file.\n"),
                  "HOSTLISTFILE", "HOSTLIST");
      return;
    }
  if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
    {
      GNUNET_free (filename);
      return;
    }
  wh = GNUNET_BIO_write_open (filename);
  if ( NULL == wh)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  _("Could not open file `%s' for writing to save hostlists: %s\n"),
                  filename,
		  STRERROR (errno));
      GNUNET_free (filename);
      return;
    }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Writing %u hostlist URIs to `%s'\n" ),
              linked_list_size, filename);
  /* add code to write hostlists to file using bio */
  ok = GNUNET_YES;
  counter = 0;
  while (NULL != (pos = linked_list_head))
    {
      if ( GNUNET_YES == shutdown)
      {
        GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
        linked_list_size--;
      }
      if (GNUNET_YES == ok)
	{
	  if ( (GNUNET_OK !=
		GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
	       (GNUNET_OK !=
		GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
	       (GNUNET_OK !=
		GNUNET_BIO_write_int64 (wh, pos->quality)) ||
               (GNUNET_OK !=
                GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value)) ||
               (GNUNET_OK !=
                GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value)) ||
	       (GNUNET_OK !=
	        GNUNET_BIO_write_int32 (wh, pos->hello_count)))
	    {
	      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
			  _("Error writing hostlist URIs to file `%s'\n"),
			  filename);
	      ok = GNUNET_NO;
	    }
	}

      if ( GNUNET_YES == shutdown)
        GNUNET_free (pos);
      counter ++;
      if ( counter >= MAX_NUMBER_HOSTLISTS) break;
    }  
  GNUNET_STATISTICS_set (stats,
                         gettext_noop("# hostlist URIs written to file"),
                         counter,
                         GNUNET_YES);

  if ( GNUNET_OK != GNUNET_BIO_write_close ( wh ) )
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
		_("Error writing hostlist URIs to file `%s'\n"),
		filename);
  GNUNET_free (filename);
}

/**
 * Start downloading hostlists from hostlist servers as necessary.
 */
int
GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
			      struct GNUNET_STATISTICS_Handle *st,
			      GNUNET_CORE_ConnectEventHandler *ch,
			      GNUNET_CORE_DisconnectEventHandler *dh,
			      GNUNET_CORE_MessageCallback *msgh,
			      int learn)
{
  char *filename;
  int result;

  if (0 != curl_global_init (CURL_GLOBAL_WIN32))
    {
      GNUNET_break (0);
      return GNUNET_SYSERR;
    }
  transport = GNUNET_TRANSPORT_connect (c, NULL, NULL, NULL, NULL, NULL);
  if (NULL == transport)
    {
      curl_global_cleanup ();
      return GNUNET_SYSERR;
    }
  cfg = c;
  stats = st;
  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_string (cfg,
					     "HOSTLIST",
					     "HTTP-PROXY", 
					     &proxy))
    proxy = NULL;
  stat_learning = learn;
  *ch = &handler_connect;
  *dh = &handler_disconnect;
  linked_list_head = NULL;
  linked_list_tail = NULL;
  stat_use_bootstrap = GNUNET_YES;
  stat_testing_hostlist = GNUNET_NO;
  stat_testing_allowed = GNUNET_YES;

  if ( GNUNET_YES == stat_learning )
  {
    *msgh = &handler_advertisement;
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Learning is enabled on this peer\n"));
    load_hostlist_file ();
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Hostlists will be saved to file again in  %llums\n"),
              (unsigned long long) SAVING_INTERVALL.rel_value);
    ti_saving_task = GNUNET_SCHEDULER_add_delayed (SAVING_INTERVALL,
						   &task_hostlist_saving,
						   NULL);
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Learning is not enabled on this peer\n"));
    *msgh = NULL;
    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                            "HOSTLIST",
                                                            "HOSTLISTFILE",
                                                            &filename))
    {
    if ( GNUNET_YES == GNUNET_DISK_file_test (filename) )
      {
        result = remove (filename);
        if (result == 0)
        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
		    _("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),
		    filename);
        else
          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
		      _("Hostlist file `%s' could not be removed\n"),
		      filename);
      }
    }
    GNUNET_free ( filename );
  }
  GNUNET_STATISTICS_get (stats,
			 "hostlist",
			 gettext_noop("# milliseconds between hostlist downloads"),
			 GNUNET_TIME_UNIT_MINUTES,
			 &primary_task,
			 &process_stat,
			 NULL);
  return GNUNET_OK;
}


/**
 * Stop downloading hostlists from hostlist servers as necessary.
 */
void
GNUNET_HOSTLIST_client_stop ()
{
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlist client shutdown\n");
#if DEBUG_HOSTLIST_CLIENT
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
	      "Hostlist client shutdown\n");
#endif
  if ( GNUNET_YES == stat_learning )
    save_hostlist_file ( GNUNET_YES );

  if (ti_saving_task != GNUNET_SCHEDULER_NO_TASK)
    {
      GNUNET_SCHEDULER_cancel (ti_saving_task);
    }

  if (ti_download_dispatcher_task != GNUNET_SCHEDULER_NO_TASK)
    {
      GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
    }
  if (ti_testing_intervall_task != GNUNET_SCHEDULER_NO_TASK)
    {
      GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
    }
  if (ti_download != GNUNET_SCHEDULER_NO_TASK)
    {
      GNUNET_SCHEDULER_cancel (ti_download);
    }
  if (ti_check_download != GNUNET_SCHEDULER_NO_TASK)
    {
      GNUNET_SCHEDULER_cancel (ti_check_download);
      curl_global_cleanup ();
    }
  if (transport != NULL)
    {
      GNUNET_TRANSPORT_disconnect (transport);
      transport = NULL;
    }
  GNUNET_assert (NULL == transport);
  GNUNET_free_non_null (proxy);
  proxy = NULL;
  cfg = NULL;
}

/* end of hostlist-client.c */