12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613 |
- /*
- This file is part of GNUnet.
- Copyright (C) 2011 - 2017 GNUnet e.V.
- GNUnet is free software: you can redistribute it and/or modify it
- under the terms of the GNU Affero General Public License as published
- by the Free Software Foundation, either version 3 of the License,
- or (at your option) any later version.
- GNUnet is distributed in the hope that it will be useful, but
- WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Affero General Public License for more details.
- You should have received a copy of the GNU Affero General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
- SPDX-License-Identifier: AGPL3.0-or-later
- */
- /**
- * @file regex/gnunet-regex-profiler.c
- * @brief Regex profiler for testing distributed regex use.
- * @author Bartlomiej Polot
- * @author Maximilian Szengel
- *
- */
- #include <string.h>
- #include "platform.h"
- #include "gnunet_applications.h"
- #include "gnunet_util_lib.h"
- #include "regex_internal_lib.h"
- #include "gnunet_arm_service.h"
- #include "gnunet_dht_service.h"
- #include "gnunet_testbed_service.h"
- #define FIND_TIMEOUT \
- GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90)
- /**
- * DLL of operations
- */
- struct DLLOperation
- {
- /**
- * The testbed operation handle
- */
- struct GNUNET_TESTBED_Operation *op;
- /**
- * Closure
- */
- void *cls;
- /**
- * The next pointer for DLL
- */
- struct DLLOperation *next;
- /**
- * The prev pointer for DLL
- */
- struct DLLOperation *prev;
- };
- /**
- * Available states during profiling
- */
- enum State
- {
- /**
- * Initial state
- */
- STATE_INIT = 0,
- /**
- * Starting slaves
- */
- STATE_SLAVES_STARTING,
- /**
- * Creating peers
- */
- STATE_PEERS_CREATING,
- /**
- * Starting peers
- */
- STATE_PEERS_STARTING,
- /**
- * Linking peers
- */
- STATE_PEERS_LINKING,
- /**
- * Matching strings against announced regexes
- */
- STATE_SEARCH_REGEX,
- /**
- * Destroying peers; we can do this as the controller takes care of stopping a
- * peer if it is running
- */
- STATE_PEERS_DESTROYING
- };
- /**
- * Peer handles.
- */
- struct RegexPeer
- {
- /**
- * Peer id.
- */
- unsigned int id;
- /**
- * Peer configuration handle.
- */
- struct GNUNET_CONFIGURATION_Handle *cfg;
- /**
- * The actual testbed peer handle.
- */
- struct GNUNET_TESTBED_Peer *peer_handle;
- /**
- * Peer's search string.
- */
- const char *search_str;
- /**
- * Set to GNUNET_YES if the peer successfully matched the above
- * search string. GNUNET_NO if the string could not be matched
- * during the profiler run. GNUNET_SYSERR if the string matching
- * timed out. Undefined if search_str is NULL
- */
- int search_str_matched;
- /**
- * Peer's DHT handle.
- */
- struct GNUNET_DHT_Handle *dht_handle;
- /**
- * Handle to a running regex search.
- */
- struct REGEX_INTERNAL_Search *search_handle;
- /**
- * Testbed operation handle for DHT.
- */
- struct GNUNET_TESTBED_Operation *op_handle;
- /**
- * Peers's statistics handle.
- */
- struct GNUNET_STATISTICS_Handle *stats_handle;
- /**
- * The starting time of a profiling step.
- */
- struct GNUNET_TIME_Absolute prof_start_time;
- /**
- * Operation timeout
- */
- struct GNUNET_SCHEDULER_Task *timeout;
- /**
- * Daemon start
- */
- struct GNUNET_TESTBED_Operation *daemon_op;
- };
- /**
- * Set when shutting down to avoid making more queries.
- */
- static int in_shutdown;
- /**
- * The array of peers; we fill this as the peers are given to us by the testbed
- */
- static struct RegexPeer *peers;
- /**
- * Host registration handle
- */
- static struct GNUNET_TESTBED_HostRegistrationHandle *reg_handle;
- /**
- * Handle to the master controller process
- */
- static struct GNUNET_TESTBED_ControllerProc *mc_proc;
- /**
- * Handle to the master controller
- */
- static struct GNUNET_TESTBED_Controller *mc;
- /**
- * Handle to global configuration
- */
- static struct GNUNET_CONFIGURATION_Handle *cfg;
- /**
- * Abort task identifier
- */
- static struct GNUNET_SCHEDULER_Task *abort_task;
- /**
- * Host registration task identifier
- */
- static struct GNUNET_SCHEDULER_Task *register_hosts_task;
- /**
- * Global event mask for all testbed events
- */
- static uint64_t event_mask;
- /**
- * The starting time of a profiling step
- */
- static struct GNUNET_TIME_Absolute prof_start_time;
- /**
- * Duration profiling step has taken
- */
- static struct GNUNET_TIME_Relative prof_time;
- /**
- * Number of peers to be started by the profiler
- */
- static unsigned int num_peers;
- /**
- * Global testing status
- */
- static int result;
- /**
- * current state of profiling
- */
- enum State state;
- /**
- * Folder where policy files are stored.
- */
- static char *policy_dir;
- /**
- * File with hostnames where to execute the test.
- */
- static char *hosts_file;
- /**
- * File with the strings to look for.
- */
- static char *strings_file;
- /**
- * Search strings (num_peers of them).
- */
- static char **search_strings;
- /**
- * How many searches are we going to start in parallel
- */
- static long long unsigned int init_parallel_searches;
- /**
- * How many searches are running in parallel
- */
- static unsigned int parallel_searches;
- /**
- * Number of strings found in the published regexes.
- */
- static unsigned int strings_found;
- /**
- * Index of peer to start next announce/search.
- */
- static unsigned int next_search;
- /**
- * Search timeout task identifier.
- */
- static struct GNUNET_SCHEDULER_Task *search_timeout_task;
- /**
- * Search timeout in seconds.
- */
- static struct GNUNET_TIME_Relative search_timeout_time = { 60000 };
- /**
- * File to log statistics to.
- */
- static struct GNUNET_DISK_FileHandle *data_file;
- /**
- * Filename to log statistics to.
- */
- static char *data_filename;
- /**
- * Prefix used for regex announcing. We need to prefix the search
- * strings with it, in order to find something.
- */
- static char *regex_prefix;
- /**
- * What's the maximum regex reannounce period.
- */
- static struct GNUNET_TIME_Relative reannounce_period_max;
- /******************************************************************************/
- /****************************** DECLARATIONS ********************************/
- /******************************************************************************/
- /**
- * DHT connect callback.
- *
- * @param cls internal peer id.
- * @param op operation handle.
- * @param ca_result connect adapter result.
- * @param emsg error message.
- */
- static void
- dht_connect_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
- void *ca_result, const char *emsg);
- /**
- * DHT connect adapter.
- *
- * @param cls not used.
- * @param cfg configuration handle.
- *
- * @return
- */
- static void *
- dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg);
- /**
- * Adapter function called to destroy a connection to
- * the DHT service
- *
- * @param cls closure
- * @param op_result service handle returned from the connect adapter
- */
- static void
- dht_da (void *cls, void *op_result);
- /**
- * Function called by testbed once we are connected to stats
- * service. Get the statistics for the services of interest.
- *
- * @param cls the 'struct RegexPeer' for which we connected to stats
- * @param op connect operation handle
- * @param ca_result handle to stats service
- * @param emsg error message on failure
- */
- static void
- stats_connect_cb (void *cls,
- struct GNUNET_TESTBED_Operation *op,
- void *ca_result,
- const char *emsg);
- /**
- * Start announcing the next regex in the DHT.
- *
- * @param cls Index of the next peer in the peers array.
- */
- static void
- announce_next_regex (void *cls);
- /******************************************************************************/
- /******************************** SHUTDOWN **********************************/
- /******************************************************************************/
- /**
- * Shutdown nicely
- *
- * @param cls NULL
- */
- static void
- do_shutdown (void *cls)
- {
- struct RegexPeer *peer;
- unsigned int peer_cnt;
- unsigned int search_str_cnt;
- char output_buffer[512];
- size_t size;
- if (NULL != abort_task)
- {
- GNUNET_SCHEDULER_cancel (abort_task);
- abort_task = NULL;
- }
- if (NULL != register_hosts_task)
- {
- GNUNET_SCHEDULER_cancel (register_hosts_task);
- register_hosts_task = NULL;
- }
- for (peer_cnt = 0; peer_cnt < num_peers; peer_cnt++)
- {
- peer = &peers[peer_cnt];
- if ((GNUNET_YES != peer->search_str_matched) && (NULL != data_file) )
- {
- prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
- size =
- GNUNET_snprintf (output_buffer,
- sizeof(output_buffer),
- "%p Search string not found: %s (%d)\n"
- "%p On peer: %u (%p)\n"
- "%p After: %s\n",
- peer, peer->search_str, peer->search_str_matched,
- peer, peer->id, peer,
- peer,
- GNUNET_STRINGS_relative_time_to_string (prof_time,
- GNUNET_NO));
- if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
- }
- if (NULL != peers[peer_cnt].op_handle)
- GNUNET_TESTBED_operation_done (peers[peer_cnt].op_handle);
- }
- if (NULL != data_file)
- {
- GNUNET_DISK_file_close (data_file);
- data_file = NULL;
- }
- for (search_str_cnt = 0;
- search_str_cnt < num_peers && NULL != search_strings;
- search_str_cnt++)
- {
- GNUNET_free (search_strings[search_str_cnt]);
- }
- GNUNET_free (search_strings);
- search_strings = NULL;
- if (NULL != reg_handle)
- {
- GNUNET_TESTBED_cancel_registration (reg_handle);
- reg_handle = NULL;
- }
- if (NULL != mc)
- {
- GNUNET_TESTBED_controller_disconnect (mc);
- mc = NULL;
- }
- if (NULL != mc_proc)
- {
- GNUNET_TESTBED_controller_stop (mc_proc);
- mc_proc = NULL;
- }
- if (NULL != cfg)
- {
- GNUNET_CONFIGURATION_destroy (cfg);
- cfg = NULL;
- }
- }
- /**
- * abort task to run on test timed out
- *
- * @param cls NULL
- */
- static void
- do_abort (void *cls)
- {
- unsigned long i = (unsigned long) cls;
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Aborting from line %lu...\n", i);
- abort_task = NULL;
- result = GNUNET_SYSERR;
- GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
- }
- /******************************************************************************/
- /********************* STATISTICS SERVICE CONNECTIONS ***********************/
- /******************************************************************************/
- /**
- * Adapter function called to establish a connection to
- * statistics service.
- *
- * @param cls closure
- * @param cfg configuration of the peer to connect to; will be available until
- * GNUNET_TESTBED_operation_done() is called on the operation returned
- * from GNUNET_TESTBED_service_connect()
- * @return service handle to return in 'op_result', NULL on error
- */
- static void *
- stats_ca (void *cls,
- const struct GNUNET_CONFIGURATION_Handle *cfg)
- {
- return GNUNET_STATISTICS_create ("<driver>", cfg);
- }
- /**
- * Adapter function called to destroy a connection to
- * statistics service.
- *
- * @param cls closure
- * @param op_result service handle returned from the connect adapter
- */
- static void
- stats_da (void *cls, void *op_result)
- {
- struct RegexPeer *peer = cls;
- GNUNET_assert (op_result == peer->stats_handle);
- GNUNET_STATISTICS_destroy (peer->stats_handle, GNUNET_NO);
- peer->stats_handle = NULL;
- }
- /**
- * Process statistic values. Write all values to global 'data_file', if present.
- *
- * @param cls closure
- * @param subsystem name of subsystem that created the statistic
- * @param name the name of the datum
- * @param value the current value
- * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
- * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
- */
- static int
- stats_iterator (void *cls,
- const char *subsystem,
- const char *name,
- uint64_t value, int is_persistent)
- {
- struct RegexPeer *peer = cls;
- char output_buffer[512];
- size_t size;
- if (NULL == data_file)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "%p -> %s [%s]: %llu\n",
- peer,
- subsystem,
- name,
- (unsigned long long) value);
- return GNUNET_OK;
- }
- size =
- GNUNET_snprintf (output_buffer,
- sizeof(output_buffer),
- "%p [%s] %llu %s\n",
- peer,
- subsystem,
- (unsigned long long) value,
- name);
- if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Unable to write to file!\n");
- return GNUNET_OK;
- }
- /**
- * Stats callback. Finish the stats testbed operation and when all stats have
- * been iterated, shutdown the profiler.
- *
- * @param cls closure
- * @param success GNUNET_OK if statistics were
- * successfully obtained, GNUNET_SYSERR if not.
- */
- static void
- stats_cb (void *cls,
- int success)
- {
- static unsigned int peer_cnt;
- struct RegexPeer *peer = cls;
- if (GNUNET_OK != success)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- "Getting statistics for peer %u failed!\n",
- peer->id);
- return;
- }
- GNUNET_assert (NULL != peer->op_handle);
- GNUNET_TESTBED_operation_done (peer->op_handle);
- peer->op_handle = NULL;
- peer_cnt++;
- peer = &peers[peer_cnt];
- fprintf (stderr, "s");
- if (peer_cnt == num_peers)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "\nCollecting stats finished. Shutting down.\n");
- GNUNET_SCHEDULER_shutdown ();
- result = GNUNET_OK;
- }
- else
- {
- peer->op_handle =
- GNUNET_TESTBED_service_connect (NULL,
- peer->peer_handle,
- "statistics",
- &stats_connect_cb,
- peer,
- &stats_ca,
- &stats_da,
- peer);
- }
- }
- /**
- * Function called by testbed once we are connected to stats
- * service. Get the statistics for the services of interest.
- *
- * @param cls the 'struct RegexPeer' for which we connected to stats
- * @param op connect operation handle
- * @param ca_result handle to stats service
- * @param emsg error message on failure
- */
- static void
- stats_connect_cb (void *cls,
- struct GNUNET_TESTBED_Operation *op,
- void *ca_result,
- const char *emsg)
- {
- struct RegexPeer *peer = cls;
- if ((NULL == ca_result) || (NULL != emsg))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- "Failed to connect to statistics service on peer %u: %s\n",
- peer->id, emsg);
- peer->stats_handle = NULL;
- return;
- }
- peer->stats_handle = ca_result;
- if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL,
- &stats_cb,
- &stats_iterator, peer))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- "Could not get statistics of peer %u!\n", peer->id);
- }
- }
- /**
- * Task to collect all statistics from all peers, will shutdown the
- * profiler, when done.
- *
- * @param cls NULL
- */
- static void
- do_collect_stats (void *cls)
- {
- struct RegexPeer *peer = &peers[0];
- GNUNET_assert (NULL != peer->peer_handle);
- peer->op_handle =
- GNUNET_TESTBED_service_connect (NULL,
- peer->peer_handle,
- "statistics",
- &stats_connect_cb,
- peer,
- &stats_ca,
- &stats_da,
- peer);
- }
- /******************************************************************************/
- /************************ REGEX FIND CONNECTIONS **************************/
- /******************************************************************************/
- /**
- * Start searching for the next string in the DHT.
- *
- * @param cls Index of the next peer in the peers array.
- */
- static void
- find_string (void *cls);
- /**
- * Method called when we've found a peer that announced a regex
- * that matches our search string. Now get the statistics.
- *
- * @param cls Closure provided in REGEX_INTERNAL_search.
- * @param id Peer providing a regex that matches the string.
- * @param get_path Path of the get request.
- * @param get_path_length Length of get_path.
- * @param put_path Path of the put request.
- * @param put_path_length Length of the put_path.
- */
- static void
- regex_found_handler (void *cls,
- const struct GNUNET_PeerIdentity *id,
- const struct GNUNET_PeerIdentity *get_path,
- unsigned int get_path_length,
- const struct GNUNET_PeerIdentity *put_path,
- unsigned int put_path_length)
- {
- struct RegexPeer *peer = cls;
- char output_buffer[512];
- size_t size;
- if (GNUNET_YES == peer->search_str_matched)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- "String %s on peer %u already matched!\n",
- peer->search_str, peer->id);
- return;
- }
- strings_found++;
- parallel_searches--;
- if (NULL != peer->timeout)
- {
- GNUNET_SCHEDULER_cancel (peer->timeout);
- peer->timeout = NULL;
- if (GNUNET_NO == in_shutdown)
- GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL);
- }
- if (NULL == id)
- {
- // FIXME not possible right now
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "String matching timed out for string %s on peer %u (%i/%i)\n",
- peer->search_str, peer->id, strings_found, num_peers);
- peer->search_str_matched = GNUNET_SYSERR;
- }
- else
- {
- prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "String %s found on peer %u after %s (%i/%i) (%u||)\n",
- peer->search_str, peer->id,
- GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO),
- strings_found, num_peers, parallel_searches);
- peer->search_str_matched = GNUNET_YES;
- if (NULL != data_file)
- {
- size =
- GNUNET_snprintf (output_buffer,
- sizeof(output_buffer),
- "%p Peer: %u\n"
- "%p Search string: %s\n"
- "%p Search duration: %s\n\n",
- peer, peer->id,
- peer, peer->search_str,
- peer,
- GNUNET_STRINGS_relative_time_to_string (prof_time,
- GNUNET_NO));
- if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
- }
- }
- GNUNET_TESTBED_operation_done (peer->op_handle);
- peer->op_handle = NULL;
- if (strings_found == num_peers)
- {
- prof_time = GNUNET_TIME_absolute_get_duration (prof_start_time);
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "All strings successfully matched in %s\n",
- GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO));
- if (NULL != search_timeout_task)
- {
- GNUNET_SCHEDULER_cancel (search_timeout_task);
- search_timeout_task = NULL;
- }
- GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats.\n");
- GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL);
- }
- }
- /**
- * Connect by string timeout task. This will cancel the profiler after the
- * specified timeout 'search_timeout'.
- *
- * @param cls NULL
- */
- static void
- search_timed_out (void *cls)
- {
- unsigned int i;
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Finding matches to all strings did not succeed after %s.\n",
- GNUNET_STRINGS_relative_time_to_string (search_timeout_time,
- GNUNET_NO));
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Found %i of %i strings\n", strings_found, num_peers);
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Search timed out after %s."
- "Collecting stats and shutting down.\n",
- GNUNET_STRINGS_relative_time_to_string (search_timeout_time,
- GNUNET_NO));
- in_shutdown = GNUNET_YES;
- for (i = 0; i < num_peers; i++)
- {
- if (NULL != peers[i].op_handle)
- {
- GNUNET_TESTBED_operation_done (peers[i].op_handle);
- peers[i].op_handle = NULL;
- }
- }
- GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL);
- }
- /**
- * Search timed out. It might still complete in the future,
- * but we should start another one.
- *
- * @param cls Index of the next peer in the peers array.
- */
- static void
- find_timed_out (void *cls)
- {
- struct RegexPeer *p = cls;
- p->timeout = NULL;
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Searching for string \"%s\" on peer %d timed out.\n",
- p->search_str,
- p->id);
- if (GNUNET_NO == in_shutdown)
- GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL);
- }
- /**
- * Start searching for a string in the DHT.
- *
- * @param cls Index of the next peer in the peers array.
- */
- static void
- find_string (void *cls)
- {
- unsigned int search_peer = (unsigned int) (long) cls;
- if ((search_peer >= num_peers) ||
- (GNUNET_YES == in_shutdown))
- return;
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Searching for string \"%s\" on peer %d (%u||)\n",
- peers[search_peer].search_str,
- search_peer,
- parallel_searches);
- peers[search_peer].op_handle =
- GNUNET_TESTBED_service_connect (NULL,
- peers[search_peer].peer_handle,
- "dht",
- &dht_connect_cb,
- &peers[search_peer],
- &dht_ca,
- &dht_da,
- &peers[search_peer]);
- GNUNET_assert (NULL != peers[search_peer].op_handle);
- peers[search_peer].timeout
- = GNUNET_SCHEDULER_add_delayed (FIND_TIMEOUT,
- &find_timed_out,
- &peers[search_peer]);
- }
- /**
- * Callback called when testbed has started the daemon we asked for.
- *
- * @param cls NULL
- * @param op the operation handle
- * @param emsg NULL on success; otherwise an error description
- */
- static void
- daemon_started (void *cls,
- struct GNUNET_TESTBED_Operation *op,
- const char *emsg)
- {
- struct RegexPeer *peer = (struct RegexPeer *) cls;
- unsigned long search_peer;
- unsigned int i;
- GNUNET_TESTBED_operation_done (peer->daemon_op);
- peer->daemon_op = NULL;
- if (NULL != emsg)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg);
- GNUNET_assert (0);
- }
- else
- {
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Daemon %u started successfully\n", peer->id);
- }
- /* Find a peer to look for a string matching the regex announced */
- search_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
- num_peers);
- for (i = 0; peers[search_peer].search_str != NULL; i++)
- {
- search_peer = (search_peer + 1) % num_peers;
- if (i > num_peers)
- GNUNET_assert (0); /* we ran out of peers, must be a bug */
- }
- peers[search_peer].search_str = search_strings[peer->id];
- peers[search_peer].search_str_matched = GNUNET_NO;
- GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_saturating_multiply (
- reannounce_period_max,
- 2),
- &find_string,
- (void *) search_peer);
- }
- /**
- * Task to start the daemons on each peer so that the regexes are announced
- * into the DHT.
- *
- * @param cls NULL
- * @param tc the task context
- */
- static void
- do_announce (void *cls)
- {
- unsigned int i;
- if (GNUNET_YES == in_shutdown)
- return;
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Starting announce.\n");
- for (i = 0; i < init_parallel_searches; i++)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- " scheduling announce %u\n",
- i);
- (void) GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL);
- }
- }
- /**
- * Start announcing the next regex in the DHT.
- *
- * @param cls Closure (unused).
- */
- static void
- announce_next_regex (void *cls)
- {
- struct RegexPeer *peer;
- if (GNUNET_YES == in_shutdown)
- return;
- if (next_search >= num_peers)
- {
- if (strings_found != num_peers)
- {
- struct GNUNET_TIME_Relative new_delay;
- if (NULL != search_timeout_task)
- GNUNET_SCHEDULER_cancel (search_timeout_task);
- new_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15);
- search_timeout_task = GNUNET_SCHEDULER_add_delayed (new_delay,
- &search_timed_out,
- NULL);
- }
- return;
- }
- GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search);
- peer = &peers[next_search];
- peer->daemon_op =
- GNUNET_TESTBED_peer_manage_service (NULL,
- peer->peer_handle,
- "regexprofiler",
- &daemon_started,
- peer,
- 1);
- next_search++;
- parallel_searches++;
- }
- /**
- * DHT connect callback. Called when we are connected to the dht service for
- * the peer in 'cls'. If successful we connect to the stats service of this
- * peer and then try to match the search string of this peer.
- *
- * @param cls internal peer id.
- * @param op operation handle.
- * @param ca_result connect adapter result.
- * @param emsg error message.
- */
- static void
- dht_connect_cb (void *cls,
- struct GNUNET_TESTBED_Operation *op,
- void *ca_result,
- const char *emsg)
- {
- struct RegexPeer *peer = (struct RegexPeer *) cls;
- if ((NULL != emsg) || (NULL == op) || (NULL == ca_result))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "DHT connect failed: %s\n", emsg);
- GNUNET_assert (0);
- }
- GNUNET_assert (NULL != peer->dht_handle);
- GNUNET_assert (peer->op_handle == op);
- GNUNET_assert (peer->dht_handle == ca_result);
- peer->search_str_matched = GNUNET_NO;
- peer->search_handle = REGEX_INTERNAL_search (peer->dht_handle,
- peer->search_str,
- ®ex_found_handler, peer,
- NULL);
- peer->prof_start_time = GNUNET_TIME_absolute_get ();
- }
- /**
- * DHT connect adapter. Opens a connection to the dht service.
- *
- * @param cls Closure (peer).
- * @param cfg Configuration handle.
- *
- * @return
- */
- static void *
- dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
- {
- struct RegexPeer *peer = cls;
- peer->dht_handle = GNUNET_DHT_connect (cfg, 32);
- return peer->dht_handle;
- }
- /**
- * Adapter function called to destroy a connection to the dht service.
- *
- * @param cls Closure (peer).
- * @param op_result Service handle returned from the connect adapter.
- */
- static void
- dht_da (void *cls, void *op_result)
- {
- struct RegexPeer *peer = (struct RegexPeer *) cls;
- GNUNET_assert (peer->dht_handle == op_result);
- if (NULL != peer->search_handle)
- {
- REGEX_INTERNAL_search_cancel (peer->search_handle);
- peer->search_handle = NULL;
- }
- if (NULL != peer->dht_handle)
- {
- GNUNET_DHT_disconnect (peer->dht_handle);
- peer->dht_handle = NULL;
- }
- }
- /**
- * Signature of a main function for a testcase.
- *
- * @param cls NULL
- * @param h the run handle
- * @param num_peers_ number of peers in 'peers'
- * @param testbed_peers handle to peers run in the testbed. NULL upon timeout (see
- * GNUNET_TESTBED_test_run()).
- * @param links_succeeded the number of overlay link connection attempts that
- * succeeded
- * @param links_failed the number of overlay link connection attempts that
- * failed
- */
- static void
- test_master (void *cls,
- struct GNUNET_TESTBED_RunHandle *h,
- unsigned int num_peers_,
- struct GNUNET_TESTBED_Peer **testbed_peers,
- unsigned int links_succeeded,
- unsigned int links_failed)
- {
- unsigned int i;
- GNUNET_assert (num_peers_ == num_peers);
- prof_time = GNUNET_TIME_absolute_get_duration (prof_start_time);
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Testbed started in %s\n",
- GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO));
- if (NULL != abort_task)
- {
- GNUNET_SCHEDULER_cancel (abort_task);
- abort_task = NULL;
- }
- for (i = 0; i < num_peers; i++)
- {
- peers[i].peer_handle = testbed_peers[i];
- }
- if (GNUNET_NO ==
- GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT"))
- {
- struct GNUNET_TIME_Relative settle_time;
- settle_time =
- GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
- 10 * num_peers);
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Waiting for DHT for %s to settle new connections.\n\n",
- GNUNET_STRINGS_relative_time_to_string (settle_time,
- GNUNET_NO));
- GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL);
- }
- else
- {
- GNUNET_SCHEDULER_add_now (&do_announce, NULL);
- }
- search_timeout_task =
- GNUNET_SCHEDULER_add_delayed (search_timeout_time, &search_timed_out, NULL);
- }
- /**
- * Function that will be called whenever something in the testbed changes.
- *
- * @param cls closure, NULL
- * @param event information on what is happening
- */
- static void
- master_controller_cb (void *cls,
- const struct GNUNET_TESTBED_EventInformation *event)
- {
- switch (event->type)
- {
- case GNUNET_TESTBED_ET_CONNECT:
- printf (".");
- break;
- case GNUNET_TESTBED_ET_PEER_START:
- printf ("#");
- break;
- default:
- break;
- }
- fflush (stdout);
- }
- /******************************************************************************/
- /*************************** TESTBED PEER SETUP *****************************/
- /******************************************************************************/
- /**
- * Process the text buffer counting the non-empty lines and separating them
- * with NULL characters, for later ease of copy using (as)printf.
- *
- * @param data Memory buffer with strings.
- * @param data_size Size of the @a data buffer in bytes.
- * @param str_max Maximum number of strings to return.
- * @return Positive number of lines found in the buffer,
- * #GNUNET_SYSERR otherwise.
- */
- static int
- count_and_separate_strings (char *data,
- uint64_t data_size,
- unsigned int str_max)
- {
- char *buf; // Keep track of last string to skip blank lines
- unsigned int offset;
- unsigned int str_cnt;
- buf = data;
- offset = 0;
- str_cnt = 0;
- while ((offset < (data_size - 1)) && (str_cnt < str_max))
- {
- offset++;
- if (((data[offset] == '\n')) &&
- (buf != &data[offset]))
- {
- data[offset] = '\0';
- str_cnt++;
- buf = &data[offset + 1];
- }
- else if ((data[offset] == '\n') ||
- (data[offset] == '\0'))
- buf = &data[offset + 1];
- }
- return str_cnt;
- }
- /**
- * Allocate a string array and fill it with the prefixed strings
- * from a pre-processed, NULL-separated memory region.
- *
- * @param data Preprocessed memory with strings
- * @param data_size Size of the @a data buffer in bytes.
- * @param strings Address of the string array to be created.
- * Must be freed by caller if function end in success.
- * @param str_cnt String count. The @a data buffer should contain
- * at least this many NULL-separated strings.
- * @return #GNUNET_OK in ase of success, #GNUNET_SYSERR otherwise.
- * In case of error @a strings must not be freed.
- */
- static int
- create_string_array (char *data, uint64_t data_size,
- char ***strings, unsigned int str_cnt)
- {
- uint64_t offset;
- uint64_t len;
- unsigned int i;
- *strings = GNUNET_malloc (sizeof(char *) * str_cnt);
- offset = 0;
- for (i = 0; i < str_cnt; i++)
- {
- len = strlen (&data[offset]);
- if (offset + len >= data_size)
- {
- GNUNET_free (*strings);
- *strings = NULL;
- return GNUNET_SYSERR;
- }
- if (0 == len) // empty line
- {
- offset++;
- i--;
- continue;
- }
- GNUNET_asprintf (&(*strings)[i],
- "%s%s",
- regex_prefix,
- &data[offset]);
- offset += len + 1;
- }
- return GNUNET_OK;
- }
- /**
- * Load search strings from given filename. One search string per line.
- *
- * @param filename filename of the file containing the search strings.
- * @param strings set of strings loaded from file. Caller needs to free this
- * if number returned is greater than zero.
- * @param limit upper limit on the number of strings read from the file
- * @return number of strings found in the file. #GNUNET_SYSERR on error.
- */
- static int
- load_search_strings (const char *filename,
- char ***strings,
- unsigned int limit)
- {
- char *data;
- uint64_t filesize;
- int str_cnt;
- /* Sanity checks */
- if (NULL == filename)
- {
- return GNUNET_SYSERR;
- }
- if (GNUNET_YES != GNUNET_DISK_file_test (filename))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Could not find search strings file %s\n", filename);
- return GNUNET_SYSERR;
- }
- if (GNUNET_OK !=
- GNUNET_DISK_file_size (filename,
- &filesize,
- GNUNET_YES,
- GNUNET_YES))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Search strings file %s cannot be read.\n",
- filename);
- return GNUNET_SYSERR;
- }
- if (0 == filesize)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Search strings file %s is empty.\n",
- filename);
- return GNUNET_SYSERR;
- }
- /* Read data into memory */
- data = GNUNET_malloc (filesize + 1);
- if (filesize != GNUNET_DISK_fn_read (filename,
- data,
- filesize))
- {
- GNUNET_free (data);
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Could not read search strings file %s.\n",
- filename);
- return GNUNET_SYSERR;
- }
- /* Process buffer and build array */
- str_cnt = count_and_separate_strings (data, filesize, limit);
- if (GNUNET_OK != create_string_array (data, filesize, strings, str_cnt))
- {
- str_cnt = GNUNET_SYSERR;
- }
- GNUNET_free (data);
- return str_cnt;
- }
- /**
- * Main function that will be run by the scheduler.
- *
- * @param cls closure
- * @param args remaining command-line arguments
- * @param cfgfile name of the configuration file used (for saving, can be NULL!)
- * @param config configuration
- */
- static void
- run (void *cls,
- char *const *args,
- const char *cfgfile,
- const struct GNUNET_CONFIGURATION_Handle *config)
- {
- unsigned int nsearchstrs;
- unsigned int i;
- struct GNUNET_TIME_Relative abort_time;
- in_shutdown = GNUNET_NO;
- /* Check config */
- if (NULL == config)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _ ("No configuration file given. Exiting\n"));
- GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
- return;
- }
- cfg = GNUNET_CONFIGURATION_dup (config);
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER",
- "REGEX_PREFIX",
- ®ex_prefix))
- {
- GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
- "regexprofiler",
- "regex_prefix");
- GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
- return;
- }
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER",
- "PARALLEL_SEARCHES",
- &init_parallel_searches))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "Configuration option \"PARALLEL_SEARCHES\" missing."
- " Using default (%d)\n", 10);
- init_parallel_searches = 10;
- }
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER",
- "REANNOUNCE_PERIOD_MAX",
- &reannounce_period_max))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "reannounce_period_max not given. Using 10 minutes.\n");
- reannounce_period_max =
- GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 10);
- }
- /* Check arguments */
- if (NULL == policy_dir)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _ (
- "No policy directory specified on command line. Exiting.\n"));
- return;
- }
- if (GNUNET_YES != GNUNET_DISK_directory_test (policy_dir, GNUNET_YES))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _ ("Specified policies directory does not exist. Exiting.\n"));
- GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
- return;
- }
- if (0 >= (int) (num_peers = GNUNET_DISK_directory_scan (policy_dir, NULL,
- NULL)))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _ ("No files found in `%s'\n"),
- policy_dir);
- return;
- }
- GNUNET_CONFIGURATION_set_value_string (cfg, "REGEXPROFILER",
- "POLICY_DIR", policy_dir);
- if (GNUNET_YES != GNUNET_DISK_file_test (strings_file))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _ ("No search strings file given. Exiting.\n"));
- GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
- return;
- }
- nsearchstrs = load_search_strings (strings_file,
- &search_strings,
- num_peers);
- if (num_peers != nsearchstrs)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- "Error loading search strings.\n");
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- "File (%s) does not contain enough strings (%u/%u).\n",
- strings_file, nsearchstrs, num_peers);
- GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
- return;
- }
- if ((0 == num_peers) || (NULL == search_strings))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _ ("Error loading search strings. Exiting.\n"));
- GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
- return;
- }
- for (i = 0; i < num_peers; i++)
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "search string: %s\n",
- search_strings[i]);
- /* Check logfile */
- if ((NULL != data_filename) &&
- (NULL == (data_file =
- GNUNET_DISK_file_open (data_filename,
- GNUNET_DISK_OPEN_READWRITE
- | GNUNET_DISK_OPEN_TRUNCATE
- | GNUNET_DISK_OPEN_CREATE,
- GNUNET_DISK_PERM_USER_READ
- | GNUNET_DISK_PERM_USER_WRITE))))
- {
- GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
- "open",
- data_filename);
- return;
- }
- /* Initialize peers */
- peers = GNUNET_malloc (sizeof(struct RegexPeer) * num_peers);
- for (i = 0; i < num_peers; i++)
- peers[i].id = i;
- GNUNET_CONFIGURATION_set_value_number (cfg,
- "TESTBED", "OVERLAY_RANDOM_LINKS",
- num_peers * 20);
- GNUNET_CONFIGURATION_set_value_number (cfg,
- "DHT", "FORCE_NSE",
- (long long unsigned)
- (log (num_peers) / log (2.0)));
- event_mask = 0LL;
- /* For feedback about the start process activate these and pass master_cb */
- event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_START);
- // event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP);
- event_mask |= (1LL << GNUNET_TESTBED_ET_CONNECT);
- // event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT);
- prof_start_time = GNUNET_TIME_absolute_get ();
- GNUNET_TESTBED_run (hosts_file,
- cfg,
- num_peers,
- event_mask,
- &master_controller_cb,
- NULL, /* master_controller_cb cls */
- &test_master,
- NULL); /* test_master cls */
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_time (cfg, "TESTBED",
- "SETUP_TIMEOUT",
- &abort_time))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "SETUP_TIMEOUT not given. Using 15 minutes.\n");
- abort_time =
- GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15);
- }
- abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES);
- abort_task =
- GNUNET_SCHEDULER_add_delayed (abort_time,
- &do_abort,
- (void *) __LINE__);
- GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- "setup_timeout: %s\n",
- GNUNET_STRINGS_relative_time_to_string (abort_time, GNUNET_YES));
- }
- /**
- * Main function.
- *
- * @param argc argument count
- * @param argv argument values
- * @return 0 on success
- */
- int
- main (int argc, char *const *argv)
- {
- struct GNUNET_GETOPT_CommandLineOption options[] = {
- GNUNET_GETOPT_option_filename ('o',
- "output-file",
- "FILENAME",
- gettext_noop (
- "name of the file for writing statistics"),
- &data_filename),
- GNUNET_GETOPT_option_relative_time ('t',
- "matching-timeout",
- "TIMEOUT",
- gettext_noop (
- "wait TIMEOUT before ending the experiment"),
- &search_timeout_time),
- GNUNET_GETOPT_option_filename ('p',
- "policy-dir",
- "DIRECTORY",
- gettext_noop ("directory with policy files"),
- &policy_dir),
- GNUNET_GETOPT_option_filename ('s',
- "strings-file",
- "FILENAME",
- gettext_noop (
- "name of file with input strings"),
- &strings_file),
- GNUNET_GETOPT_option_filename ('H',
- "hosts-file",
- "FILENAME",
- gettext_noop (
- "name of file with hosts' names"),
- &hosts_file),
- GNUNET_GETOPT_OPTION_END
- };
- int ret;
- if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
- return 2;
- result = GNUNET_SYSERR;
- ret =
- GNUNET_PROGRAM_run (argc, argv,
- "gnunet-regex-profiler",
- _ ("Profiler for regex"),
- options, &run, NULL);
- if (GNUNET_OK != ret)
- return ret;
- if (GNUNET_OK != result)
- return 1;
- return 0;
- }
|