/* How it works. A blackhole server establish contact with horizon servers all over the network. The horizon server runs on the host, which in turn run vservers. Each host setups a bunch of IPs, say 192.168.1.1, 192.168.1.2, etc... called ip0, ip1 and so on. In the vserver, all contacts made with outside is done using one of the ip1,ip2. For example, the DNS may ba ip1. All vservers will share the same DNS (so they think). Then the database server may be on ip2. Again, all vservers (in all hosts) think they are using the same database server. Then when a request comes to the horizon server, it identifies the origin (a vserver can't lie about its source adress) and the destination (ip1, ip2, ...). The connection is then forwarded to the backhole server. This server has a configuration going like that vserverX is allowed to talk to vserverY on port 4101 (for example). and so on. Using this, you end up with a simple distributed configuration (all server have the same ip1, ip2 definitions) and a simple central configuration. Further, because communication is initiated from the blackhole, you can use all kind of routing,nat,firewall. As long as the blackhole can reach all the horizons, they can all comunicate. This program was inspired by the SDN concept. Note that this program may be rolled out step by step. One vserver may access its database server normally, while another one in the same host may use this program to reach its database server. Also, you may install one star server for a whole network if you see fit. Said completly differently, you can roll out this without changing your networking at all. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fdpass.h" using namespace std; static DEBUG_KEY D_BLACKHOLE("blackhole","blackhole main program"); static DEBUG_KEY D_MATCH("match","matching connection rules"); enum CONNECT_TYPE { TYPE_NONE, TYPE_CONTROL, TYPE_HORIZON, TYPE_WORMHOLE, TYPE_BLACKHOLE, TYPE_PARENT, TYPE_FORK, TYPE_IDLE }; static const char *tbtype[]={ "none","control","Horizon","Wormhole","Blackhole","Parent","Fork","Idle" }; static int maxfork = 1000; static int nbfork = 0; static bool logall = false; static unsigned logid = 0; static FILE *fstatfile = NULL; static int blackhole_nbone (unsigned long mask) { int ret; unsigned long cmp = 0xffffffff; for (ret = 32; ret > 0; ret--){ if (mask == cmp) break; cmp = (cmp << 1) & 0xffffffff; } return ret; } struct FLAGTIME{ time_t since; bool is_enabled() const{ return since == (time_t)0; } FLAGTIME(){ since = (time_t)0; } void enable(){ since = (time_t)0; } void disable(){ since = time(NULL); } int disabled_since() const { return is_enabled() ? 0 : time(NULL) - since; } }; #include "connectrules.h" struct HANDLE_INFO: public ARRAY_OBJ{ CONNECT_TYPE type; string host; CONNECT_RULE_FROM from; // Info for active connection CONNECT_RULE_TO to; // Used to manage load balancer stats CONNECT_RULE_TO check; // Same thing for protocol checker int nbping; // Number unanswered sent ping time_t lastping; string lasterror; time_t intruder; // This connection failed to provide proper secret // From now on, we ignere any request // This field hold the time it happened or 0 bool connecting = false; // Copy of HORIZON::connecting HANDLE_INFO(){ type = TYPE_NONE; nbping = 0; lastping = 0; intruder = (time_t)0; } // This tells that the horizon,wormhole,sub-blackhole is alive // and knows the protocol (it pings) // and knows the secret (it pings with the secret) bool available() const { return intruder == (time_t)0 && lastping != (time_t)0; } }; #include "proto/blackhole_horizon.protoh" #include "proto/blackhole_wormhole.protoh" #include "proto/blackhole_blackhole.protoh" #include "proto/blackhole_control.protoh" #include "proto/blackhole_parent.protoh" struct REJECT_STAT{ time_t lasttry; int nbtry; string original_target; // In case of an open_client connection, the IP number is translated // to a network name. This is what is used for rules and this // is also what will show up in the rejected connection. // The original IP is not preserved, but we keep here // the original IP of the last rejected connection. REJECT_STAT(){ lasttry = (time_t)0; nbtry = 0; } REJECT_STAT(int _nbtry, time_t _lasttry, const char *_original_target){ lasttry = _lasttry; nbtry = _nbtry; original_target = _original_target; } }; struct HORIZON{ int fd=-1; // Connected handle int nbretry=0; // How many time we tried to connect bool is_far = false; // Used to tell the horizon this blackhole is far // so it should be used (to request connections) // only if no other is available time_t since = 0; // Connected since string port; // Connect the horizon using this TCP port or unix path bool connecting = false; // The connection is not established yet // This flag is maintained in HANDLE_INFO for convenience HORIZON (){ } HORIZON(bool _is_far, const string &_port) :is_far(_is_far),port(_port){ } }; /* Transform a date in the format yyyy/mm/dd and hh:mm:ss If the date is not valid, return 0. */ static time_t blackhole_mktime (const char *datestr, const char *hourstr) { time_t ret = 0; if (strlen(datestr) == 10 && datestr[4] == '/' && datestr[7] == '/' && strlen(hourstr) == 8 && hourstr[2] == ':' && hourstr[5] == ':'){ struct tm t; t.tm_year = atoi(datestr) - 1900; t.tm_mon = atoi(datestr+5) -1; t.tm_mday = atoi(datestr+8); t.tm_hour = atoi(hourstr); t.tm_min = atoi(hourstr+3); t.tm_sec = atoi(hourstr+6); t.tm_isdst= -1; ret = mktime (&t); }else{ tlmp_error ("Invalid date/time definition %s %s\n",datestr,hourstr); } return ret; } /* Does the opposite of fdpass_asctime */ static time_t blackhole_cnvtime (const char *s) { time_t ret = (time_t)0; if (strlen(s)==19 && s[0] != '-'){ char date[20],hour[9]; strcpy (date,s); date[10] = '\0'; strcpy (hour,s+11); ret = blackhole_mktime (date,hour); } return ret; } static void blackhole_horizon_connect( _F_TCPSERVER_V1 *c, const map &secrets, map &horizons, CONNECT_TYPE type, bool paused) { for (map::iterator it = horizons.begin(); it != horizons.end(); it++){ if (it->second.fd == -1){ debug_printf (D_EXTRA,"Trying to connect to %s server %s\n" ,tbtype[type] ,it->first.c_str()); bool unix_port = it->second.port[0] == '/'; int fd = unix_port ? fdpass_tcpconnect (it->first.c_str(),it->second.port.c_str()) : fdpass_tcpconnect_async (it->first.c_str(),it->second.port.c_str()); if (fd == -1){ if (it->second.nbretry == 0){ tlmp_error ("Can't connect to horizon server %s\n",it->first.c_str()); } it->second.nbretry++; }else{ HANDLE_INFO *n = new HANDLE_INFO; n->type = type; n->host = it->first; int fail = 0; if (unix_port){ fail = c->inject (fd,n); if (type == TYPE_HORIZON){ fdpass_sendmaster (n->host,secrets,fd,it->second.is_far); } }else{ it->second.connecting = true; // Note that inject_connecting may immediatly trigger // the event calling blackhole_horizon_connect_ok() // which resets connecting to false; fail = c->inject_connecting (fd,n); n->connecting = it->second.connecting; } if (fail == -1){ it->second.nbretry++; close (fd); }else{ it->second.fd = fd; if (unix_port) it->second.nbretry = 0; it->second.since = time(NULL); if (paused) c->setlisten(fd,false); } } } } } // Function called to complete the asynchronous connection static void blackhole_horizon_connect_ok( map &horizons, const string &host, int no, CONNECT_TYPE type, const map &secrets, const char *servtype) // string to identify the connection type { auto it = horizons.find(host); if (it != horizons.end()){ it->second.connecting = false; it->second.nbretry = 0; tlmp_warning ("async connection succeeded to %s %s\n",servtype,host.c_str()); if (type == TYPE_HORIZON){ fdpass_sendmaster (host,secrets,no,it->second.is_far); } } } static void blackhole_horizon_connect_fail( map &horizons, const string &host, const char *servtype) // string to identify the connection type { auto it = horizons.find(host); if (it != horizons.end()){ if (it->second.nbretry == 0) tlmp_error ("async connection failed to %s %s\n",servtype,host.c_str()); it->second.connecting = false; it->second.nbretry++; it->second.fd = -1; } } static const char *TARGET_KEEP="KEEP"; static const char *TARGET_SOURCE="SOURCE"; static const char *TARGET_DROP="DROP"; /* Replace keyword in s with repl Return a new string with the result. */ static string blackhole_replace (const char *s, const char *keyword, const char *repl) { const char *pt = strstr(s,keyword); if (pt != NULL){ int len = strlen(keyword); if (pt != s){ return string(s,pt-s) + string(repl) + (pt+len); }else{ return string(repl) + (pt+len); } }else{ return s; } } /* Remap the target of a connection using keyword in vserver name The vserver string may be -a vserver name. Normal case -an IP address (horizon --open_client mode) -The keyword KEEP, which will be translated to the original target of the connection made in a vserver (intercepted using iptable REDIRECT rules) -a vserver name with the suffix /SOURCE. In this case, the function will return the vserver name followed by / and the source address This is used to pass the original internet client address to a vserver */ static string blackhole_settarget (const char *target, const char *source, const char *vserver) { string tmp = blackhole_replace (vserver,TARGET_KEEP,target); return blackhole_replace (tmp.c_str(),TARGET_SOURCE,source); } struct CONNECT_REQUEST{ const char *horizon; const char *source; const char *target; const char *dstport; const char *link; }; static bool donot_optim_request=false; // This allows the test environment with a single horizon to // test the various cases /* Establish a connection between two horizon servers. It connect to each one. One will receive a link command and the other a connect command. In the case where both horizon servers are the same, only one connect command is issued and the new process ends. It means the horizon server does everything on its own. Return true if the connection will be optimised (handled completly by the horizon) */ static bool blackhole_connect ( _F_TCPSERVER_V1 *c, const map &secrets, CONNECT_RULE_TO *check, const CONNECT_RULE_FROM &from, const CONNECT_RULE_TO &to, // Modified target rule (SOURCE and KEEP processed) const CONNECT_RULE_TO &ruleto, // Original target rule const CONNECT_REQUEST &req, const char *horizon1_port, const char *horizon2_port, const char *check_port, const char *description, int do_not_close, // Pipe: It will be closed when the sub-process ends // The parent will know the sub-process ended. bool logging) { int logging_id = -1; if (logging || logall){ logging_id = logid++; if (fstatfile != NULL){ DATEASC tstr; time_t t = time(NULL); fdpass_asctime (t,tstr); fprintf (fstatfile,"\"%d\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n" ,logging_id,tstr.buf ,from.master.c_str(),from.vserver.c_str(),from.target.c_str(),from.dstport.c_str() ,to.master.c_str(),to.vserver.c_str(),to.dstport.c_str() ,req.source,req.target,req.dstport); fflush (fstatfile); } } if (!donot_optim_request && to.check.empty()){ // Special cases we can handle without forking by using the already established channel between // the horizon and the blackhole. We just reply on the same socket on which we received the // request if (from.master == to.master){ char logging_str[100]; snprintf (logging_str,sizeof(logging_str)-1,"%d",logging_id); fdpass_sendf (c,from.master,secrets,PROTO_CONNECT_LINK,to.vserver.c_str(),to.dstport.c_str(),req.link ,ruleto.vserver.c_str(),ruleto.dstport.c_str(),logging_str); // Since the horizon server talks to itself, no need to wait. debug_printf (D_BLACKHOLE,"horizon %s handle both connections: connect %s %s %s\n",from.master.c_str() ,to.vserver.c_str(),to.dstport.c_str(),req.link); return true; }else if (to.master == TARGET_DROP){ // Special case. The DROP pseudo host means to simply reject the connection fdpass_sendf (c,from.master,secrets,PROTO_REJECT,req.link); return false; } } // End of special cases pid_t pid = fork(); if (pid == (pid_t)-1){ tlmp_error ("Can't fork (%s)\n",strerror(errno)); // Things are not going well, we can't fork. To avoid slowing down, we don't send // a PROTO_REJECT to the caller. Pending connections are going away after 60 seconds. // So the following line is not needed. // fdpass_sendf (from.master,port,secrets,PROTO_REJECT,req.link); }else if (pid == (pid_t)0){ if (to.master == TARGET_DROP){ // Special case. The DROP pseudo host means to simply reject the connection fdpass_sendf (from.master,horizon1_port,secrets,PROTO_REJECT,req.link); }else{ int fd1 = fdpass_tcpconnect (from.master.c_str(),horizon1_port); if (fd1 == -1){ tlmp_error ("Can't connect to horizon server %s (%s)\n",from.master.c_str(),strerror(errno)); // We could not connect to from.master, so no need to try to connect again // to send PROTO_REJECT. Pending connections are going away after 60 seconds anyway. // So the following line is not needed. // fdpass_sendf (from.master,port,secrets,PROTO_REJECT,req.link); }else if (from.master == to.master && to.check.empty()){ SSTRING tmp; tmp.setfromf ("connect %s %s %s",to.vserver.c_str(),to.dstport.c_str(),req.link); fdpass_sendtof (fd1,from.master,secrets,PROTO_CONNECT_LINK,to.vserver.c_str(),to.dstport.c_str(),req.link ,ruleto.vserver.c_str(),ruleto.dstport.c_str(),"-1"); // Since the horizon server talks to itself, no need to wait. debug_printf (D_BLACKHOLE,"horizon %s handle both connections: %s\n",from.master.c_str(),tmp.c_str()); }else{ int opt = 1; setsockopt (fd1,SOL_TCP,TCP_NODELAY,&opt,sizeof(opt)); int fdcheck = -1; if (check != NULL){ // Before establishing the final connection, we establish a connection to the protocol checker. fdcheck = fdpass_tcpconnect (check->master.c_str(),check_port); if (fdcheck == -1){ tlmp_error ("protocheck: Can't connect to horizon server %s (%s)\n",check->master.c_str(),strerror(errno)); fdpass_sendf (from.master,horizon1_port,secrets,PROTO_REJECT,req.link); _exit (0); }else{ setsockopt (fdcheck,SOL_TCP,TCP_NODELAY,&opt,sizeof(opt)); fdpass_sendtof (fdcheck,check->master,secrets,PROTO_CONNECT_LINK,check->vserver.c_str(),check->dstport.c_str(),"_" ,"-","-","-1"); // We wait for a "data" reply on the link debug_printf (D_BLACKHOLE,"protocheck waiting for data command\n"); if (fdpass_waitdata(fdcheck)==-1){ debug_printf (D_BLACKHOLE,"protocheck waiting for data command ... failed\n"); _exit (0); }else{ SSTRING tmp; tmp.setfromf ("proto %s %s %s %s\n",req.horizon,req.source,req.target,req.dstport); write (fdcheck,tmp.c_str(),tmp.getlen()); } } } { debug_printf (D_BLACKHOLE,"Connecting to second horizon server %s\n",to.master.c_str()); int fd2 = fdpass_tcpconnect (to.master.c_str(),horizon2_port); if (fd2 == -1){ tlmp_error ("Can't connect to horizon server %s (%s)\n",to.master.c_str(),strerror(errno)); fdpass_sendf (from.master,horizon1_port,secrets,PROTO_REJECT,req.link); }else{ setsockopt (fd2,SOL_TCP,TCP_NODELAY,&opt,sizeof(opt)); { SSTRING tmp; tmp.setfromf ("link %s",req.link); fdpass_sendto (fd1,from.master,secrets,tmp.c_str()); if (fdpass_waitdata(fd1)==-1){ _exit (0); } } fdpass_sendtof (fd2,to.master,secrets,PROTO_CONNECT_LINK,to.vserver.c_str(),to.dstport.c_str(),"_" ,"-","-","-1"); // We wait for a "data" reply on each link debug_printf (D_BLACKHOLE,"Waiting for data command\n"); if (fdpass_waitdata(fd2)!=-1){ string tmp; if (logging_id != -1){ char logstr[100]; snprintf (logstr,sizeof(logstr),"LOGID=%d ",logging_id); tmp = string(logstr)+description; description = tmp.c_str(); } if (fdcheck != -1 || fdpass_sendfd2proxy (fd1,fd2,do_not_close,description)==-1){ // The connection proxy is not running, so we handle the // the job here. fdpass_loop (fd1,fd2,fdcheck,do_not_close); } } } } } } _exit(0); } return false; } struct NETWORK{ unsigned long net; unsigned long mask; NETWORK(unsigned long _net, unsigned long _mask){ net = _net; mask = _mask; } bool operator < (const NETWORK &r) const { return blackhole_nbone(mask) < blackhole_nbone(r.mask); } }; struct IPLIST{ set ips; vector networks; }; /* Find all the iplist matching a source and put them in the out vector If the source is not an IP, it ends up copied in the out vector as is. */ static void blackhole_findiplists ( const char *source, const map &iplists, vector &out) { unsigned long addr = ipnum_aip2l (source); if (addr != 0xffffffffl){ // The source is an IP address. It is either in the set or part of a network // We find all matching iplists for (map::const_iterator it = iplists.begin(); it != iplists.end(); it++){ if (it->second.ips.count(addr) > 0){ debug_printf (D_BLACKHOLE,"Horizon request IP addr %s is part of iplist %s\n" ,source,it->first.c_str()); out.push_back(it->first); continue; }else{ for (vector::const_iterator nit=it->second.networks.begin(); nit != it->second.networks.end(); nit++){ if ((addr&nit->mask) == nit->net){ debug_printf (D_BLACKHOLE,"Horizon request IP addr %s is part of network %s\n" ,source,it->first.c_str()); out.push_back(it->first); break; } } } } }else{ out.push_back(source); } } static void blackhole_record_reject ( CONNECT_RULE_FROM &from, const char *original_target, map &rejects, time_t now) { map::iterator it = rejects.find(from); if (it == rejects.end()){ // We avoid growing the map too much. This could be used as a denial of service on the blackhole server. if (rejects.size() < 1000){ // 1000 is arbitrary. If we have so many refused connection, it's bad. rejects[from] = REJECT_STAT(1,now,original_target); } }else{ it->second.nbtry++; it->second.lasttry = now; it->second.original_target = original_target; } } /* Remove one file handle from a vector */ static void blackhole_remove (vector &v, int fd) { for (vector::iterator it = v.begin(); it != v.end(); it++){ if (*it == fd){ v.erase(it); break; } } } /* Report any error condition on horizons, wormholes and sub-blackholes */ static void blackhole_reporterr (_F_TCPSERVER_V1 *c, map &tb, const char *title) { for (map::iterator it=tb.begin(); it != tb.end(); it++){ int fd = it->second.fd; const char *host = it->first.c_str(); if (fd == -1){ c->sendf ("%s %s not connected\n",title,host); }else{ HANDLE_INFO *n = (HANDLE_INFO*)c->getclientdata(fd); if (n->nbping > 1){ c->sendf ("%s %s not pinging (nbping=%d)\n",title,host,n->nbping); } if (n->intruder != 0){ DATEASC date; fdpass_asctime (n->intruder,date); c->sendf ("%s %s may be an intruder since %s, ignoring all requests\n",title,host,date.buf); } } } } /* Check if we have a secret for all items in the map */ static void blackhole_checksecret (_F_TCPSERVER_V1 *c, const map &secrets, map &tb, const char *title) { for (map::iterator it=tb.begin(); it != tb.end(); it++){ map::const_iterator sit = secrets.find (it->first); if (sit == secrets.end()){ c->sendf ("%s %s has no secret\n",title,it->first.c_str()); } } } static void blackhole_reportstatus (_F_TCPSERVER_V1 *c, map &tb, const char *title) { for (map::iterator it=tb.begin(); it != tb.end(); it++){ int fd = it->second.fd; int nbping=0; const char *lasterror = ""; time_t intruder = (time_t)0; if (fd != -1){ HANDLE_INFO *n = (HANDLE_INFO*)c->getclientdata(fd); nbping = n->nbping; lasterror = n->lasterror.c_str(); intruder = n->intruder; } DATEASC since; fdpass_asctime (it->second.since,since); DATEASC intruder_date; fdpass_asctime (intruder,intruder_date); c->sendf ("%s %-20s fd=%-2d%s nbretry=%d nbping=%d is_far=%d since=%s intruder=%s lasterror=%s\n" ,title,it->first.c_str(),fd,it->second.connecting ? "(connecting)" : "" ,it->second.nbretry,nbping,it->second.is_far,since.buf ,intruder_date.buf,lasterror); } } struct LOADSTAT{ unsigned connected; // How many connected to that destination unsigned weight; // Weight of this destination for load balancing unsigned alt_con; // How many connections made using alternate blackholes // Set by the setaltcon blackhole-control command unsigned max_con; // Maximum concurent connections LOADSTAT(){ connected = 0; weight = 100; alt_con = 0; max_con = 0; } void add_connection(){ connected++; if (connected > max_con) max_con = connected; } bool is_default()const { return connected == 0 && weight == 100 && alt_con == 0; } }; static int blackhole_ruleconnect_one( _F_TCPSERVER_V1 *c, const map &secrets, CONNECT_RULE_TO *check, // Access to the protocol checker or NULL const CONNECT_RULE_FROM &rulefrom, CONNECT_RULE_TO &ruleto, const CONNECT_REQUEST &req, map > &holes, map &wormholes, int &nbavoided, map &active_cons, bool logging, const char *horizon1_port, const char *horizon2_port, const char *horizon_check_port) { int ret = -1; // The original target of the connection was maybe an IP number (transparent proxy) // and not ip0,ip1,... // The vserver in the CONNECT_RULE_TO is potentially a keyword explaining // how to translate the IP address (or keep it unchanged). CONNECT_RULE_TO to = ruleto; to.vserver = blackhole_settarget (req.target,req.source,ruleto.vserver.c_str()); if (to.dstport == TARGET_KEEP) to.dstport = req.dstport; // We try to find if one wormhole is able to connect to both horizon server bool found = false; if (rulefrom.master != ruleto.master){ // If the two vserver are on the same master(host), blackhole_connect handles this case // and request a local connection completly managed by the horizon on that host. // If the master are different, we check if a wormhole server can handle the connection // since it will be more efficient network wise. // If there is no wormhole server (with connectivity to both masters), then // the blackhole server will act as the relay (handled also by blackhole_connect(). for (map >::iterator ht = holes.begin(); ht != holes.end(); ht++){ int count_from = ht->second.count(rulefrom.master); int count_to = ht->second.count(ruleto.master); if (count_from !=0 && count_to != 0){ // One wormhole may reach both horizons server // Now is it connected int fd = wormholes[ht->first].fd; if (fd == -1){ nbavoided++; }else{ HANDLE_INFO *n = (HANDLE_INFO*)c->getclientdata(fd); if (!n->available() && n->nbping > 1){ // Avoid this wormhole because it is not responding nbavoided++; }else{ debug_printf (D_BLACKHOLE,"Connect using wormhole %s: %s %s -> %s %s %s\n" ,ht->first.c_str() ,rulefrom.master.c_str(),req.link ,to.master.c_str(),to.vserver.c_str(),to.dstport.c_str()); found = true; fdpass_sendtof (fd,ht->first,secrets,"connect-link %s %s %s %s %s" ,to.master.c_str(),to.vserver.c_str(),to.dstport.c_str() ,rulefrom.master.c_str(),req.link); ret = 0; break; } } } #if 0 // This functionality is useless since horizon has --open_client else if (count_to != 0){ if (strcmp(TARGET_KEEP,ruleto.vserver.c_str())==0){ int fd = wormholes[ht->first].fd; if (fd == -1){ nbavoided++; }else{ HANDLE_INFO *n = (HANDLE_INFO*)c->getclientdata(fd); if (n->nbping > 1){ // Avoid this wormhole because it is not responding nbavoided++; }else{ debug_printf (D_BLACKHOLE,"Using wormhole %s to connect to %s port %s\n" ,ht->first.c_str(),to.vserver.c_str(),to.dstport.c_str()); to.master = ht->first; // We cheat here. We are using the wormhole as a proxy to a network. // We replace the master (which was a network name) by the wormhole // name and let the rest of this code connects as if it was talking // to two horizon servers. break; } } } } #endif } } if (!found){ if (nbfork >= maxfork){ tlmp_error ("Too many sub-process, rejecting valid connection from %s\n",req.horizon); fdpass_sendf (rulefrom.master,horizon1_port,secrets,PROTO_REJECT,link); }else{ int tb[2]; if (pipe(tb)==-1){ tlmp_error ("Can't setup pipe (%s)\n",strerror(errno)); fdpass_sendf (rulefrom.master,horizon1_port,secrets,PROTO_REJECT,link); }else{ SSTRING comment; comment.setfromf ("%s %s %s %s -> %s %s %s" ,rulefrom.master.c_str(),rulefrom.vserver.c_str(),req.target,rulefrom.dstport.c_str() ,to.master.c_str(),to.vserver.c_str(),to.dstport.c_str()); bool optim = blackhole_connect (c,secrets,check,rulefrom,to,ruleto,req ,horizon1_port,horizon2_port,horizon_check_port,comment.c_str(),tb[1],logging); active_cons[ruleto].add_connection(); if (check != NULL) active_cons[*check].add_connection(); if (optim){ // The connection is completly handled by the horizon // (was received by one horizon and processed by the same) // so the PIPE handle is not used to monitor the end of // the connection. Instead, the horizon is sending and "endcon" // request to indicate this. close (tb[1]); close (tb[0]); }else{ HANDLE_INFO *d = new HANDLE_INFO; d->type = TYPE_FORK; d->from = rulefrom; d->to = ruleto; if (check != NULL) d->check = *check; close (tb[1]); c->inject (tb[0],d); nbfork++; } ret = 0; } } } return ret; } /* Load balancer, selection of the best target Return -1 if not found Return the index in tos[] */ static int blackhole_besttarget( _F_TCPSERVER_V1 *c, vector &tos, map &horizons, map &active_cons, time_t now, bool &one_possible_rule, string &horizon_port) { int found=-1; const double default_load_factor = 1000000; double min_load_factor = default_load_factor; one_possible_rule = false; for (unsigned i=0; i now){ one_possible_rule = true; if (to.master == TARGET_DROP){ // No need to search more, the connection will be dropped in blackhole_connect() found = i; break; } // We have to check if the corresponding horizon do exist (invalid rule) // We also have to check if it is currently connected. // If all true, we check if it is slow to respond to pings. // nbping is the amount of unanswered pings. // If an horizon is slow to respond, it may be dead or there is // some overload somewhere. So if nbping is > 1, we raise te load_factor // way up. If it is the only horizon available, it will be picked. map::iterator ithor = horizons.find(to.master); if (ithor != horizons.end() && ithor->second.fd != -1 && !ithor->second.connecting){ double load_factor = 0; map::iterator itcon = active_cons.find(to); if (itcon != active_cons.end()){ unsigned weight = itcon->second.weight; if (weight == 0){ // This is larger than min_load_factor // so this will never be picked load_factor = default_load_factor+10; }else{ load_factor = (double)(itcon->second.connected + itcon->second.alt_con) /weight; if (weight == 1){ // A weight of one is a fallback to target // you normally do not want to use. // If all the other normal targets are not // available (weight=0), you will use // target with weight 1 load_factor += 1000; } } } HANDLE_INFO *n = (HANDLE_INFO*)c->getclientdata(ithor->second.fd); if (n->available()){ if (n->nbping > 1) load_factor += 500000; if (load_factor < min_load_factor){ found = i; min_load_factor = load_factor; horizon_port = ithor->second.port; } } } } } return found; } static int blackhole_ruleconnect( _F_TCPSERVER_V1 *c, const string &myname, const map &secrets, map &checks, const CONNECT_REQUEST &req, map &rules, map > &holes, map &wormholes, map &horizons, map &rejects, map &iplists, vector &parents, int &nbavoided, map &active_cons) { int ret = -1; const string &horizon1_port = horizons[req.horizon].port; map::iterator rule = rules.end(); CONNECT_RULE_FROM from(req.horizon,req.source,req.target,req.dstport); { /* Here is how it works. we receive a connection request that looks like this typical: host vserver ipname port open_network: host ip-addr ipname port open_client: host vserver ip-addr port IP addresses are convert to iplist name. One IP address may be part of several iplists. So we have to try them all The function blackhole_findiplists () finds all matching iplists for the vserver and the target IP. Using two nested for it tries to find a matching connection rule. The typical case is not just a vserver, but when the findproc system is used, a fully described client: vserver,command,user,group,env,xid The blackhole_findmatch has to locate the rule with the highest score. And to make it more general, once we have tried everything, if there is no match, we change the dstport for _. This is a wildcard rule. */ CONNECT_RULE_FROM testfrom(from); vector targets; blackhole_findiplists(req.target,iplists,targets); vector sources; blackhole_findiplists (from.vserver.c_str(),iplists,sources); for (unsigned h=0; h<2 && rule == rules.end(); h++){ testfrom.dstport = h==0 ? req.dstport : "_"; for (unsigned i=0; i %d\n" ,from.master.c_str(),from.vserver.c_str() ,from.target.c_str(),from.dstport.c_str() ,testfrom.vserver.c_str(),testfrom.target.c_str() ,rule!=rules.end()); } } } } time_t now = time(NULL); if (rule == rules.end()){ if (parents.size() > 0){ debug_printf (D_BLACKHOLE,"No rule, sending to parent %d: connect %s %s %s %s %s\n" ,parents[0],req.horizon,req.source,req.target,req.dstport,req.link); fdpass_sendtof (parents[0],myname,secrets,"connect %s %s %s %s %s",req.horizon,req.source,req.target,req.dstport,req.link); ret = 0; }else{ tlmp_error ("Reject connection (no rule): %s %s %s %s\n",from.master.c_str() ,from.vserver.c_str(),from.target.c_str(),from.dstport.c_str()); fdpass_sendf (c,from.master,secrets,PROTO_REJECT,req.link); //fdpass_sendf (from.master,horizon_port,secrets,PROTO_REJECT,req.link); blackhole_record_reject (from,req.target,rejects,now); } }else{ // Load balancer bool one_possible_rule = false; string horizon2_port; int found = blackhole_besttarget (c,rule->second.tos,horizons,active_cons,now,one_possible_rule,horizon2_port); if (found == -1){ tlmp_error ("Rejected connection (%s): %s %s %s %s\n",from.master.c_str() ,one_possible_rule ? "No available horizon" : "Expired rule" ,from.vserver.c_str(),from.target.c_str(),from.dstport.c_str()); fdpass_sendf (c,from.master,secrets,PROTO_REJECT,req.link); //fdpass_sendf (from.master,horizon_port,secrets,PROTO_REJECT,req.link); blackhole_record_reject (from,req.target,rejects,now); }else{ CONNECT_RULE_TO &to = rule->second.tos[found]; CONNECT_RULE_TO *check = NULL; string horizon_check_port; bool ok = true; if (to.check.size() > 0){ map::iterator itcheck = checks.find(to.check); if (itcheck == checks.end()){ tlmp_error ("Missing protocheck definition %s, can't handle connection to horizon server %s (%s)\n" ,to.check.c_str(),to.master.c_str(),strerror(errno)); fdpass_sendf (c,from.master,secrets,PROTO_REJECT,req.link); //fdpass_sendf (from.master,horizon_port,secrets,PROTO_REJECT,req.link); ok = false; }else{ bool one_possible_rule; int found = blackhole_besttarget (c,itcheck->second.tos,horizons,active_cons,now,one_possible_rule,horizon_check_port); if (found == -1){ tlmp_error ("protocheck %s: no server available\n",to.check.c_str()); fdpass_sendf (c,from.master,secrets,PROTO_REJECT,req.link); //fdpass_sendf (from.master,horizon_port,secrets,PROTO_REJECT,req.link); ok = false; }else{ check = &itcheck->second.tos[found]; check->lastcon = now; check->nbcon++; check->last_source = req.source; check->last_target = req.target; } } } if (ok){ // Log some stats to.lastcon = now; to.nbcon++; to.last_source = req.source; to.last_target = req.target; ret = blackhole_ruleconnect_one (c,secrets,check,rule->first,to,req,holes ,wormholes,nbavoided,active_cons,rule->second.logging ,horizon1_port.c_str(),horizon2_port.c_str(),horizon_check_port.c_str()); } } } return ret; } static void blackhole_pauseresume (_F_TCPSERVER_V1 *c, map &tb, bool listening) { for (map::iterator it=tb.begin(); it != tb.end(); it++){ if (it->second.fd != -1){ c->setlisten (it->second.fd,listening); } } } static void blackhole_lostconnection(const char *name, const string &host, map &tb) { const char *pthost = host.c_str(); debug_printf (D_BLACKHOLE,"Lost connection with %s server %s\n",name,pthost); syslog (LOG_NOTICE,"Lost connectionw with %s server %s\n",name,pthost); // Will try to reconnect using the idle loop tb[host].fd = -1; } /* Forget about servers (horizons, wormholes, blackholes) and close their connections */ static void blackhole_resetservers (_F_TCPSERVER_V1 *c, map &tb) { for (map::iterator it=tb.begin(); it != tb.end(); it++){ if (it->second.fd != -1){ c->closeclient (it->second.fd); } } tb.clear(); } static void blackhole_checkrule ( SSTRING &msg, const CONNECT_RULE_FROM &from, const CONNECT_RULE_TO &to, const map &horizons, const map &checks) { if (horizons.find(from.master)==horizons.end()){ msg.appendf ("Unknown horizon %s: rules **%s** %s %s %s -> %s %s %s\n" ,from.master.c_str() ,from.master.c_str(),from.vserver.c_str(),from.target.c_str(),from.dstport.c_str() ,to.master.c_str(),to.vserver.c_str(),to.dstport.c_str()); } if (to.master != TARGET_DROP && horizons.find(to.master)==horizons.end()){ msg.appendf ("Unknown horizon %s: rules %s %s %s %s -> **%s** %s %s\n" ,to.master.c_str() ,from.master.c_str(),from.vserver.c_str(),from.target.c_str(),from.dstport.c_str() ,to.master.c_str(),to.vserver.c_str(),to.dstport.c_str()); } if (to.check.size() > 0 && checks.find(to.check)==checks.end()){ msg.appendf ("Unknown protocol check %s: rules %s %s %s %s -> %s:%s %s %s\n" ,to.check.c_str() ,from.master.c_str(),from.vserver.c_str(),from.target.c_str(),from.dstport.c_str() ,to.check.c_str(),to.master.c_str(),to.vserver.c_str(),to.dstport.c_str()); } } static void blackhole_checkrule ( _F_TCPSERVER_V1 *c, const CONNECT_RULE_FROM &from, const CONNECT_RULE_TO &to, const map &horizons, const map &checks) { SSTRING msg; blackhole_checkrule (msg,from,to,horizons,checks); if (msg.is_filled()) c->send (msg.c_str()); } static void blackhole_checkrules ( _F_TCPSERVER_V1 *c, map &rules, map &horizons, const map &checks) { for (map::iterator it=rules.begin(); it != rules.end(); it++){ for (unsigned i=0; isecond.tos.size(); i++){ blackhole_checkrule (c,it->first,it->second.tos[i],horizons,checks); } } } static void blackhole_setlogging ( _F_TCPSERVER_V1 *c, map &rules, const char *server, const char *vserver, const char *ipname, const char *port, bool logging) { CONNECT_RULE_FROM from (server,vserver,ipname,port); map::iterator it = rules.find(from); if (it == rules.end()){ c->send ("No matching rules\n"); }else{ it->second.logging = logging; } } int main (int argc, char *argv[]) { glocal int connectdelay=0; // To help the testrace.tlcc program glocal const char *pidfile = "/run/blackhole.pid"; glocal const char *user = "blackhole"; glocal bool daemon = false; glocal const char *control = "/var/run/blackhole/blackhole.sock"; glocal const char *parent = NULL; glocal const char *secretfile = NULL; glocal const char *blackname = NULL; // Name used for this blackhole instead of the hostname // Used when this blackhole is a sub-blackhole glocal const char *statfile = "/var/log/blackhole-connect.log"; glocal const char *errorfile = NULL; glocal FILE *ferror = NULL; glocal const char *horizon_port = "8000"; glocal const char *blackhole_port = "9000"; glocal int ret; glocal.ret = (argc,argv); setproginfo ("blackhole",VERSION,"Proxy TCP request, main authority"); setarg (' ',"control","Unix control port",glocal.control,false); setarg (' ',"horizon_port","TCP port used to contact horizon servers",glocal.horizon_port,false); setarg (' ',"blackhole_port","TCP port used to contact sub-blackhole servers",glocal.blackhole_port,false); setarg (' ',"secretfile","File holding secrets",glocal.secretfile,false); setarg (' ',"name","Name to use instead of the hostname",glocal.blackname,false); fdpass_setarg (this); setarg (' ',"parent","Used by blackhole parent: ip,port",glocal.parent,false); setarg (' ',"maxfork","Max number of sub-process",maxfork,false); setarg (' ',"statfile","File holding connection statistics",glocal.statfile,false); setgrouparg ("Daemon options"); setarg (' ',"daemon","Run in background",glocal.daemon,false); setarg (' ',"pidfile","Write the daemon pid in this file",glocal.pidfile,false); setarg (' ',"user","Run as this user",glocal.user,false); setgrouparg ("Test options"); setarg (' ',"connectdelay","Slow down connect request for testrace (microseconds)",glocal.connectdelay,false); setarg (' ',"donotoptimrequest","Do not reuse the same connection for connect request",donot_optim_request,false); setarg (' ',"errorfile","Error message will be written to this file",glocal.errorfile,false); if (glocal.ferror != NULL){ fprintf (glocal.ferror,"ERR: %s",msg); fflush (glocal.ferror); }else if (glocal.daemon){ syslog (LOG_ERR,"%s",msg); }else{ fprintf (stderr,"%s",msg); } if (glocal.ferror != NULL){ fprintf (glocal.ferror,"WARN: %s",msg); fflush (glocal.ferror); }else if (glocal.daemon){ syslog (LOG_WARNING,"%s",msg); }else{ fprintf (stderr,"WARN: %s",msg); } int ret = -1; glocal time_t started; glocal time_t reloaded; glocal map rules; glocal map rejects; glocal map horizons; // Connected horizon servers glocal map wormholes; // Connected wormhole servers glocal map blackholes; // Connected blackhole sub servers // Used by the pause,resume trick to perform rules reload // We must not loose connections to horizons and blackholes. // There is a window where new connections may be lost. // (Established connections are not a problem though) glocal map old_horizons; // Connected horizon servers glocal map old_blackholes; // Connected blackhole sub servers glocal map > holes; // Gives the list of all reachable horizon servers // from a given wormhome. glocal map iplists; // Lists of IP and network (used for blacklisting) glocal bool paused = true; // Do not listen for request by default // wait for a resume command. // It allows us to start, set the various definitions // and then start processing. glocal time_t pausedtime=time(NULL); glocal vector parents; // blackhole parents used to delegate invalid request glocal set allows; // IP of allowed parents glocal int nbavoided=0; glocal map active_cons; // Track connections glocal map secrets; // Secrets of the horizon, sub-blackhole and wormhole glocal string myname; // Name used to identify this blackhole and lookup its secret glocal FLAGTIME pingmode; // Control if we send pings to horizons, wormholes and sub-blackholes glocal unsigned pinginterval = 5; // Interval between pings glocal time_t lastping = (time_t)0; // Last time we sent a ping glocal map checks; if (glocal.blackname != NULL){ glocal.myname = glocal.blackname; }else{ char buf[1000]; if (gethostname(buf,sizeof(buf))!=-1){ char *pt = strchr(buf,'.'); if (pt != NULL) *pt = '\0'; glocal.myname = buf; } } fdpass_readsecrets (glocal.secretfile,glocal.secrets); fdpass_checkservice (glocal.control); fstatfile = fopen (glocal.statfile,"a"); if (fstatfile == NULL){ tlmp_error ("Can't open statfile %s (%s)\n",glocal.statfile,strerror(errno)); } glocal.started = time(NULL); glocal.reloaded = 0; if (glocal.errorfile != NULL){ glocal.ferror = fopen (glocal.errorfile,"a"); if (glocal.ferror == NULL){ tlmp_error ("Can't open errorfile %s (%s), aborting\n",glocal.errorfile,strerror(errno)); exit (-1); } } (); debug_printf (D_BLACKHOLE,"newclient no=%d port=%s\n",no,info.port); HANDLE_INFO *c = new HANDLE_INFO; info.data = c; if (strncmp(info.port,"unix:",5)==0){ c->type = TYPE_CONTROL; }else{ char addr[20]; ipnum_ip2a (from,addr); // We are contacted by a parent blackhole if (glocal.allows.count(addr)==0){ endclient = true; tlmp_error ("Rejected parent connection from ip %s\n",addr); }else{ c->type = TYPE_PARENT; c->host = addr; glocal.parents.push_back(no); } } HANDLE_INFO *c = (HANDLE_INFO*)info.data; if (c->type == TYPE_FORK){ debug_printf (D_BLACKHOLE,"sub-process ending: %s %s %s %s -> %s %s %s\n" ,c->from.master.c_str(),c->from.vserver.c_str(),c->from.target.c_str(),c->from.dstport.c_str() ,c->to.master.c_str(),c->to.vserver.c_str(),c->to.dstport.c_str()); nbfork--; // Manage load balancer stats map::iterator it = glocal.active_cons.find (c->to); if (it != glocal.active_cons.end() && it->second.connected > 0){ it->second.connected--; } // Do the same for the protocol checker it = glocal.active_cons.find (c->check); if (it != glocal.active_cons.end() && it->second.connected > 0){ it->second.connected--; } }else if (c->type == TYPE_HORIZON){ blackhole_lostconnection ("horizon",c->host,glocal.horizons); }else if (c->type == TYPE_WORMHOLE){ blackhole_lostconnection ("wormhole",c->host,glocal.wormholes); }else if (c->type == TYPE_BLACKHOLE){ blackhole_lostconnection ("blackhole",c->host,glocal.blackholes); }else if (c->type == TYPE_PARENT){ debug_printf (D_EXTRA,"Parent %d disconnected\n",no); blackhole_remove (glocal.parents,no); }else if (c->type == TYPE_IDLE){ tlmp_error ("Lost connection with idle process, ending\n"); endserver = true; } HANDLE_INFO *n = (HANDLE_INFO*)data; if (ev == TCPSERVER_CONNECTED){ if (n->type == TYPE_HORIZON){ blackhole_horizon_connect_ok(glocal.horizons,n->host,no,n->type,glocal.secrets,"horizon"); }else if (n->type == TYPE_WORMHOLE){ blackhole_horizon_connect_ok(glocal.wormholes,n->host,no,n->type,glocal.secrets,"wormhole"); }else if (n->type == TYPE_BLACKHOLE){ blackhole_horizon_connect_ok(glocal.blackholes,n->host,no,n->type,glocal.secrets,"blackhole"); } n->connecting = false; }else if (ev == TCPSERVER_CONNECTFAIL){ if (n->type == TYPE_HORIZON){ blackhole_horizon_connect_fail(glocal.horizons,n->host,"horizon"); }else if (n->type == TYPE_WORMHOLE){ blackhole_horizon_connect_fail(glocal.wormholes,n->host,"wormhole"); }else if (n->type == TYPE_BLACKHOLE){ blackhole_horizon_connect_fail(glocal.blackholes,n->host,"blackhole"); } } HANDLE_INFO *c = (HANDLE_INFO*)info.data; debug_printf (D_EXTRA,"receive %s: %s\n",c->host.c_str(),line); if (c->type == TYPE_CONTROL){ (this,line,endserver,endclient,no,c); endserver = true; bool ison = atoi(on)==1; if (ison){ debug_seton(); }else{ debug_setoff(); } debug_setfdebug (filename); CONNECT_RULE_FROM from (from_server,from_vserver,ipname,port); CONNECT_RULE_TO to(to_server,to_vserver,to_port); glocal.rules[from].tos.clear(); // It is a replacement rule glocal.rules[from].tos.push_back(to); blackhole_checkrule (&glocal.TCPSERVER,from,to,glocal.horizons,glocal.checks); CONNECT_RULE_FROM from (from_server,from_vserver,ipname,port); CONNECT_RULE_TO to(to_server,to_vserver,to_port); glocal.rules[from].tos.push_back(to); blackhole_checkrule (&glocal.TCPSERVER,from,to,glocal.horizons,glocal.checks); time_t endtime = blackhole_mktime (datestr,hourstr); if (endtime == (time_t)0){ glocal.TCPSERVER.sendf ("Invalid date/time definition, expect yyyy/mm/dd hh:mm:ss: %s\n",line); }else{ CONNECT_RULE_FROM from (from_server,from_vserver,ipname,port); CONNECT_RULE_TO to(to_server,to_vserver,to_port,endtime); glocal.rules[from].tos.clear(); // It is a replacement rule glocal.rules[from].tos.push_back(to); blackhole_checkrule (&glocal.TCPSERVER,from,to,glocal.horizons,glocal.checks); } time_t endtime = blackhole_mktime (datestr,hourstr); if (endtime == (time_t)0){ glocal.TCPSERVER.sendf ("Invalid date/time definition, expect yyyy/mm/dd hh:mm:ss: %s\n",line); }else{ CONNECT_RULE_FROM from (from_server,from_vserver,ipname,port); CONNECT_RULE_TO to(to_server,to_vserver,to_port,endtime); glocal.rules[from].tos.push_back(to); blackhole_checkrule (&glocal.TCPSERVER,from,to,glocal.horizons,glocal.checks); } // Reload old stats for rules. // This is done after a service reload or restart CONNECT_RULE_FROM from (from_server,from_vserver,ipname,port); CONNECT_RULE_TO to(to_server,to_vserver,to_port); map::iterator it = glocal.rules.find(from); if (it != glocal.rules.end()){ for (unsigned i=0; isecond.tos.size(); i++){ if (it->second.tos[i] == to){ CONNECT_RULE_TO &t = it->second.tos[i]; t.nbcon = atoi(nbcon); t.lastcon = blackhole_cnvtime (lastcon); if (strcmp(last_source,"_")==0) last_source = ""; if (strcmp(last_target,"_")==0) last_target = ""; t.last_source = last_source; t.last_target = last_target; break; } } } // Reload old stats for protochecs. // This is done after a service reload or restart CONNECT_RULE_TO to(to_server,to_vserver,to_port); map::iterator it = glocal.checks.find(name); if (it != glocal.checks.end()){ for (unsigned i=0; isecond.tos.size(); i++){ if (it->second.tos[i] == to){ CONNECT_RULE_TO &t = it->second.tos[i]; t.nbcon = atoi(nbcon); t.lastcon = blackhole_cnvtime (lastcon); if (strcmp(last_source,"_")==0) last_source = ""; if (strcmp(last_target,"_")==0) last_target = ""; t.last_source = last_source; t.last_target = last_target; break; } } } // Reload the stats of rejected connections. // Done after a server restart CONNECT_RULE_FROM from (server,vserver,ipname,port); map::iterator it=glocal.rejects.find(from); int nbtry = atoi(nbtry_str); time_t lasttry = blackhole_cnvtime (lasttry_str); if (it != glocal.rejects.end()){ it->second.nbtry += nbtry; // We do not keep the old lasttry // We assume that if there was already an entry, it must have occured // just after the restart so is more current. }else{ glocal.rejects[from] = REJECT_STAT(nbtry,lasttry,original_target); } unsigned net = ipnum_aip2l (netaddr); unsigned mask = ipnum_aip2l (netmask); if ((net & mask) != net){ unsigned netm = net & mask; char netmstr[20]; ipnum_ip2a (netm,netmstr); tlmp_error ("Invalid network definition net=%s mask=%s net&mask=%s\n",netaddr,netmask,netmstr); glocal.TCPSERVER.sendf ("Invalid network definition: net=%s mask=%s net&mask=%s\n",netaddr,netmask,netmstr); }else{ bool found = false; IPLIST &ipl = glocal.iplists[name]; for (vector::iterator it=ipl.networks.begin(); it != ipl.networks.end(); it++){ if (it->net == net && it->mask == mask){ found = true; break; } } if (!found){ ipl.networks.push_back (NETWORK(net,mask)); } debug_printf (D_BLACKHOLE,"Adding network %s %s %s -> %08x %08x\n",name ,netaddr,netmask,net,mask); sort (ipl.networks.rbegin(),ipl.networks.rend()); } const char *distance = nearfar[0] == '\0' ? "far" : nearfar; string tmp_name,port(glocal.horizon_port); const char *pt = strchr(name,':'); if (pt != NULL){ tmp_name = string(name,pt-name); port = pt+1; name = tmp_name.c_str(); } bool is_far = strcmp(distance,"far")==0; if (strcmp(distance,"near")==0 || is_far){ map::iterator it = glocal.horizons.find(name); if (it == glocal.horizons.end()){ it = glocal.old_horizons.find(name); if (it == glocal.old_horizons.end()){ glocal.horizons[name] = HORIZON(is_far,port); blackhole_horizon_connect (&glocal.TCPSERVER,glocal.secrets,glocal.horizons,TYPE_HORIZON,glocal.paused); }else{ it->second.is_far = is_far; it->second.port = port; glocal.horizons[name] = it->second; glocal.old_horizons.erase (it); } } }else{ glocal.TCPSERVER.send ("Invalid horizon command, expect near or far\n"); endclient = true; } map::iterator it = glocal.wormholes.find(name); if (it == glocal.wormholes.end()){ glocal.wormholes[name] = HORIZON(false,glocal.horizon_port); blackhole_horizon_connect (&glocal.TCPSERVER,glocal.secrets,glocal.wormholes,TYPE_WORMHOLE,glocal.paused); } map::iterator it = glocal.blackholes.find(name); if (it == glocal.blackholes.end()){ it = glocal.old_blackholes.find(name); if (it == glocal.old_blackholes.end()){ glocal.blackholes[name] = HORIZON(false,glocal.blackhole_port); blackhole_horizon_connect (&glocal.TCPSERVER,glocal.secrets,glocal.blackholes,TYPE_BLACKHOLE,glocal.paused); }else{ glocal.blackholes[name] = it->second; glocal.old_blackholes.erase (it); } } glocal.allows.insert (ipaddr); debug_printf (D_BLACKHOLE,"Allow parent connection from ip %s\n",ipaddr); glocal.holes[wormhole_name].insert(horizon_name); glocal.checks.clear(); glocal.rules.clear(); glocal.holes.clear(); glocal.allows.clear(); glocal.reloaded = time(NULL); fdpass_readsecrets (glocal.secretfile,glocal.secrets); glocal.old_horizons = glocal.horizons; glocal.horizons.clear(); blackhole_resetservers (&glocal.TCPSERVER,glocal.wormholes); glocal.old_blackholes = glocal.blackholes; glocal.blackholes.clear(); glocal.rejects.clear(); CONNECT_RULE_FROM from (server,vserver,ipname,port); map::iterator it = glocal.rejects.find(from); if (it != glocal.rejects.end()){ glocal.rejects.erase(it); }else{ glocal.TCPSERVER.send ("No match\n"); } blackhole_pauseresume(&glocal.TCPSERVER,glocal.horizons,false); blackhole_pauseresume(&glocal.TCPSERVER,glocal.wormholes,false); blackhole_pauseresume(&glocal.TCPSERVER,glocal.blackholes,false); glocal.paused = true; glocal.pausedtime = time(NULL); blackhole_pauseresume(&glocal.TCPSERVER,glocal.horizons,true); blackhole_pauseresume(&glocal.TCPSERVER,glocal.wormholes,true); blackhole_pauseresume(&glocal.TCPSERVER,glocal.blackholes,true); glocal.paused = false; glocal.TCPSERVER.sendf ("blackhole version %s\n",VERSION); { DATEASC tmp; fdpass_asctime (glocal.started,tmp); glocal.TCPSERVER.sendf ("started=%s\n",tmp.buf); fdpass_asctime (glocal.reloaded,tmp); glocal.TCPSERVER.sendf ("reloaded=%s\n",tmp.buf); } glocal.TCPSERVER.sendf ("myname=%s\n",glocal.myname.c_str()); glocal.TCPSERVER.sendf ("nbfork=%d\n",nbfork); glocal.TCPSERVER.sendf ("paused=%d%s\n",glocal.paused,glocal.paused ? " ****** attention ********" : ""); glocal.TCPSERVER.sendf ("pingmode=%d\n",glocal.pingmode.is_enabled()); glocal.TCPSERVER.sendf ("pinginterval=%u seconds\n",glocal.pinginterval); glocal.TCPSERVER.sendf ("nbavoided=%d\n",glocal.nbavoided); glocal.TCPSERVER.sendf ("secrets %zu\n",glocal.secrets.size()); glocal.TCPSERVER.sendf ("logall=%d\n",logall); glocal.TCPSERVER.sendf ("logid=%d\n",logid); for (set::iterator it=glocal.allows.begin(); it != glocal.allows.end(); it++){ glocal.TCPSERVER.sendf ("allow %s\n",it->c_str()); } blackhole_reportstatus (&glocal.TCPSERVER,glocal.horizons,"Horizon"); blackhole_reportstatus (&glocal.TCPSERVER,glocal.old_horizons,"Old_horizon"); blackhole_reportstatus (&glocal.TCPSERVER,glocal.wormholes,"Wormhole"); blackhole_reportstatus (&glocal.TCPSERVER,glocal.blackholes,"Blackhole"); time_t now = time(NULL); for (map::iterator it=glocal.checks.begin(); it != glocal.checks.end(); it++){ const char *keyword = "protocheck "; for (unsigned i=0; isecond.tos.size(); i++){ CONNECT_RULE_TO &to = it->second.tos[i]; DATEASC lastcon; fdpass_asctime (to.lastcon,lastcon); glocal.TCPSERVER.sendf ("%s %s %s %s %s nbcon=%d last=%s last_source=%s last_target=%s\n",keyword,it->first.c_str() ,to.master.c_str(),to.vserver.c_str(),to.dstport.c_str() ,to.nbcon,lastcon.buf ,to.last_source.c_str(),to.last_target.c_str()); keyword = "aprotocheck"; } } for (map::iterator it=glocal.rules.begin(); it != glocal.rules.end(); it++){ const char *rule = "rule "; for (unsigned i=0; isecond.tos.size(); i++){ CONNECT_RULE_TO &to = it->second.tos[i]; DATEASC lastcon; fdpass_asctime (to.lastcon,lastcon); glocal.TCPSERVER.sendf ("%s %s %s %s %s -> %s%s%s %s %s nbcon=%d last=%s last_source=%s last_target=%s logging=%d",rule ,it->first.master.c_str(),it->first.vserver.c_str(),it->first.target.c_str(),it->first.dstport.c_str() ,to.check.c_str(), to.check.size() > 0 ? ":" : "" ,to.master.c_str(),to.vserver.c_str(),to.dstport.c_str() ,to.nbcon,lastcon.buf ,to.last_source.c_str(),to.last_target.c_str(),it->second.logging); if (to.end != (time_t)0){ DATEASC datet; fdpass_asctime (to.end,datet); glocal.TCPSERVER.sendf (" (until %s%s)",datet.buf,to.end < now ? ",expired" : ""); } glocal.TCPSERVER.send ("\n"); rule = "arule"; } } for (map >::iterator it=glocal.holes.begin(); it != glocal.holes.end(); it++){ glocal.TCPSERVER.sendf ("hole %s:",it->first.c_str()); for (set::iterator st=it->second.begin(); st != it->second.end(); st++){ glocal.TCPSERVER.sendf (" %s",st->c_str()); } glocal.TCPSERVER.send ("\n"); } for (map::iterator it = glocal.iplists.begin(); it != glocal.iplists.end(); it++){ glocal.TCPSERVER.sendf ("iplist %s %zu IP %zu NET\n",it->first.c_str(),it->second.ips.size(),it->second.networks.size()); } void *data; int fd = glocal.TCPSERVER.iter_init(data); while (fd != -1){ HANDLE_INFO *n = (HANDLE_INFO*)data; glocal.TCPSERVER.sendf ("Connection %d %s",fd,n->host.c_str()); if (n->type == TYPE_CONTROL){ glocal.TCPSERVER.send ("(control)"); }else if (n->type == TYPE_IDLE){ glocal.TCPSERVER.send ("(idle)"); }else if (n->type == TYPE_HORIZON){ glocal.TCPSERVER.send ("(horizon)"); }else if (n->type == TYPE_WORMHOLE){ glocal.TCPSERVER.send ("(wormhole)"); }else if (n->type == TYPE_BLACKHOLE){ glocal.TCPSERVER.send ("(sub-blackhole)"); }else if (n->type == TYPE_PARENT){ glocal.TCPSERVER.sendf ("(parent lastping %lds ago)",now-n->lastping); }else if (n->type == TYPE_FORK){ glocal.TCPSERVER.sendf ("(fork) %s %s %s %s -> %s %s %s" ,n->from.master.c_str(),n->from.vserver.c_str(),n->from.target.c_str(),n->from.dstport.c_str() ,n->to.master.c_str(),n->to.vserver.c_str(),n->to.dstport.c_str()); } glocal.TCPSERVER.send ("\n"); fd = glocal.TCPSERVER.iter_next(data); } // Being paused is not an error, but being paused for a long time // is not normal. int ping_disabled_since = glocal.pingmode.disabled_since(); if (ping_disabled_since > 30){ glocal.TCPSERVER.sendf ("ping disable since %d seconds\n",ping_disabled_since); } if (glocal.paused) glocal.TCPSERVER.sendf ("paused=%ld\n",time(NULL)-glocal.pausedtime); blackhole_reporterr (&glocal.TCPSERVER,glocal.horizons,"Horizon"); blackhole_reporterr (&glocal.TCPSERVER,glocal.wormholes,"Wormhole"); blackhole_reporterr (&glocal.TCPSERVER,glocal.blackholes,"Blackhole"); blackhole_checkrules (&glocal.TCPSERVER,glocal.rules,glocal.horizons,glocal.checks); if (glocal.secretfile != NULL){ // Check if we have a secret for all horizons, wormholes and sub-blackholes blackhole_checksecret (&glocal.TCPSERVER,glocal.secrets,glocal.horizons,"Horizon"); blackhole_checksecret (&glocal.TCPSERVER,glocal.secrets,glocal.wormholes,"Wormhole"); blackhole_checksecret (&glocal.TCPSERVER,glocal.secrets,glocal.blackholes,"Blackhole"); } void *data; int fd = glocal.TCPSERVER.iter_init(data); while (fd != -1){ HANDLE_INFO *n = (HANDLE_INFO*)data; if (n->type == TYPE_FORK){ glocal.TCPSERVER.sendf ("fd=%d %s %s %s %s -> %s %s %s\n",fd ,n->from.master.c_str(),n->from.vserver.c_str(),n->from.target.c_str(),n->from.dstport.c_str() ,n->to.master.c_str(),n->to.vserver.c_str(),n->to.dstport.c_str()); } fd = glocal.TCPSERVER.iter_next (data); } for (map::iterator it=glocal.rejects.begin(); it != glocal.rejects.end(); it++){ DATEASC datet; fdpass_asctime (it->second.lasttry,datet); glocal.TCPSERVER.sendf ("%s %s %s %s: nbtry=%d last=%s last_ip=%s\n" ,it->first.master.c_str(),it->first.vserver.c_str(),it->first.target.c_str(),it->first.dstport.c_str() ,it->second.nbtry,datet.buf,it->second.original_target.c_str()); } for (map::iterator it=glocal.active_cons.begin(); it != glocal.active_cons.end(); it++){ glocal.TCPSERVER.sendf ("%s %s %s %u %u %u %u\n" ,it->first.master.c_str(),it->first.vserver.c_str(),it->first.dstport.c_str() ,it->second.connected,it->second.weight,it->second.alt_con,it->second.max_con); } // Cleanup of the active_cons for (map::iterator it=glocal.active_cons.begin(); it != glocal.active_cons.end(); ){ if (it->second.is_default()){ glocal.active_cons.erase (it++); }else{ ++it; } } CONNECT_RULE_TO to(server,vserver,port); glocal.active_cons[to].weight = atoi(weight); CONNECT_RULE_TO to(server,vserver,port); glocal.active_cons[to].alt_con = atoi(nbcon); CONNECT_RULE_TO to(server,vserver,port); glocal.active_cons[to].connected = atoi(nbcon); unsigned long addr = ipnum_aip2l(ipaddr); glocal.iplists[name].ips.insert(addr); map::iterator it = glocal.iplists.find(name); if (it != glocal.iplists.end()) glocal.iplists.erase (it); IPLIST &list = glocal.iplists[name]; for (set::iterator it=list.ips.begin(); it != list.ips.end(); it++){ char ip[20]; ipnum_ip2a (*it,ip); glocal.TCPSERVER.sendf ("%s\n",ip); } for (vector::iterator it=list.networks.begin(); it != list.networks.end(); it++){ char net[20],mask[20]; ipnum_ip2a (it->net,net); ipnum_ip2a (it->mask,mask); glocal.TCPSERVER.sendf ("network %s %s\n",net,mask); } unsigned nb = atoi(seconds); if (nb < 5){ glocal.TCPSERVER.send ("pinginterval: minimum value is 5 seconds\n"); }else{ glocal.pinginterval = nb; } glocal.pingmode.disable(); glocal.pingmode.enable(); logall = true; logall = false; // server vserver ipname port blackhole_setlogging (&glocal.TCPSERVER,glocal.rules,server,vserver,ipname,port,true); // server vserver ipname port blackhole_setlogging (&glocal.TCPSERVER,glocal.rules,server,vserver,ipname,port,false); // num logid = atoi(num); if (fstatfile != NULL) fclose (fstatfile); fstatfile = fopen (filename,"a"); if (fstatfile == NULL){ tlmp_error ("Can't open statfile %s (%s)\n",filename,strerror(errno)); } glocal.checks[name].tos.clear(); glocal.checks[name].tos.push_back(CONNECT_RULE_TO(server,vserver,port)); glocal.checks[name].tos.push_back(CONNECT_RULE_TO(server,vserver,port)); glocal.TCPSERVER.sendf ("Invalid command: %s\n",line); endclient = true; }else if (c->type == TYPE_IDLE){ if (glocal.paused) debug_printf (D_BLACKHOLE,"Server is paused\n"); blackhole_horizon_connect (this,glocal.secrets,glocal.horizons,TYPE_HORIZON,glocal.paused); blackhole_horizon_connect (this,glocal.secrets,glocal.wormholes,TYPE_WORMHOLE,glocal.paused); blackhole_horizon_connect (this,glocal.secrets,glocal.blackholes,TYPE_BLACKHOLE,glocal.paused); if (glocal.pingmode.is_enabled()){ time_t now = time(nullptr); if ((unsigned)(now - glocal.lastping) > glocal.pinginterval){ glocal.lastping = now; void *data; int fd = iter_init(data); while (fd != -1){ HANDLE_INFO *n = (HANDLE_INFO*)data; // We must not send a ping to a handle not connected. It will block. // and for horizons, it won't be accepted until the master command has been sent. if (!n->connecting && (n->type == TYPE_HORIZON || n->type == TYPE_WORMHOLE || n->type == TYPE_BLACKHOLE)){ fdpass_sendto (fd,n->host,glocal.secrets,"ping"); n->nbping++; } fd = iter_next (data); } } } if (!glocal.paused){ blackhole_resetservers (this,glocal.old_horizons); blackhole_resetservers (this,glocal.old_blackholes); } }else if (c->type == TYPE_PARENT){ string secret = fdpass_findsecret (glocal.secrets,glocal.myname); (this,line,endserver,endclient,secret,c->intruder,no,c); debug_printf (D_EXTRA,"Ping from parent %d\n",no); blackhole_remove (glocal.parents,no); glocal.parents.insert(glocal.parents.begin(),no); c->lastping = time(NULL); fdpass_send (&glocal.TCPSERVER,glocal.myname,glocal.secrets,"pong"); tlmp_error ("Invalid request from parent: %s\n",line); endclient = true; }else if (c->type == TYPE_BLACKHOLE){ string secret = fdpass_findsecret (glocal.secrets,c->host); (this,line,endserver,endclient,secret,c->intruder,no,c,c->host.c_str()); debug_printf (D_BLACKHOLE,"sub-blackhole request %s: %s\n",host,line); CONNECT_REQUEST req; req.horizon = horizon; req.source = source; req.target = target; req.dstport = dstport; req.link = linkstr; blackhole_ruleconnect (&glocal.TCPSERVER,glocal.myname,glocal.secrets,glocal.checks,req ,glocal.rules,glocal.holes,glocal.wormholes,glocal.horizons,glocal.rejects ,glocal.iplists,glocal.parents ,glocal.nbavoided,glocal.active_cons); debug_printf (D_EXTRA,"Pong from sub-blackhole %s, nbping=%d\n",host,c->nbping); c->nbping--; c->lastping = time(NULL); if (c->nbping < 0){ tlmp_error ("Too many pong from sub-blackhole server %s\n",host); } tlmp_error ("Invalid command from sub-blackhole server %s: %s\n",host ,line); }else if (c->type == TYPE_HORIZON){ glocal const char *server = c->host.c_str(); string secret = fdpass_findsecret (glocal.secrets,c->host); (this,line,endserver,endclient,secret,c->intruder,no,c,c->host.c_str()); debug_printf (D_BLACKHOLE,"Horizon request %s: %s\n",host,line); if (glocal.connectdelay > 0){ // This feature was done to help the testrace program //fprintf (stderr,"avant delay %d\n",glocal.connectdelay); struct timespec tm; tm.tv_nsec = glocal.connectdelay*1000; tm.tv_sec = 0; nanosleep (&tm,NULL); //fprintf (stderr,"apres delay\n"); } CONNECT_REQUEST req; req.horizon = host; req.link = link; req.source = source; req.target = target; req.dstport = dstport; blackhole_ruleconnect (&glocal.TCPSERVER,glocal.myname,glocal.secrets,glocal.checks,req ,glocal.rules,glocal.holes,glocal.wormholes,glocal.horizons,glocal.rejects ,glocal.iplists,glocal.parents ,glocal.nbavoided,glocal.active_cons); debug_printf (D_EXTRA,"Pong from horizon %s, nbping=%d\n",host,c->nbping); c->nbping--; c->lastping = time(NULL); if (c->nbping < 0){ tlmp_error ("Too many pong from horizon server %s\n",c->host.c_str()); } if (extra[0] != '\0'){ // horizon are using the pong protocol to report the last error. // This is a quick way to debug a network c->lasterror = extra; }else{ c->lasterror.clear(); } CONNECT_RULE_TO to(glocal.server,vserver,port); map::iterator it = glocal.active_cons.find (to); bool expected = it != glocal.active_cons.end(); debug_printf (D_EXTRA,"Endcon from horizon %s: %s %s, expected %d\n" ,glocal.server,vserver,port,expected); if (expected){ it->second.connected--; }else{ tlmp_error ("Unmatched endcon from horizon %s: %s %s\n",glocal.server,vserver,port); } tlmp_error ("Invalid command from horizon server %s: %s\n",host ,line); }else if (c->type == TYPE_WORMHOLE){ string secret = fdpass_findsecret (glocal.secrets,c->host); (this,line,endserver,endclient,secret,c->intruder,no,c,c->host.c_str()); debug_printf (D_EXTRA,"Pong from wormhole %s, nbping=%d\n",host,c->nbping); c->nbping--; c->lastping = time(NULL); if (c->nbping < 0){ tlmp_error ("Too many pong from wormhole server %s\n",host); } tlmp_error ("Command received from wormhole %s: %s\n",c->host.c_str(),line); } bool some_errors = false; if (fdpass_setcontrol(o,glocal.control,glocal.user)==-1){ some_errors = true; } if(glocal.parent != NULL){ SSTRING name,port; if(fdpass_splitbind(glocal.parent,name,port)==-1){ some_errors = true; }else if (o.listen (name.c_str(),port.c_str()) == -1){ some_errors = true; } } if (some_errors){ tlmp_error ("Some errors, ending\n"); }else{ signal (SIGPIPE,SIG_IGN); signal (SIGCHLD,SIG_IGN); if (glocal.daemon){ daemon_init (glocal.pidfile,glocal.user); } { // A fork to send a message to the main process every 5 seconds. int tb[2]; if (pipe(tb)==-1){ tlmp_error ("Can't set pipe for idle loop (%s), ending\n",strerror(errno)); exit (-1); }else{ pid_t pid = fork(); if (pid == (pid_t)0){ for (int i=0; i<1024; i++){ // This closes as well tb[0] if (i != tb[1]) close (i); } while (1){ sleep (5); if (write (tb[1],"hello\n",6)!=6){ tlmp_error ("Can't write to idle pipe, ending\n"); break; } } _exit (0); }else if (pid == (pid_t)-1){ tlmp_error ("Can't fork (%s), ending\n",strerror(errno)); exit (-1); }else{ close (tb[1]); HANDLE_INFO *c = new HANDLE_INFO; c->type = TYPE_IDLE; o.inject (tb[0],c); } } } o.loop(); ret = 0; } if (fstatfile != NULL) fclose (fstatfile); return ret; return glocal.ret; }