#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <limits.h>
#include <errno.h>
#include <sys/wait.h>
#include <misc.h>
#include <tlmplib.h>
#include <string>
#include <set>
#include <map>
#include <vector>
#include <algorithm>
#include <sys/time.h>
#include "lasuite.h"

static long long getnow ()
{
	struct timeval tv;
	gettimeofday (&tv,NULL);
	return tv.tv_sec *(long long)1000000 + tv.tv_usec;
}

using namespace std;




#if 1
struct COMMON{
	int nb;
	unsigned noexpr;
	COMMON(int _nb, unsigned _noexpr){
		nb = _nb;
		noexpr = _noexpr;
	};
	bool operator < (const COMMON &o) const{
		return nb < o.nb;
	}
};
static void lasuite_classify (vector<EXPR> &exprs)
{
	map<string,vector<unsigned> > ckeys;
	for (unsigned i=0; i<exprs.size(); i++){
		string ckey = exprs[i].getckey();
		// printf ("ckey = %s\n",ckey.c_str());
		ckeys[ckey].push_back(i);
	}
	vector<COMMON> commons;
	for (map<string,vector<unsigned> >::iterator it = ckeys.begin();
		it != ckeys.end(); it++){
		vector<unsigned> &v = it->second;
		unsigned size = v.size();
		//printf ("key %s %lu\n",it->first.c_str(),it->second.size());
		//exprs[v[0]].dumptop();
		int tb[size];
		memset (tb,0,sizeof(tb));
		unsigned size_1 = size-1;
		for (unsigned i=0; i<size_1; i++){
			for (unsigned j=i+1; j<size; j++){
				if (tb[j] == 0){
					if (exprs[v[i]].cmpstruct(&exprs[v[j]])){
						tb[i]++;
						tb[j] = -1;
					}
				}
			}
		}
		for (unsigned i=0; i<size; i++){
			if (tb[i] > 0) commons.push_back(COMMON(tb[i],v[i])); 
		}
	}
	sort (commons.rbegin(),commons.rend());
	for (unsigned i=0; i<commons.size(); i++){
		printf ("NBREPEAT=%d\n",commons[i].nb);
		exprs[commons[i].noexpr].dumptop();
	}
}

#endif

struct ALIKE{
	EXPR query;
	int nbalike;
	int retlines;
	double duree;
	string sql;
	int id;			// Used to compare all alike query (for debugging)
	ALIKE (const EXPR &expr, const string &_sql, double _duree, int _retlines, int _nbalike, int _id){
		query = expr;
		sql = _sql;
		nbalike = _nbalike;
		duree = _duree;
		retlines = _retlines;
		id = _id;
	}
	ALIKE(const ALIKE &a){
		query = a.query;
		sql = a.sql;
		duree = a.duree;
		nbalike = a.nbalike;
		retlines = a.retlines;
		id = a.id;
	}
};
struct RESULT{
	int nb;
	int retlines;
	double duree;
	const EXPR *query;
	const string *sql;
	RESULT(int _nb, int _retlines, double _duree, const EXPR &_q, const string &_sql){
		nb = _nb;
		retlines = _retlines;
		duree = _duree;
		query = &_q;
		sql = &_sql;
	}
	bool operator < (const RESULT &o) const{
		return nb < o.nb;
	}
};

struct RESCMPUSAGE{
	bool operator () (const RESULT &r1, const RESULT &r2) const {
		bool ret = false;
		int idiff = r1.nb - r2.nb;
		if (idiff < 0){
			ret = true;
		}else if (idiff == 0){
			double diff = r1.duree - r2.duree;
			if (diff < 0){
				ret = true;
			}else if (diff == 0){
				idiff = r1.retlines - r2.retlines;
				if (idiff < 0){
					ret = true;
				}
			}
		}
		return ret;
	}
};
struct RESCMPTIME{
	bool operator () (const RESULT &r1, const RESULT &r2) const {
		bool ret = false;
		double diff = r1.duree - r2.duree;
		if (diff < 0){
			ret = true;
		}else if (diff == 0){
			int idiff = r1.nb - r2.nb;
			if (idiff < 0){
				ret = true;
			}else if (idiff == 0){
				idiff = r1.retlines - r2.retlines;
				if (idiff < 0){
					ret = true;
				}
			}
		}
		return ret;
	}
};

static int lasuite_getint (const char *buf, const char *key)
{
	int ret = -1;
	const char *pt = strstr(buf,key);
	if (pt != NULL){
		pt += strlen(key);
		if (*pt == '('){
			pt++;
			ret = atoi(pt);
		}
	}
	return ret;
}
static double lasuite_getdbl (const char *buf, const char *key)
{
	double ret = -1;
	const char *pt = strstr(buf,key);
	if (pt != NULL){
		pt += strlen(key);
		if (*pt == '('){
			pt++;
			ret = atof(pt);
		}
	}
	return ret;
}
#if 0
static string lasuite_getstr (const char *buf, const char *key)
{
	string ret;
	const char *pt = strstr(buf,key);
	if (pt != NULL){
		pt += strlen(key);
		if (*pt == '('){
			pt++;
			const char *start = pt;
			while (*pt != '\0' && *pt != ')') pt++;
			ret = string(start,pt-start);
		}
	}
	return ret;
}
#endif

static FILE *lasuite_opensniff(const char *snifffile)
{
	FILE *fin = NULL;
	if (strstr(snifffile,".gz")!=NULL){
		string tmp("zcat ");
		tmp += snifffile;
		fin = popen (tmp.c_str(),"r");
		if (fin == NULL){
			tlmp_error ("Can't execute %s (%s)\n",tmp.c_str(),strerror(errno));
			exit (-1);
		}
	}else if ((fin = fopen (snifffile,"r"))==NULL){
		tlmp_error ("Can't open snifffile %s (%s)\n",snifffile,strerror(errno));
		exit (-1);
	}
	return fin;
}
static void lasuite_closesniff(const char *snifffile, FILE *fin)
{
	if (strstr(snifffile,".gz")!=NULL){
		pclose (fin);
	}else{
		fclose (fin);
	}
}

static void lasuite_printif (FILE *ftty, const char *ctl, ...)
{
	if (ftty != NULL){
		va_list list;
		va_start (list,ctl);
		vfprintf (ftty,ctl,list);
		va_end (list);
	}
}

<mod>
int main (int argc, char *argv[])
{
	glocal int ret = -1;
	glocal const char *ftest = NULL;
	glocal SSTRINGS snifffiles;
	glocal int stop_after = 1;
	glocal bool show_query = false;
	glocal bool print_error = false;
	glocal const char *classify = NULL;		// path/file prefix to save the result
	glocal bool stop = false;
	glocal bool showprogress = false;
	glocal int workers = 4;
	glocal bool batch = false;
	glocal const char *batchdir = "/tmp/batch";
	glocal const char *argv0 = argv[0];
	glocal const char *logquery = NULL;

	parser_initfunctions();
	parser_inittokens();
	glocal.ret = <call tlmpprogram>(argc,argv);
	<f init>
		setproginfo ("lasuite",VERSION
			,"Analyse SQL statement in sniffer and produce statistics\n");
		setarg (' ',"classify","Organise queries",glocal.classify,false);
		setarg (' ',"snifffile","Log produced by the SQL sniffer",glocal.snifffiles,false);
		setarg (' ',"logquery","Save original query with the alikeid",glocal.logquery,false);

		setgrouparg ("General");
		setarg (' ',"ftest","Test file",glocal.ftest,false);
		setarg (' ',"print_error","Prints query with syntax error",glocal.print_error,false);
		setarg (' ',"showprogress","Show some stats during processing",glocal.showprogress,false);
		setarg (' ',"show_query","Prints query while they are parsed",glocal.show_query,false);
		setarg (' ',"stop","Stop after processing all queries (to look a memory usage)",glocal.stop,false);
		setarg (' ',"stop_after","Stop after N fails (sqlfile)",glocal.stop_after,false);

		setgrouparg ("Multi-worker mode");
		setarg (' ',"batch","Split the snifffiles to N workers",glocal.batch,false);
		setarg (' ',"batchdir","Tmp directory to produce results",glocal.batchdir,false);
		setarg (' ',"workers","Number of workers to start",glocal.workers,false);
	</f>
	<f main_noarg>
		glocal int ret = 0;
		FILE *ftty = NULL;
		if (glocal.showprogress){
			ftty = fopen ("/dev/tty","w");
			if (ftty == NULL){
				tlmp_error ("Can't open /dev/tty (--showprogress) (%s)\n",strerror(errno));
				exit (-1);
			}
		}
		if (!glocal.batch && glocal.snifffiles.getnb() > 0){
			glocal map<string,ALIKE> generics;
			long long start = getnow();
			int nbfail = 0;
			int nbquery = 0;
			int alikeid=0;
			FILE *flog = NULL;
			if (glocal.logquery != NULL){
				flog = fopen (glocal.logquery,"w");
				if (flog == NULL){
					tlmp_error ("Can't open logquery file %s (%), aborting\n",glocal.logquery,strerror(errno));
					exit (-1);
				}
			}
			for (int i=0; i<glocal.snifffiles.getnb(); i++){
				const char *snifffile = glocal.snifffiles.getitem(i)->c_str();
				FILE *fin = lasuite_opensniff (snifffile);
				if (fin != NULL){
					char buf[100000];
					string onesql;
					double realtime = 0;
					int retlines = 0;
					#if 0
					string ipnum;
					int port = 0;
					#endif
					int nbalike = 0;
					while (fgets(buf,sizeof(buf),fin)!=NULL){
						strip_end (buf);
						if (strncmp(buf,"/*",2)==0){
							// Details of the request
							// /* 2014-02-13 06:00:31 : realtime(0.022) --status(0)-- ++numretLines(2442)++  --IP(192.168.99.53) PORT(46777)-- queryNo(2)
							// comment[TO: Too much time]  TYPE[LOGIN 4.2]  */
							realtime = lasuite_getdbl(buf,"realtime");
							retlines = lasuite_getint(buf,"numretLines");
							#if 0
							ipnum = lasuite_getstr(buf,"IP");
							port = lasuite_getint(buf,"PORT");
							#endif
							// nbalike is not part of the sniffer out, but it is added when this program rewrite the output of its
							// classify function
							nbalike = lasuite_getint(buf,"nbalike");
							if (nbalike == -1) nbalike = 1;
							// printf ("Stats: realtime=%lf retlines=%d IP=%s port=%d nbalike=%d\n",realtime,retlines,ipnum.c_str(),port,nbalike);
							onesql.clear();
						}else if (strcmp(buf,"GO")==0){
							nbquery++;
							PARSE p (onesql.c_str());
							EXPR expr;
							if (f_main(p,expr)==-1){
								nbfail++;
								if (glocal.print_error){
									printf ("line: %s",onesql.c_str());
									fprintf (stderr,"\tfail\n");
									p.showlast();
									expr.dumptop();
								}
								glocal.ret = -1;
								glocal.stop_after--;
								if (glocal.stop_after <= 0) break;
							}else{
								if (glocal.classify != NULL){
									string ckey = expr.getckey();
									map<string,ALIKE>::iterator it = glocal.generics.find(ckey);
									if (it == glocal.generics.end()){
										if (flog != NULL){
											// Using sort on this file will yield a classification key
											// followed by all query considered equivalent
											fprintf (flog,"%06dA: %s\n",alikeid,ckey.c_str());
											fprintf (flog,"%06dB: %s\n",alikeid,onesql.c_str());
										}
										glocal.generics.insert (pair<string,ALIKE>(ckey,ALIKE(expr,onesql,realtime,retlines,nbalike,alikeid++)));
									}else{
										if (flog != NULL){
											fprintf (flog,"%06dB: %s\n",it->second.id,onesql.c_str());
										}
										it->second.nbalike++;
										it->second.duree += realtime;
										it->second.retlines += retlines;
									}
								}
								if (glocal.show_query){
									printf ("line: %s",buf);
									expr.dumptop();
								}
							}
							if (ftty != NULL && nbquery % 100000 == 0){
								long long now = getnow();
								fprintf (ftty,"%s: nbquery=%d nbfail=%d %lf q/s\n"
									,snifffile,nbquery,nbfail
									,nbquery/((now-start)/1000000.0));
							}
						}else{
							onesql += string (" ") + buf;
						}	
					}
					lasuite_closesniff (snifffile,fin);
					if (ftty != NULL) fprintf (ftty,"NBFAIL=%d   NBQUERY=%d\n",nbfail,nbquery);
				}
				if (flog != NULL) fclose (flog);
				if (glocal.stop){
					printf ("enter");
					char buf[10];
					fgets(buf,sizeof(buf),stdin);
				}
				if (glocal.classify != NULL){
					glocal vector<RESULT> results;
					string tmp = string(glocal.classify) + ".original";
					lasuite_printif (ftty,"generics.size=%lu\n",glocal.generics.size());
					<call savefile>(tmp.c_str(),false);
					<f dowrite>
						for (map<string,ALIKE>::iterator it = glocal.generics.begin();
							it != glocal.generics.end(); it++){
							ALIKE &al = it->second;
							glocal.results.push_back(RESULT(al.nbalike,al.retlines,al.duree,al.query,al.sql));
							fprintf (fout,"/* realtime(%lf) nbalike(%d) numretLines(%d) */\n"
								,al.duree,al.nbalike,al.retlines);
							fprintf (fout,"%s\n",al.sql.c_str());
							fprintf (fout,"GO\n");
						}
						return 0;
					</f>
					</call>
					for (int i=0; i<2; i++){
						if (i==0){
							tmp = string(glocal.classify) + ".byusage";
							sort (glocal.results.rbegin(),glocal.results.rend(),RESCMPUSAGE());
						}else{
							tmp = string(glocal.classify) + ".bytime";
							sort (glocal.results.rbegin(),glocal.results.rend(),RESCMPTIME());
						}
						<call savefile>(tmp.c_str(),false);
						<f dowrite>
							int total = 0;
							for (unsigned i=0; i<glocal.results.size(); i++){
								RESULT &r = glocal.results[i];
								int nb = r.nb;
								total += nb;
								fprintf (fout,"NBALIKE=%d total=%d duree=%lf retlines=%d\n"
									,nb,total,r.duree,r.retlines);
								#if 0
									dumpfout = fout;
									r.query->dumptop();
									dumpfout = stdout;
								#else
									fprintf (fout,"%s\n",r.sql->c_str());
								#endif
							}
							fprintf (fout,"TOTAL=%d\n",total);
							return 0;
						</f>
						</call>
					}
				}
			}	
		}else if (glocal.ftest != NULL){
			glocal string onesql;
			glocal int nbquery = 0;
			glocal int nbfail = 0;
			glocal vector<EXPR> exprs;
			<call loadfile>(glocal.ftest,true);
			<f oneline>
				if (strcmp(line,"go")==0){
					const char *onesql = glocal.onesql.c_str();
					glocal.nbquery++;
					PARSE p (onesql);
					EXPR expr;
					if (f_main(p,expr)==-1){
						printf ("test: %s\n",onesql);
						fprintf (stdout,"\tfail\n");
						p.showlast();
						expr.dumptop();

						glocal.nbfail++;
						if (glocal.nbfail >= glocal.stop_after) return -1;
					}else{
						glocal.exprs.push_back(expr);
						if (glocal.show_query){
							printf ("test: %s\n",onesql);
							printf ("    : %s\n",expr.getckey().c_str());
							expr.dumptop();
						}
					}
					glocal.onesql.clear();
				}else{
					glocal.onesql += "\n";
					glocal.onesql += line;
				}
				return 0;
			</f>
			</call>
			fprintf (stdout,"NBFAIL=%d   NBQUERY=%d\n",glocal.nbfail,glocal.nbquery);
			if (glocal.nbfail > 0) glocal.ret = -1;
			if (glocal.classify){
				lasuite_classify (glocal.exprs);
			}
		}else if (glocal.batch){
			mkdir (glocal.batchdir,0700);
			FILE *f[glocal.workers];
			for (int i=0; i<glocal.workers; i++){
				char tmp[PATH_MAX];
				snprintf (tmp,sizeof(tmp),"%s/pipe-%d",glocal.batchdir,i);
				mknod (tmp,S_IFIFO|0600,0);
				pid_t pid = fork();
				if (pid == (pid_t)0){
					char tmpres[PATH_MAX];
					snprintf (tmpres,sizeof(tmpres),"%s/result-%d",glocal.batchdir,i);
					const char *args[]={
						glocal.argv0,
						"--snifffile", tmp,
						"--classify", tmpres,
						"--showprogress",
						"--stop_after", "10000000",
						NULL
					};
					execv (glocal.argv0,(char**)args);
					tlmp_error ("Can't exec worker %d (%s)\n",i,strerror(errno));
					_exit (-1);
				}else if (pid == (pid_t)-1){
					tlmp_error ("Can't fork worker %i (%s)\n",i,strerror(errno));
					_exit (-1);
				}else{
					f[i] = fopen (tmp,"w");
				}
			}
			int curwork = 0;
			for (int i=0; i<glocal.snifffiles.getnb(); i++){
				const char *snifffile = glocal.snifffiles.getitem(i)->c_str();
				FILE *fin = lasuite_opensniff (snifffile);
				if (fin != NULL){
					char buf[100000];
					while (fgets(buf,sizeof(buf),fin)!=NULL){
						if (buf[0] == '/' && buf[1] == '*'){
							curwork = (curwork+1)%glocal.workers;
						}
						fputs (buf,f[curwork]);
					}
					lasuite_closesniff(snifffile,fin);
				}
			}
			for (int i=0; i<glocal.workers; i++) fclose (f[i]);
			lasuite_printif (ftty,"Done, waiting for workers to end\n");
	
			int status;
			int val;
			while ((val=wait(&status)) != -1){
				lasuite_printif (ftty,"Worker %d ended\n",val);
			}
			// Now we can put the workers output together
			// We could optimise the processing by feeding the output of the last workers into the work of the first to same
			// a little time. Maybe one day.
			string cmd = glocal.argv0 + string(" --showprogress --classify ") + glocal.batchdir + "/all";
			for (int i=0; i<glocal.workers; i++){
				char tmpres[PATH_MAX];
				snprintf (tmpres,sizeof(tmpres),"%s/result-%d.original",glocal.batchdir,i);
				cmd += string(" --snifffile ") + tmpres;
			}
			glocal.ret = system (cmd.c_str());	
		}else{
			usage();
			glocal.ret = -1;
		}
		if (ftty != NULL) fclose (ftty);
		return glocal.ret;
	</f>
	</call>
	return glocal.ret;
}
</mod>