#!/hive/groups/encode/dcc/bin/python
import sys, os, re, argparse, subprocess, math, datetime
from ucscgenomics import ra, track, mkChangeNotes, qa

def getMethods(qaDir, args, user, d):
	cmd = "curl -X GET http://genomewiki.cse.ucsc.edu/genecats/index.php/ENCODE_QA 2>/dev/null"
	p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
	output = p.stdout.read().split("\n")
	r1 = re.compile('<li class="toclevel.*="tocnumber">(\S+)<.*="toctext">(.*)<\/span>')
	r2 = re.compile('&amp;')
	if os.path.exists(qaDir + '/methods'):
		return
	
	f = open(qaDir + '/methods', 'w')
	f.write(args.composite + "\n")
	f.write(args.database + "\n")
	f.write("Release " + args.release + "\n")
	f.write("Redmine #" + args.redmine + "\n")
        f.write(str(d) + "\n")
	f.write(user + "\n")
	f.write("\n")
	for line in output:
		
		m1 = r1.match(line)
		
		if m1:
			tabs = ""
			for char in m1.group(1):
				if char == ".":
					tabs = tabs + "    "
			if tabs == "":
				f.write("\n")
			m2 = r2.search(line)
			second = r2.sub('&', m1.group(2))
			f.write("%s%s %s\n" % (tabs, m1.group(1), second))
	return

def writeClaimMail(args, user, qaDir):
	f = open(qaDir + '/claimMail', 'w')
	mail = """Hi Kate,

I am claiming the %s (Release %s) %s track.
Please let me know if there is another track you'd like me to work on instead.
http://redmine.soe.ucsc.edu/issues/%s

%s
""" % (args.composite, args.release, args.database, args.redmine, user)
	
	f.write(mail)
	f.close()
	return
	
def makeLinks(c, args, qaDir):

	notesfile = c._notesDirectory + "%s.release%s.notes" % (args.composite, args.release)
	downloads = c.downloadsDirectory + "release%s" % args.release

	if not os.path.exists(notesfile):
		return ("""
FATAL ERROR: Notes file for %s release %s doesn't exist in %s
You may need to 'git pull' or tell the wrangler to check in notes file
""" % (args.composite, args.release, c._notesDirectory))
	if os.path.exists(qaDir + "/notes.file"):
		os.remove(qaDir + "/notes.file")
	os.symlink(notesfile, qaDir + "/notes.file")
	
	if os.path.exists(qaDir + "/downloads"):
		os.remove(qaDir + "/downloads")
	os.symlink(downloads, qaDir + "/downloads")
	
	return notesfile

def parseNotes(lines):
	tables = set()
	gbdbs = set()
	files = set()
	supplemental = set()
	others = set()
	size = ""
	switch = 0
	p1 = re.compile('New Tables')
	p2 = re.compile('New Download Files')
	p3 = re.compile('New Gbdb Files')
	p4 = re.compile('New Supplemental Files')
	p5 = re.compile('New Other Files')
	s = re.compile('Total: (\d+) MB')
	e = re.compile('^$')
	for i in lines:
		if p1.search(i):
			switch = 1
			continue
		if p2.search(i):
			switch = 2
			continue
		if p3.search(i):
			switch = 3
			continue
		if p4.search(i):
			switch = 4
			continue
		if p5.search(i):
			switch = 5
			continue
		
		sm = s.match(i)
		if sm:
			size = sm.group(1)
			
		if e.match(i):
			switch = 0
			#print "empty line"	
			
		if switch == 1:
			tables.add(i)
		if switch == 2:
			files.add(i)
		if switch == 3:
			gbdbs.add(i)
		if switch == 4:
			supplemental.add(i)
		if switch == 5:
			others.add(i)

	return tables, gbdbs, files, supplemental, others, str(size)

def writeTableMail(tables, args, user, qaDir):
	tablestr = ""
	if tables:
		sep = ""
		tablestr = sep.join(list(sorted(tables)))
		mail = """Hi Pushers,

For the *%s* database (2 part request):

1) Please push trackDb and friends

2) Please push these %s tables:

%s

from mysqlbeta -> mysqlrr

Reason: releasing ENCODE %s track on %s to the RR http://redmine.soe.ucsc.edu/issues/%s

Thank you!

%s
""" % (args.database, len(tables), tablestr, args.composite, args.database, args.redmine, user)
	else:
		mail = """Hi Pushers,

For the *%s* database:

1) Please push trackDb and friends

from mysqlbeta -> mysqlrr

Reason: releasing _____(fill in metadata or trackDb changes)____ for ENCODE %s track on %s to the RR

Thank you!

%s
""" % (args.database, args.composite, args.database, user)

	f = open(qaDir + "/pushTableMail", "w")
	f.write(mail)
	f.close
	
	if not tablestr:
		return ""

	f = open(qaDir + "/tableList", "w")
	f.write(tablestr)
	f.close
	
	return tablestr
	
def writeGbdbMail(gbdbs, args, user, qaDir):
	sep = ""
	gbdbstr = sep.join(list(sorted(gbdbs)))
	mail = """
Hi Pushers,

Please push these %s gbdb files:

%s

\tfrom hgwdev --> hgnfs1

Reason: staging %s %s track on beta, http://redmine.soe.ucsc.edu/issues/%s

Thank You!

%s
""" % (len(gbdbs), gbdbstr, args.composite, args.database, args.redmine, user)
	f = open(qaDir + "/pushGbdbsMail", "w")
	f.write(mail)
	f.close()
	return gbdbstr
	
def writeFileMail(files, others, args, user, qaDir, c):
	sep = ""
	filestr = sep.join(list(sorted(files | others)))
	mail = """Hi Pushers,

Please push these %s files:

%s

from hgwdev to hgdownload:
%s

Please note the destination on hgdownload is *one directory above the location on dev*

Reason: releasing ENCODE %s on %s to the RR http://redmine.soe.ucsc.edu/issues/%s

Thanks!

%s
""" % (len(files|others), filestr, c._rrHttpDir, args.composite, args.database, args.redmine, user)

	f = open(qaDir + "/pushFilesMail", "w")
	f.write(mail)
	f.close()
	subIds = dict()
	mdb = c.alphaMetaDb
	p = re.compile('.*(wgE.*)')
	for i in files:
		m = p.match(i)
		file = m.group(1)
		filestanza = mdb.filter(lambda s: re.match(".*%s.*" % file, s['fileName']), lambda s: s)
		if filestanza:
			for j in filestanza:
				if 'subId' in j:
					if not j['subId'] in subIds:
						subIds[j['subId']] = 1

	f = open(qaDir + "/subIds", "w")
	for i in subIds.keys():
		f.write(i + "\n")
	f.close()
            
	return filestr, set(sorted(subIds.keys()))

def writeHtml(args, c, qaDir):
	f = open(c._trackDbPath, "r")
	lines = f.readlines()
	f.close
	short = ""
	long = ""
	for i in lines:
		m = re.match('((long|short)Label)\s+(.*)', i)
		
		if m:
			if m.group(1) == 'longLabel':
				long = m.group(3)
			if m.group(1) == 'shortLabel':
				short = m.group(3)
				
	html = """<TR>
        <TD><A HREF="http://hgdownload.cse.ucsc.edu/goldenPath/%s/encodeDCC/%s/"
        TARGET=_BLANK>%s</A></TD>
        <TD>%s</TD>
</TR>""" % (args.database, args.composite, short, long)

	f = open(qaDir + "/htmlDownloadSnippet", "w")
	f.write(html)
	f.close()
	return short, long

def writeScriptOut(qaDir, output):
	f = open(qaDir + "/script.output", "w")
	sep = "\n"
	outstr = sep.join(output)
	f.write(outstr)
	f.close()
	return

def lastInitDate(sqlfile):
	if not os.path.exists(sqlfile):
		return 0
	else:
		f = open(sqlfile, "r")
		lines = f.readlines()
		f.close()
		grab = 0
		for i in lines:
			if grab == 1:
				return i.rstrip()
			if re.match('initdate={', i):
				grab = 1

def writeBetaMdb(mdb, qaDir):

	f = open(qaDir + "/beta.mdb.ra", "w")
	for i in sorted(mdb):
		for j in mdb[i]:
			f.write("%s %s\n" % (j, mdb[i][j]))
		f.write("\n")

	f.close()

def writeSql(tablestr, filestr, gbdbstr, d, short, long, args, notes, size, user, qaDir, initDate):
	newYN = "Y"
	if int(args.release) > 1:
		newYN = "N"

	sql = """tbls={
%s
}
files={
%s
}
sizeMB={
%s
}
currLoc={
hgwbeta
}
reviewer={
%s
}
initdate={
%s
}
lastdate={
%s
}
track={
%s
}
releaseLog={
%s
}
priority={
L
}
dbs={
%s
}
newYN={
%s
}
rank={
0
}
ndxYN={
Y
}
joinerYN={
X
}
makeDocYN={
X
}
openIssues={
http://redmine.soe.ucsc.edu/issues/%s
}
bounces={
0
}
notes={
%s
}
releaseLogUrl={
../../cgi-bin/hgTrackUi?db=%s&g=%s
}
pushState={
D
}""" % (tablestr, filestr + "\n" + gbdbstr, size, user, str(initDate), str(d), short, long, args.database, newYN, args.redmine, notes, args.database, args.composite) 
	f = open(qaDir + "/release.sql", "w")
	f.write(sql)
	f.close()
	return
    
def writePushHtmlMail(args, user, qaDir):

	downloads = 'downloads.html'
	if re.match('mm\S+', args.database):
		downloads = 'downloadsMouse.html'
	
	mail = """Hi Pushers,

Please push the following file:

/usr/local/apache/htdocs/ENCODE/%s

from hgwbeta --> RR.
Reason: added the newly released ENCODE %s %s

Thanks!

%s
""" % (downloads, args.composite, args.database, user)
	f = open(qaDir + "/pushHtmlMail", "w")
	f.write(mail)
	f.close()
	return



def changeStatus(subIds):
	for i in subIds:
		cmd = "encodeStatus.pl %s reviewing 2>&1" % (i)
		p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
		output = p.stdout.read()
	return

def checkWithMkNotes(ourset, mkNotesSet, status):
	compset = set()
	errors = []
	for i in ourset:
		compset.add(i.rstrip())
	if mkNotesSet != compset:
		errors.append("""
FATAL ERROR: The list of %s in the notes file is out of date.
You may need to do a 'git pull'.
""" % status)
	return errors


def main():
	parser = argparse.ArgumentParser(
        prog='encodeQaInit',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description='Initializes QA directory for claiming a release',
        epilog=
"""Example:
qaInit hg19 wgEncodeSydhTfbs 1 69
qaInit hg18 wgEncodeHudsonalphaChipSeq 3 504
"""
        )
	parser.add_argument('database', help='The database, typically hg19 or mm9')
	parser.add_argument('composite', help='The composite name, wgEncodeCshlLongRnaSeq for instance')
	parser.add_argument('release', help='The new release to be released')
	parser.add_argument('redmine', help='The Redmine issue number')
	parser.add_argument('-t', '--test', action="store_true", default=0, help="Test mode:doesn't change status to reviewing, outputs to test qa Directory")
	parser.add_argument('-m', '--mdb', default=0, help='use a different mdb composite name')



	if len(sys.argv) == 1:
		parser.print_help()
		return

	args = parser.parse_args(sys.argv[1:])

	releaseOld = 'solo'
	if not args.release.isdigit():
		parser.print_help()
	if int(args.release) > 1:
		releaseOld = int(args.release) - 1
	specialMdb = None
	if args.mdb:
		specialMdb = args.mdb
		
	mkNotesArgs = { 'database': args.database,
					'composite': args.composite,
					'releaseNew': args.release,
					'releaseOld': str(releaseOld),
					'ignore' : 0,
					'loose': 1,
					'summary': 0,
					'specialMdb': specialMdb}
	errors = []
	c = track.CompositeTrack(args.database, args.composite, None, specialMdb)
	qaDir = ""
	if args.test:
		qaDir = c.qaInitDirTest + "release%s" % args.release 
	else:
		qaDir = c.qaInitDir + "release%s" % args.release 
	d = datetime.date.today()
	user = os.environ['USER']
	
	if not os.path.exists(qaDir):
		os.makedirs(qaDir)

	print "Making QA directory: %s" % qaDir
	notesFile = makeLinks(c, args, qaDir)

	
	print "Getting methods"
	getMethods(qaDir, args, user, d)
	
	print "Checking notes file"
	if re.search('FATAL ERROR', notesFile):
		print notesFile
		return
	m = re.match('.*(kent/.*)', notesFile)
	notes = m.group(1)
	f = open(notesFile, "r")
	lines = f.readlines()
	if not re.match('mkChangeNotes v2', lines[0]):
		print "notes files is not the correct version"
		return
	
	writeClaimMail(args, user, qaDir)
	print "Making local mkChangeNotes"
	genNotes = mkChangeNotes.makeNotes(mkNotesArgs)
	if genNotes.errors:
		print "NON-FATAL ERROR: Local mkChangeNotes found errors in the release"
	#	return

	betaMdb = genNotes.newMdb
	betaMdb[args.composite] = c.alphaMetaDb[args.composite]
	writeBetaMdb(betaMdb, qaDir)
	

	print "Parsing notes file"
	(tables, gbdbs, files, supplemental, others, size) = parseNotes(lines)
	print "Comparing notes file to local notes"
	errors.extend(checkWithMkNotes(tables, genNotes.newTables, "new tables"))
	errors.extend(checkWithMkNotes(gbdbs, genNotes.newGbdbs, "new gbdbs"))
	errors.extend(checkWithMkNotes(files, genNotes.newFiles, "new download files"))
	errors.extend(checkWithMkNotes(supplemental, genNotes.newSupplemental, "new supplemental files"))
	errors.extend(checkWithMkNotes(others, genNotes.newOthers, "new other files"))
	
	if errors:
		for i in errors:
			print i
		return
	
	print "Writing mails and friends"
	tablestr = writeTableMail(tables, args, user, qaDir)
	gbdbstr = writeGbdbMail(gbdbs, args, user, qaDir)
	(filestr, subIds) = writeFileMail(files, supplemental | others, args, user, qaDir, c)
	(short, long) = writeHtml(args, c, qaDir)
	
	writePushHtmlMail(args, user, qaDir)

	initDate = lastInitDate(qaDir + "/release.sql")
	if not initDate:
		initdate = d
	writeSql(tablestr, filestr, gbdbstr, d, short, long, args, notes, size, user, qaDir, initDate)

	if not args.test:
		print "Changing subIds' statuses to reviewing"
		changeStatus(subIds)
	
	output = []
	
	tables = genNotes.newTables
	
	print "Running qa tests"
	output.append("Checking for missing table descriptions:")
        (tableDescOutput, noDescription) = qa.checkTableDescriptions(args.database, tables)
	output.extend(tableDescOutput)

        output.append("Checking for missing table indexes:")
	(tableIndexOut, missingIndex) = qa.checkTableIndex(args.database, tables)
	output.extend(tableIndexOut)
	
        output.append("Checking table names for errors:")
	(tableNameOut, badTableNames) = qa.checkTableName(tables)
	output.extend(tableNameOut)
	
	trackDbPath = c.currentTrackDb
	if trackDbPath:
                output.append("Checking short and long labels in trackDb:")
		(labelOut, badLabels) = qa.checkLabels(trackDbPath)
		output.extend(labelOut)
	else:
		print "Could not determine track Db to use, does the composite have the alpha tag in trackDb.wgEncode.ra?"
	
        output.append("Checking tables for coordinate errors:")
	(coordsOut, badCoords) = qa.checkTableCoords(args.database, tables)
	output.extend(coordsOut)
	
        output.append("Checking tables for positional errors:")
	(posOut, badPos) = qa.positionalTblCheck(args.database, tables)
	output.extend(posOut)

        output.append("Counts per chromosome:")
        output.append("")
	(countChromOut, tableCounts) = qa.countPerChrom(args.database, tables)
	output.extend(countChromOut)

	writeScriptOut(qaDir, output)

	print "Done!"
	

if __name__ == "__main__":
	main()
