#!/usr/bin/python 
# -*- coding: utf-8  -*-

#
# (c) Dispenser, 2007
#

import re, os, cgi
import checklink, wikipedia
import cgitb; cgitb.enable(display=1, logdir="/home/philip/scripts/tmp")

textFiles = './text/'
count	= 0
WikiUrl = ''
blanktext = """<!--
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
                                                                               
-->"""

def printu(ustr):
	#print ustr.encode('raw_unicode_escape')
	try:
		print ustr
	except:
		print ustr.encode('utf-8')

def printWikiFile(name):
	f = open('%s%s.txt' % (textFiles, name,) )
	print wikiToHTML(f.read())
	f.close()
def printFile(name, arg1="$1"):
	f = open('%s%s.txt' % (textFiles, name,) )
	print f.read().replace('$1', arg1.encode('utf-8'))
	f.close()

def wikiToHTML(text):
	# Strip HTML comments
	text = re.sub(r"<!--.*?-->", r'', text)
	
	# Headers
	text = re.sub(r'\n=====([^\n]*)=====\n',r'\n<h5>\1</h5>\n', text)
	text = re.sub(r'\n====([^\n]*?)====\n',  r'\n<h4>\1</h4>\n', text)
	text = re.sub(r'\n===([^\n]*?)===\n',    r'\n<h3>\1</h3>\n', text)
	text = re.sub(r'\n==([^\n]*?)==\n',      r'\n<h2>\1</h2>\n', text)
	#text = re.sub(r'\n=([^\n]*?)=\n',        r'\n<h1>\1</h1>\n', text)
	
	# definition lists
	text = re.sub(r"\n;([^:\n]*)\n?", r'\n<dt>\1</dt>\n', text)
	text = re.sub(r"\n:([^\n]*)\n?", r'\n<dd>\1</dd>', text)
	
	# bullet and numbered lists
	text = re.sub(r"\n ?\* *([^\n]*)", r'<ul>\n\t<li>\1</li>\n</ul>', text)
	text = re.sub(r"\n ?\# *([^\n]*)", r'<ol>\n\t<li>\1</li>\n</ol>', text)
	
	# Merge list together
	text = text.replace("</ul><ul>\n", r'')
	text = text.replace("</ol><ol>\n", r'')
	
	# Horizonal line
	text = text.replace("\n----", '<hr />')
	
	# Bold and Illatics
	text = re.sub(r"'''([^\n]*?)'''", r'<b>\1</b>', text)
	text = re.sub(r"''([^\n]*?)''", r'<i>\1</i>', text)	
	
	# space infront to <pre> (simple ver.)
#	text = re.sub(r'\n ([^\n]*)',	r'\n<pre>\1</pre>', text)
	
	# Internal links
	text = re.compile(r'\[\[([^][]*?)\|([^][]*)\]\](\w*)', re.UNICODE).sub(r'<a href="http://%s/wiki/\1" class="extiw">\2\3</a>' % hostname, text)
	text = re.compile(r'\[\[([^][|]*?)\]\](\w*)', re.UNICODE).sub(r'<a href="http://%s/wiki/\1" class="extiw">\1\2</a>' % hostname, text)
	
	'''
	# Simple Wikitable support
	text = re.sub(r'\n\{\|([^\n]*)', r'\n<table \1><tr>', text)
	text = re.sub(r'\n\|-([^\n]*)',	 r'\n</tr></tr \1>', text)
	text = re.sub(r'\n\|\}',		 r'\n</tr></table>', text)
	text = re.sub(r'\n!([^][\n|]*\|)?([^\n]*)',  r'\n<th \1>\2</th>', text)
	text = re.sub(r'\n\|([^][\n|]*\|)?([^\n]*)', r'\n<td \1>\2</td>', text) #'''
	
	# External links
	#text = re.sub(r'\[(\w+://[^][><" \n]+)\s*([^][]*?]*)\]', r'<a href="\1" class="external">\2</a>', text)
	text = re.sub(r'\[(\w+://[^][><" \n]+)\s*([^]]*?]*)\]', r'<a href="\1" class="external">\2</a>', text)
	global count
	while(text.find(r'"></a>') != -1):
		count += 1
		text = re.sub(r'"></a>', r'">[%s]</a>' % count, text, 1)
		
	text = re.sub(r'(\A|[]<>\n])(\w+://[^][><\s"]*)', r'\1<a href="\2" class="external breakme"> \2 </a>', text)
	
	# Encode & -> &amp;
	text = text.replace('&', '&amp;')
	text = re.sub(r'\&amp;(#\d+|#x\w+|\w{1,6});', r'&amp;\1;', text)
	
	# New paragraphs
	text = re.sub(r"\n\n+([^\n<>][^\n]*)", r'\n\n<p>\1</p>', text)
	
	return text

def HTMLheader(title=''):
	print '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">'
	print '<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr">'
	printFile ('head', title)
	print """
<body  class="mediawiki ns-0 ltr">
	<div id="globalWrapper">
		<div id="column-content">
	<div id="content">"""
	printWikiFile('siteNotice')
	print """
		<a name="top" id="top"></a>
		<h1 class="firstHeading">%s</h1>
		<div id="bodyContent">
			<h3 id="siteSub"></h3>
			<div id="contentSub"></div>
			<!-- start content -->""" % title.encode('utf-8')
	
def endContent():
	print """
<!-- end content -->
			<div class="visualClear"></div>
		</div>
	</div>
		</div>"""

#
#
# Readability code
#
def printStat(name, str1 = '', str2 = '', str3 = ''):
	printu('<tr><td>%s</td>' % wikiToHTML(name))
	printu('<td>%s</td>' % str1)
	printu('<td>%s</td>' % str2)
	printu('<td>%s</td>' % str3)
	printu('</tr>')
def average(list):
	return float(sum(list))/float(len(list))
def doStats(page):
	try:
		wikiText = page.get()
		wikiText = wikiText.replace('\r\n', '\n')
	except wikipedia.NoPage:
		print '<div style="margin-left:2em;">Page is empty/non-existant</div>'
		return
	except wikipedia.IsRedirectPage:
		print '<div style="margin-left:2em;">Did you mean <a href="?page=%s">%s</a>?</div>' % (page._redirarg, page._redirarg)
		return
	if not wikiText.strip():
		print 'Page does not exist'
		return

	#text = wikipedia.removeDisabledParts(wikiText)
	text = re.compile(r'<nowiki>.*?</nowiki>|<!--.*?-->|<pre>.*?</pre>|<includeonly>.*?</includeonly>|<math>.*?</math>', re.IGNORECASE | re.DOTALL).sub('', wikiText)
	print '<!-- Removed disabled parts -->'
	# Remove links
	while re.search(r'\[\[[^][]+\]\]', text):
		text = re.sub(r'\[\[(Image|Category|[A-Za-z]{2,3}(-[A-Za-z]+)*|simple):[^][]*\]\]', '', text, re.IGNORECASE | re.DOTALL | re.UNICODE)
		text = re.sub(r'\[\[([^]|[]+\|)?([^][]+)\]\]', r'\2', text, re.IGNORECASE|re.DOTALL)
		text = re.sub(r'\[\[Image:[^][]*\]\]', '', text, re.IGNORECASE|re.DOTALL)
	print '<!-- Wikilink removal complete -->'

    # Remove templates
	#while '{{' in text and '}}' in text:
	while re.findall(r'\{\{[^{}]*?\}\}', text, re.DOTALL):
		text = re.compile(r'\{\{[^{}]*?\}\} *', re.DOTALL).sub(r'', text)
	print '<!-- template removal complete -->'

	# Remove italics and bolding
	text = re.sub(r"''(.*?)''|'''(.*?)'''", r'\1', text)
	
	# Remove External links
	text = re.sub(r'\[\w+:/*[^][<>\s"]* *(.*?)\]', r'\1', text)

	# Get "readable prose"
	prose = re.compile('|'.join((
                r'\{\|.*?\|\}',
                r'<ref[^<>/]*>.*?</ref>',
                r'\{\{.*?\}\}',
                r'^=+[^\n]+=+ *$',
                r'^ *\*[^\n]*$',
                r'^ *\#[^\n]*$',
		r'^;+[^\n]*$',
		#r'^:+[^\n]{1,200}$', # can do full definition list
		r"^:+ *''\w[^\n]*$", # remove dab like lines
                r'&[^;]{2,6};')), re.IGNORECASE | re.DOTALL | re.MULTILINE | re.UNICODE).sub(r'', text)
	# Remove HTML
	text = re.sub(r'</?\w+[^<>]*>', r'', text)
	prose= re.sub(r'</?\w+[^<>]*>', r'', prose)

	#print '<!--'
	#wikipedia.output(text)
	#print '-->'
	WordCounter 	= re.compile(r'\b[^\W\d]+\b', re.UNICODE)
	sentenceFinder	= re.compile(r'\w*\w\w[ \'"\])]*[\.\!\?]+[\s\'"\]()]', re.UNICODE)
	#sentenceFinder	= re.compile(r'\.\s') # Tests other test

#	lettersProse	= len( re.findall(r'[^\W\d]', prose) )
	wordsWiki	= len(	WordCounter.findall(wikiText)) or 1
	wordsText	= len(	WordCounter.findall(text    )) or 1
	wordsProse	= len(	WordCounter.findall(prose   )) or 1
	sentencesWiki	= len(	sentenceFinder.findall(wikiText)) or 1
	sentencesText	= len(	sentenceFinder.findall(text    )) or 1
	sentencesProse	= len(	sentenceFinder.findall(prose   )) or 1

	report		= prose
	# hightlight end of sentences
	for s in sentenceFinder.findall(prose):
		report = report.replace(s, '<span style="background:#fee;">%s</span>%s' % (s[:-1],s[-1]))
	print '<!-- senetence counter complete -->'
	
	# The actual staticstics stuff, should be branched off into an object/class
	IgnoreWords	= re.split(r'\W', page.title().lower())
	IgnoreWords.append(u'wikipedia')
	PropNouns	= re.findall(r'(?<!\w.[\.\n])\s+\b((?:Mc|)[A-Z][a-z\-]+)\b', prose)
	HardWords	= 0
	lettersProse	= 0
	syllable_list	= []
	
	printu("<!-- Proper nouns:\n %r\n -->" % PropNouns)
	print '<!-- polysyllabic identified:'
	for s in WordCounter.findall(prose):
		word = s
	#	if s.lower() in IgnoreWords:
	#		continue
		if len(s) > 1 and (s.endswith('e')):
			s = s[:-1]
		elif len(s) > 2 and (s.endswith('es') or s.endswith('ed')):
			s = s[:-2]
		elif len(s) > 3 and (s.endswith('ing') or s.endswith('ies')):
			s = s[:-3]
		elif len(s) > 3 and (s.endswith('ely')):
			s = s[:-3]+s[-2:]
		vowels = re.findall(r'(?i)[aeiouy]+', s)
		if vowels:
			lettersProse += len(word)	# Make sure we're adding words
			syllable_list.append(len(vowels))
		if len(vowels) >= 3 and not s.lower() in IgnoreWords and not word in PropNouns:
			HardWords += 1
			report = re.sub(r'(%s)\b([^<>])' % word, r'<u>\1</u>\2', report)
	#print syllable_list

	print ( 0.39 * float(wordsProse) / float(sentencesProse) ) 
	print ( 11.8 * average(syllable_list) )
	print 'syllable list length: %d' % len(syllable_list)
	print 'sum of syllable:%d' % sum(syllable_list)
	print '-->'
	
	# Calculate stats
	Flesch_Ease	= 206.835 - (1.015 * float(wordsProse) / float(sentencesProse)) - (84.6 * average(syllable_list))
	AR_Index	= ( 4.71 * float(lettersProse) / float(wordsProse)) + ( 0.5 * float(wordsProse) / float(sentencesProse) ) - 21.43
	Kincaid_level	= ( 0.39 * float(wordsProse) / float(sentencesProse) ) + ( 11.8 * average(syllable_list) ) - 15.59
	Cloeman_Liau	= ( 5.89 * float(lettersProse) / float(wordsProse) ) - ( 0.3 * float(sentencesProse) / float(wordsProse) ) - 15.8
	Gunning_index	= 0.4 * ( (float(wordsProse) / float(sentencesProse)) + 100 * (float(HardWords) / float(wordsProse)) )
	import math
	SMOG_Index	= 1.0430 * math.sqrt(30.0 * float(HardWords) / float(sentencesProse) ) + 3.1291
        # TODO: add per paragraph readability scores

        print '<table border="1" cellpadding="4" cellspacing="0" style="border:solid 1px gray; background:#f7f7f7; margin:auto; border-collapse:collapse;">'
       	print '<caption><big><a href="http://%s/wiki/%s" class="extiw">%s</a></big></caption>' % (page.hostname(), page.urlname(), page.title().encode('utf-8'))
	print '<tr><td></td><th>Wiki text</th><th>Text</th><th>Prose</th></tr>'

	printStat('Links',	len(re.findall(r'\[\[[^][:]+\]\]',	wikiText)))
	printStat('Images',	len(re.findall(r'\[\[ *[Ii]mage:',	wikiText)))
	printStat('Categories',	len(re.findall(r'\[\[ *[Cc]ategory:',	wikiText)))
	printStat('Interwikis',	len(re.findall(r'\[\[ *[a-z\-]{2,4}:',	wikiText)))
	printStat('Weblinks',	len(re.findall(r'\w+://\w+\.\w+',	wikiText)))
	print '<tr><td colspan="4"></td></tr>'
	printStat('Size (KiB)', '%#.3g' % (len(wikiText)/1024.), '%#.3g' % (len(text)/1024.), '%#.3g' % (len(prose)/1024.))
	printStat('Words', wordsWiki, wordsText, wordsProse)
	printStat('Sentences', sentencesWiki, sentencesText, sentencesProse)
	printStat('Letters', 			str3 = lettersProse)
	printStat('Polysyllabic words',			str3 =  HardWords)
	printStat('Characters per word',		str3 = '%.2f' % (float(lettersProse)/float(wordsProse)))
	printStat('Syllables per word', 		str3 = '%.2f' % average(syllable_list))
	printStat('Words per sentence', 		str3 = '%.2f' % (float(wordsProse)/float(sentencesProse)))
	print '<tr><td colspan="4"></td></tr>'
	printStat('[[Flesch Reading Ease]]',		str3 = '%.1f' % Flesch_Ease)
	printStat('[[Automated Readability Index]]',	str3 = '%.2f' % AR_Index)
	printStat('[[Flesch-Kincaid Grade Level]]',	str3 = '%.2f' % Kincaid_level)
# way out of whack
	printStat('[[Coleman-Liau Index]]',		str3 = '%.2f' % Cloeman_Liau)
	printStat('[[Gunning fog index]]',		str3 = '%.2f' % Gunning_index)
	printStat('[[SMOG Index]] (&sigma; = +/-1.5)',	str3 = '%.2f' % SMOG_Index)
	
	# Suggestion
	print '<tr><td colspan="4"><b>Notes:</b>'
	if wordsProse < 300:
		print "This article seems to be a stub."
	elif wordsProse > 10000:
		print "This article seems to be too long and probably needs to be split."
	elif SMOG_Index <= 8:
		print "Longer words and more compound or complex sentences may make this article more interesting, more precise and less wordy."
	elif SMOG_Index >= 16:
		print "This article seems to have too many long words and sentences for even most university graduates to easily read and understand."
	elif len(text) > 75e3:
		print "Almost certainly should be divided up"
	elif len(text) > 50e3:
		print "Probably should be divided (although the scope of a topic can sometimes justify the added reading time)"
	else:
		print '-'

	print '</td></tr></table>'
	print '<div>The follow is what the software to recognized as "<a href="http://en.wikipedia.org/wiki/Wikipedia:Article_length#What_is_and_is_not_included_as_.22readable_prose.22">Readable Prose</a>", underlined words are polysyllabic words (syllables &#8805; 3) and highlighed words indicate puncuation.  No direct action should result from the information provided here.</div>'
	print '<hr />'
	printu(wikiToHTML(report))
#	printu('<pre>%r</pre>' % report)

Languages = {
	'en':	"English",
	'simple':	"Simple English",
}
Families = {
	'wikibooks':	"Wikibooks",
	'wikipedia':	"Wikipedia",
	'wikisource':	"Wikisource",
	'wikinews':	"Wikinews",
	'wiktionary':	"Wiktionary",
	'wikiquote':	"Wikiquote",
	'wikiversity':	"Wikiversity",
}
def printOptions(dict, default):
	for key in dict:
		if key == default:
			print '<option value="%s" selected="selected">%s</option>' % (key, dict[key])
		else:
			print '<option value="%s">%s</option>' % (key, dict[key])

#
#  Here are the function that are stubed to reduce complexity
#
def main():
	global threshold, hostname

	form = cgi.FieldStorage()
	host     = form.getfirst('hostname', 'en.wikipedia.org').split('.')
	lang     = form.getfirst('lang',     host[0])
	family   = form.getfirst('family',   host[1])
	site     = wikipedia.Site(lang, family)
	page     = wikipedia.Page(site, form.getfirst('page', ''))
	hostname     = page.hostname()
	
	#wikipedia.startContent()
	HTMLheader(u'Readability for %s' % page.title())
	print '<form action="readability.py" style="text-align:center;"><input name="page" id="page" size="40" value="%s" />' % page.title().encode('utf-8')
	print '<select name="lang">'
	printOptions(Languages, lang)
	print '</select>'
	print '<select name="family">'
	printOptions(Families, family)
	print '</select>'
	print '<input type="submit" value="Check page" /></form>'
	print '<small>New version in the works <a href="readability-new.py?page=%s">check it out</a></small>' % page.urlname()
	if page.title():
		doStats(page)
		
if __name__ == "__main__" and wikipedia.handleUrlAndHeader():
	try:
		main()
	finally:
		wikipedia.endContent()
		wikipedia.stopme()