#!/usr/bin/env python # -*- coding: utf-8 -*- """ This module can do slight modifications to a wiki page source code such that the code looks cleaner. The changes are not supposed to change the look of the rendered wiki page. The following parameters are supported: ¶ms; All other parameters will be regarded as part of the title of a single page, and the bot will only work on that single page. &warning; For regular use, it is recommended to put this line into your user-config.py: cosmetic_changes = True There is another config variable: You can set cosmetic_changes_mylang_only = False if you're running a bot on multiple sites and want to do cosmetic changes on all of them, but be careful if you do. """ __version__ = '$Id$' import wikipedia, pagegenerators, isbn import sys import re warning = """ATTENTION: You can run this script as a stand-alone for testing purposes. However, the changes are that are made are only minor, and other users might get angry if you fill the version histories and watchlists with such irrelevant changes.""" docuReplacements = { '¶ms;': pagegenerators.parameterHelp, '&warning;': warning, } # Summary message when using this module as a stand-alone script msg_standalone = { 'ar': u'روبوت: تغييرات تجميلية', 'bg': u'Робот козметични промени', 'ca': u'Robot: Canvis cosmètics', 'da': u'Bot: Kosmetiske ændringer', 'de': u'Bot: Kosmetische Änderungen', 'el': u'Ρομπότ: διακοσμητικές αλλαγές', 'en': u'Robot: Cosmetic changes', 'es': u'Robot: Cambios triviales', 'et': u'robot: kosmeetilised muudatused', 'fa': u'ربات: ویرایش جزئی', 'fi': u'Botti kosmeettisia muutoksia', 'fr': u'Robot : Changement de type cosmétique', 'gl': u'bot Cambios estética', 'he': u'בוט: שינויים קוסמטיים', 'hi': u'Bot: अंगराग परिवर्तन', 'hr': u'robot kozmetičke promjene', 'hu': u'Bot: kozmetikai változtatások', 'id': u'bot kosmetik perubahan', 'it': u'Bot: Modifiche estetiche', 'ja': u'ロボットによる: 細部の編集', 'ko': u'로봇: 예쁘게 바꿈', 'lt': u'robotas: smulkūs taisymai', 'lv': u'robots kosmētiskās izmaiņas', 'mt': u'Bot: kosmetiċi bidliet', 'nl': u'Bot: cosmetische wijzigingen', 'no': u'Bot: Kosmetiske endringer', 'pl': u'Robot dokonuje poprawek kosmetycznych', 'pt': u'Bot: Mudanças triviais', 'ro': u'robot modificări cosmetice', 'ru': u'робот косметические изменения', 'sk': u'robot kozmetické zmeny', 'sl': u'robot kozmetične spremembe', 'sr': u'Бот козметичке промене', 'sv': u'Bot: Kosmetiska ändringar', 'th': u'โรบอต ประทิ่นเปลี่ยนแปลง', 'tl': u'robot Kosmetiko pagbabago', 'tr': u'Bot Kozmetik değişiklikler', 'uk': u'робот косметичні зміни', 'vi': u'robot: Sửa cách trình bày', 'war': u'Robot: Kosmetiko nga mga pagbag-o', 'zh': u'機器人: 細部更改', } # Summary message that will be appended to the normal message when # cosmetic changes are made on the fly msg_append = { 'ar': u'; تغييرات تجميلية', 'bg': u'; козметични промени', 'ca': u'; canvis cosmètics', 'da': u'; kosmetiske ændringer', 'de': u'; kosmetische Änderungen', 'el': u'; διακοσμητικές αλλαγές', 'en': u'; cosmetic changes', 'es': u'; cambios triviales', 'et': u'; kosmeetilised muudatused', 'fa': u'; ویرایش جزئی', 'fi': u'; kosmeettisia muutoksia', 'fr': u'; changement de type cosmétique', 'gl': u'; cambios estética', 'he': u'; שינויים קוסמטיים', 'hi': u'; अंगराग परिवर्तन', 'hr': u'; kozmetičke promjene', 'hu': u'; kozmetikai változtatások', 'id': u'; kosmetik perubahan', 'it': u'; modifiche estetiche', 'ja': u'; 細部の編集', 'ko': u'; 예쁘게 바꿈', 'lt': u'; smulkūs taisymai', 'lv': u'; kosmētiskās izmaiņas', 'mt': u'; kosmetiċi bidliet', 'nl': u'; cosmetische veranderingen', 'no': u'; kosmetiske endringer', 'pl': u'; zmiany kosmetyczne', 'pt': u'; mudanças triviais', 'ro': u'; modificări cosmetice', 'ru': u'; косметические изменения', 'sk': u'; kozmetické zmeny', 'sl': u'; kozmetične spremembe', 'sr': u'; козметичке промене', 'sv': u'; kosmetiska ändringar', 'th': u'; ประทิ่นเปลี่ยนแปลง', 'tl': u'; Kosmetiko pagbabago', 'tr': u'; Kozmetik değişiklikler', 'uk': u'; косметичні зміни', 'vi': u'; sửa cách trình bày', 'war': u'; kosmetiko nga mga pagbag-o', 'zh': u'; 細部更改', } deprecatedTemplates = { 'wikipedia': { 'de': [ u'Stub', ] } } class CosmeticChangesToolkit: def __init__(self, site, debug = False): self.site = site self.debug = debug def change(self, text): """ Given a wiki source code text, returns the cleaned up version. """ oldText = text text = self.fixSelfInterwiki(text) text = self.standardizeInterwiki(text) text = self.standardizeCategories(text) text = self.cleanUpLinks(text) text = self.cleanUpSectionHeaders(text) # Disabled because of a bug, and because its usefulness is disputed # text = self.putSpacesInLists(text) # text = self.translateAndCapitalizeNamespaces(text) text = self.removeDeprecatedTemplates(text) text = self.resolveHtmlEntities(text) text = self.validXhtml(text) text = self.removeUselessSpaces(text) text = self.removeNonBreakingSpaceBeforePercent(text) try: text = isbn.hyphenateIsbnNumbers(text) except isbn.InvalidIsbnException, error: pass if self.debug: wikipedia.showDiff(oldText, text) return text def fixSelfInterwiki(self, text): """ Interwiki links to the site itself are displayed like local links. Remove their language code prefix. """ interwikiR = re.compile(r'\[\[%s\s?:([^\[\]\n]*)\]\]' % self.site.lang) text = interwikiR.sub(r'[[\1]]', text) return text def standardizeInterwiki(self, text): """ Makes sure that interwiki links are put to the correct position and into the right order. """ interwikiLinks = wikipedia.getLanguageLinks(text, insite = self.site) text = wikipedia.replaceLanguageLinks(text, interwikiLinks, site = self.site) return text def standardizeCategories(self, text): """ Makes sure that categories are put to the correct position, but does not sort them. """ # The PyWikipediaBot is no longer allowed to touch categories on the German Wikipedia. See http://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/bis_2006#Position_der_Personendaten_am_.22Artikelende.22 if self.site != wikipedia.getSite('de', 'wikipedia'): categories = wikipedia.getCategoryLinks(text, site = self.site) text = wikipedia.replaceCategoryLinks(text, categories, site = self.site) return text def translateAndCapitalizeNamespaces(self, text): """ Makes sure that localized namespace names are used. """ family = self.site.family # wiki links aren't parsed here. exceptions = ['nowiki', 'comment', 'math', 'pre'] for nsNumber in family.namespaces: if not family.isDefinedNSLanguage(nsNumber, self.site.lang): # Skip undefined namespaces continue namespaces = list(family.namespace(self.site.lang, nsNumber, all = True)) thisNs = namespaces.pop(0) # skip main (article) namespace if thisNs and namespaces: text = wikipedia.replaceExcept(text, r'\[\[\s*(' + '|'.join(namespaces) + ') *:(?P.*?)\]\]', r'[[' + thisNs + ':\g]]', exceptions) return text def cleanUpLinks(self, text): # helper function which works on one link and either returns it # unmodified, or returns a replacement. def handleOneLink(match): titleWithSection = match.group('titleWithSection') label = match.group('label') trailingChars = match.group('linktrail') if not self.site.isInterwikiLink(titleWithSection): # The link looks like this: # [[page_title|link_text]]trailing_chars # We only work on namespace 0 because pipes and linktrails work # differently for images and categories. try: page = wikipedia.Page(self.site, titleWithSection) except wikipedia.InvalidTitle: return match.group() if page.namespace() == 0: # Replace underlines by spaces, also multiple underlines titleWithSection = re.sub('_+', ' ', titleWithSection) # Remove double spaces titleWithSection = re.sub(' +', ' ', titleWithSection) # Remove unnecessary leading spaces from title, # but remember if we did this because we eventually want # to re-add it outside of the link later. titleLength = len(titleWithSection) titleWithSection = titleWithSection.lstrip() hadLeadingSpaces = (len(titleWithSection) != titleLength) hadTrailingSpaces = False # Remove unnecessary trailing spaces from title, # but remember if we did this because it may affect # the linktrail and because we eventually want to # re-add it outside of the link later. if not trailingChars: titleLength = len(titleWithSection) titleWithSection = titleWithSection.rstrip() hadTrailingSpaces = (len(titleWithSection) != titleLength) # Convert URL-encoded characters to unicode titleWithSection = wikipedia.url2unicode(titleWithSection, site = self.site) if titleWithSection == '': # just skip empty links. return match.group() # Remove unnecessary initial and final spaces from label. # Please note that some editors prefer spaces around pipes. (See [[en:Wikipedia:Semi-bots]]). We remove them anyway. if label is not None: # Remove unnecessary leading spaces from label, # but remember if we did this because we want # to re-add it outside of the link later. labelLength = len(label) label = label.lstrip() hadLeadingSpaces = (len(label) != labelLength) # Remove unnecessary trailing spaces from label, # but remember if we did this because it affects # the linktrail. if not trailingChars: labelLength = len(label) label = label.rstrip() hadTrailingSpaces = (len(label) != labelLength) else: label = titleWithSection if trailingChars: label += trailingChars if titleWithSection == label or titleWithSection[0].lower() + titleWithSection[1:] == label: newLink = "[[%s]]" % label # Check if we can create a link with trailing characters instead of a pipelink elif len(titleWithSection) <= len(label) and label[:len(titleWithSection)] == titleWithSection and re.sub(trailR, '', label[len(titleWithSection):]) == '': newLink = "[[%s]]%s" % (label[:len(titleWithSection)], label[len(titleWithSection):]) else: # Try to capitalize the first letter of the title. # Maybe this feature is not useful for languages that # don't capitalize nouns... #if not self.site.nocapitalize: if self.site.sitename() == 'wikipedia:de': titleWithSection = titleWithSection[0].upper() + titleWithSection[1:] newLink = "[[%s|%s]]" % (titleWithSection, label) # re-add spaces that were pulled out of the link. # Examples: # text[[ title ]]text -> text [[title]] text # text[[ title | name ]]text -> text [[title|name]] text # text[[ title |name]]text -> text[[title|name]]text # text[[title| name]]text -> text [[title|name]]text if hadLeadingSpaces: newLink = ' ' + newLink if hadTrailingSpaces: newLink = newLink + ' ' return newLink # don't change anything return match.group() trailR = re.compile(self.site.linktrail()) # The regular expression which finds links. Results consist of four groups: # group title is the target page title, that is, everything before | or ]. # group section is the page section. It'll include the # to make life easier for us. # group label is the alternative link title, that's everything between | and ]. # group linktrail is the link trail, that's letters after ]] which are part of the word. # note that the definition of 'letter' varies from language to language. linkR = re.compile(r'\[\[(?P[^\]\|]+)(\|(?P