Benutzer:Delilahblue

mein freier Python Dump Parser, realisiert mit einem Zustandsautomat:

#!/bin/env python

import sys
import re
import codecs

def unicode_sequence(seq):
  return u""

class respider:
  def __init__(self):
    self.states = {}
    self.current = None
    self.cache = 
    self.filters = {}
    self.filename = ""

  def add_filter(self, name, match):
    self.filters = re.compile(match)

  def add_state(self, state):
    self.states] = state

  def run(self, inputfile, outputfile):
    fi = codecs.open(inputfile, "r", encoding="utf-8")
    fo = codecs.open(outputfile, "w", encoding="utf-8")
    self.start()
    for line in fi:
      fo.writelines(unicode_sequence(outputline) + u"\n" for outputline in self.step(line))
    self.stop()
    fi.close()
    fo.close()

  def __iter__(self):
    fi = codecs.open(self.filename, "r", encoding="utf-8")
    self.start()
    for line in fi:
      result = self.step(line)
      if len(result) > 0:
        yield result
    self.stop()
    fi.close()

  def start(self):
    for state in self.states:
      if self.states.has_key("start"):
        self.current = self.states

  def stop(self):
    del self.current

  def step(self, input):
    result = 
    result_step = 
    visited = 
    current_delta = 0
    if self.current is not None:
      while current_delta < len(self.current) or self.current not in visited:
        if self.current not in visited:
          visited.append(self.current)
        if self.current.has_key("action"):
          if "flush" in self.current:
            self.cache = 
            result_step = 
          if "uncache" in self.current:
            result += self.cache
            result += result_step
        m = self.filters].search(input)
        if m:
          #result_step.append((self.current, self.current, ))
          result_step.append((self.current, self.current, m.groups()))
          if self.current.has_key("action") and "cut" in self.current:
            input = input
          if self.current is not None:
            self.current = self.states]
            current_delta = 0
          else:
            current_delta += 1
        else:
          if self.current is not None:
            self.current = self.states]
            current_delta = 0
          else:
            current_delta += 1
    self.cache += result_step
    return result

#Beim Aufruf muss der erste Parameter die Eingabedatei enthalten
if len(sys.argv) > 1:
  filename = sys.argv
else:
  filename = "dewiktionary-latest-pages-articles.xml"

my_spider = respider()
my_spider.filename = filename

#Hier folgen die Filterausdrücke, reguläre Ausdrücke (regex):
my_spider.add_filter("any", "")
my_spider.add_filter("all", "(.*)")
my_spider.add_filter("page", "<page>")
my_spider.add_filter("endpage", "</page>")
my_spider.add_filter("title", "<title>({2,}?)</title>")
my_spider.add_filter("text", r"<text.*?>== (*?) \(")
my_spider.add_filter("lang", r"\{\{Sprache\|Deutsch\}\}")
my_spider.add_filter("type", r"\{Wortart\|(.*?)\|Deutsch\}\}")
my_spider.add_filter("singular", r"SINGULAR=(.*)|Wer oder was.*?\(Einzahl(?: )?\)=(?:\s*)(?:der|die|das)?(?:\s*)(.*)")
my_spider.add_filter("plural", r"PLURAL=(.*)|Wer oder was.*?\(Mehrzahl(?: )?\)=(?:\s*)(?:die)?(?:\s*)(.*)")
my_spider.add_filter("phonetic", r":\\]:(?:(.*?)\{\{Lautschrift\|(.+?)\}\})*")

#Folgende Zustände werden vom endlichen Automaten benutzt
#Dabei wird vor allem ein Titel und ein Textabschnitt gesucht und dann die notwendigen Angaben extrahiert.
my_spider.add_state(dict(name="start", delta=, start=True, action=))
my_spider.add_state(dict(name="getdata", delta=))
my_spider.add_state(dict(name="getdata2", delta=))
my_spider.add_state(dict(name="end", delta=, action=))

#Welche Angaben werden ausgewertet:
parse = 

#Welche werden benötigt:

#a, für alle Wortarten
essential_all = #, 'phonetic']
essential = {}

#b, nur für spezielle Wortarten
essential = 

#wurden spezielle Erwartungen angegeben muss die Wortart hier aufgeführt werden:
known_types = 

unknown_types = {}

for es in essential:
  essential = essential_all

#Die Ausgabe erfolgt in die Dateien, die als zweite und dritte Parameter angegeben wurden
fo = codecs.open(sys.argv, "w", encoding="utf-8")
ferr = codecs.open(sys.argv, "w", encoding="utf-8")

complete = 0
count = 0
for result in my_spider:
  count += 1
  buffer = dict((item,  for data in result if data == item]) for item in parse)
  if len(buffer) > 0:
    is_known_type = buffer in known_types
    if u"{" in buffer or u"}" in buffer:
      ferr.write(u"TypeError: " + buffer + " - " + buffer + u"\n")
    elif not is_known_type:
      if buffer in unknown_types:
        unknown_types] += 1
      else:
        unknown_types] = 1
    if all(len(buffer) > 0 and any(any(grouppart is not None for grouppart in match) for match in buffer) for item in (essential] if is_known_type else essential_all)):
      fo.write(buffer + u": " + unicode_sequence(buffer.items()) + u"\n")
      complete += 1
    else:
      ferr.write(u"Missing: " + buffer + u" - " + unicode_sequence((item, data) for (item, data) in buffer.items() if all(all(grouppart is None for grouppart in match) for match in data)) + u"\n")

fo.close()
ferr.close()

print complete, "/", count, "complete"
print unknown_types