Revision 7741 (by gradha, 2007/02/14 19:11:15) Added minimal support for custom defines through allegro.ini configuration
file and basic #else support. Required for the talula mirror, there is
not much space left and the online documentation generation fails. So
I better get rid of it and point it at the sourceforge version.
#!/usr/bin/env python
# vim: tabstop=3 shiftwidth=3 expandtab
"""webmake.py, or another silly html generator.

This program was created to fulfill the following tasks for the Allegro web
page:

- Avoid centralization of 'power'. Web page updates cannot depend on a
  single person, no matter how is he/she reliable.
- Provide a single place/outlook for all the Allegro web translations,
  which lived unsincronized and sometimes unmantained.
- Let web translators work without conflict: hence, the outlook and the
  contets of the web pages are separated in different directories.
  Write once the outlook, translate as many times as you wish.
- Keep the whole source in a single place (ie: Sourceforge cvs repository)
  to which all people have access and can use it to mirror the web page.
- All the web pages have to be plain html files, to ensure they can be
  mirrored anywhere without restriction (ie: host not supporting php or
  whatever you prefer). Hence the need of this script to generate them.

This script parses files in the src directory, which contain special
html-like tags transformed by the script. These can be simple commands
like include a translation file or print a timestamp in a specific
language.

There should be some kind of document lying around in the same directory
explaining correct usage of the src and data directories, as well as the
script itself, please take a look at it.

This script was written by Grzegorz Adam Hankiewicz and is Giftware: you
are free to do what you want with it without any restriction. I do not
accept responsibility for any effects, adverse or otherwise, that this
script may have on you, your computer, your sanity, your dog, and
anything else that you can think of. Use it at your own risk.

The file doesn't use tabs, each identation level is three space characters.

$Id: webmake.py 7741 2007-02-14 19:11:15Z gradha $
"""


import re, sys, os, stat, getopt, localize_time, glob, ConfigParser


include_path_data = "data/"
include_path_source = "src/"
expresion_include = None
simple_substitutions = []
defines = []
allowed_includes = []
langcode = "en"
short_arguments = "hvi:o:l:M,m:c:w:"
long_arguments = ["help", "version", "input=", "output=", "localize=",
   "depend", "mirror-timestamp=", "mirror-conf=", "warnings-to="]
input_filename = ""
output_filename = ""
filename_tag_substitution = ""
last_timestamp = 0
mirror_timestamp_filename = ""
mirror_data = []
kill_warnings = {}
file_dictionaries = {}
generate_dependency = 0
redirect_warnings_to = ""
version = "Webmake.py 0.2"

def show_program_usage():
   """Simple function which explains the basic switches you can use."""

   print "Usage: %s -i filename -o filename [-hv -l x -m x]\n" % os.path.split(sys.argv[0])[1]
   print "-h, --help                 Shows this help message"
   print "-v, --version              Displays version and exits"
   print "-i x, --input=x            Use xxx as input file"
   print "-o x, --output=x           Write output to file xxx, otherwise stdout"
   print "-l x, --localize=x         Use x language code (ie: es, fr, etc)"
   print "-M, --depend               Generate dependency info for make. Ignores -i"
   print "-m x, --mirror-timestamp=x Use x as mirror timestamp (only with -M)"
   print "-c x, --mirror-conf=x      Use mirror configuration from x"
   print "-w x, --warnings-to=x      Redirect warnings about missing translations to x"



def retrieve_definitions(line):
   """This function is used as a filter of line agains the external
   world. It's purpose is to detect definitions in the form of
   #!- <tag> text, where '#' has to be the first character of the line,
   '<tag>' can be any text without spaces, and 'text' is the text which
   will substitute '<tag>' whenever it's found. line with such definitions
   will be erased (with this function returning 0 to filter), but the
   definitions will be added to simple_substitutions.
   """
   if line[:4] == "#!- ":
      result = line[4:].rstrip().split(None, 1)
      assert len(result) == 2
      # delete previos definitions if found
      f = 0
      while f < len(simple_substitutions):
         if simple_substitutions[f][0] == result[0]:
            del simple_substitutions[f]
         else:
            f = f + 1
      simple_substitutions.append((result[0], result[1]))
      return 0
   else:
      return 1



def save_open_file_timestamp(file):
   """During the process of recursion through files, these are opened and
   read from. To help calculating the timestamp of the oldes used file,
   pass an open file this function and the global last_timestamp will be
   updated if needed. You can later use this variable to put a timestamp
   in written form somewhere in the output (preferably when all files
   have been parsed.
   """
   global last_timestamp
   last_timestamp = max(last_timestamp, os.fstat(file.fileno())[stat.ST_MTIME])



def clean_whitespace_lines(lines):
   """Since the new algorithm processes single lies, the old rstrip method
   of cleaning trailing whitespace doesn't work any more. This is something
   similar for a list containing single lines; the result is not exactly
   what it used to be, but the html output is the same. This function
   doesn't return anything, the parameter is transformed, whitspace lines
   at the beginning and end will be deleted.
   """
   while len(lines) and len(lines[0].rstrip()) < 1:
      del lines[0]
   while len(lines) and len(lines[-1].rstrip()) < 1:
      del lines[-1]
   if len(lines) < 2:
      try: lines[-1] = lines[-1].rstrip()
      except IndexError: pass



def report_missing_localization(missing_filename, section):
   """Looks up the filename in the kill_file list, and if it's not there,
   reports a warning message through stderr. Otherwise, keeps quiet.
   """
   try: val = kill_warnings["%s:%s" %(missing_filename, section)]
   except KeyError:
      if redirect_warnings_to:
         file = open(redirect_warnings_to, "at")
         file.write('%s:%s\n' % (missing_filename, section))
         file.close()
      else:
         sys.stderr.write("Didn't find '%s:%s'\n" % (missing_filename, section))



def generate_dictionary_sections(lines):
   """Given a list of lines, creates a sectioned dictionary with them.
   Lines which aren't in a section or comments will be deleted.
   """
   dic = {"none": []}
   section = "none"
   for f in lines:
      if f and f[0] == "#":
         if f[1] == "-":
            section = f[2:].strip()
         elif f[1:4] == "!- ":
            try: dic[section].append(f)
            except KeyError: dic[section] = [f]
      else:
         try: dic[section].append(f)
         except KeyError: dic[section] = [f]

   del dic["none"]

   for f in dic.keys():
      clean_whitespace_lines(dic[f])

   return dic



def load_file_dictionary(filename):
   """Loads the specified file, creates a dictionary with it's sections
   and adds it to the global file_dictionaries variable. If the file
   doesn't exist, an empty dictionary is added
   """
   assert filename
   try:
      file = open(filename, "rt")
      lines = map(substitute_mirror_links, file.readlines())
      dic = generate_dictionary_sections(lines)
      dic["TIMESTAMP"] = os.fstat(file.fileno())[stat.ST_MTIME]
      file_dictionaries[filename] = dic
      file.close()
   except IOError:
      file_dictionaries[filename] = {"TIMESTAMP": 0}



def insert_localization(filename, section, langcode):
   """Given a langcode and a base filename, this function modifies the
   path to insert the langode as a subdirectory before the filename, and
   then tries to open the file. If it fails, or langcode is empty, it
   will try to save the day using the default 'en' langcode. It returns
   the file as a list of strings without trailing whitespace. The
   section indicates that only files after a #-section line will be
   included.
   """
   global last_timestamp
   assert filename
   assert langcode
   assert section

   new_path = "%s.%s" % (filename, langcode)

   try:
      dic = file_dictionaries[new_path]
   except KeyError:
      load_file_dictionary(new_path)
      dic = file_dictionaries[new_path]

   try:
      lines = filter(retrieve_definitions, dic[section])
      last_timestamp = max(last_timestamp, dic["TIMESTAMP"])
      return lines
   except KeyError:
      pass

   if langcode != "en":
      report_missing_localization(new_path, section)
      return insert_localization(filename, section, "en")
   else:
      return []



def insert_file(filename):
   """A much simple version of insert_localization. Just opens the file
   and returns it as a list of strings without trailing whitespace.
   """
   assert filename

   try:
      file = open(filename, "rt")
      save_open_file_timestamp(file)
      temp = filter(retrieve_definitions, map(substitute_mirror_links, file.readlines()))
      file.close()
      if temp:
         clean_whitespace_lines(temp)
         return temp

   except IOError, msg:
      sys.stderr.write("Error while trying to include '%s'\n" % filename)
      sys.stderr.write("%s\n" % msg)

   return []



def get_data_for_mirror_file(mirror_link):
   """Returns the data for the specified mirrored link -> (url, ksize, md5)

   Searches for nick in the mirror data, and returns the apropiate local url
   if the file is there locally, otherwise get's the world's url. This only
   works as long as output filenames are created in the base directory
   (limitation due to avoiding another silly parameter). ValueError raised
   if nick is not in the data.
   """
   def ksize(number):
      number = int(number/1024.0) + 1
      if number == 1:
         return '?'
      else:
         return number

   for nick, url, size, hash, location, type in mirror_data:
      if nick == mirror_link:
         try: # See if file is there
            file_size = os.path.getsize(os.path.join(os.path.split(output_filename)[0], location))
            if size == file_size or size < 0:
               return location, ksize(file_size), hash
         except OSError:
            pass

         return url, ksize(size), hash

   msg = """%s not found in mirror data. This usually means that
an automatic link is using a filename not defined in
external/binary-files.txt.  Check that out, and also 'Mirrored
binary files' in docs/instructions.txt."""
   raise ValueError, msg % mirror_link


def substitute_mirror_links(text_line):
   """Substitutes instances of <mirror-x=xxx> by apropiate text."""

   # the mirror_data check is for dependency generation
   if len(mirror_data) < 1 or text_line.find("<mirror-") < 0:
      return text_line

   # reminder note: compress the following four into one with a table
   s = text_line.find("<mirror-ahref=")
   while s >= 0:
      e = text_line.find(">", s + 14)
      name = text_line[s + 14:e]
      info = get_data_for_mirror_file(name)
      text_line = '%s<a href="%s" title="md5: %s">%s' % (text_line[:s],
         info[0], info[2], text_line[e + 1:])
      s = text_line.find("<mirror-href=")

   s = text_line.find("<mirror-url=")
   while s >= 0:
      e = text_line.find(">", s + 12)
      name = text_line[s + 12:e]
      text_line = "%s%s%s" % (text_line[:s],
         get_data_for_mirror_file(name)[0], text_line[e + 1:])
      s = text_line.find("<mirror-url=")

   s = text_line.find("<mirror-ksize=")
   while s >= 0:
      e = text_line.find(">", s + 14)
      name = text_line[s + 14:e]
      text_line = "%s%s%s" % (text_line[:s],
         get_data_for_mirror_file(name)[1], text_line[e + 1:])
      s = text_line.find("<mirror-ksize=")

   s = text_line.find("<mirror-link=")
   while s >= 0:
      e = text_line.find(">", s + 13)
      name = text_line[s + 13:e]
      info = get_data_for_mirror_file(name)
      clean_url = info[0].replace("?download", "")
      text_line = '%s<a href="%s" title="md5: %s">%s</a>%s' % (text_line[:s],
         info[0], info[2], os.path.basename(clean_url), text_line[e + 1:])
      s = text_line.find("<mirror-link=")

   return text_line



def transform_line(text_line):
   """Transformation function. Given a text line (which can be multiline)
   transforms all the ocurrences of <include-whatever> tags, replacing them
   with src files or localized data files.
   """
   if not text_line: return []
   assert expresion_include

   result = expresion_include.search(text_line)
   if not result:
      return [text_line]

   # First see if there is an inclusion condition
   _cond = result.group("cond")
   if _cond: # Now see if we have to stop right here
      word = _cond[1:]
      if(word[0] != "!" and word not in allowed_includes) or (word[0] == "!" and word[1:] in allowed_includes):
         return mix_string_with_list(result.group("pre"),
            transform_line(text_line[result.start("post"):]))

   # Data includes are easy because they can't contain recursive includes
   if result.group("type") == "data":
      pre = result.group("pre")

      include = result.group("filename").split(":", 1)
      assert len(include) == 2, "was processing '%s', code '%s'" % (text_line.strip(), langcode)
      middle = insert_localization("%s%s" % (include_path_data, include[0]), include[1], langcode)

      post = transform_line(text_line[result.start("post"):])

      lines = mix_string_with_list(pre, middle)
      if len(lines) < 1:
         return post

      lines[-1:] = mix_string_with_list(lines[-1], post)
   else:
      assert result.group("type") == "source"
      lines = transform_line(result.group("pre"))
      for f in insert_file("%s%s" % (include_path_source, result.group("filename"))):
         lines.extend(transform_line(f))
      lines.extend(transform_line(text_line[result.start("post"):]))

   return lines



def mix_string_with_list(string, list):
   """This will return list, with string preppended to the first element
   of the list (which can be an empty list). If both are null, and empty
   list will be returned."""
   if not string:
      return list
   if not list:
      return [string]
   list[0] = "%s%s" % (string, list[0])
   return list



def transform_next_link_terminator(lines, match, line_number):
   """This is a helper of delete_self_links. Since <a href="..."> tags have
   to be closed, this functions starts searching iterativly in lines for
   the </a> tag, starting at line_number. </a> will be changed
   to </strong>. Only the first match will be replaced.
   A special case will be made for self links and self language flags, which
   will be erased from the output. This presumes that the language self link
   is all in the same line and has a specific format embedded in this script.
   """
   assert lines
   assert match

   if line_number >= len(lines): return
   found = match.match(lines[line_number])
   if found:
      lines[line_number] = "%s</strong>%s" % (found.group("pre"),
         found.string[found.start("post"):])
   else: # search next lines...
      transform_next_link_terminator(lines, match, line_number + 1)



def delete_self_links(lines):
   """This function processes all the lines and removes the <a href="...">
   tags which point to the self html file, and replaces them with bold tags.
   It cannot be done with a simple regular expression because <a href...>
   and </a> could be in separated lines.
   """
   assert lines
   assert input_filename

   # First discover which self name should we replace
   if langcode == "en":
      self_file = input_filename
   else:
      self_file = "%s.%s%s" % (os.path.splitext(input_filename)[0], langcode,
         os.path.splitext(input_filename)[1])

   if self_file[:len(include_path_source)] == include_path_source:
      self_file = self_file [len(include_path_source):]

   #sys.stderr.write ("using self name : %s\n" % self_file)
   regex = r'(?P<pre>.*?)(?P<self><a href="' + self_file + r'".*?>)(?P<post>.*)'
   start = re.compile(regex, re.IGNORECASE)
   end = re.compile(r"(?P<pre>.*?)(?P<self></a>)(?P<post>.*)", re.IGNORECASE)

   line_number = 0
   while line_number < len(lines):
      found = start.match(lines[line_number])
      if found:
         lines[line_number:line_number + 1] = ["%s<strong>" % found.group("pre"), found.group("post")]
         line_number = line_number + 1
         transform_next_link_terminator(lines, end, line_number)
      else:
         line_number = line_number + 1



def remove_negated_ifs(lines):
   """Ugly hack to handle #ifdef/#ifndef lines. At the moment only the
   langcode works as a posible define.
   """
   line_number = 0
   removing_lines = 0
   while line_number < len(lines):
      if lines[line_number].find("#ifndef ") == 0:
         if lines[line_number][8:].strip() in defines:
            removing_lines = 1
         del lines[line_number]
      elif lines[line_number].find("#ifdef ") == 0:
         if lines[line_number][7:].strip() not in defines:
            removing_lines = 1
         del lines[line_number]
      elif lines[line_number].find("#else") == 0:
         removing_lines = not removing_lines
         del lines[line_number]
      elif lines[line_number].find("#endif") == 0:
         removing_lines = 0
         del lines[line_number]
      else:
         if removing_lines:
            del lines[line_number]
         else:
            line_number = line_number + 1



def post_process_file_lines(lines):
   """Before the file reaches the external world, we want to process
   it to fix different simple text substitutions like the <time>
   filestamp thing. This goes through the list of lines and changes
   whatever is neccesary.
   """
   assert lines

   # Now we translate the timestamps
   pretty_date = localize_time.localize_time(time_in_seconds = last_timestamp,
      localization_file = "data/localize_time/%s" % langcode)

   substitutions = len(simple_substitutions)
   for line_number in range(len(lines)):
      lines[line_number] = lines[line_number].replace("<time>", pretty_date)
      f = 0
      while f < substitutions:
         if lines[line_number].find(simple_substitutions[f][0]) >= 0:
            lines[line_number] = lines[line_number].replace(
               simple_substitutions[f][0], simple_substitutions[f][1])
            # Damn!!! The ugliest hack of all to avoid recursive substituion!
            if simple_substitutions[f][0][0] == '<':
               f = 0
            else:
               f += 1
         else:
            f += 1

   delete_self_links(lines)

   remove_negated_ifs(lines)



def process_file(filename):
   """This could be the real main function of the script: the specified
   filename is opened and traversed recursively until all file dependencies
   have been processed. It also transforms several tags into text.
   If output_filename is not set, everything will be dumped in the standard
   output.
   """
   assert filename
   assert filename_tag_substitution
   assert simple_substitutions

   try:
      file = open(filename, "rt")
      save_open_file_timestamp(file)

      # First we read the complete input file
      buffer = []
      for line in filter(retrieve_definitions, map(substitute_mirror_links, file.readlines())):
         for f in transform_line(line):
            buffer.append(f)

      file.close()

      # Do some dirty work
      post_process_file_lines(buffer)

      # And finally print it
      output = sys.stdout
      if output_filename:
         output = open(output_filename, "wt")

      for f in range(len(buffer)):
         output.write(buffer[f])

      if output_filename:
         output.close()

   except IOError, msg:
      sys.stderr.write("Error processing '%s'\n%s\n" % (filename, msg))



def what_file_would_be_included(filename, section, langcode):
   """Checks the path and takes a look at what files would be included by
   the algorithm used in the insert_localization function. Returns a
   string with the included file or nothing if no file is found.
   """
   assert filename
   assert section
   assert langcode

   new_path = "%s.%s" % (filename, langcode)

   try:
      dic = file_dictionaries[new_path]
   except KeyError:
      load_file_dictionary(new_path)
      dic = file_dictionaries[new_path]

   try:
      lines = dic[section]
      return new_path
   except KeyError:
      pass

   if langcode != "en":
      return what_file_would_be_included(filename, section, "en")

   return None



def scan_for_include_tags(text_line, list, langcode):
   """This is like the transformation function, but doesn't include/return
   any text line, it just adds to the list the files which would be included
   if the program was really ran.
   """
   if not text_line: return
   assert expresion_include

   if text_line.find("<mirror-") >= 0:
      list.append(mirror_timestamp_filename)

   if text_line.find("<include") < 0: return
   result = expresion_include.search(text_line)
   if not result: return

   scan_for_include_tags(result.group("pre"), list, langcode)
   scan_for_include_tags(result.group("post"), list, langcode)

   # See if there is an inclusion condition
   _cond = result.group("cond")
   if _cond: # Now see if we have to stop right here
      word = _cond[1:]
      if (word[0] != "!" and word not in allowed_includes) or (word[0] == "!" and word[1:] in allowed_includes):
         return

   if result.group("type") == "data":
      include = result.group("filename").split(":", 1)
      assert len(include) == 2, "was processing '%s', code '%s'" % (text_line.strip(), langcode)
      test = (what_file_would_be_included
         ("%s%s" % (include_path_data, include[0]), include[1], langcode))
      if test:
         list.append(test)
   else:
      assert result.group("type") == "source"
      name = "%s%s" % (include_path_source, result.group("filename"))
      list.append(name)
      list.extend(extract_dependencies_from(name, langcode))



def string_list_without_duplicates(list):
   """Pass this function a list of strings, and it will return a new
   one without duplicate entries.
   """
   assert list
   dic = {}
   for f in list: dic[f] = 1
   return dic.keys()



def extract_dependencies_from(filename, langcode):
   """This is somehow similar to process_file, but doesn't recurse into the
   file and checks manually in the datafile for all available localization
   files, which is better than having to specify them manually.
   """
   assert filename

   list = []
   try:
      file = open(filename, "rt")
      line = file.readline()
      while line:
         scan_for_include_tags(line, list, langcode)
         line = file.readline()

      file.close()

   except IOError, msg:
      sys.stderr.write("Error extracting dependency info from '%s'\n%s\n" % (filename, msg))

   if list: return string_list_without_duplicates(list)
   else: return []



def filter_charset_encoding(langcode):
   """Given a string in '<langcode>,<charset>' form, returns a tuple
   in (<langcode>, <charset>) form. If <charset> is empty it returns
   iso-8859-1.
   """
   list = langcode.split(",", 1)
   if len(list) < 2: list.append("iso-8859-1")
   return list[0], list[1]



def generate_dependency_for(list_of_files):
   """Wrapper to generate dependency of a list of files."""
   assert list_of_files
   assert langcode

   langcodes = map(filter_charset_encoding, langcode.split(":"))

   final_dependencies = []
   output = sys.stdout
   _join = os.path.join
   for filename in list_of_files:
      basefilename = filename [len(include_path_source):]
      for code, charset in langcodes:
         if code == "en":
            temp = basefilename
         else:
            temp = os.path.splitext(basefilename)[0] + "." + code + os.path.splitext(basefilename)[1]

         out = _join(output_filename, temp).replace("\\", "/")
         final_dependencies.append(out)

         # Write here output, be nice and split long lines for humans
         written = len("%s: src/force_update %s" % (out, filename))
         output.write("%s: src/force_update %s" % (out, filename))
         for f in extract_dependencies_from(filename, code):
            f = f.replace("\\", "/")
            if written > 50:
               output.write(" \\\n\t")
               written = 0
            output.write(" %s" % f)
            written = written + len(f) + 1
         output.write("\n")

   # Generate variable with final dependency information
   output.write("\n\nDEPS = ")
   written = 0
   for f in final_dependencies:
      written = written + len("%s " % f)
      output.write("%s " % f)
      if written > 50:
         output.write(" \\\n\t")
         written = 0

   output.write("\n\n")

   # Generate automatic rules for the specified langcodes
   source_path = os.path.dirname(os.path.commonprefix(list_of_files).replace("\\", "/"))
   for code, charset in langcodes:
      if code == "en":
         target = _join(output_filename, "%.html")
      else:
         target = _join(output_filename, "%." + code + ".html")
      target = target.replace("\\", "/")

      output.write("%s: %s/%%.html\n\t$(PRE_FILE_GENERATION)\n" % (target, source_path))
      output.write("\t$(WEBMAKE) --mirror-conf=$(MIRROR_DATA) --input=$< --localize=%s,%s --output=$(STDOUTPUT)\n" % (code, charset))
      output.write("\t$(POST_FILE_GENERATION)\n\n")

   # Generate rules for automatic cleanup
   dic = {}
   for f in final_dependencies:
      dic[os.path.basename(f).split(".", 1)[0]] = os.path.dirname(f)

   assert dic.keys()

   output.write("automatic_cleanup_one:\n");
   for f in dic.keys():
      output.write("\t$(RM) %s*.html\n" % _join(dic[f], f))

   output.write("\nAUTOMATIC_CLEANUP_FILES += automatic_cleanup_one\n\n")

   # test for directory existance
   try:
      name = _join(output_filename, "test")
      file = open(name, "wt")
      file.write("test")
      file.close()
      os.unlink(name)
   except IOError:
      try: os.makedirs(output_filename, 0775)
      except OSError: pass



def expand_file_patterns(files_to_process):
   """There's a problem under non-Linux platforms: there's no shell
   expansion, and this breaks the dependency generation. It would be nice
   to pass all parameters expanded, but I fear then for the commandline to
   be broken at some point if it's too long, so let's expand them manually.
   """
   list = []
   for f in files_to_process:
      for g in glob.glob(f):
         list.append(g.replace("\\", "/"))

   return list



def add_local_href_substitutions(substitution_list, config_file):
   """Adds to substitution_list the substitutions found at config_file.

   Also, this has been tweaked to accept new defines."""
   assert substitution_list
   assert config_file

   conf = ConfigParser.ConfigParser()
   conf.read(config_file)

   # Ok, start adding tags
   try:
      val = conf.get("global", "hosturl", 1)
   except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
      val = "http://alleg.sourceforge.net/"
   substitution_list.append(["<hosturl>", val])

   # Substitutions for external language codes
   try:
      val = conf.get("global", "external_languages", 1)
      link = conf.get("global", "external_language_link", 1)
   except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
      val = ""
      link = "http://alleg.sourceforge.net/"

   for f in val.split(":"):
      assert f != "en", "You simply can't exclude english page generation!"
      substitution_list.append(["%s.%s.html" % (filename_tag_substitution, f),
         "%s%s.%s.html" % (link, filename_tag_substitution, f)])

   # Substitutions for external online docs
   try:
      val = conf.get("global", "external_online_docs", 1)
      link = conf.get("global", "external_online_docs_link", 1)
   except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
      val = ""
      link = "http://alleg.sourceforge.net/"

   for f in val.split(":"):
      substitution_list.append(["latestdocs/%s/index.html" % f,
         "%slatestdocs/%s/index.html" % (link, f)])

   # Conditional includes to be allowed:
   try: val = conf.get("global", "allowed_includes", 1)
   except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): val = ""

   for f in val.split(":"):
      if f: allowed_includes.append(f)

   # Additional defines specified by the configuration file.
   try: val = conf.get("global", "defines", 1)
   except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): val = ""

   for f in val.split():
      defines.append(f)


def process_arguments(arguments):
   """Reads commandline arguments and modifies program behaviour according
   to them.
   """
   global input_filename, output_filename, langcode, filename_tag_substitution
   global generate_dependency, mirror_timestamp_filename, redirect_warnings_to

   try:
      options, files_to_process = getopt.getopt(arguments, short_arguments, long_arguments)
   except getopt.error, msg:
      print "Error parsing arguments:\n", msg, "\n"
      show_program_usage()
      sys.exit(1)

   for op, value in options:
      if op in ("--input", "-i"):
         input_filename = value
      elif op in ("--output", "-o"):
         output_filename = value
      elif op in ("--version", "-v"):
         print version
         sys.exit(0)
      elif op in ("--help", "-h"):
         show_program_usage()
         sys.exit(0)
      elif op in ("--localize", "-l"):
         langcode = value
      elif op in ("--depend", "-M"):
         generate_dependency = 1
      elif op in ("--mirror-timestamp", "-m"):
         if os.path.isfile(value):
            mirror_timestamp_filename = value
      elif op in ("--warnings-to", "-w"):
         redirect_warnings_to = value
      elif op in ("--mirror-conf", "-c"):
         import mirror
         mirror_data.extend(mirror.load_mirror_data(value))

   if not input_filename and not generate_dependency:
      print "You must supply an input filename."
      show_program_usage()
      sys.exit(1)

   if not output_filename:
      print "You must supply an output filename."
      show_program_usage()
      sys.exit(1)

   # Silly stupid check for the generate_dependency info
   if not input_filename:
      input_filename = expand_file_patterns(files_to_process)
   else:
      filename_tag_substitution = os.path.splitext(os.path.split(input_filename)[1])[0]



def main(argv):
   """Entry point of the script, prepares all regex variables after
   processing the arguments.
   """
   global expresion_include, langcode
   # Some initialization
   process_arguments(argv[1:])

   expresion_include = re.compile(r"(?P<pre>.*?)(<include-(?P<type>data|source)(?P<cond>[^=]*)=(?P<filename>\S+?)>)(?P<post>.*)")

   if generate_dependency:
      add_local_href_substitutions(["<nothing>", "<nothing>"], "allegro.ini")

      generate_dependency_for(input_filename)
   else:
      simple_substitutions.append(["<charset>", filter_charset_encoding(langcode)[1]])
      langcode = filter_charset_encoding(langcode)[0]
      simple_substitutions.append(["<filename>", filename_tag_substitution])
      simple_substitutions.append(["<langcode>", langcode])
      if langcode == "en":
         simple_substitutions.append(["<dotlangcode>", ""])
      else:
         simple_substitutions.append(["<dotlangcode>", "." + langcode])
      defines.append(langcode)

      add_local_href_substitutions(simple_substitutions, "allegro.ini")

      process_file(input_filename)



if __name__ == "__main__":
   main(sys.argv)