path: root/tools/3rdparty/sitemap_gen-1.4/test_sitemap_gen.py

#!/usr/bin/env python
#
# Copyright (c) 2004, 2005 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of Google nor the names of its contributors may
#   be used to endorse or promote products derived from this software
#   without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# The sitemap_gen.py script is written in Python 2.2 and released to
# the open source community for continuous improvements under the BSD
# 2.0 new license, which can be found at:
#
#   http://www.opensource.org/licenses/bsd-license.php
#

"""Unit tests for sitemap_gen.py, a script for generating sitemaps
for a web server.
"""

# Please be careful that all syntax used in this file can be parsed on
# Python 1.5 -- this version check is not evaluated until after the
# entire file has been parsed.
import sys
if sys.hexversion < 0x02020000:
  print 'This script requires Python 2.2 or later.'
  print 'Currently run with version: %s' % sys.version
  sys.exit(1)

import binascii
import fnmatch
import gzip
import os
import tempfile
import unittest
import xml.dom.minidom
import sitemap_gen

# True and False were introduced in Python2.2.2
try:
  testTrue=True
  del testTrue
except NameError:
  True=1
  False=0


class URLCounter(object):
  """Counts returned URLs, determines how many valid v. invalid we get.
  This is a helper for consuming what the many Input* objects produce."""
  def __init__(self, root, print_invalid, expected):
    """Input:
      root          :: root URL for calling the URL's own Validate()
      print_invalid :: print to output all invalid URLs
      expected      :: sequence of wildcard filters to validate against
    """
    self._root     = root
    self._print    = print_invalid
    self._expected = expected
    self._valid    = 0
    self._invalid  = 0
  #end def __init__

  def Reset(self):
    """Reset our counts without harming the validity filters."""
    self._valid   = 0
    self._invalid = 0
  #end def Reset

  def Valid(self):
    """Returns number of valid URLs."""
    return self._valid
  #end def Valid
 
  def Invalid(self):
    """Returns number of invalid URLs."""
    return self._invalid
  #end def Valid

  def Count(self, url, allow_fragment):
    """The 'please consume this URL' function called by the URL producer."""
    valid = True
    if valid:
      valid = url.Validate(self._root, allow_fragment)
    if valid:
      for filter in self._expected:
        valid = fnmatch.fnmatchcase(url.loc, filter)
        if valid:
          break
    if valid:
      self._valid = self._valid + 1
    else:
      if self._print:
        url.Log(prefix='URLCounter', level=0)
      self._invalid = self._invalid + 1
  #end def Count
#end class URLCounter


class TestSiteMap(unittest.TestCase):
  """Tests the sitemap_gen application."""

  def testTimestampISO8601(self):
    """ Checks whether the TimestampISO8601 function works. """
    self.assertEqual(sitemap_gen.TimestampISO8601(23),
                     '1970-01-01T00:00:23Z')
    self.assertEqual(sitemap_gen.TimestampISO8601(549876543),
                     '1987-06-05T07:29:03Z')
  #end def testTimestampISO8601

  def testExpandPathAttribute(self):
    """ Verifies our path globbing function works. """
    temppath = tempfile.mktemp()
    tempwild = tempfile.tempdir
    if tempwild:
      tempwild = tempwild + os.sep
    tempwild = tempwild + '*'
    try:
      open(temppath, 'w').close()

      dict1 = {}
      dict2 = {'alpha' : 'beta', 'path' : 'DoesNotExist987654321.xyz'}
      dict3 = {'alpha' : 'beta', 'path' : tempwild}

      res1  = sitemap_gen.ExpandPathAttribute(dict1, 'path')
      res2  = sitemap_gen.ExpandPathAttribute(dict2, 'path')
      res3  = sitemap_gen.ExpandPathAttribute(dict3, 'path')

      self.assertEqual(len(res1), 1)
      self.assertEqual(res1[0], dict1)

      self.assertEqual(len(res2), 1)
      self.assertEqual(res2[0], dict2)
   
      self.assert_(len(res3) >= 1)
      anymatch = False
      for res in res3:
        path = res['path']
        if path.find(temppath) >= 0:
          anymatch = True
        self.assertEqual(res['alpha'], 'beta')
      self.assert_(anymatch)

    finally:
      os.unlink(temppath)
  #end def testExpandPathAttribute

  def testEncoder(self):
    """ Tests minimal functionality of the learning Unicode codec """
    ENC_UTF8      = 'UTF-8'
    ENC_LATIN1    = 'ISO-8859-1'
    ENC_CYRILLIC  = 'ISO-8859-5'

    STR1_LATIN1   = 'has an ' + binascii.a2b_hex('FC') + 'mlat'
    STR1_UTF8     = 'has an ' + binascii.a2b_hex('C3BC') + 'mlat'
    STR1_UCS2     = 'has an ' + unichr(252) + 'mlat'

    STR2_LATIN1   = 'DRAGON' + binascii.a2b_hex('A7') + '!'
    STR2_CYRILLIC = 'DRAGON' + binascii.a2b_hex('FD') + '!'
    STR2_UCS2     = 'DRAGON' + unichr(167) + '!'
    
    # Spawn our own encoder instance so we don't abuse the module one.
    encoder = sitemap_gen.Encoder()

    # Convert Latin-1 to UTF-8, by way of Unicode
    encoder.SetUserEncoding(ENC_LATIN1)
    self.assertEqual(encoder.WidenText(STR1_LATIN1, None), STR1_UCS2)
    self.assertEqual(encoder.NarrowText(STR1_UCS2, ENC_UTF8), STR1_UTF8)

    # Test learning.  STR1 has no Cyrillic equivalent, STR2 just changes.
    encoder.SetUserEncoding(None)
    encoder._learned = []
    self.assertEqual(encoder.WidenText(STR2_CYRILLIC, ENC_CYRILLIC), STR2_UCS2)
    self.assertEqual(encoder.WidenText(STR2_CYRILLIC, None), STR2_UCS2)
    self.assertEqual(encoder.NarrowText(STR1_UCS2, None), STR1_UTF8)
    self.assert_(not encoder._learned)
    self.assertEqual(encoder.NarrowText(STR1_UCS2, ENC_LATIN1), STR1_LATIN1)
    self.assertEqual(encoder.NarrowText(STR1_UCS2, None), STR1_LATIN1)
    self.assertEqual(encoder.NarrowText(STR2_UCS2, None), STR2_LATIN1)
  #end def testEncoder

  def testURL(self):
    """ Vigorously tests our URL attribute processing. """

    # Test the IsAbsolute method
    self.assert_(sitemap_gen.URL.IsAbsolute('http://a.b.c/d/e.txt?f=g#h'))
    self.assert_(sitemap_gen.URL.IsAbsolute('http://a.b.c'))
    self.assert_(not sitemap_gen.URL.IsAbsolute('http:///d/e.txt?f=g#h'))
    self.assert_(not sitemap_gen.URL.IsAbsolute('http:a.b.c/d/e.txt?f=g#h'))
    self.assert_(not sitemap_gen.URL.IsAbsolute('a.b.c/d/e.txt?f=g#h'))
    self.assert_(not sitemap_gen.URL.IsAbsolute('/d/e.txt?f=g#h'))

    # Canonicalize our base URL
    BASE_R = 'http://www.example.com/f' + binascii.a2b_hex('F6F6') + '/'
    BASE_C = 'http://www.example.com/f%F6%F6/'
    sitemap_gen.encoder.SetUserEncoding('ISO-8859-1')
    self.assertEqual(sitemap_gen.URL.Canonicalize(BASE_R), BASE_C)

    # Test how canonicalization handles pre-quoted values
    self.assertEqual(sitemap_gen.URL.Canonicalize(
      'http://www.example.com/my%25thing'),
      'http://www.example.com/my%25thing')
    self.assertEqual(sitemap_gen.URL.Canonicalize(
      'http://www.example.com/my%thing'),
      'http://www.example.com/my%25thing')

    # Test IDNA encoding
    # The generator can only do the "right thing" on Python 2.3 or higher
    warn = sitemap_gen.output.num_warns
    if sys.hexversion >= 0x02030000:
      self.assertEqual(sitemap_gen.URL.Canonicalize(
        'http://www.' + unichr(252) + 'mlat.com/' + unichr(252) + 'mlat.txt'),
        'http://www.xn--mlat-zra.com/%FCmlat.txt')
      self.assertEqual(sitemap_gen.output.num_warns, warn)
    else:
      self.assertEqual(sitemap_gen.URL.Canonicalize(
        'http://www.' + unichr(252) + 'mlat.com/' + unichr(252) + 'mlat.txt'),
        'http://www.%FCmlat.com/%FCmlat.txt')
      self.assertEqual(sitemap_gen.output.num_warns, warn + 2)

    # All valid data
    warn = sitemap_gen.output.num_warns
    url1 = sitemap_gen.URL()
    url1.TrySetAttribute('loc', BASE_R + 'bar.html')
    url1.TrySetAttribute('lastmod', '1987-06-05T07:29:03Z')
    url1.TrySetAttribute('changefreq', 'daily')
    url1.TrySetAttribute('priority', '0.3')
    self.assert_(url1.Validate(BASE_C, True))
    self.assertEqual(sitemap_gen.output.num_warns, warn)

    # Valid ref, all else invalid
    warn = sitemap_gen.output.num_warns
    url2 = sitemap_gen.URL()
    url2.TrySetAttribute('loc', BASE_C + 'bar.html')
    url2.TrySetAttribute('lastmod', 'June 1, 2005')
    url2.TrySetAttribute('changefreq', 'every second')
    url2.TrySetAttribute('priority', 'infinite')
    url2.TrySetAttribute('badattr', 'Nope!')
    self.assert_(url2.Validate(BASE_C, True))
    self.assertEqual(sitemap_gen.output.num_warns, warn + 4)

    # Two URLs with same ref should compare equal
    self.assertEqual(url1, url2)

    # A ref not based
    warn = sitemap_gen.output.num_warns
    url3 = sitemap_gen.URL()
    url3.TrySetAttribute('loc', 'http://www.example.com/bar/foo.html')
    self.assert_(not url3.Validate(BASE_C, True))
    self.assertEqual(sitemap_gen.output.num_warns, warn + 1)

    # A fragmentary URL
    warn = sitemap_gen.output.num_warns
    url4 = sitemap_gen.URL()
    url4.TrySetAttribute('loc', '/foo.html')
    self.assert_(not url4.Validate(BASE_C, False))
    self.assertEqual(sitemap_gen.output.num_warns, warn + 1)
    url4.TrySetAttribute('loc', '/xyzzy/foo.html')
    self.assert_(url4.Validate('http://www.example.com/', True))
    self.assertEqual(url4.loc, 'http://www.example.com/xyzzy/foo.html')
    self.assertEqual(sitemap_gen.output.num_warns, warn + 1)

    # Test a whole sequence of good and bad timestamp values
    timestamps_good = [
      '2001',
      '2001-01',
      '2001-01-02',
      '2001-01-03T01:02Z',
      '2001-01-03T01:02:03Z',
      '2001-01-03T01:02:03.0123Z',
      '2001-01-03T01:02+00:00',
      '2001-01-03T01:02:03-99:99',
      '2001-01-03T01:02:03.0123+88:88',
      ]
    timestamps_bad = [
      '2001:01:03T01:02Z',
      '2001-01-03T01:02:03.Z',
      'a2001-01-06T01:02:05-99:99',
      '2001-01-06T01:02:05-99:99Z',
      '2001-1-6T01:02:05-99:99',
      'xyzzy',
      '2001-01-03T01:02:03.1.2Z',
      ]
    warn = sitemap_gen.output.num_warns
    url3.TrySetAttribute('loc', BASE_C + 'foo.html')
    for ts in timestamps_good:
      url3.TrySetAttribute('lastmod', ts)
      self.assert_(url3.Validate(BASE_C, True))
    self.assertEqual(sitemap_gen.output.num_warns, warn)
    for ts in timestamps_bad:
      url3.TrySetAttribute('lastmod', ts)
      self.assert_(url3.Validate(BASE_C, True))
    self.assertEqual(sitemap_gen.output.num_warns, warn + len(timestamps_bad))
  #end def testURL

  def testFilter(self):
    """ Test the filtering object """
    url1 = sitemap_gen.URL()
    url2 = sitemap_gen.URL()
    url1.TrySetAttribute('loc', 'http://www.example.com/foo/bar.html')
    url2.TrySetAttribute('loc', 'http://www.example.com/bar/foo.html')
    url1.Validate('http://www.example.com', True)
    url2.Validate('http://www.example.com', True)

    # Arguments
    error = sitemap_gen.output.num_errors
    args_bad = [
      {},
      {'pattern' : '*', 'type' : 'unknown'},
      {'pattern' : '*', 'type' : 'wildcard', 'action' : 'look pretty'},
      {'pattern' : '*', 'type' : 'regexp'},
      ]
    error = sitemap_gen.output.num_errors
    for args in args_bad:
      sitemap_gen.Filter(args)
    self.assertEqual(sitemap_gen.output.num_errors, error + len(args_bad))

    # Wildcard
    filt_w = sitemap_gen.Filter({'pattern' : '*/foo/*', 'type' : 'wildcard' })
    self.assertEqual(filt_w.Apply(url1), False)
    self.assertEqual(filt_w.Apply(url2), None)

    # Regexp
    filt_r = sitemap_gen.Filter({'pattern' : '/bar/[^/]+$', 'type' : 'REGEXP',
                                 'action' : 'PASS'})
    self.assertEqual(filt_r.Apply(url1), None)
    self.assertEqual(filt_r.Apply(url2), True)
  #end def testFilter

  def Count(self, url, allow_fragment):
    if url.Validate('http://www.example.com/', allow_fragment):
      self.valid_urls = self.valid_urls + 1
    else:
      self.invalid_urls = self.invalid_urls + 1
  #end def Count
  valid_urls   = 0
  invalid_urls = 0

  def testInputURL(self):
    """ Test one of the Input mechanisms: InputURL """
    
    # Feed a couple URLs.  Make sure we get an error on extra attributes.
    self.valid_urls   = 0
    self.invalid_urls = 0
    error = sitemap_gen.output.num_errors
    warn = sitemap_gen.output.num_warns
    generator1 = sitemap_gen.InputURL({'href' : 'http://www.example.com/1',
                                      'priority' : '0.3',
                                      'lastmod' : '2004-11-14T01:00-07:00',
                                      'changefreq' : 'hourly',
                                      'unknownInURL' : 'attribute'})
    generator2 = sitemap_gen.InputURL({'href' : 'http://www.example.com/2',
                                      'priority' : '0.3',
                                      'lastmod' : '2004-11-14T01:00-07:00',
                                      'changefreq' : 'hourly'})
    generator1.ProduceURLs(self.Count)
    generator2.ProduceURLs(self.Count)
    self.assertEqual(self.valid_urls, 1)
    self.assertEqual(self.invalid_urls, 0)
    self.assertEqual(sitemap_gen.output.num_errors, error + 1)
    self.assertEqual(sitemap_gen.output.num_warns, warn)
  #end def testInputURL

  def testInputURLList(self):
    """ Test one of the Input mechanisms: InputURLList """
    path = tempfile.mktemp()
    file = open(path, 'w')

    try:
      # Create a temp file we can read
      testText = """
http://www.example.com/foo/bar unknownInURLList=attribute
http://www.example.com/foo/xxx.pdf lastmod=2003-12-31T14:05:06+00:00
http://www.example.com/foo/yyy?x=12&y=23   changefreq=weekly   priority=0.3
      """
      file.write(testText)
      file.close()
      
      # Feed in the data.  Make sure we get a warning on the bad attribute.
      self.valid_urls   = 0
      self.invalid_urls = 0
      warn = sitemap_gen.output.num_warns
      generator = sitemap_gen.InputURLList({'path' : path})
      generator.ProduceURLs(self.Count)
      self.assertEqual(self.valid_urls, 3)
      self.assertEqual(self.invalid_urls, 0)
      self.assertEqual(sitemap_gen.output.num_warns, warn + 1)

    finally:
      os.unlink(path)
  #end def testInputURLList

  def testInputDirectory(self):
    """Test one of the Input mechanisms: InputDirectory.
    I've seen a subtle path-bug appear when going into sub-sub-directories
    that didn't under just sub-directories.  So we go to the trouble to
    make a whole little directory tree to read.
    """
    counter = URLCounter('http://www.example.com/', True, (
      'http://www.example.com/',
      'http://www.example.com/one.html',
      'http://www.example.com/two.html',
      'http://www.example.com/xyzzy/',
      'http://www.example.com/xyzzy/thr.html',
      'http://www.example.com/xyzzy/zyxxy/',
      'http://www.example.com/xyzzy/zyxxy/fiv.html',
      ))
    path = tempfile.mktemp()
    subpath = os.path.join(path, 'xyzzy')
    subsubpath = os.path.join(subpath, 'zyxxy')
   
    try:
      # Create some dummy empty files
      os.mkdir(path)
      os.mkdir(subpath)
      os.mkdir(subsubpath)
      path_one = os.path.join(path, 'one.html')
      path_two = os.path.join(path, 'two.html')
      path_thr = os.path.join(subpath, 'thr.html')
      path_for = os.path.join(subpath, 'default.html')
      path_fiv = os.path.join(subsubpath, 'fiv.html')
      open(path_one, 'w').close()
      open(path_two, 'w').close()
      open(path_thr, 'w').close()
      open(path_for, 'w').close()
      open(path_fiv, 'w').close()

      # Feed in the data.  There should be no warnings.
      warn = sitemap_gen.output.num_warns
      generator = sitemap_gen.InputDirectory({'path' : path,
        'url' : 'http://www.example.com/', 'default_file' : 'default.html'},
        'http://www.example.com/')
      generator.ProduceURLs(counter.Count)
      self.assertEqual(counter.Valid(), 7)
      self.assertEqual(counter.Invalid(), 0)
      self.assertEqual(sitemap_gen.output.num_warns, warn)

    finally:
      os.unlink(path_one)
      os.unlink(path_two)
      os.unlink(path_thr)
      os.unlink(path_for)
      os.unlink(path_fiv)
      os.rmdir(subsubpath)
      os.rmdir(subpath)
      os.rmdir(path)
  #end def testInputDirectory

  def testInputAccessLogCLF(self):
    """ Test one of the Input mechanisms: InputAccessLog (Common logfile) """
    path = tempfile.mktemp()
    file = open(path, 'w')

    try:
      # Create a temp file we can read
      testText = '''
msnbot.msn.com - - [15/May/2005:07:46:50 -0700] "GET /~guest/main/ HTTP/1.0" 200 5670
221.216.237.71 - - [15/May/2005:07:59:25 -0700] "GET /~guest/bookmark/ HTTP/1.1" 200 39195
221.216.237.71 - - [15/May/2005:07:59:27 -0700] "GET /favicon.ico HTTP/1.1" 404 217
c-67-161-121-105.hsd1.wa.comcast.net - - [15/May/2005:11:17:23 -0700] "GET /picts/top.jpg HTTP/1.1" 200 10044
cpe-65-24-155-46.columbus.res.rr.com - - [16/May/2005:22:53:07 -0700] "HEAD http://www.example.com/~guest HTTP/1.1" 200 0
      '''
      file.write(testText)
      file.close()
      
      # Feed in the data
      self.valid_urls   = 0
      self.invalid_urls = 0
      warn = sitemap_gen.output.num_warns
      generator = sitemap_gen.InputAccessLog({'path' : path})
      generator.ProduceURLs(self.Count)
      self.assertEqual(self.valid_urls, 4)
      self.assertEqual(self.invalid_urls, 0)
      self.assertEqual(sitemap_gen.output.num_warns, warn)

    finally:
      os.unlink(path)
  #end def testInputAccessLogCLF

  def testInputAccessLogELF(self):
    """ Test one of the Input mechanisms: InputAccessLog (Extended logfile) """
    path = tempfile.mktemp()
    file = open(path, 'w')

    try:
      # Create a temp file we can read
      testText = '''
#Software: Microsoft Internet Information Services 6.0
#Version: 1.0
#Date: 2004-03-22 09:20:36
#Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) sc-status sc-substatus sc-w
in32-status
2004-03-22 09:20:36 192.168.0.58 GET /Default.htm - 80 - 4.5.11.3 Mozilla/4.0+(compatible;+MSIE+5.5;+Windows+98) 200 0 64
2004-03-22 09:22:58 192.168.0.58 GET /Default.htm - 80 - 24.87.160.82 Mozilla/4.0+(compatible;+MSIE+5.5;+Windows+98) 200 0 6
4
      '''
      file.write(testText)
      file.close()
      
      # Feed in the data
      self.valid_urls   = 0
      self.invalid_urls = 0
      warn = sitemap_gen.output.num_warns
      generator = sitemap_gen.InputAccessLog({'path' : path})
      generator.ProduceURLs(self.Count)
      self.assertEqual(self.valid_urls, 2)
      self.assertEqual(self.invalid_urls, 0)
      self.assertEqual(sitemap_gen.output.num_warns, warn)

    finally:
      os.unlink(path)
  #end def testInputAccessLogELF

  def testInputSitemap(self):
    """ Test one of the Input mechanisms: InputSitemap """
    path1 = tempfile.mktemp('.xml')
    path2 = tempfile.mktemp('.xml')
    path3 = tempfile.mktemp('.xml')
    path4 = tempfile.mktemp('.xml')
    file1 = None
    file2 = None
    file3 = None
    file4 = None

    index = '''<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex
  xmlns="http://www.google.com/schemas/sitemap/0.84"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://www.google.com/schemas/sitemap/0.84
                     http://www.google.com/schemas/sitemap/0.84/siteindex.xsd">
 <sitemap>
  <loc>http://www.example.com/path/to/%(PATH2)s</loc>
  <lastmod>2005-07-15T17:41:22Z</lastmod>
 </sitemap>
 <sitemap>
  <loc>http://www.example.com/path/to/%(PATH3)s</loc>
  <lastmod>2005-07-15T17:41:22Z</lastmod>
 </sitemap>
</sitemapindex>
'''
    content1 = '''<?xml version="1.0" encoding="UTF-8"?>
<urlset
  xmlns="http://www.google.com/schemas/sitemap/0.84"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://www.google.com/schemas/sitemap/0.84
                      http://www.google.com/schemas/sitemap/0.84/sitemap.xsd">
 <url>
  <loc>http://www.example.com/another/path/to/samplefile1.html</loc>
  <lastmod>2005-07-13T00:00:12Z</lastmod>
  <priority>0.5000</priority>
 </url>
 <url>
  <loc>http://www.example.com/another/path/to/samplefile2.html</loc>
  <lastmod>2004-11-16T20:22:06Z</lastmod>
  <priority>0.5000</priority>
 </url>
</urlset>
'''
    content2 = '''<?xml version="1.0" encoding="UTF-8"?>
<urlset
  xmlns="http://www.google.com/schemas/sitemap/0.84"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://www.google.com/schemas/sitemap/0.84
                      http://www.google.com/schemas/sitemap/0.84/sitemap.xsd">
 <url badSitemapAttr="Hello, World!">
  <loc>http://www.example.com/another/path/to/samplefile3.html</loc>
  <lastmod>2005-07-13T00:00:12Z</lastmod>
  <priority>0.5000</priority>
 </url>
 <url>
  <loc>http://www.example.com/another/path/to/samplefile4.html</loc>
  <lastmod>2004-11-16T20:22:06Z</lastmod>
  <priority>0.5000</priority>
 </url>
</urlset>
'''
    # This index is invalid because it points to another index file.
    badindex = '''<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex
  xmlns="http://www.google.com/schemas/sitemap/0.84"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://www.google.com/schemas/sitemap/0.84
                     http://www.google.com/schemas/sitemap/0.84/siteindex.xsd">
 <sitemap>
  <loc>http://www.example.com/path/to/%(PATH2)s</loc>
  <lastmod>2005-07-15T17:41:22Z</lastmod>
 </sitemap>
 <sitemap>
  <loc>http://www.example.com/path/to/%(PATH1)s</loc>
  <lastmod>2005-07-15T17:41:22Z</lastmod>
 </sitemap>
</sitemapindex>
'''

    # Make a nice complicated set of two index files and two sitemaps.
    try:
      file1 = open(path1, 'wt')
      file2 = open(path2, 'wt')
      file3 = open(path3, 'wt')
      file4 = open(path4, 'wt')
      file1.write(index % {
        'PATH1' : os.path.basename(path1),
        'PATH2' : os.path.basename(path2),
        'PATH3' : os.path.basename(path3)})
      file2.write(content1)
      file3.write(content2)
      file4.write(badindex % {
        'PATH1' : os.path.basename(path1),
        'PATH2' : os.path.basename(path2),
        'PATH3' : os.path.basename(path3)})
      file1.close()
      file1 = None
      file2.close()
      file2 = None
      file3.close()
      file3 = None
      file4.close()
      file4 = None

      # Feed in the good data.  Make sure we get warned on the bad attribute.
      self.valid_urls   = 0
      self.invalid_urls = 0
      warn = sitemap_gen.output.num_warns
      generator = sitemap_gen.InputSitemap({'path' : path1})
      generator.ProduceURLs(self.Count)
      self.assertEqual(self.valid_urls, 4)
      self.assertEqual(self.invalid_urls, 0)
      self.assertEqual(sitemap_gen.output.num_warns, warn + 1)

      # Feed in the bad data.  Should error once on the bad index and once
      # because it aborts processing the XML.
      self.valid_urls   = 0
      self.invalid_urls = 0
      errors = sitemap_gen.output.num_errors
      generator = sitemap_gen.InputSitemap({'path' : path4})
      generator.ProduceURLs(self.Count)
      self.assertEqual(self.valid_urls, 2)
      self.assertEqual(self.invalid_urls, 0)
      self.assertEqual(sitemap_gen.output.num_errors, errors + 2)

    finally:
      if file1 is not None:
        file1.close()
      if file2 is not None:
        file2.close()
      if file3 is not None:
        file3.close()
      if os.path.exists(path1):
        os.unlink(path1)
      if os.path.exists(path2):
        os.unlink(path2)
      if os.path.exists(path3):
        os.unlink(path3)
  #end def testInputSitemap

  def testFilePathGenerator(self):
    """ Test our iteration of filenames """
    gen1 = sitemap_gen.FilePathGenerator()
    gen2 = sitemap_gen.FilePathGenerator()
    gen3 = sitemap_gen.FilePathGenerator()
    self.assert_(gen1.Preload('/tmp/bar/foo.xml'))
    self.assert_(gen2.Preload('foo.xml.gz'))
    self.assert_(gen3.Preload('/foo.gz'))
    self.assert_(not gen1.is_gzip)
    self.assert_(    gen2.is_gzip)
    self.assert_(    gen3.is_gzip)
    self.assertEqual(gen1.GeneratePath(0),
                     os.path.normpath('/tmp/bar/foo.xml'))
    self.assertEqual(gen2.GeneratePath(1),'foo1.xml.gz')
    self.assertEqual(gen1.GeneratePath('_index.xml'),
                     os.path.normpath('/tmp/bar/foo_index.xml'))
    self.assertEqual(gen1.GenerateURL('_index.xml', 'http://www.example.com/'),
                     'http://www.example.com/foo_index.xml')
    self.assertEqual(gen1.GenerateURL(2, 'http://www.example.com/'),
                     'http://www.example.com/foo2.xml')
    self.assertEqual(gen2.GenerateWildURL('http://www.example.com/'),
                     'http://www.example.com/foo*.xml.gz')
  #end def testFilePathGenerator

  def testSitemap(self):
    """Test a basic config of the overall sitemap class."""
    path1 = tempfile.mktemp()
    path2 = tempfile.mktemp(".xml.gz")
    file = open(path1, 'w')

    try:
      # Create a temp file we can read
      testText = '''<?xml version="1.0" encoding="UTF-8"?>
<site
  base_url="http://www.example.com/"
  store_into="%(OUTPUTFILENAME)s"
  default_encoding="UTF-8"
  verbose="3"
>
  <url href="http://www.example.com/.htaccess" />
  <url href="http://www.example.com/foo/bar.html" />
  <url href="http://www.example.com/foo/bar.gif" />
  <url href="http://www.example.com/foo/bar.html" />
  <url href="http://www.example.com/percent%%%%percent.html" />
  <url href="http://www.example.com/&#252;mlat.html" />
  <filter action="drop" type="regexp" pattern="/\.[^/]*$" />
</site>
'''
      file.write(testText % {'OUTPUTFILENAME' : path2})
      file.close()

      # Bring up the engine
      warn = sitemap_gen.output.num_warns
      error = sitemap_gen.output.num_errors
      sitemap = sitemap_gen.CreateSitemapFromFile(path1, True)
      self.assert_(sitemap)
      sitemap.Generate()
      self.assertEqual(sitemap_gen.output.num_warns, warn)
      self.assertEqual(sitemap_gen.output.num_errors, error)

      # Verify we got readable XML out of it
      file = gzip.open(path2, mode='rb')
      result = file.read()
      file.close()
      dom = xml.dom.minidom.parseString(result)
      self.assertEqual(len(dom.getElementsByTagName('url')), 4)
      self.assert_(result.find('http://www.example.com/foo/bar.html') > 0)
      self.assert_(result.find('http://www.example.com/foo/bar.gif') > 0)
      self.assert_(result.find('%25%25') > 0)
      self.assert_(result.find('%C3%BC') > 0)
    finally:
      if os.path.exists(path2):
        os.unlink(path2)
      os.unlink(path1)
  #end def testSitemap
    
#end class TestSiteMap

#
# __main__
#

if __name__ == '__main__':
  unittest.main()