python Permutation 和 unittest 耗时测试隔离

在python unittest TestSuite 框架实践和几个问题中，提到"按照任意的顺序去调用 _set* 和 test* 方法，始终保证后面的操作不会对前面的已有结果造成破坏"这样一个需求，当时并没有给出一个结论。

实际上，这里涉及到一个组合数的问题。比如在 test_tree.TestSetAttr 中，有这样几个带 _set* 的测试：
["testCreateNew", "testAssignNonTreeToExisted", "testAssignTreeToExisted", "testReserveSomeAttrWhenReplaceNode"]。这里 TestSetAttr 虽然是从 TestTree 继承来的，包含 testBaseCase() 方法，但因为 testBaseCase() 是基础，在 setUp() 里面设定，所以应该排除。这样一来，就有 P(4, 4)=24 种组合情况。

为了实现这一点，首先编写一个简单的组合数函数：test/caxes/support.py：

def full_permutate(items):
    if len(items) <=1:
        yield items
    else:
        for P in full_permutate(items[1:]):
            for i in range(len(P) + 1):
                yield P[:i] + items[0:1] + P[i:]

可以定义相应的单元测试 test/caxes/test_support.py：

#!/usr/bin/python
# -*- encoding: utf-8 -*-

__author__ = "Roc Zhou  #周鹏"
__date__ = "12 November 2007"
__version__ = "0.2"
__license__ = "GPL v2.0"

import unittest

from support import full_permutate

class TestPermute(unittest.TestCase):
    def testFullPermutate(self):
        self.failUnlessEqual([ x for x in full_permutate("") ], [""])
        self.failUnlessEqual([ x for x in full_permutate([]) ], [[]])
        self.failUnlessEqual([ x for x in full_permutate(['a']) ], [['a']])
        self.failUnlessEqual([ x for x in full_permutate(['a', 'b']) ], [['a', 'b'], ['b', 'a']])
        result = []
        for x in full_permutate(['a', 'b', 'c']): result.append(x)
        expect = [
            ['a', 'b', 'c'],
            ['a', 'c', 'b'],
            ['b', 'a', 'c'],
            ['b', 'c', 'a'],
            ['c', 'a', 'b'],
            ['c', 'b', 'a']  ]
        result.sort()
        expect.sort()
        self.failUnlessEqual(result, expect)
        result = []
        for x in full_permutate(['a', 'b', 'c', 'd']): result.append(x)
        expect = [
            ['a', 'b', 'c', 'd'],
            ['a', 'b', 'd', 'c'],
            ['a', 'c', 'b', 'd'],
            ['a', 'c', 'd', 'b'],
            ['a', 'd', 'b', 'c'],
            ['a', 'd', 'c', 'b'],
            ['b', 'a', 'c', 'd'],
            ['b', 'a', 'd', 'c'],
            ['b', 'c', 'a', 'd'],
            ['b', 'c', 'd', 'a'],
            ['b', 'd', 'a', 'c'],
            ['b', 'd', 'c', 'a'],
            ['c', 'a', 'b', 'd'],
            ['c', 'a', 'd', 'b'],
            ['c', 'b', 'a', 'd'],
            ['c', 'b', 'd', 'a'],
            ['c', 'd', 'a', 'b'],
            ['c', 'd', 'b', 'a'],
            ['d', 'a', 'b', 'c'],
            ['d', 'a', 'c', 'b'],
            ['d', 'b', 'a', 'c'],
            ['d', 'b', 'c', 'a'],
            ['d', 'c', 'a', 'b'],
            ['d', 'c', 'b', 'a']  ]
        result.sort()
        expect.sort()
        self.failUnlessEqual(result, expect)

if __name__ == "__main__":
    unittest.main()

关于 Python 的排列组合，有不少参考资料，讨论的人也很多。例如可以参考：
http://snippets.dzone.com/posts/show/753
http://www.pyzen.cn/subject/2966/
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/474124

然后，在 test_tree.TestTree 中增加这样的定义：

from caxes import support

class TestTree(unittest.TestCase):
    ...
    def _testPrevious(self, test_name):
        self._setToggle = 0
        test_method = getattr(self, test_name)
        test_method()
        self._setToggle = 1

    def testPermutation(self):
        names = []
        for member in inspect.getmembers(self, inspect.ismethod):
            name = member[0]
            if name.startswith("_set"): names.append(name[4:])
        names.remove("BaseCase")
        cond_len = len(names)
        for cond in support.full_permutate(names):
            self.tearDown()
            self.setUp()
            previous_names = ["testBaseCase"]
            for i in range(cond_len):
                test_name = "test%s" % cond[0]
                testMethod = getattr(self, test_name)
                testMethod()
                for pre_test_name in previous_names:
                    self._testPrevious(pre_test_name)
                previous_names.append(test_name)

...
if __name__ == "__main__":
    # #2:
    import __main__
    suite = unittest.TestLoader().loadTestsFromModule(__main__)
    unittest.TextTestRunner(verbosity=2).run(suite

这样运行时可以的。但是有一个问题，在后面的 TestKeyIndex 测试中，_set* 方法有 6 个，则 P(6, 6)=720 中排列，这意味着需要花比较长的时间才能完成一次测试，在一台 2.4GHz*2 Xeon, 1G MEM 的主机上，花费了 ~45s。

《Pragmatic Unit Testing》中提到要讲耗时的测试隔离。那么在这里具体应该怎么做呢。设定条件从 tests list 删除实际上不太现实：

suite = unittest.TestLoader().loadTestsFromTestCase(TestKeyIndex)
# for test in suite: print test
# suite._tests.remove(TestKeyIndex("testUnordered"))

remove() 最终会抛出 IndexError。而且这样又会破坏正交性。

也许有很多解法。目前我使用一个类 TestTreeComplex 从 TestTree 集成，将 testPermutation() 只定义到 TestTreeComplex 中，同时定义 TestSetAttrComplex 和 TestKeyIndexComplex 从 TestTreeComplex 和 TestSetAttr/TestKeyIndex 双重集成：

sh# vi test/test_tree_complex.py
#!/usr/bin/python
# -*- encoding: utf-8 -*-

__author__ = "Roc Zhou  #周鹏"
__date__ = "13 November 2007"
__version__ = "0.2"
__license__ = "GPL v2.0"

import unittest
import inspect

from caxes import support
import test_tree

import tree
from tree import Tree,TreeExc,TreeTypeExc,TreePathConvExc

class TestTreeComplex(test_tree.TestTree):
    def _testPrevious(self, test_name):
        self._setToggle = 0
        test_method = getattr(self, test_name)
        test_method()
        self._setToggle = 1

    def testPermutation(self):
        names = []
        for member in inspect.getmembers(self, inspect.ismethod):
            name = member[0]
            if name.startswith("_set"): names.append(name[4:])
        names.remove("BaseCase")
        cond_len = len(names)
        for cond in support.full_permutate(names):
            self.tearDown()
            self.setUp()
            previous_names = ["testBaseCase"]
            for i in range(cond_len):
                test_name = "test%s" % cond[0]
                testMethod = getattr(self, test_name)
                testMethod()
                for pre_test_name in previous_names:
                    self._testPrevious(pre_test_name)
                previous_names.append(test_name)

class TestSetAttrComplex(TestTreeComplex, test_tree.TestSetAttr):
    pass

class TestKeyIndexComplex(TestTreeComplex, test_tree.TestKeyIndex):
    pass

if __name__ == "__main__":
    suite = unittest.TestSuite()
    suite.addTest(TestSetAttrComplex("testPermutation"))
    suite.addTest(TestKeyIndexComplex("testPermutation"))
    unittest.TextTestRunner(verbosity=2).run(suite)

这里 TestSetAttrComplex 和 TestKeyIndexComplex 实际上不需要定义任何其他方法和属性。这样可以有效隔离耗时测试，将其放到项目自动化构建的每时或每日构建中。

__package__ = "caxes"
__revision__ = [258:~]