gh-157144: Let unittest discovery and regrtest find tests inside archives on sys.path

Doc/library/unittest.rst

@@ -2027,6 +2027,10 @@ Loading and running tests
       .. versionchanged:: 3.14
          *start_dir* can once again be a :term:`namespace package`.
 
+      .. versionchanged:: next
+         *start_dir* can be inside an archive on :data:`sys.path`, such as
+         a zip file, whose path entry finder lists its contents.
+
    The following attributes of a :class:`TestLoader` can be configured either by
    subclassing or assignment on an instance:
 

Lib/test/libregrtest/findtests.py

@@ -37,13 +37,24 @@ def findtestdir(path: StrPath | None = None) -> StrPath:
     return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir
 
 
+def _listdir(testdir: StrPath) -> list[str]:
+    if not os.path.isdir(testdir):
+        # The test package may be inside an archive on sys.path, such as a
+        # zip file, which the import system can still list.
+        import pkgutil
+        if pkgutil.get_importer(testdir) is not None:
+            return [name if ispkg else f"{name}.py"
+                    for _, name, ispkg in pkgutil.iter_modules([testdir])]
+    return os.listdir(testdir)
+
+
 def findtests(*, testdir: StrPath | None = None, exclude: Container[str] = (),
               split_test_dirs: set[TestName] = SPLITTESTDIRS,
               base_mod: str = "") -> TestList:
     """Return a list of all applicable test modules."""
     testdir = findtestdir(testdir)
     tests = []
-    for name in os.listdir(testdir):
+    for name in _listdir(testdir):
         mod, ext = os.path.splitext(name)
         if (not mod.startswith("test_")) or (mod in exclude):
             continue

Lib/test/test_regrtest.py

@@ -24,9 +24,11 @@
 import textwrap
 import unittest
 import unittest.mock
+import zipfile
+import zipimport
 from xml.etree import ElementTree
 
-from test.libregrtest.findtests import collect_cases
+from test.libregrtest.findtests import collect_cases, findtests
 from test.libregrtest.filter import set_match_tests
 from test.libregrtest.run_workers import MultiprocessIterator
 from test import support
@@ -2638,6 +2640,28 @@ def test_collect_cases_skiptest(self):
         self.assertEqual(cases_by_module, {})
         self.assertIn(testname, skipped)
 
+    def test_findtests_not_on_file_system(self):
+        # gh-157144: the test directory may not be on the file system,
+        # e.g. the standard library is a zip archive on sys.path.
+        zip_path = os.path.join(self.tmptestdir, 'tests.zip')
+        with zipfile.ZipFile(zip_path, 'w') as zf:
+            zf.writestr('tests/test_a.py', '')
+            zf.writestr('tests/test_b.py', '')
+            zf.writestr('tests/not_a_test.py', '')
+            zf.writestr('tests/test_data.txt', '')
+            zf.writestr('tests/test_pkg/__init__.py', '')
+            zf.writestr('tests/test_pkg/test_c.py', '')
+        testdir = os.path.join(zip_path, 'tests')
+        self.addCleanup(zipimport._zip_directory_cache.pop, zip_path, None)
+        for path in (testdir, os.path.join(testdir, 'test_pkg')):
+            self.addCleanup(sys.path_importer_cache.pop, path, None)
+
+        tests = findtests(testdir=testdir)
+        self.assertEqual(tests, ['test_a', 'test_b', 'test_pkg'])
+
+        tests = findtests(testdir=testdir, split_test_dirs={'test_pkg'})
+        self.assertEqual(tests, ['test.test_pkg.test_c', 'test_a', 'test_b'])
+
 
 class MultiprocessIteratorTestCase(unittest.TestCase):
     def test_yields_all_groups_once(self):

Lib/test/test_support.py

@@ -16,7 +16,10 @@
 import tempfile
 import textwrap
 import unittest
+import unittest.mock
 import warnings
+import zipfile
+import zipimport
 
 from test import support
 from test.support import isolation
@@ -381,6 +384,53 @@ def test_DirsOnSysPath(self):
         self.assertNotIn("foo", sys.path)
         self.assertNotIn("bar", sys.path)
 
+    def test_load_package_tests_not_on_file_system(self):
+        # gh-157144: load_package_tests() loads the tests of a package
+        # which is not on the file system, e.g. inside a zip archive.
+        package = textwrap.dedent('''
+            import os
+            from test.support import load_package_tests
+
+            def load_tests(*args):
+                return load_package_tests(os.path.dirname(__file__), *args)
+        ''')
+        test_module = textwrap.dedent('''
+            import unittest
+
+            class Tests(unittest.TestCase):
+                def test_zip(self):
+                    pass
+        ''')
+        tmpdir = self.enterContext(os_helper.temp_dir())
+        zip_path = os.path.join(tmpdir, 'zpkg.zip')
+        with zipfile.ZipFile(zip_path, 'w') as zf:
+            zf.writestr('zpkg/__init__.py', package)
+            zf.writestr('zpkg/test_x.py', test_module)
+            # Does not match the default "test*" pattern.
+            zf.writestr('zpkg/other.py', test_module)
+        self.enterContext(import_helper.DirsOnSysPath(zip_path))
+        # As when the standard library itself is a zip archive.
+        self.enterContext(unittest.mock.patch.object(
+            support, 'STDLIB_DIR', zip_path))
+        self.addCleanup(zipimport._zip_directory_cache.pop, zip_path, None)
+        for path in (zip_path, os.path.join(zip_path, 'zpkg')):
+            self.addCleanup(sys.path_importer_cache.pop, path, None)
+        for name in ('zpkg', 'zpkg.test_x', 'zpkg.other'):
+            self.addCleanup(import_helper.unload, name)
+
+        zpkg = importlib.import_module('zpkg')
+        loader = unittest.TestLoader()
+        suite = loader.loadTestsFromModule(zpkg)
+        self.assertEqual(loader.errors, [])
+        self.assertEqual(suite.countTestCases(), 1)
+        self.assertIn('zpkg.test_x', sys.modules)
+        self.assertNotIn('zpkg.other', sys.modules)
+
+        result = unittest.TestResult()
+        suite.run(result)
+        self.assertEqual(result.testsRun, 1)
+        self.assertTrue(result.wasSuccessful(), result.errors)
+
     def test_captured_stdout(self):
         with support.captured_stdout() as stdout:
             print("hello")

Lib/test/test_unittest/test_discovery.py

@@ -7,9 +7,12 @@
 from importlib._bootstrap_external import NamespaceLoader
 from test import support
 from test.support import import_helper
+from test.support import os_helper
 
 import unittest
 import unittest.mock
+import zipfile
+import zipimport
 import test.test_unittest
 from test.test_importlib import util as test_util
 
@@ -920,5 +923,109 @@ def _import(packagename, *args, **kwargs):
                                      .format(package))
 
 
+class TestDiscoveryInArchive(unittest.TestCase):
+    """Discovery of tests inside a zip archive on sys.path (gh-157144)."""
+
+    TEST_MODULE = """\
+import unittest
+
+class Test%s(unittest.TestCase):
+    def test_%s(self):
+        pass
+"""
+
+    def setUp(self):
+        self.tmpdir = self.enterContext(os_helper.temp_dir())
+        self.zip_path = os.path.join(self.tmpdir, 'tests.zip')
+        with zipfile.ZipFile(self.zip_path, 'w') as zf:
+            zf.writestr('zpkg/__init__.py', '')
+            zf.writestr('zpkg/test_a.py', self.TEST_MODULE % ('A', 'a'))
+            zf.writestr('zpkg/test_b.py', self.TEST_MODULE % ('B', 'b'))
+            # Does not match the pattern.
+            zf.writestr('zpkg/other.py', self.TEST_MODULE % ('X', 'x'))
+            # A sub-package is recursed into.
+            zf.writestr('zpkg/sub/__init__.py', '')
+            zf.writestr('zpkg/sub/test_c.py', self.TEST_MODULE % ('C', 'c'))
+            # A sub-package with load_tests() is not recursed into.
+            zf.writestr('zpkg/loaded/__init__.py',
+                        'def load_tests(loader, tests, pattern):\n'
+                        '    return tests\n')
+            zf.writestr('zpkg/loaded/test_d.py', self.TEST_MODULE % ('D', 'd'))
+            # A directory which is not a package is ignored.
+            zf.writestr('zpkg/data/test_e.py', self.TEST_MODULE % ('E', 'e'))
+        self.enterContext(import_helper.DirsOnSysPath())
+        self.addCleanup(self.forget_archive)
+
+    def forget_archive(self):
+        for name in list(sys.modules):
+            if name == 'zpkg' or name.startswith('zpkg.'):
+                del sys.modules[name]
+        for path in list(sys.path_importer_cache):
+            if path.startswith(self.zip_path):
+                del sys.path_importer_cache[path]
+        zipimport._zip_directory_cache.pop(self.zip_path, None)
+
+    def discover(self, start_dir, **kwargs):
+        loader = unittest.TestLoader()
+        suite = loader.discover(start_dir, **kwargs)
+        self.assertEqual(loader.errors, [])
+        return suite
+
+    def suite_ids(self, suite):
+        ids = []
+        for test in suite:
+            if isinstance(test, unittest.TestSuite):
+                ids.extend(self.suite_ids(test))
+            else:
+                ids.append(test.id())
+        return sorted(ids)
+
+    def test_discover_package_in_archive(self):
+        suite = self.discover(os.path.join(self.zip_path, 'zpkg'),
+                              top_level_dir=self.zip_path)
+        self.assertEqual(self.suite_ids(suite), [
+            'zpkg.sub.test_c.TestC.test_c',
+            'zpkg.test_a.TestA.test_a',
+            'zpkg.test_b.TestB.test_b',
+        ])
+        self.assertNotIn('zpkg.other', sys.modules)
+        self.assertNotIn('zpkg.loaded.test_d', sys.modules)
+        result = unittest.TestResult()
+        suite.run(result)
+        self.assertEqual(result.testsRun, 3)
+        self.assertTrue(result.wasSuccessful(), result.errors)
+
+    def test_discover_archive_root(self):
+        suite = self.discover(self.zip_path)
+        self.assertEqual(self.suite_ids(suite), [
+            'zpkg.sub.test_c.TestC.test_c',
+            'zpkg.test_a.TestA.test_a',
+            'zpkg.test_b.TestB.test_b',
+        ])
+
+    def test_discover_pattern(self):
+        suite = self.discover(os.path.join(self.zip_path, 'zpkg'),
+                              pattern='test_[ac]*',
+                              top_level_dir=self.zip_path)
+        self.assertEqual(self.suite_ids(suite), [
+            'zpkg.sub.test_c.TestC.test_c',
+            'zpkg.test_a.TestA.test_a',
+        ])
+
+    def test_discover_from_dotted_name_in_archive(self):
+        sys.path.insert(0, self.zip_path)
+        suite = self.discover('zpkg.sub')
+        self.assertEqual(self.suite_ids(suite),
+                         ['zpkg.sub.test_c.TestC.test_c'])
+
+    def test_discover_not_a_package(self):
+        loader = unittest.TestLoader()
+        for start_dir in ('zpkg/data', 'zpkg/nonexistent'):
+            with self.subTest(start_dir=start_dir):
+                with self.assertRaisesRegex(ImportError, 'not importable'):
+                    loader.discover(os.path.join(self.zip_path, start_dir),
+                                    top_level_dir=self.zip_path)
+
+
 if __name__ == '__main__':
     unittest.main()