py3: conditionalize test-demandimport.py for Python 3

py3: conditionalize test-demandimport.py for Python 3

The Python 3 lazy importer uses the LazyLoader that is part of
importlib.

On Python 3 and later, LazyLoader is implemented using a custom module
type that defines a __getattribute__ which triggers module loading.

Furthermore, there are additional differences as well. For example,
it appears that Python 3 will return an existing sys.modules
entry instead of constructing a new module object.

This commit adds additional test coverage for lazy importing
behavior to cover the differences between Python 2 and 3. This
reveals that the test and some lazy import functionality is kinda
busted. For example, the test assumes "contextlib" will be lazy.
But in reality an import before it has already imported contextlib!

There's definitely room to improve the behavior of the demand
importer code, both for Python 2 and 3. But at least the test
passes on Python 3 now.

Differential Revision: https://phab.mercurial-scm.org/D5796

--- a/tests/test-demandimport.py	Fri Feb 01 12:09:05 2019 -0800
+++ b/tests/test-demandimport.py	Fri Feb 01 16:47:29 2019 -0800
@@ -6,6 +6,10 @@
 import os
 import subprocess
 import sys
+import types
+
+# Don't import pycompat because it has too many side-effects.
+ispy3 = sys.version_info[0] >= 3
 
 # Only run if demandimport is allowed
 if subprocess.call(['python', '%s/hghave' % os.environ['TESTDIR'],
@@ -16,6 +20,16 @@
 if sys.flags.optimize:
     sys.exit(80)
 
+if ispy3:
+    from importlib.util import _LazyModule
+
+    try:
+        from importlib.util import _Module as moduletype
+    except ImportError:
+        moduletype = types.ModuleType
+else:
+    moduletype = types.ModuleType
+
 if os.name != 'nt':
     try:
         import distutils.msvc9compiler
@@ -43,15 +57,26 @@
 
 # We use assert instead of a unittest test case because having imports inside
 # functions changes behavior of the demand importer.
-assert f(node) == "<module 'mercurial.node' from '?'>", f(node)
+if ispy3:
+    assert not isinstance(node, _LazyModule)
+else:
+    assert f(node) == "<module 'mercurial.node' from '?'>", f(node)
 
 # now enable it for real
 del os.environ['HGDEMANDIMPORT']
 demandimport.enable()
 
 # Test access to special attributes through demandmod proxy
+assert 'mercurial.error' not in sys.modules
 from mercurial import error as errorproxy
-assert f(errorproxy) == "<unloaded module 'error'>", f(errorproxy)
+
+if ispy3:
+    # unsure why this isn't lazy.
+    assert not isinstance(f, _LazyModule)
+    assert f(errorproxy) == "<module 'mercurial.error' from '?'>", f(errorproxy)
+else:
+    assert f(errorproxy) == "<unloaded module 'error'>", f(errorproxy)
+
 doc = ' '.join(errorproxy.__doc__.split()[:3])
 assert doc == 'Mercurial exceptions. This', doc
 assert errorproxy.__name__ == 'mercurial.error', errorproxy.__name__
@@ -61,50 +86,122 @@
 name = errorproxy.__dict__['__name__']
 assert name == 'mercurial.error', name
 
-assert f(errorproxy) == "<proxied module 'error'>", f(errorproxy)
+if ispy3:
+    assert not isinstance(errorproxy, _LazyModule)
+    assert f(errorproxy) == "<module 'mercurial.error' from '?'>", f(errorproxy)
+else:
+    assert f(errorproxy) == "<proxied module 'error'>", f(errorproxy)
 
 import os
 
-assert f(os) == "<unloaded module 'os'>", f(os)
+if ispy3:
+    assert not isinstance(os, _LazyModule)
+    assert f(os) == "<module 'os' from '?'>", f(os)
+else:
+    assert f(os) == "<unloaded module 'os'>", f(os)
+
 assert f(os.system) == '<built-in function system>', f(os.system)
 assert f(os) == "<module 'os' from '?'>", f(os)
 
+assert 'mercurial.utils.procutil' not in sys.modules
 from mercurial.utils import procutil
 
-assert f(procutil) == "<unloaded module 'procutil'>", f(procutil)
+if ispy3:
+    assert isinstance(procutil, _LazyModule)
+    assert f(procutil) == "<module 'mercurial.utils.procutil' from '?'>", f(
+        procutil
+    )
+else:
+    assert f(procutil) == "<unloaded module 'procutil'>", f(procutil)
+
 assert f(procutil.system) == '<function system at 0x?>', f(procutil.system)
+assert procutil.__class__ == moduletype, procutil.__class__
 assert f(procutil) == "<module 'mercurial.utils.procutil' from '?'>", f(
     procutil
 )
 assert f(procutil.system) == '<function system at 0x?>', f(procutil.system)
 
+assert 'mercurial.hgweb' not in sys.modules
 from mercurial import hgweb
-assert f(hgweb) == "<unloaded module 'hgweb'>", f(hgweb)
-assert f(hgweb.hgweb_mod) == "<unloaded module 'hgweb_mod'>", f(hgweb.hgweb_mod)
+
+if ispy3:
+    assert not isinstance(hgweb, _LazyModule)
+    assert f(hgweb) == "<module 'mercurial.hgweb' from '?'>", f(hgweb)
+    assert isinstance(hgweb.hgweb_mod, _LazyModule)
+    assert (
+        f(hgweb.hgweb_mod) == "<module 'mercurial.hgweb.hgweb_mod' from '?'>"
+    ), f(hgweb.hgweb_mod)
+else:
+    assert f(hgweb) == "<unloaded module 'hgweb'>", f(hgweb)
+    assert f(hgweb.hgweb_mod) == "<unloaded module 'hgweb_mod'>", f(
+        hgweb.hgweb_mod
+    )
+
 assert f(hgweb) == "<module 'mercurial.hgweb' from '?'>", f(hgweb)
 
 import re as fred
-assert f(fred) == "<unloaded module 're'>", f(fred)
+
+if ispy3:
+    assert not isinstance(fred, _LazyModule)
+    assert f(fred) == "<module 're' from '?'>"
+else:
+    assert f(fred) == "<unloaded module 're'>", f(fred)
 
 import re as remod
-assert f(remod) == "<unloaded module 're'>", f(remod)
+
+if ispy3:
+    assert not isinstance(remod, _LazyModule)
+    assert f(remod) == "<module 're' from '?'>"
+else:
+    assert f(remod) == "<unloaded module 're'>", f(remod)
 
 import sys as re
-assert f(re) == "<unloaded module 'sys'>", f(re)
+
+if ispy3:
+    assert not isinstance(re, _LazyModule)
+    assert f(re) == "<module 'sys' (built-in)>"
+else:
+    assert f(re) == "<unloaded module 'sys'>", f(re)
 
-assert f(fred) == "<unloaded module 're'>", f(fred)
+if ispy3:
+    assert not isinstance(fred, _LazyModule)
+    assert f(fred) == "<module 're' from '?'>", f(fred)
+else:
+    assert f(fred) == "<unloaded module 're'>", f(fred)
+
 assert f(fred.sub) == '<function sub at 0x?>', f(fred.sub)
-assert f(fred) == "<proxied module 're'>", f(fred)
+
+if ispy3:
+    assert not isinstance(fred, _LazyModule)
+    assert f(fred) == "<module 're' from '?'>", f(fred)
+else:
+    assert f(fred) == "<proxied module 're'>", f(fred)
 
 remod.escape  # use remod
 assert f(remod) == "<module 're' from '?'>", f(remod)
 
-assert f(re) == "<unloaded module 'sys'>", f(re)
-assert f(re.stderr) == "<open file '<whatever>', mode 'w' at 0x?>", f(re.stderr)
-assert f(re) == "<proxied module 'sys'>", f(re)
+if ispy3:
+    assert not isinstance(re, _LazyModule)
+    assert f(re) == "<module 'sys' (built-in)>"
+    assert f(type(re.stderr)) == "<class '_io.TextIOWrapper'>", f(
+        type(re.stderr)
+    )
+    assert f(re) == "<module 'sys' (built-in)>"
+else:
+    assert f(re) == "<unloaded module 'sys'>", f(re)
+    assert f(re.stderr) == "<open file '<whatever>', mode 'w' at 0x?>", f(
+        re.stderr
+    )
+    assert f(re) == "<proxied module 'sys'>", f(re)
 
 import contextlib
-assert f(contextlib) == "<unloaded module 'contextlib'>", f(contextlib)
+
+if ispy3:
+    assert not isinstance(contextlib, _LazyModule)
+    assert f(contextlib) == "<module 'contextlib' from '?'>"
+else:
+    assert f(contextlib) == "<unloaded module 'contextlib'>", f(contextlib)
+
 try:
     from contextlib import unknownattr
 
@@ -113,7 +210,9 @@
         'module:\ncontextlib.unknownattr = %s' % f(unknownattr)
     )
 except ImportError as inst:
-    assert rsub(r"'", '', str(inst)) == 'cannot import name unknownattr'
+    assert rsub(r"'", '', str(inst)).startswith(
+        'cannot import name unknownattr'
+    )
 
 from mercurial import util