Mercurial > hg
changeset 42806:50e25f30da9c
merge with stable
| author | Augie Fackler <augie@google.com> |
|---|---|
| date | Fri, 23 Aug 2019 17:03:42 -0400 |
| parents | 7b80ad5af239 (diff) 7521e6d18057 (current diff) |
| children | 383fdfa6bba9 |
| files | |
| diffstat | 90 files changed, 4529 insertions(+), 1077 deletions(-) [+] |
line wrap: on
line diff
--- a/contrib/automation/hgautomation/aws.py Wed Aug 21 17:56:50 2019 +0200 +++ b/contrib/automation/hgautomation/aws.py Fri Aug 23 17:03:42 2019 -0400 @@ -970,7 +970,7 @@ 'DeviceName': image.block_device_mappings[0]['DeviceName'], 'Ebs': { 'DeleteOnTermination': True, - 'VolumeSize': 8, + 'VolumeSize': 12, 'VolumeType': 'gp2', }, }
--- a/contrib/automation/hgautomation/linux.py Wed Aug 21 17:56:50 2019 +0200 +++ b/contrib/automation/hgautomation/linux.py Fri Aug 23 17:03:42 2019 -0400 @@ -28,11 +28,11 @@ INSTALL_PYTHONS = r''' PYENV2_VERSIONS="2.7.16 pypy2.7-7.1.1" -PYENV3_VERSIONS="3.5.7 3.6.8 3.7.3 3.8-dev pypy3.5-7.0.0 pypy3.6-7.1.1" +PYENV3_VERSIONS="3.5.7 3.6.9 3.7.4 3.8-dev pypy3.5-7.0.0 pypy3.6-7.1.1" git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv pushd /hgdev/pyenv -git checkout 3faeda67bb33e07750d1a104271369a7384ca45c +git checkout 17f44b7cd6f58ea2fa68ec0371fb9e7a826b8be2 popd export PYENV_ROOT="/hgdev/pyenv" @@ -65,6 +65,18 @@ '''.lstrip().replace('\r\n', '\n') +INSTALL_RUST = r''' +RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076 +wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init +echo "${RUSTUP_INIT_SHA256} rustup-init" | sha256sum --check - + +chmod +x rustup-init +sudo -H -u hg -g hg ./rustup-init -y +sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup install 1.31.1 1.34.2 +sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup component add clippy +''' + + BOOTSTRAP_VIRTUALENV = r''' /usr/bin/virtualenv /hgdev/venv-bootstrap @@ -286,6 +298,8 @@ # Will be normalized to hg:hg later. sudo chown `whoami` /hgdev +{install_rust} + cp requirements-py2.txt /hgdev/requirements-py2.txt cp requirements-py3.txt /hgdev/requirements-py3.txt @@ -309,6 +323,7 @@ sudo chown -R hg:hg /hgdev '''.lstrip().format( + install_rust=INSTALL_RUST, install_pythons=INSTALL_PYTHONS, bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV ).replace('\r\n', '\n')
--- a/contrib/automation/linux-requirements-py2.txt Wed Aug 21 17:56:50 2019 +0200 +++ b/contrib/automation/linux-requirements-py2.txt Fri Aug 23 17:03:42 2019 -0400 @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -U --generate-hashes --output-file contrib/automation/linux-requirements-py2.txt contrib/automation/linux-requirements.txt.in +# pip-compile --generate-hashes --output-file=contrib/automation/linux-requirements-py2.txt contrib/automation/linux-requirements.txt.in # astroid==1.6.6 \ --hash=sha256:87de48a92e29cedf7210ffa853d11441e7ad94cb47bacd91b023499b51cbc756 \ @@ -22,10 +22,10 @@ --hash=sha256:509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48 \ --hash=sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00 \ # via vcrpy -docutils==0.14 \ - --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \ - --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \ - --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 +docutils==0.15.2 \ + --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \ + --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \ + --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 enum34==1.1.6 \ --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \ --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ @@ -36,83 +36,70 @@ --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \ # via mock -futures==3.2.0 \ - --hash=sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265 \ - --hash=sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1 \ +futures==3.3.0 \ + --hash=sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16 \ + --hash=sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794 \ # via isort fuzzywuzzy==0.17.0 \ --hash=sha256:5ac7c0b3f4658d2743aa17da53a55598144edbc5bee3c6863840636e6926f254 \ --hash=sha256:6f49de47db00e1c71d40ad16da42284ac357936fa9b66bea1df63fed07122d62 -isort==4.3.17 \ - --hash=sha256:01cb7e1ca5e6c5b3f235f0385057f70558b70d2f00320208825fa62887292f43 \ - --hash=sha256:268067462aed7eb2a1e237fcb287852f22077de3fb07964e87e00f829eea2d1a \ +isort==4.3.21 \ + --hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \ + --hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd \ # via pylint -lazy-object-proxy==1.3.1 \ - --hash=sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33 \ - --hash=sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39 \ - --hash=sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019 \ - --hash=sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088 \ - --hash=sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b \ - --hash=sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e \ - --hash=sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6 \ - --hash=sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b \ - --hash=sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5 \ - --hash=sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff \ - --hash=sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd \ - --hash=sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7 \ - --hash=sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff \ - --hash=sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d \ - --hash=sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2 \ - --hash=sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35 \ - --hash=sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4 \ - --hash=sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514 \ - --hash=sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252 \ - --hash=sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109 \ - --hash=sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f \ - --hash=sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c \ - --hash=sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92 \ - --hash=sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577 \ - --hash=sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d \ - --hash=sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d \ - --hash=sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f \ - --hash=sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a \ - --hash=sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b \ +lazy-object-proxy==1.4.1 \ + --hash=sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661 \ + --hash=sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f \ + --hash=sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13 \ + --hash=sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821 \ + --hash=sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71 \ + --hash=sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e \ + --hash=sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea \ + --hash=sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229 \ + --hash=sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4 \ + --hash=sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e \ + --hash=sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20 \ + --hash=sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16 \ + --hash=sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b \ + --hash=sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7 \ + --hash=sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c \ + --hash=sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a \ + --hash=sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e \ + --hash=sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1 \ # via astroid mccabe==0.6.1 \ --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \ # via pylint -mock==2.0.0 \ - --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ - --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba \ +mock==3.0.5 \ + --hash=sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3 \ + --hash=sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8 \ # via vcrpy -pbr==5.1.3 \ - --hash=sha256:8257baf496c8522437e8a6cfe0f15e00aedc6c0e0e7c9d55eeeeab31e0853843 \ - --hash=sha256:8c361cc353d988e4f5b998555c88098b9d5964c2e11acf7b0d21925a66bb5824 \ - # via mock pyflakes==2.1.1 \ --hash=sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0 \ --hash=sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2 -pygments==2.3.1 \ - --hash=sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a \ - --hash=sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d -pylint==1.9.4 \ - --hash=sha256:02c2b6d268695a8b64ad61847f92e611e6afcff33fd26c3a2125370c4662905d \ - --hash=sha256:ee1e85575587c5b58ddafa25e1c1b01691ef172e139fc25585e5d3f02451da93 +pygments==2.4.2 \ + --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \ + --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 +pylint==1.9.5 \ + --hash=sha256:367e3d49813d349a905390ac27989eff82ab84958731c5ef0bef867452cfdc42 \ + --hash=sha256:97a42df23d436c70132971d1dcb9efad2fe5c0c6add55b90161e773caf729300 python-levenshtein==0.12.0 \ --hash=sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1 -pyyaml==5.1 \ - --hash=sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c \ - --hash=sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 \ - --hash=sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2 \ - --hash=sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4 \ - --hash=sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad \ - --hash=sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba \ - --hash=sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1 \ - --hash=sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e \ - --hash=sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673 \ - --hash=sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13 \ - --hash=sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19 \ +pyyaml==5.1.2 \ + --hash=sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9 \ + --hash=sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4 \ + --hash=sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8 \ + --hash=sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696 \ + --hash=sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34 \ + --hash=sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9 \ + --hash=sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73 \ + --hash=sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299 \ + --hash=sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b \ + --hash=sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae \ + --hash=sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681 \ + --hash=sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41 \ + --hash=sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8 \ # via vcrpy singledispatch==3.4.0.3 \ --hash=sha256:5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c \ @@ -125,6 +112,10 @@ vcrpy==2.0.1 \ --hash=sha256:127e79cf7b569d071d1bd761b83f7b62b2ce2a2eb63ceca7aa67cba8f2602ea3 \ --hash=sha256:57be64aa8e9883a4117d0b15de28af62275c001abcdb00b6dc2d4406073d9a4f -wrapt==1.11.1 \ - --hash=sha256:4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533 \ +wrapt==1.11.2 \ + --hash=sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1 \ # via astroid, vcrpy + +# WARNING: The following packages were not pinned, but pip requires them to be +# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. +# setuptools==41.0.1 # via python-levenshtein
--- a/contrib/automation/linux-requirements-py3.txt Wed Aug 21 17:56:50 2019 +0200 +++ b/contrib/automation/linux-requirements-py3.txt Fri Aug 23 17:03:42 2019 -0400 @@ -2,16 +2,16 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile -U --generate-hashes --output-file contrib/automation/linux-requirements-py3.txt contrib/automation/linux-requirements.txt.in +# pip-compile --generate-hashes --output-file=contrib/automation/linux-requirements-py3.txt contrib/automation/linux-requirements.txt.in # astroid==2.2.5 \ --hash=sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4 \ --hash=sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4 \ # via pylint -docutils==0.14 \ - --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \ - --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \ - --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 +docutils==0.15.2 \ + --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \ + --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \ + --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 fuzzywuzzy==0.17.0 \ --hash=sha256:5ac7c0b3f4658d2743aa17da53a55598144edbc5bee3c6863840636e6926f254 \ --hash=sha256:6f49de47db00e1c71d40ad16da42284ac357936fa9b66bea1df63fed07122d62 @@ -19,40 +19,29 @@ --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ # via yarl -isort==4.3.17 \ - --hash=sha256:01cb7e1ca5e6c5b3f235f0385057f70558b70d2f00320208825fa62887292f43 \ - --hash=sha256:268067462aed7eb2a1e237fcb287852f22077de3fb07964e87e00f829eea2d1a \ +isort==4.3.21 \ + --hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \ + --hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd \ # via pylint -lazy-object-proxy==1.3.1 \ - --hash=sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33 \ - --hash=sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39 \ - --hash=sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019 \ - --hash=sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088 \ - --hash=sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b \ - --hash=sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e \ - --hash=sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6 \ - --hash=sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b \ - --hash=sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5 \ - --hash=sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff \ - --hash=sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd \ - --hash=sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7 \ - --hash=sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff \ - --hash=sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d \ - --hash=sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2 \ - --hash=sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35 \ - --hash=sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4 \ - --hash=sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514 \ - --hash=sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252 \ - --hash=sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109 \ - --hash=sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f \ - --hash=sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c \ - --hash=sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92 \ - --hash=sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577 \ - --hash=sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d \ - --hash=sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d \ - --hash=sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f \ - --hash=sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a \ - --hash=sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b \ +lazy-object-proxy==1.4.1 \ + --hash=sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661 \ + --hash=sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f \ + --hash=sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13 \ + --hash=sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821 \ + --hash=sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71 \ + --hash=sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e \ + --hash=sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea \ + --hash=sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229 \ + --hash=sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4 \ + --hash=sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e \ + --hash=sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20 \ + --hash=sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16 \ + --hash=sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b \ + --hash=sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7 \ + --hash=sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c \ + --hash=sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a \ + --hash=sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e \ + --hash=sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1 \ # via astroid mccabe==0.6.1 \ --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ @@ -92,57 +81,54 @@ pyflakes==2.1.1 \ --hash=sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0 \ --hash=sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2 -pygments==2.3.1 \ - --hash=sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a \ - --hash=sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d +pygments==2.4.2 \ + --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \ + --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 pylint==2.3.1 \ --hash=sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09 \ --hash=sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1 python-levenshtein==0.12.0 \ --hash=sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1 -pyyaml==5.1 \ - --hash=sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c \ - --hash=sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 \ - --hash=sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2 \ - --hash=sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4 \ - --hash=sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad \ - --hash=sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba \ - --hash=sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1 \ - --hash=sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e \ - --hash=sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673 \ - --hash=sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13 \ - --hash=sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19 \ +pyyaml==5.1.2 \ + --hash=sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9 \ + --hash=sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4 \ + --hash=sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8 \ + --hash=sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696 \ + --hash=sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34 \ + --hash=sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9 \ + --hash=sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73 \ + --hash=sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299 \ + --hash=sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b \ + --hash=sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae \ + --hash=sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681 \ + --hash=sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41 \ + --hash=sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8 \ # via vcrpy six==1.12.0 \ --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ # via astroid, vcrpy -typed-ast==1.3.4 ; python_version >= "3.0" and platform_python_implementation != "PyPy" \ - --hash=sha256:04894d268ba6eab7e093d43107869ad49e7b5ef40d1a94243ea49b352061b200 \ - --hash=sha256:16616ece19daddc586e499a3d2f560302c11f122b9c692bc216e821ae32aa0d0 \ - --hash=sha256:252fdae740964b2d3cdfb3f84dcb4d6247a48a6abe2579e8029ab3be3cdc026c \ - --hash=sha256:2af80a373af123d0b9f44941a46df67ef0ff7a60f95872412a145f4500a7fc99 \ - --hash=sha256:2c88d0a913229a06282b285f42a31e063c3bf9071ff65c5ea4c12acb6977c6a7 \ - --hash=sha256:2ea99c029ebd4b5a308d915cc7fb95b8e1201d60b065450d5d26deb65d3f2bc1 \ - --hash=sha256:3d2e3ab175fc097d2a51c7a0d3fda442f35ebcc93bb1d7bd9b95ad893e44c04d \ - --hash=sha256:4766dd695548a15ee766927bf883fb90c6ac8321be5a60c141f18628fb7f8da8 \ - --hash=sha256:56b6978798502ef66625a2e0f80cf923da64e328da8bbe16c1ff928c70c873de \ - --hash=sha256:5cddb6f8bce14325b2863f9d5ac5c51e07b71b462361fd815d1d7706d3a9d682 \ - --hash=sha256:644ee788222d81555af543b70a1098f2025db38eaa99226f3a75a6854924d4db \ - --hash=sha256:64cf762049fc4775efe6b27161467e76d0ba145862802a65eefc8879086fc6f8 \ - --hash=sha256:68c362848d9fb71d3c3e5f43c09974a0ae319144634e7a47db62f0f2a54a7fa7 \ - --hash=sha256:6c1f3c6f6635e611d58e467bf4371883568f0de9ccc4606f17048142dec14a1f \ - --hash=sha256:b213d4a02eec4ddf622f4d2fbc539f062af3788d1f332f028a2e19c42da53f15 \ - --hash=sha256:bb27d4e7805a7de0e35bd0cb1411bc85f807968b2b0539597a49a23b00a622ae \ - --hash=sha256:c9d414512eaa417aadae7758bc118868cd2396b0e6138c1dd4fda96679c079d3 \ - --hash=sha256:f0937165d1e25477b01081c4763d2d9cdc3b18af69cb259dd4f640c9b900fe5e \ - --hash=sha256:fb96a6e2c11059ecf84e6741a319f93f683e440e341d4489c9b161eca251cf2a \ - --hash=sha256:fc71d2d6ae56a091a8d94f33ec9d0f2001d1cb1db423d8b4355debfe9ce689b7 +typed-ast==1.4.0 ; python_version >= "3.0" and platform_python_implementation != "PyPy" \ + --hash=sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e \ + --hash=sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e \ + --hash=sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0 \ + --hash=sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c \ + --hash=sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631 \ + --hash=sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4 \ + --hash=sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34 \ + --hash=sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b \ + --hash=sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a \ + --hash=sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233 \ + --hash=sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1 \ + --hash=sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36 \ + --hash=sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d \ + --hash=sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a \ + --hash=sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12 vcrpy==2.0.1 \ --hash=sha256:127e79cf7b569d071d1bd761b83f7b62b2ce2a2eb63ceca7aa67cba8f2602ea3 \ --hash=sha256:57be64aa8e9883a4117d0b15de28af62275c001abcdb00b6dc2d4406073d9a4f -wrapt==1.11.1 \ - --hash=sha256:4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533 \ +wrapt==1.11.2 \ + --hash=sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1 \ # via astroid, vcrpy yarl==1.3.0 \ --hash=sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9 \ @@ -157,3 +143,7 @@ --hash=sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8 \ --hash=sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1 \ # via vcrpy + +# WARNING: The following packages were not pinned, but pip requires them to be +# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. +# setuptools==41.0.1 # via python-levenshtein
--- a/contrib/byteify-strings.py Wed Aug 21 17:56:50 2019 +0200 +++ b/contrib/byteify-strings.py Fri Aug 23 17:03:42 2019 -0400 @@ -78,23 +78,69 @@ already been done. """ - st = tokens[j] - if st.type == token.STRING and st.string.startswith(("'", '"')): - sysstrtokens.add(st) + k = j + currtoken = tokens[k] + while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL): + k += 1 + if ( + currtoken.type == token.STRING + and currtoken.string.startswith(("'", '"')) + ): + sysstrtokens.add(currtoken) + try: + currtoken = tokens[k] + except IndexError: + break + + def _isitemaccess(j): + """Assert the next tokens form an item access on `tokens[j]` and that + `tokens[j]` is a name. + """ + try: + return ( + tokens[j].type == token.NAME + and _isop(j + 1, '[') + and tokens[j + 2].type == token.STRING + and _isop(j + 3, ']') + ) + except IndexError: + return False + + def _ismethodcall(j, *methodnames): + """Assert the next tokens form a call to `methodname` with a string + as first argument on `tokens[j]` and that `tokens[j]` is a name. + """ + try: + return ( + tokens[j].type == token.NAME + and _isop(j + 1, '.') + and tokens[j + 2].type == token.NAME + and tokens[j + 2].string in methodnames + and _isop(j + 3, '(') + and tokens[j + 4].type == token.STRING + ) + except IndexError: + return False coldelta = 0 # column increment for new opening parens coloffset = -1 # column offset for the current line (-1: TBD) - parens = [(0, 0, 0)] # stack of (line, end-column, column-offset) + parens = [(0, 0, 0, -1)] # stack of (line, end-column, column-offset, type) + ignorenextline = False # don't transform the next line + insideignoreblock = False # don't transform until turned off for i, t in enumerate(tokens): # Compute the column offset for the current line, such that # the current line will be aligned to the last opening paren # as before. if coloffset < 0: - if t.start[1] == parens[-1][1]: - coloffset = parens[-1][2] - elif t.start[1] + 1 == parens[-1][1]: + lastparen = parens[-1] + if t.start[1] == lastparen[1]: + coloffset = lastparen[2] + elif ( + t.start[1] + 1 == lastparen[1] + and lastparen[3] not in (token.NEWLINE, tokenize.NL) + ): # fix misaligned indent of s/util.Abort/error.Abort/ - coloffset = parens[-1][2] + (parens[-1][1] - t.start[1]) + coloffset = lastparen[2] + (lastparen[1] - t.start[1]) else: coloffset = 0 @@ -103,11 +149,26 @@ yield adjusttokenpos(t, coloffset) coldelta = 0 coloffset = -1 + if not insideignoreblock: + ignorenextline = ( + tokens[i - 1].type == token.COMMENT + and tokens[i - 1].string == "# no-py3-transform" + ) + continue + + if t.type == token.COMMENT: + if t.string == "# py3-transform: off": + insideignoreblock = True + if t.string == "# py3-transform: on": + insideignoreblock = False + + if ignorenextline or insideignoreblock: + yield adjusttokenpos(t, coloffset) continue # Remember the last paren position. if _isop(i, '(', '[', '{'): - parens.append(t.end + (coloffset + coldelta,)) + parens.append(t.end + (coloffset + coldelta, tokens[i + 1].type)) elif _isop(i, ')', ']', '}'): parens.pop() @@ -129,8 +190,10 @@ # components touching docstrings need to handle unicode, # unfortunately. if s[0:3] in ("'''", '"""'): - yield adjusttokenpos(t, coloffset) - continue + # If it's assigned to something, it's not a docstring + if not _isop(i - 1, '='): + yield adjusttokenpos(t, coloffset) + continue # If the first character isn't a quote, it is likely a string # prefixing character (such as 'b', 'u', or 'r'. Ignore. @@ -149,8 +212,10 @@ fn = t.string # *attr() builtins don't accept byte strings to 2nd argument. - if (fn in ('getattr', 'setattr', 'hasattr', 'safehasattr') and - not _isop(i - 1, '.')): + if fn in ( + 'getattr', 'setattr', 'hasattr', 'safehasattr', 'wrapfunction', + 'wrapclass', 'addattr' + ) and (opts['allow-attr-methods'] or not _isop(i - 1, '.')): arg1idx = _findargnofcall(1) if arg1idx is not None: _ensuresysstr(arg1idx) @@ -169,6 +234,12 @@ yield adjusttokenpos(t._replace(string=fn[4:]), coloffset) continue + if t.type == token.NAME and t.string in opts['treat-as-kwargs']: + if _isitemaccess(i): + _ensuresysstr(i + 2) + if _ismethodcall(i, 'get', 'pop', 'setdefault', 'popitem'): + _ensuresysstr(i + 4) + # Looks like "if __name__ == '__main__'". if (t.type == token.NAME and t.string == '__name__' and _isop(i + 1, '==')): @@ -207,14 +278,23 @@ def main(): ap = argparse.ArgumentParser() + ap.add_argument('--version', action='version', + version='Byteify strings 1.0') ap.add_argument('-i', '--inplace', action='store_true', default=False, help='edit files in place') ap.add_argument('--dictiter', action='store_true', default=False, help='rewrite iteritems() and itervalues()'), + ap.add_argument('--allow-attr-methods', action='store_true', + default=False, + help='also handle attr*() when they are methods'), + ap.add_argument('--treat-as-kwargs', nargs="+", default=[], + help="ignore kwargs-like objects"), ap.add_argument('files', metavar='FILE', nargs='+', help='source file') args = ap.parse_args() opts = { 'dictiter': args.dictiter, + 'treat-as-kwargs': set(args.treat_as_kwargs), + 'allow-attr-methods': args.allow_attr_methods, } for fname in args.files: if args.inplace:
--- a/contrib/import-checker.py Wed Aug 21 17:56:50 2019 +0200 +++ b/contrib/import-checker.py Fri Aug 23 17:03:42 2019 -0400 @@ -31,6 +31,7 @@ 'mercurial.node', # for revlog to re-export constant to extensions 'mercurial.revlogutils.constants', + 'mercurial.revlogutils.flagutil', # for cffi modules to re-export pure functions 'mercurial.pure.base85', 'mercurial.pure.bdiff',
--- a/contrib/perf.py Wed Aug 21 17:56:50 2019 +0200 +++ b/contrib/perf.py Fri Aug 23 17:03:42 2019 -0400 @@ -126,16 +126,18 @@ getargspec = pycompat.getargspec # added to module after 4.5 _byteskwargs = pycompat.byteskwargs # since 4.1 (or fbc3f73dc802) _sysstr = pycompat.sysstr # since 4.0 (or 2219f4f82ede) + _bytestr = pycompat.bytestr # since 4.2 (or b70407bd84d5) _xrange = pycompat.xrange # since 4.8 (or 7eba8f83129b) fsencode = pycompat.fsencode # since 3.9 (or f4a5e0e86a7e) if pycompat.ispy3: _maxint = sys.maxsize # per py3 docs for replacing maxint else: _maxint = sys.maxint -except (ImportError, AttributeError): +except (NameError, ImportError, AttributeError): import inspect getargspec = inspect.getargspec _byteskwargs = identity + _bytestr = str fsencode = identity # no py3 support _maxint = sys.maxint # no py3 support _sysstr = lambda x: x # no py3 support @@ -144,12 +146,12 @@ try: # 4.7+ queue = pycompat.queue.Queue -except (AttributeError, ImportError): +except (NameError, AttributeError, ImportError): # <4.7. try: queue = pycompat.queue - except (AttributeError, ImportError): - queue = util.queue + except (NameError, AttributeError, ImportError): + import Queue as queue try: from mercurial import logcmdutil @@ -241,6 +243,37 @@ configitem = mercurial.registrar.configitem(configtable) configitem(b'perf', b'presleep', default=mercurial.configitems.dynamicdefault, + experimental=True, + ) + configitem(b'perf', b'stub', + default=mercurial.configitems.dynamicdefault, + experimental=True, + ) + configitem(b'perf', b'parentscount', + default=mercurial.configitems.dynamicdefault, + experimental=True, + ) + configitem(b'perf', b'all-timing', + default=mercurial.configitems.dynamicdefault, + experimental=True, + ) + configitem(b'perf', b'pre-run', + default=mercurial.configitems.dynamicdefault, + ) + configitem(b'perf', b'profile-benchmark', + default=mercurial.configitems.dynamicdefault, + ) + configitem(b'perf', b'run-limits', + default=mercurial.configitems.dynamicdefault, + experimental=True, + ) +except (ImportError, AttributeError): + pass +except TypeError: + # compatibility fix for a11fd395e83f + # hg version: 5.2 + configitem(b'perf', b'presleep', + default=mercurial.configitems.dynamicdefault, ) configitem(b'perf', b'stub', default=mercurial.configitems.dynamicdefault, @@ -260,8 +293,6 @@ configitem(b'perf', b'run-limits', default=mercurial.configitems.dynamicdefault, ) -except (ImportError, AttributeError): - pass def getlen(ui): if ui.configbool(b"perf", b"stub", False): @@ -352,16 +383,16 @@ % item)) continue try: - time_limit = float(pycompat.sysstr(parts[0])) + time_limit = float(_sysstr(parts[0])) except ValueError as e: ui.warn((b'malformatted run limit entry, %s: %s\n' - % (pycompat.bytestr(e), item))) + % (_bytestr(e), item))) continue try: - run_limit = int(pycompat.sysstr(parts[1])) + run_limit = int(_sysstr(parts[1])) except ValueError as e: ui.warn((b'malformatted run limit entry, %s: %s\n' - % (pycompat.bytestr(e), item))) + % (_bytestr(e), item))) continue limits.append((time_limit, run_limit)) if not limits: @@ -3056,7 +3087,7 @@ def doprogress(): with ui.makeprogress(topic, total=total) as progress: - for i in pycompat.xrange(total): + for i in _xrange(total): progress.increment() timer(doprogress)
--- a/contrib/python3-whitelist Wed Aug 21 17:56:50 2019 +0200 +++ b/contrib/python3-whitelist Fri Aug 23 17:03:42 2019 -0400 @@ -124,6 +124,7 @@ test-convert-hg-sink.t test-convert-hg-source.t test-convert-hg-startrev.t +test-convert-identity.t test-convert-mtn.t test-convert-splicemap.t test-convert-svn-sink.t
--- a/hgext/fix.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/fix.py Fri Aug 23 17:03:42 2019 -0400 @@ -36,6 +36,15 @@ {first} The 1-based line number of the first line in the modified range {last} The 1-based line number of the last line in the modified range +Deleted sections of a file will be ignored by :linerange, because there is no +corresponding line range in the version being fixed. + +By default, tools that set :linerange will only be executed if there is at least +one changed line range. This is meant to prevent accidents like running a code +formatter in such a way that it unexpectedly reformats the whole file. If such a +tool needs to operate on unchanged files, it should set the :skipclean suboption +to false. + The :pattern suboption determines which files will be passed through each configured tool. See :hg:`help patterns` for possible values. If there are file arguments to :hg:`fix`, the intersection of these patterns is used. @@ -102,6 +111,13 @@ mapping fixer tool names to lists of metadata values returned from executions that modified a file. This aggregates the same metadata previously passed to the "postfixfile" hook. + +Fixer tools are run the in repository's root directory. This allows them to read +configuration files from the working copy, or even write to the working copy. +The working copy is not updated to match the revision being fixed. In fact, +several revisions may be fixed in parallel. Writes to the working copy are not +amended into the revision being fixed; fixer tools should always write fixed +file content back to stdout as documented above. """ from __future__ import absolute_import @@ -119,6 +135,7 @@ from mercurial.utils import ( procutil, + stringutil, ) from mercurial import ( @@ -152,10 +169,10 @@ FIXER_ATTRS = { 'command': None, 'linerange': None, - 'fileset': None, 'pattern': None, 'priority': 0, - 'metadata': False, + 'metadata': 'false', + 'skipclean': 'true', } for key, default in FIXER_ATTRS.items(): @@ -233,7 +250,7 @@ for rev, path in items: ctx = repo[rev] olddata = ctx[path].data() - metadata, newdata = fixfile(ui, opts, fixers, ctx, path, + metadata, newdata = fixfile(ui, repo, opts, fixers, ctx, path, basectxs[rev]) # Don't waste memory/time passing unchanged content back, but # produce one result per item either way. @@ -530,7 +547,7 @@ basectxs[rev].add(pctx) return basectxs -def fixfile(ui, opts, fixers, fixctx, path, basectxs): +def fixfile(ui, repo, opts, fixers, fixctx, path, basectxs): """Run any configured fixers that should affect the file in this context Returns the file content that results from applying the fixers in some order @@ -539,21 +556,22 @@ (i.e. they will only avoid lines that are common to all basectxs). A fixer tool's stdout will become the file's new content if and only if it - exits with code zero. + exits with code zero. The fixer tool's working directory is the repository's + root. """ metadata = {} newdata = fixctx[path].data() for fixername, fixer in fixers.iteritems(): if fixer.affects(opts, fixctx, path): - rangesfn = lambda: lineranges(opts, path, basectxs, fixctx, newdata) - command = fixer.command(ui, path, rangesfn) + ranges = lineranges(opts, path, basectxs, fixctx, newdata) + command = fixer.command(ui, path, ranges) if command is None: continue ui.debug('subprocess: %s\n' % (command,)) proc = subprocess.Popen( procutil.tonativestr(command), shell=True, - cwd=procutil.tonativestr(b'/'), + cwd=repo.root, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -702,14 +720,20 @@ for name in fixernames(ui): fixers[name] = Fixer() attrs = ui.configsuboptions('fix', name)[1] - if 'fileset' in attrs and 'pattern' not in attrs: - ui.warn(_('the fix.tool:fileset config name is deprecated; ' - 'please rename it to fix.tool:pattern\n')) - attrs['pattern'] = attrs['fileset'] for key, default in FIXER_ATTRS.items(): setattr(fixers[name], pycompat.sysstr('_' + key), attrs.get(key, default)) fixers[name]._priority = int(fixers[name]._priority) + fixers[name]._metadata = stringutil.parsebool(fixers[name]._metadata) + fixers[name]._skipclean = stringutil.parsebool(fixers[name]._skipclean) + # Don't use a fixer if it has no pattern configured. It would be + # dangerous to let it affect all files. It would be pointless to let it + # affect no files. There is no reasonable subset of files to use as the + # default. + if fixers[name]._pattern is None: + ui.warn( + _('fixer tool has no pattern configuration: %s\n') % (name,)) + del fixers[name] return collections.OrderedDict( sorted(fixers.items(), key=lambda item: item[1]._priority, reverse=True)) @@ -727,13 +751,14 @@ def affects(self, opts, fixctx, path): """Should this fixer run on the file at the given path and context?""" - return scmutil.match(fixctx, [self._pattern], opts)(path) + return (self._pattern is not None and + scmutil.match(fixctx, [self._pattern], opts)(path)) def shouldoutputmetadata(self): """Should the stdout of this fixer start with JSON and a null byte?""" return self._metadata - def command(self, ui, path, rangesfn): + def command(self, ui, path, ranges): """A shell command to use to invoke this fixer on the given file/lines May return None if there is no appropriate command to run for the given @@ -743,8 +768,7 @@ parts = [expand(ui, self._command, {'rootpath': path, 'basename': os.path.basename(path)})] if self._linerange: - ranges = rangesfn() - if not ranges: + if self._skipclean and not ranges: # No line ranges to fix, so don't run the fixer. return None for first, last in ranges:
--- a/hgext/fsmonitor/__init__.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/fsmonitor/__init__.py Fri Aug 23 17:03:42 2019 -0400 @@ -112,6 +112,7 @@ import os import stat import sys +import tempfile import weakref from mercurial.i18n import _ @@ -166,6 +167,7 @@ ) configitem('fsmonitor', 'verbose', default=True, + experimental=True, ) configitem('experimental', 'fsmonitor.transaction_notify', default=False, @@ -175,6 +177,23 @@ # and will disable itself when encountering one of these: _blacklist = ['largefiles', 'eol'] +def debuginstall(ui, fm): + fm.write("fsmonitor-watchman", + _("fsmonitor checking for watchman binary... (%s)\n"), + ui.configpath("fsmonitor", "watchman_exe")) + root = tempfile.mkdtemp() + c = watchmanclient.client(ui, root) + err = None + try: + v = c.command("version") + fm.write("fsmonitor-watchman-version", + _(" watchman binary version %s\n"), v["version"]) + except watchmanclient.Unavailable as e: + err = str(e) + fm.condwrite(err, "fsmonitor-watchman-error", + _(" watchman binary missing or broken: %s\n"), err) + return 1 if err else 0 + def _handleunavailable(ui, state, ex): """Exception handler for Watchman interaction exceptions""" if isinstance(ex, watchmanclient.Unavailable): @@ -780,7 +799,7 @@ return try: - client = watchmanclient.client(repo) + client = watchmanclient.client(repo.ui, repo._root) except Exception as ex: _handleunavailable(ui, fsmonitorstate, ex) return
--- a/hgext/fsmonitor/watchmanclient.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/fsmonitor/watchmanclient.py Fri Aug 23 17:03:42 2019 -0400 @@ -33,12 +33,12 @@ super(WatchmanNoRoot, self).__init__(msg) class client(object): - def __init__(self, repo, timeout=1.0): + def __init__(self, ui, root, timeout=1.0): err = None if not self._user: err = "couldn't get user" warn = True - if self._user in repo.ui.configlist('fsmonitor', 'blacklistusers'): + if self._user in ui.configlist('fsmonitor', 'blacklistusers'): err = 'user %s in blacklist' % self._user warn = False @@ -47,8 +47,8 @@ self._timeout = timeout self._watchmanclient = None - self._root = repo.root - self._ui = repo.ui + self._root = root + self._ui = ui self._firsttime = True def settimeout(self, timeout):
--- a/hgext/largefiles/overrides.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/largefiles/overrides.py Fri Aug 23 17:03:42 2019 -0400 @@ -459,7 +459,7 @@ lfiles = set() for f in actions: splitstandin = lfutil.splitstandin(f) - if splitstandin in p1: + if splitstandin is not None and splitstandin in p1: lfiles.add(splitstandin) elif lfutil.standin(f) in p1: lfiles.add(f)
--- a/hgext/lfs/wrapper.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/lfs/wrapper.py Fri Aug 23 17:03:42 2019 -0400 @@ -169,7 +169,7 @@ # Wrapping may also be applied by remotefilelog def filelogrenamed(orig, self, node): if _islfs(self, node): - rawtext = self._revlog.revision(node, raw=True) + rawtext = self._revlog.rawdata(node) if not rawtext: return False metadata = pointer.deserialize(rawtext) @@ -183,7 +183,7 @@ def filelogsize(orig, self, rev): if _islfs(self, rev=rev): # fast path: use lfs metadata to answer size - rawtext = self._revlog.revision(rev, raw=True) + rawtext = self._revlog.rawdata(rev) metadata = pointer.deserialize(rawtext) return int(metadata['size']) return orig(self, rev)
--- a/hgext/remotefilelog/__init__.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/remotefilelog/__init__.py Fri Aug 23 17:03:42 2019 -0400 @@ -219,7 +219,7 @@ configitem('remotefilelog', 'gcrepack', default=False) configitem('remotefilelog', 'repackonhggc', default=False) -configitem('repack', 'chainorphansbysize', default=True) +configitem('repack', 'chainorphansbysize', default=True, experimental=True) configitem('packs', 'maxpacksize', default=0) configitem('packs', 'maxchainlen', default=1000)
--- a/hgext/remotefilelog/contentstore.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/remotefilelog/contentstore.py Fri Aug 23 17:03:42 2019 -0400 @@ -264,7 +264,7 @@ self._repackstartlinkrev = 0 def get(self, name, node): - return self._revlog(name).revision(node, raw=True) + return self._revlog(name).rawdata(node) def getdelta(self, name, node): revision = self.get(name, node)
--- a/hgext/remotefilelog/fileserverclient.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/remotefilelog/fileserverclient.py Fri Aug 23 17:03:42 2019 -0400 @@ -569,7 +569,7 @@ node = bin(id) rlog = self.repo.file(file) if rlog.flags(node) & revlog.REVIDX_EXTSTORED: - text = rlog.revision(node, raw=True) + text = rlog.rawdata(node) p = _lfsmod.pointer.deserialize(text) oid = p.oid() if not store.has(oid):
--- a/hgext/remotefilelog/remotefilelog.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/remotefilelog/remotefilelog.py Fri Aug 23 17:03:42 2019 -0400 @@ -262,7 +262,7 @@ revision = None delta = self.revdiff(basenode, node) else: - revision = self.revision(node, raw=True) + revision = self.rawdata(node) delta = None yield revlog.revlogrevisiondelta( node=node, @@ -277,8 +277,8 @@ ) def revdiff(self, node1, node2): - return mdiff.textdiff(self.revision(node1, raw=True), - self.revision(node2, raw=True)) + return mdiff.textdiff(self.rawdata(node1), + self.rawdata(node2)) def lookup(self, node): if len(node) == 40: @@ -324,6 +324,9 @@ text, verifyhash = self._processflags(rawtext, flags, 'read') return text + def rawdata(self, node): + return self.revision(node, raw=False) + def _processflags(self, text, flags, operation, raw=False): # mostly copied from hg/mercurial/revlog.py validatehash = True
--- a/hgext/remotefilelog/remotefilelogserver.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/remotefilelog/remotefilelogserver.py Fri Aug 23 17:03:42 2019 -0400 @@ -335,7 +335,7 @@ text = filectx.data() else: # lfs, read raw revision data - text = flog.revision(frev, raw=True) + text = flog.rawdata(frev) repo = filectx._repo
--- a/hgext/remotefilelog/shallowbundle.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/remotefilelog/shallowbundle.py Fri Aug 23 17:03:42 2019 -0400 @@ -124,7 +124,7 @@ def nodechunk(self, revlog, node, prevnode, linknode): prefix = '' if prevnode == nullid: - delta = revlog.revision(node, raw=True) + delta = revlog.rawdata(node) prefix = mdiff.trivialdiffheader(len(delta)) else: # Actually uses remotefilelog.revdiff which works on nodes, not revs @@ -267,7 +267,7 @@ if not available(f, node, f, deltabase): continue - base = fl.revision(deltabase, raw=True) + base = fl.rawdata(deltabase) text = mdiff.patch(base, delta) if not isinstance(text, bytes): text = bytes(text)
--- a/hgext/sqlitestore.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/sqlitestore.py Fri Aug 23 17:03:42 2019 -0400 @@ -90,7 +90,8 @@ # experimental config: storage.sqlite.compression configitem('storage', 'sqlite.compression', - default='zstd' if zstd else 'zlib') + default='zstd' if zstd else 'zlib', + experimental=True) # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should @@ -549,6 +550,9 @@ return fulltext + def rawdata(self, *args, **kwargs): + return self.revision(*args, **kwargs) + def read(self, node): return storageutil.filtermetadata(self.revision(node)) @@ -653,8 +657,7 @@ # patch operation. if baserev != nullrev and self.iscensored(baserev): hlen = struct.calcsize('>lll') - oldlen = len(self.revision(deltabase, raw=True, - _verifyhash=False)) + oldlen = len(self.rawdata(deltabase, _verifyhash=False)) newlen = len(delta) - hlen if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen): @@ -663,7 +666,7 @@ if (not (storeflags & FLAG_CENSORED) and storageutil.deltaiscensored( - delta, baserev, lambda x: len(self.revision(x, raw=True)))): + delta, baserev, lambda x: len(self.rawdata(x)))): storeflags |= FLAG_CENSORED linkrev = linkmapper(linknode) @@ -716,7 +719,7 @@ # This restriction is cargo culted from revlogs and makes no sense for # SQLite, since columns can be resized at will. - if len(tombstone) > len(self.revision(censornode, raw=True)): + if len(tombstone) > len(self.rawdata(censornode)): raise error.Abort(_('censor tombstone must be no longer than ' 'censored data'))
--- a/hgext/transplant.py Wed Aug 21 17:56:50 2019 +0200 +++ b/hgext/transplant.py Fri Aug 23 17:03:42 2019 -0400 @@ -412,6 +412,17 @@ # this is kept only to reduce changes in a patch. pass + def stop(self, ui, repo): + """logic to stop an interrupted transplant""" + if self.canresume(): + startctx = repo['.'] + hg.updaterepo(repo, startctx.node(), overwrite=True) + ui.status(_("stopped the interrupted transplant\n")) + ui.status(_("working directory is now at %s\n") % + startctx.hex()[:12]) + self.unlog() + return 0 + def readseries(self): nodes = [] merges = [] @@ -559,6 +570,7 @@ _('parent to choose when transplanting merge'), _('REV')), ('e', 'edit', False, _('invoke editor on commit messages')), ('', 'log', None, _('append transplant info to log message')), + ('', 'stop', False, _('stop interrupted transplant')), ('c', 'continue', None, _('continue last transplant session ' 'after fixing conflicts')), ('', 'filter', '', @@ -646,6 +658,11 @@ raise error.Abort(_('--continue is incompatible with ' '--branch, --all and --merge')) return + if opts.get('stop'): + if opts.get('branch') or opts.get('all') or opts.get('merge'): + raise error.Abort(_('--stop is incompatible with ' + '--branch, --all and --merge')) + return if not (opts.get('source') or revs or opts.get('merge') or opts.get('branch')): raise error.Abort(_('no source URL, branch revision, or revision ' @@ -675,6 +692,10 @@ if opts.get('continue'): if not tp.canresume(): raise error.Abort(_('no transplant to continue')) + elif opts.get('stop'): + if not tp.canresume(): + raise error.Abort(_('no interrupted transplant found')) + return tp.stop(ui, repo) else: cmdutil.checkunfinished(repo) cmdutil.bailifchanged(repo) @@ -734,6 +755,13 @@ if cleanupfn: cleanupfn() +def continuecmd(ui, repo): + """logic to resume an interrupted transplant using + 'hg continue'""" + with repo.wlock(): + tp = transplanter(ui, repo, {}) + return tp.resume(repo, repo, {}) + revsetpredicate = registrar.revsetpredicate() @revsetpredicate('transplanted([set])') @@ -760,9 +788,10 @@ def extsetup(ui): statemod.addunfinished ( 'transplant', fname='transplant/journal', clearable=True, + continuefunc=continuecmd, statushint=_('To continue: hg transplant --continue\n' - 'To abort: hg update'), - cmdhint=_("use 'hg transplant --continue' or 'hg update' to abort") + 'To stop: hg transplant --stop'), + cmdhint=_("use 'hg transplant --continue' or 'hg transplant --stop'") ) # tell hggettext to extract docstrings from these functions:
--- a/mercurial/bundlerepo.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/bundlerepo.py Fri Aug 23 17:03:42 2019 -0400 @@ -105,8 +105,8 @@ elif rev1 <= self.repotiprev and rev2 <= self.repotiprev: return revlog.revlog.revdiff(self, rev1, rev2) - return mdiff.textdiff(self.revision(rev1, raw=True), - self.revision(rev2, raw=True)) + return mdiff.textdiff(self.rawdata(rev1), + self.rawdata(rev2)) def revision(self, nodeorrev, _df=None, raw=False): """return an uncompressed revision of a given node or revision @@ -146,11 +146,14 @@ self._revisioncache = (node, rev, rawtext) return text + def rawdata(self, nodeorrev, _df=None): + return self.revision(nodeorrev, _df=_df, raw=True) + def baserevision(self, nodeorrev): # Revlog subclasses may override 'revision' method to modify format of # content retrieved from revlog. To use bundlerevlog with such class one # needs to override 'baserevision' and make more specific call here. - return revlog.revlog.revision(self, nodeorrev, raw=True) + return revlog.revlog.rawdata(self, nodeorrev) def addrevision(self, *args, **kwargs): raise NotImplementedError @@ -181,7 +184,7 @@ oldfilter = self.filteredrevs try: self.filteredrevs = () - return changelog.changelog.revision(self, nodeorrev, raw=True) + return changelog.changelog.rawdata(self, nodeorrev) finally: self.filteredrevs = oldfilter @@ -206,7 +209,7 @@ if node in self.fulltextcache: result = '%s' % self.fulltextcache[node] else: - result = manifest.manifestrevlog.revision(self, nodeorrev, raw=True) + result = manifest.manifestrevlog.rawdata(self, nodeorrev) return result def dirlog(self, d): @@ -224,7 +227,7 @@ cgunpacker, linkmapper) def baserevision(self, nodeorrev): - return filelog.filelog.revision(self, nodeorrev, raw=True) + return filelog.filelog.rawdata(self, nodeorrev) class bundlepeer(localrepo.localpeer): def canpush(self):
--- a/mercurial/commands.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/commands.py Fri Aug 23 17:03:42 2019 -0400 @@ -1872,6 +1872,7 @@ for section, name, value in ui.walkconfig(untrusted=untrusted): source = ui.configsource(section, name, untrusted) value = pycompat.bytestr(value) + defaultvalue = ui.configdefault(section, name) if fm.isplain(): source = source or 'none' value = value.replace('\n', '\\n') @@ -1885,6 +1886,7 @@ fm.write('value', '%s\n', value) else: fm.write('name value', '%s=%s\n', entryname, value) + fm.data(defaultvalue=defaultvalue) matched = True fm.end() if matched:
--- a/mercurial/configitems.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/configitems.py Fri Aug 23 17:03:42 2019 -0400 @@ -39,13 +39,14 @@ """ def __init__(self, section, name, default=None, alias=(), - generic=False, priority=0): + generic=False, priority=0, experimental=False): self.section = section self.name = name self.default = default self.alias = list(alias) self.generic = generic self.priority = priority + self.experimental = experimental self._re = None if generic: self._re = re.compile(self.name) @@ -166,6 +167,7 @@ ) coreconfigitem('censor', 'policy', default='abort', + experimental=True, ) coreconfigitem('chgserver', 'idletimeout', default=3600, @@ -184,9 +186,11 @@ ) coreconfigitem('cmdserver', 'max-repo-cache', default=0, + experimental=True, ) coreconfigitem('cmdserver', 'message-encodings', default=list, + experimental=True, ) coreconfigitem('cmdserver', 'track-log', default=lambda: ['chgserver', 'cmdserver', 'repocache'], @@ -207,6 +211,7 @@ ) coreconfigitem('commands', 'grep.all-files', default=False, + experimental=True, ) coreconfigitem('commands', 'resolve.confirm', default=False, @@ -226,6 +231,7 @@ ) coreconfigitem('commands', 'status.skipstates', default=[], + experimental=True, ) coreconfigitem('commands', 'status.terse', default='', @@ -314,6 +320,7 @@ ) coreconfigitem('convert', 'ignoreancestorcheck', default=False, + experimental=True, ) coreconfigitem('convert', 'localtimezone', default=False, @@ -415,6 +422,9 @@ coreconfigitem('devel', 'debug.peer-request', default=False, ) +coreconfigitem('devel', 'discovery.randomize', + default=True, +) _registerdiffopts(section='diff') coreconfigitem('email', 'bcc', default=None, @@ -684,18 +694,22 @@ ) coreconfigitem('format', 'chunkcachesize', default=None, + experimental=True, ) coreconfigitem('format', 'dotencode', default=True, ) coreconfigitem('format', 'generaldelta', default=False, + experimental=True, ) coreconfigitem('format', 'manifestcachesize', default=None, + experimental=True, ) coreconfigitem('format', 'maxchainlen', default=dynamicdefault, + experimental=True, ) coreconfigitem('format', 'obsstore-version', default=None, @@ -718,6 +732,7 @@ ) coreconfigitem('format', 'internal-phase', default=False, + experimental=True, ) coreconfigitem('fsmonitor', 'warn_when_unused', default=True, @@ -823,6 +838,7 @@ ) coreconfigitem('merge', 'preferancestor', default=lambda: ['*'], + experimental=True, ) coreconfigitem('merge', 'strict-capability-check', default=False, @@ -1007,6 +1023,7 @@ ) coreconfigitem('storage', 'new-repo-backend', default='revlogv1', + experimental=True, ) coreconfigitem('storage', 'revlog.optimize-delta-parent-choice', default=True, @@ -1117,6 +1134,7 @@ ) coreconfigitem('sparse', 'missingwarning', default=True, + experimental=True, ) coreconfigitem('subrepos', 'allowed', default=dynamicdefault, # to make backporting simpler @@ -1463,6 +1481,7 @@ ) coreconfigitem('web', 'view', default='served', + experimental=True, ) coreconfigitem('worker', 'backgroundclose', default=dynamicdefault,
--- a/mercurial/context.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/context.py Fri Aug 23 17:03:42 2019 -0400 @@ -24,6 +24,7 @@ wdirhex, ) from . import ( + copies, dagop, encoding, error, @@ -274,23 +275,7 @@ @propertycache def _copies(self): - p1copies = {} - p2copies = {} - p1 = self.p1() - p2 = self.p2() - narrowmatch = self._repo.narrowmatch() - for dst in self.files(): - if not narrowmatch(dst) or dst not in self: - continue - copied = self[dst].renamed() - if not copied: - continue - src, srcnode = copied - if src in p1 and p1[src].filenode() == srcnode: - p1copies[dst] = src - elif src in p2 and p2[src].filenode() == srcnode: - p2copies[dst] = src - return p1copies, p2copies + return copies.computechangesetcopies(self) def p1copies(self): return self._copies[0] def p2copies(self): @@ -474,24 +459,14 @@ (source == 'compatibility' and self._changeset.filesadded is not None)): return self._changeset.filesadded or [] - - added = [] - for f in self.files(): - if not any(f in p for p in self.parents()): - added.append(f) - return added + return scmutil.computechangesetfilesadded(self) def filesremoved(self): source = self._repo.ui.config('experimental', 'copies.read-from') if (source == 'changeset-only' or (source == 'compatibility' and self._changeset.filesremoved is not None)): return self._changeset.filesremoved or [] - - removed = [] - for f in self.files(): - if f not in self: - removed.append(f) - return removed + return scmutil.computechangesetfilesremoved(self) @propertycache def _copies(self): @@ -1078,7 +1053,7 @@ filelog=self._filelog, changeid=changeid) def rawdata(self): - return self._filelog.revision(self._filenode, raw=True) + return self._filelog.rawdata(self._filenode) def rawflags(self): """low-level revlog flags"""
--- a/mercurial/copies.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/copies.py Fri Aug 23 17:03:42 2019 -0400 @@ -809,3 +809,28 @@ continue if dst in wctx: wctx[dst].markcopied(src) + +def computechangesetcopies(ctx): + """return the copies data for a changeset + + The copies data are returned as a pair of dictionnary (p1copies, p2copies). + + Each dictionnary are in the form: `{newname: oldname}` + """ + p1copies = {} + p2copies = {} + p1 = ctx.p1() + p2 = ctx.p2() + narrowmatch = ctx._repo.narrowmatch() + for dst in ctx.files(): + if not narrowmatch(dst) or dst not in ctx: + continue + copied = ctx[dst].renamed() + if not copied: + continue + src, srcnode = copied + if src in p1 and p1[src].filenode() == srcnode: + p1copies[dst] = src + elif src in p2 and p2[src].filenode() == srcnode: + p2copies[dst] = src + return p1copies, p2copies
--- a/mercurial/debugcommands.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/debugcommands.py Fri Aug 23 17:03:42 2019 -0400 @@ -562,7 +562,7 @@ raise error.CommandError('debugdata', _('invalid arguments')) r = cmdutil.openstorage(repo, 'debugdata', file_, opts) try: - ui.write(r.revision(r.lookup(rev), raw=True)) + ui.write(r.rawdata(r.lookup(rev))) except KeyError: raise error.Abort(_('invalid revision identifier %s') % rev) @@ -1383,6 +1383,11 @@ fm.condwrite(err, 'usernameerror', _("checking username...\n %s\n" " (specify a username in your configuration file)\n"), err) + for name, mod in extensions.extensions(): + handler = getattr(mod, 'debuginstall', None) + if handler is not None: + problems += handler(ui, fm) + fm.condwrite(not problems, '', _("no problems detected\n")) if not problems:
--- a/mercurial/dirstate.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/dirstate.py Fri Aug 23 17:03:42 2019 -0400 @@ -28,7 +28,7 @@ ) parsers = policy.importmod(r'parsers') -dirstatemod = policy.importrust(r'dirstate', default=parsers) +rustmod = policy.importrust(r'dirstate') propertycache = util.propertycache filecache = scmutil.filecache @@ -652,7 +652,8 @@ delaywrite = self._ui.configint('debug', 'dirstate.delaywrite') if delaywrite > 0: # do we have any files to delay for? - for f, e in self._map.iteritems(): + items = self._map.iteritems() + for f, e in items: if e[0] == 'n' and e[3] == now: import time # to avoid useless import # rather than sleep n seconds, sleep until the next @@ -663,6 +664,12 @@ time.sleep(end - clock) now = end # trust our estimate that the end is near now break + # since the iterator is potentially not deleted, + # delete the iterator to release the reference for the Rust + # implementation. + # TODO make the Rust implementation behave like Python + # since this would not work with a non ref-counting GC. + del items self._map.write(st, now) self._lastnormaltime = 0 @@ -1475,7 +1482,7 @@ # parsing the dirstate. # # (we cannot decorate the function directly since it is in a C module) - parse_dirstate = util.nogc(dirstatemod.parse_dirstate) + parse_dirstate = util.nogc(parsers.parse_dirstate) p = parse_dirstate(self._map, self.copymap, st) if not self._dirtyparents: self.setparents(*p) @@ -1486,8 +1493,8 @@ self.get = self._map.get def write(self, st, now): - st.write(dirstatemod.pack_dirstate(self._map, self.copymap, - self.parents(), now)) + st.write(parsers.pack_dirstate(self._map, self.copymap, + self.parents(), now)) st.close() self._dirtyparents = False self.nonnormalset, self.otherparentset = self.nonnormalentries() @@ -1516,3 +1523,186 @@ for name in self._dirs: f[normcase(name)] = name return f + + +if rustmod is not None: + class dirstatemap(object): + def __init__(self, ui, opener, root): + self._ui = ui + self._opener = opener + self._root = root + self._filename = 'dirstate' + self._parents = None + self._dirtyparents = False + + # for consistent view between _pl() and _read() invocations + self._pendingmode = None + + def addfile(self, *args, **kwargs): + return self._rustmap.addfile(*args, **kwargs) + + def removefile(self, *args, **kwargs): + return self._rustmap.removefile(*args, **kwargs) + + def dropfile(self, *args, **kwargs): + return self._rustmap.dropfile(*args, **kwargs) + + def clearambiguoustimes(self, *args, **kwargs): + return self._rustmap.clearambiguoustimes(*args, **kwargs) + + def nonnormalentries(self): + return self._rustmap.nonnormalentries() + + def get(self, *args, **kwargs): + return self._rustmap.get(*args, **kwargs) + + @propertycache + def _rustmap(self): + self._rustmap = rustmod.DirstateMap(self._root) + self.read() + return self._rustmap + + @property + def copymap(self): + return self._rustmap.copymap() + + def preload(self): + self._rustmap + + def clear(self): + self._rustmap.clear() + self.setparents(nullid, nullid) + util.clearcachedproperty(self, "_dirs") + util.clearcachedproperty(self, "_alldirs") + util.clearcachedproperty(self, "dirfoldmap") + + def items(self): + return self._rustmap.items() + + def keys(self): + return iter(self._rustmap) + + def __contains__(self, key): + return key in self._rustmap + + def __getitem__(self, item): + return self._rustmap[item] + + def __len__(self): + return len(self._rustmap) + + def __iter__(self): + return iter(self._rustmap) + + # forward for python2,3 compat + iteritems = items + + def _opendirstatefile(self): + fp, mode = txnutil.trypending(self._root, self._opener, + self._filename) + if self._pendingmode is not None and self._pendingmode != mode: + fp.close() + raise error.Abort(_('working directory state may be ' + 'changed parallelly')) + self._pendingmode = mode + return fp + + def setparents(self, p1, p2): + self._rustmap.setparents(p1, p2) + self._parents = (p1, p2) + self._dirtyparents = True + + def parents(self): + if not self._parents: + try: + fp = self._opendirstatefile() + st = fp.read(40) + fp.close() + except IOError as err: + if err.errno != errno.ENOENT: + raise + # File doesn't exist, so the current state is empty + st = '' + + try: + self._parents = self._rustmap.parents(st) + except ValueError: + raise error.Abort(_('working directory state appears ' + 'damaged!')) + + return self._parents + + def read(self): + # ignore HG_PENDING because identity is used only for writing + self.identity = util.filestat.frompath( + self._opener.join(self._filename)) + + try: + fp = self._opendirstatefile() + try: + st = fp.read() + finally: + fp.close() + except IOError as err: + if err.errno != errno.ENOENT: + raise + return + if not st: + return + + parse_dirstate = util.nogc(self._rustmap.read) + parents = parse_dirstate(st) + if parents and not self._dirtyparents: + self.setparents(*parents) + + def write(self, st, now): + parents = self.parents() + st.write(self._rustmap.write(parents[0], parents[1], now)) + st.close() + self._dirtyparents = False + + @propertycache + def filefoldmap(self): + """Returns a dictionary mapping normalized case paths to their + non-normalized versions. + """ + return self._rustmap.filefoldmapasdict() + + def hastrackeddir(self, d): + self._dirs # Trigger Python's propertycache + return self._rustmap.hastrackeddir(d) + + def hasdir(self, d): + self._dirs # Trigger Python's propertycache + return self._rustmap.hasdir(d) + + @propertycache + def _dirs(self): + return self._rustmap.getdirs() + + @propertycache + def _alldirs(self): + return self._rustmap.getalldirs() + + @propertycache + def identity(self): + self._rustmap + return self.identity + + @property + def nonnormalset(self): + nonnorm, otherparents = self._rustmap.nonnormalentries() + return nonnorm + + @property + def otherparentset(self): + nonnorm, otherparents = self._rustmap.nonnormalentries() + return otherparents + + @propertycache + def dirfoldmap(self): + f = {} + normcase = util.normcase + for name in self._dirs: + f[normcase(name)] = name + return f
--- a/mercurial/filelog.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/filelog.py Fri Aug 23 17:03:42 2019 -0400 @@ -90,6 +90,9 @@ def revision(self, node, _df=None, raw=False): return self._revlog.revision(node, _df=_df, raw=raw) + def rawdata(self, node, _df=None): + return self._revlog.rawdata(node, _df=_df) + def emitrevisions(self, nodes, nodesorder=None, revisiondata=False, assumehaveparentrevisions=False, deltamode=repository.CG_DELTAMODE_STD):
--- a/mercurial/localrepo.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/localrepo.py Fri Aug 23 17:03:42 2019 -0400 @@ -1942,6 +1942,12 @@ **pycompat.strkwargs(tr.hookargs)) def releasefn(tr, success): repo = reporef() + if repo is None: + # If the repo has been GC'd (and this release function is being + # called from transaction.__del__), there's not much we can do, + # so just leave the unfinished transaction there and let the + # user run `hg recover`. + return if success: # this should be explicitly invoked here, because # in-memory changes aren't written out at closing @@ -2214,6 +2220,16 @@ self.tags() self.filtered('served').tags() + # The `full` arg is documented as updating even the lazily-loaded + # caches immediately, so we're forcing a write to cause these caches + # to be warmed up even if they haven't explicitly been requested + # yet (if they've never been used by hg, they won't ever have been + # written, even if they're a subset of another kind of cache that + # *has* been used). + for filt in repoview.filtertable.keys(): + filtered = self.filtered(filt) + filtered.branchmap().write(filtered) + def invalidatecaches(self): if r'_tagscache' in vars(self):
--- a/mercurial/manifest.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/manifest.py Fri Aug 23 17:03:42 2019 -0400 @@ -1620,6 +1620,9 @@ def revision(self, node, _df=None, raw=False): return self._revlog.revision(node, _df=_df, raw=raw) + def rawdata(self, node, _df=None): + return self._revlog.rawdata(node, _df=_df) + def revdiff(self, rev1, rev2): return self._revlog.revdiff(rev1, rev2)
--- a/mercurial/match.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/match.py Fri Aug 23 17:03:42 2019 -0400 @@ -25,7 +25,7 @@ stringutil, ) -rustmod = policy.importrust('filepatterns') +rustmod = policy.importrust(r'filepatterns') allpatternkinds = ('re', 'glob', 'path', 'relglob', 'relpath', 'relre', 'rootglob',
--- a/mercurial/merge.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/merge.py Fri Aug 23 17:03:42 2019 -0400 @@ -2025,7 +2025,8 @@ raise error.Abort(_("outstanding uncommitted merge")) ms = mergestate.read(repo) if list(ms.unresolved()): - raise error.Abort(_("outstanding merge conflicts")) + raise error.Abort(_("outstanding merge conflicts"), + hint=_("use 'hg resolve' to resolve")) if branchmerge: if pas == [p2]: raise error.Abort(_("merging with a working directory ancestor"
--- a/mercurial/repository.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/repository.py Fri Aug 23 17:03:42 2019 -0400 @@ -597,6 +597,10 @@ consumers should use ``read()`` to obtain the actual file data. """ + def rawdata(node): + """Obtain raw data for a node. + """ + def read(node): """Resolve file fulltext data. @@ -1164,6 +1168,9 @@ def revision(node, _df=None, raw=False): """Obtain fulltext data for a node.""" + def rawdata(node, _df=None): + """Obtain raw data for a node.""" + def revdiff(rev1, rev2): """Obtain a delta between two revision numbers. @@ -1195,7 +1202,7 @@ def rawsize(rev): """Obtain the size of tracked data. - Is equivalent to ``len(m.revision(node, raw=True))``. + Is equivalent to ``len(m.rawdata(node))``. TODO this method is only used by upgrade code and may be removed. """
--- a/mercurial/revlog.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/revlog.py Fri Aug 23 17:03:42 2019 -0400 @@ -38,13 +38,6 @@ from .revlogutils.constants import ( FLAG_GENERALDELTA, FLAG_INLINE_DATA, - REVIDX_DEFAULT_FLAGS, - REVIDX_ELLIPSIS, - REVIDX_EXTSTORED, - REVIDX_FLAGS_ORDER, - REVIDX_ISCENSORED, - REVIDX_KNOWN_FLAGS, - REVIDX_RAWTEXT_CHANGING_FLAGS, REVLOGV0, REVLOGV1, REVLOGV1_FLAGS, @@ -54,6 +47,14 @@ REVLOG_DEFAULT_FORMAT, REVLOG_DEFAULT_VERSION, ) +from .revlogutils.flagutil import ( + REVIDX_DEFAULT_FLAGS, + REVIDX_ELLIPSIS, + REVIDX_EXTSTORED, + REVIDX_FLAGS_ORDER, + REVIDX_ISCENSORED, + REVIDX_RAWTEXT_CHANGING_FLAGS, +) from .thirdparty import ( attr, ) @@ -70,6 +71,7 @@ ) from .revlogutils import ( deltas as deltautil, + flagutil, ) from .utils import ( interfaceutil, @@ -94,7 +96,6 @@ REVIDX_EXTSTORED REVIDX_DEFAULT_FLAGS REVIDX_FLAGS_ORDER -REVIDX_KNOWN_FLAGS REVIDX_RAWTEXT_CHANGING_FLAGS parsers = policy.importmod(r'parsers') @@ -108,11 +109,6 @@ _maxinline = 131072 _chunksize = 1048576 -# Store flag processors (cf. 'addflagprocessor()' to register) -_flagprocessors = { - REVIDX_ISCENSORED: None, -} - # Flag processors for REVIDX_ELLIPSIS. def ellipsisreadprocessor(rl, text): return text, False @@ -129,45 +125,6 @@ ellipsisrawprocessor, ) -def addflagprocessor(flag, processor): - """Register a flag processor on a revision data flag. - - Invariant: - - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER, - and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext. - - Only one flag processor can be registered on a specific flag. - - flagprocessors must be 3-tuples of functions (read, write, raw) with the - following signatures: - - (read) f(self, rawtext) -> text, bool - - (write) f(self, text) -> rawtext, bool - - (raw) f(self, rawtext) -> bool - "text" is presented to the user. "rawtext" is stored in revlog data, not - directly visible to the user. - The boolean returned by these transforms is used to determine whether - the returned text can be used for hash integrity checking. For example, - if "write" returns False, then "text" is used to generate hash. If - "write" returns True, that basically means "rawtext" returned by "write" - should be used to generate hash. Usually, "write" and "read" return - different booleans. And "raw" returns a same boolean as "write". - - Note: The 'raw' transform is used for changegroup generation and in some - debug commands. In this case the transform only indicates whether the - contents can be used for hash integrity checks. - """ - _insertflagprocessor(flag, processor, _flagprocessors) - -def _insertflagprocessor(flag, processor, flagprocessors): - if not flag & REVIDX_KNOWN_FLAGS: - msg = _("cannot register processor on unknown flag '%#x'.") % (flag) - raise error.ProgrammingError(msg) - if flag not in REVIDX_FLAGS_ORDER: - msg = _("flag '%#x' undefined in REVIDX_FLAGS_ORDER.") % (flag) - raise error.ProgrammingError(msg) - if flag in flagprocessors: - msg = _("cannot register multiple processors on flag '%#x'.") % (flag) - raise error.Abort(msg) - flagprocessors[flag] = processor - def getoffset(q): return int(q >> 16) @@ -175,7 +132,7 @@ return int(q & 0xFFFF) def offset_type(offset, type): - if (type & ~REVIDX_KNOWN_FLAGS) != 0: + if (type & ~flagutil.REVIDX_KNOWN_FLAGS) != 0: raise ValueError('unknown revlog index flags') return int(int(offset) << 16 | type) @@ -384,7 +341,7 @@ # Make copy of flag processors so each revlog instance can support # custom flags. - self._flagprocessors = dict(_flagprocessors) + self._flagprocessors = dict(flagutil.flagprocessors) # 2-tuple of file handles being used for active writing. self._writinghandles = None @@ -442,7 +399,7 @@ # revlog v0 doesn't have flag processors for flag, processor in opts.get(b'flagprocessors', {}).iteritems(): - _insertflagprocessor(flag, processor, self._flagprocessors) + flagutil.insertflagprocessor(flag, processor, self._flagprocessors) if self._chunkcachesize <= 0: raise error.RevlogError(_('revlog chunk cache size %r is not ' @@ -679,7 +636,7 @@ if l >= 0: return l - t = self.revision(rev, raw=True) + t = self.rawdata(rev) return len(t) def size(self, rev): @@ -687,7 +644,7 @@ # fast path: if no "read" flag processor could change the content, # size is rawsize. note: ELLIPSIS is known to not change the content. flags = self.flags(rev) - if flags & (REVIDX_KNOWN_FLAGS ^ REVIDX_ELLIPSIS) == 0: + if flags & (flagutil.REVIDX_KNOWN_FLAGS ^ REVIDX_ELLIPSIS) == 0: return self.rawsize(rev) return len(self.revision(rev, raw=False)) @@ -1639,8 +1596,8 @@ if rev1 != nullrev and self.deltaparent(rev2) == rev1: return bytes(self._chunk(rev2)) - return mdiff.textdiff(self.revision(rev1, raw=True), - self.revision(rev2, raw=True)) + return mdiff.textdiff(self.rawdata(rev1), + self.rawdata(rev2)) def revision(self, nodeorrev, _df=None, raw=False): """return an uncompressed revision of a given node or revision @@ -1651,6 +1608,10 @@ treated as raw data when applying flag transforms. 'raw' should be set to True when generating changegroups or in debug commands. """ + return self._revisiondata(nodeorrev, _df, raw=raw) + + def _revisiondata(self, nodeorrev, _df=None, raw=False): + # deal with <nodeorrev> argument type if isinstance(nodeorrev, int): rev = nodeorrev node = self.node(rev) @@ -1658,65 +1619,88 @@ node = nodeorrev rev = None - cachedrev = None - flags = None - rawtext = None + # fast path the special `nullid` rev if node == nullid: return "" - if self._revisioncache: - if self._revisioncache[0] == node: - # _cache only stores rawtext - if raw: - return self._revisioncache[2] - # duplicated, but good for perf - if rev is None: - rev = self.rev(node) - if flags is None: - flags = self.flags(rev) - # no extra flags set, no flag processor runs, text = rawtext - if flags == REVIDX_DEFAULT_FLAGS: - return self._revisioncache[2] - # rawtext is reusable. need to run flag processor - rawtext = self._revisioncache[2] - - cachedrev = self._revisioncache[1] - - # look up what we need to read - if rawtext is None: - if rev is None: - rev = self.rev(node) - - chain, stopped = self._deltachain(rev, stoprev=cachedrev) - if stopped: - rawtext = self._revisioncache[2] - - # drop cache to save memory - self._revisioncache = None - - targetsize = None - rawsize = self.index[rev][2] - if 0 <= rawsize: - targetsize = 4 * rawsize - - bins = self._chunks(chain, df=_df, targetsize=targetsize) - if rawtext is None: - rawtext = bytes(bins[0]) - bins = bins[1:] - - rawtext = mdiff.patches(rawtext, bins) - self._revisioncache = (node, rev, rawtext) - - if flags is None: - if rev is None: - rev = self.rev(node) - flags = self.flags(rev) + + # The text as stored inside the revlog. Might be the revision or might + # need to be processed to retrieve the revision. + rawtext = None + + rev, rawtext, validated = self._rawtext(node, rev, _df=_df) + + if raw and validated: + # if we don't want to process the raw text and that raw + # text is cached, we can exit early. + return rawtext + if rev is None: + rev = self.rev(node) + # the revlog's flag for this revision + # (usually alter its state or content) + flags = self.flags(rev) + + if validated and flags == REVIDX_DEFAULT_FLAGS: + # no extra flags set, no flag processor runs, text = rawtext + return rawtext text, validatehash = self._processflags(rawtext, flags, 'read', raw=raw) if validatehash: self.checkhash(text, node, rev=rev) + if not validated: + self._revisioncache = (node, rev, rawtext) return text + def _rawtext(self, node, rev, _df=None): + """return the possibly unvalidated rawtext for a revision + + returns (rev, rawtext, validated) + """ + + # revision in the cache (could be useful to apply delta) + cachedrev = None + # An intermediate text to apply deltas to + basetext = None + + # Check if we have the entry in cache + # The cache entry looks like (node, rev, rawtext) + if self._revisioncache: + if self._revisioncache[0] == node: + return (rev, self._revisioncache[2], True) + cachedrev = self._revisioncache[1] + + if rev is None: + rev = self.rev(node) + + chain, stopped = self._deltachain(rev, stoprev=cachedrev) + if stopped: + basetext = self._revisioncache[2] + + # drop cache to save memory, the caller is expected to + # update self._revisioncache after validating the text + self._revisioncache = None + + targetsize = None + rawsize = self.index[rev][2] + if 0 <= rawsize: + targetsize = 4 * rawsize + + bins = self._chunks(chain, df=_df, targetsize=targetsize) + if basetext is None: + basetext = bytes(bins[0]) + bins = bins[1:] + + rawtext = mdiff.patches(basetext, bins) + del basetext # let us have a chance to free memory early + return (rev, rawtext, False) + + def rawdata(self, nodeorrev, _df=None): + """return an uncompressed raw data of a given node or revision number. + + _df - an existing file handle to read from. (internal-only) + """ + return self._revisiondata(nodeorrev, _df, raw=True) + def hash(self, text, p1, p2): """Compute a node hash. @@ -1754,9 +1738,9 @@ raise error.ProgrammingError(_("invalid '%s' operation") % operation) # Check all flags are known. - if flags & ~REVIDX_KNOWN_FLAGS: + if flags & ~flagutil.REVIDX_KNOWN_FLAGS: raise error.RevlogError(_("incompatible revision flag '%#x'") % - (flags & ~REVIDX_KNOWN_FLAGS)) + (flags & ~flagutil.REVIDX_KNOWN_FLAGS)) validatehash = True # Depending on the operation (read or write), the order might be # reversed due to non-commutative transforms. @@ -2461,13 +2445,14 @@ # the revlog chunk is a delta. cachedelta = None rawtext = None - if destrevlog._lazydelta: + if (deltareuse != self.DELTAREUSEFULLADD + and destrevlog._lazydelta): dp = self.deltaparent(rev) if dp != nullrev: cachedelta = (dp, bytes(self._chunk(rev))) if not cachedelta: - rawtext = self.revision(rev, raw=True) + rawtext = self.rawdata(rev) if deltareuse == self.DELTAREUSEFULLADD: @@ -2545,7 +2530,7 @@ 'revision having delta stored')) rawtext = self._chunk(rev) else: - rawtext = self.revision(rev, raw=True) + rawtext = self.rawdata(rev) newrl.addrawrevision(rawtext, tr, self.linkrev(rev), p1, p2, node, self.flags(rev)) @@ -2603,8 +2588,8 @@ # rawtext[0:2]=='\1\n'| False | True | True | ? # # "rawtext" means the raw text stored in revlog data, which - # could be retrieved by "revision(rev, raw=True)". "text" - # mentioned below is "revision(rev, raw=False)". + # could be retrieved by "rawdata(rev)". "text" + # mentioned below is "revision(rev)". # # There are 3 different lengths stored physically: # 1. L1: rawsize, stored in revlog index @@ -2614,7 +2599,7 @@ # # L1 should be equal to L2. L3 could be different from them. # "text" may or may not affect commit hash depending on flag - # processors (see revlog.addflagprocessor). + # processors (see flagutil.addflagprocessor). # # | common | rename | meta | ext # ------------------------------------------------- @@ -2646,7 +2631,7 @@ self.revision(node) l1 = self.rawsize(rev) - l2 = len(self.revision(node, raw=True)) + l2 = len(self.rawdata(node)) if l1 != l2: yield revlogproblem(
--- a/mercurial/revlogutils/constants.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/revlogutils/constants.py Fri Aug 23 17:03:42 2019 -0400 @@ -11,7 +11,6 @@ from .. import ( repository, - util, ) # revlog header flags @@ -48,7 +47,7 @@ REVIDX_ELLIPSIS, REVIDX_EXTSTORED, ] -REVIDX_KNOWN_FLAGS = util.bitsfrom(REVIDX_FLAGS_ORDER) + # bitmark for flags that could cause rawdata content change REVIDX_RAWTEXT_CHANGING_FLAGS = REVIDX_ISCENSORED | REVIDX_EXTSTORED
--- a/mercurial/revlogutils/deltas.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/revlogutils/deltas.py Fri Aug 23 17:03:42 2019 -0400 @@ -925,7 +925,7 @@ header = mdiff.replacediffheader(revlog.rawsize(base), len(t)) delta = header + t else: - ptext = revlog.revision(base, _df=fh, raw=True) + ptext = revlog.rawdata(base, _df=fh) delta = mdiff.textdiff(ptext, t) return delta
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/revlogutils/flagutil.py Fri Aug 23 17:03:42 2019 -0400 @@ -0,0 +1,80 @@ +# flagutils.py - code to deal with revlog flags and their processors +# +# Copyright 2016 Remi Chaintron <remi@fb.com> +# Copyright 2016-2019 Pierre-Yves David <pierre-yves.david@ens-lyon.org> +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +from ..i18n import _ + +from .constants import ( + REVIDX_DEFAULT_FLAGS, + REVIDX_ELLIPSIS, + REVIDX_EXTSTORED, + REVIDX_FLAGS_ORDER, + REVIDX_ISCENSORED, + REVIDX_RAWTEXT_CHANGING_FLAGS, +) + +from .. import ( + error, + util +) + +# blanked usage of all the name to prevent pyflakes constraints +# We need these name available in the module for extensions. +REVIDX_ISCENSORED +REVIDX_ELLIPSIS +REVIDX_EXTSTORED +REVIDX_DEFAULT_FLAGS +REVIDX_FLAGS_ORDER +REVIDX_RAWTEXT_CHANGING_FLAGS + +REVIDX_KNOWN_FLAGS = util.bitsfrom(REVIDX_FLAGS_ORDER) + +# Store flag processors (cf. 'addflagprocessor()' to register) +flagprocessors = { + REVIDX_ISCENSORED: None, +} + +def addflagprocessor(flag, processor): + """Register a flag processor on a revision data flag. + + Invariant: + - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER, + and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext. + - Only one flag processor can be registered on a specific flag. + - flagprocessors must be 3-tuples of functions (read, write, raw) with the + following signatures: + - (read) f(self, rawtext) -> text, bool + - (write) f(self, text) -> rawtext, bool + - (raw) f(self, rawtext) -> bool + "text" is presented to the user. "rawtext" is stored in revlog data, not + directly visible to the user. + The boolean returned by these transforms is used to determine whether + the returned text can be used for hash integrity checking. For example, + if "write" returns False, then "text" is used to generate hash. If + "write" returns True, that basically means "rawtext" returned by "write" + should be used to generate hash. Usually, "write" and "read" return + different booleans. And "raw" returns a same boolean as "write". + + Note: The 'raw' transform is used for changegroup generation and in some + debug commands. In this case the transform only indicates whether the + contents can be used for hash integrity checks. + """ + insertflagprocessor(flag, processor, flagprocessors) + +def insertflagprocessor(flag, processor, flagprocessors): + if not flag & REVIDX_KNOWN_FLAGS: + msg = _("cannot register processor on unknown flag '%#x'.") % (flag) + raise error.ProgrammingError(msg) + if flag not in REVIDX_FLAGS_ORDER: + msg = _("flag '%#x' undefined in REVIDX_FLAGS_ORDER.") % (flag) + raise error.ProgrammingError(msg) + if flag in flagprocessors: + msg = _("cannot register multiple processors on flag '%#x'.") % (flag) + raise error.Abort(msg) + flagprocessors[flag] = processor
--- a/mercurial/revset.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/revset.py Fri Aug 23 17:03:42 2019 -0400 @@ -1695,7 +1695,7 @@ parent. (EXPERIMENTAL) """ if x is None: - stacks = stackmod.getstack(repo, x) + stacks = stackmod.getstack(repo) else: stacks = smartset.baseset([]) for revision in getset(repo, fullreposet(repo), x):
--- a/mercurial/scmutil.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/scmutil.py Fri Aug 23 17:03:42 2019 -0400 @@ -1984,3 +1984,21 @@ "ancestors(head() and not bookmark(%s)) - " "ancestors(bookmark() and not bookmark(%s))", mark, mark, mark) + +def computechangesetfilesadded(ctx): + """return the list of files added in a changeset + """ + added = [] + for f in ctx.files(): + if not any(f in p for p in ctx.parents()): + added.append(f) + return added + +def computechangesetfilesremoved(ctx): + """return the list of files removed in a changeset + """ + removed = [] + for f in ctx.files(): + if f not in ctx: + removed.append(f) + return removed
--- a/mercurial/setdiscovery.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/setdiscovery.py Fri Aug 23 17:03:42 2019 -0400 @@ -52,6 +52,7 @@ ) from . import ( error, + policy, util, ) @@ -92,11 +93,19 @@ dist.setdefault(p, d + 1) visit.append(p) -def _limitsample(sample, desiredlen): - """return a random subset of sample of at most desiredlen item""" - if len(sample) > desiredlen: - sample = set(random.sample(sample, desiredlen)) - return sample +def _limitsample(sample, desiredlen, randomize=True): + """return a random subset of sample of at most desiredlen item. + + If randomize is False, though, a deterministic subset is returned. + This is meant for integration tests. + """ + if len(sample) <= desiredlen: + return sample + if randomize: + return set(random.sample(sample, desiredlen)) + sample = list(sample) + sample.sort() + return set(sample[:desiredlen]) class partialdiscovery(object): """an object representing ongoing discovery @@ -110,7 +119,7 @@ (all tracked revisions are known locally) """ - def __init__(self, repo, targetheads, respectsize): + def __init__(self, repo, targetheads, respectsize, randomize=True): self._repo = repo self._targetheads = targetheads self._common = repo.changelog.incrementalmissingrevs() @@ -118,6 +127,7 @@ self.missing = set() self._childrenmap = None self._respectsize = respectsize + self.randomize = randomize def addcommons(self, commons): """register nodes known as common""" @@ -222,7 +232,7 @@ sample = set(self._repo.revs('heads(%ld)', revs)) if len(sample) >= size: - return _limitsample(sample, size) + return _limitsample(sample, size, randomize=self.randomize) _updatesample(None, headrevs, sample, self._parentsgetter(), quicksamplesize=size) @@ -249,12 +259,21 @@ if not self._respectsize: size = max(size, min(len(revsroots), len(revsheads))) - sample = _limitsample(sample, size) + sample = _limitsample(sample, size, randomize=self.randomize) if len(sample) < size: more = size - len(sample) - sample.update(random.sample(list(revs - sample), more)) + takefrom = list(revs - sample) + if self.randomize: + sample.update(random.sample(takefrom, more)) + else: + takefrom.sort() + sample.update(takefrom[:more]) return sample +partialdiscovery = policy.importrust(r'discovery', + member=r'PartialDiscovery', + default=partialdiscovery) + def findcommonheads(ui, local, remote, initialsamplesize=100, fullsamplesize=200, @@ -376,7 +395,9 @@ # full blown discovery - disco = partialdiscovery(local, ownheads, remote.limitedarguments) + randomize = ui.configbool('devel', 'discovery.randomize') + disco = partialdiscovery(local, ownheads, remote.limitedarguments, + randomize=randomize) # treat remote heads (and maybe own heads) as a first implicit sample # response disco.addcommons(knownsrvheads)
--- a/mercurial/shelve.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/shelve.py Fri Aug 23 17:03:42 2019 -0400 @@ -177,6 +177,7 @@ _nokeep = 'nokeep' # colon is essential to differentiate from a real bookmark name _noactivebook = ':no-active-bookmark' + _interactive = 'interactive' @classmethod def _verifyandtransform(cls, d): @@ -247,6 +248,7 @@ obj.activebookmark = '' if d.get('activebook', '') != cls._noactivebook: obj.activebookmark = d.get('activebook', '') + obj.interactive = d.get('interactive') == cls._interactive except (error.RepoLookupError, KeyError) as err: raise error.CorruptedState(pycompat.bytestr(err)) @@ -254,7 +256,7 @@ @classmethod def save(cls, repo, name, originalwctx, pendingctx, nodestoremove, - branchtorestore, keep=False, activebook=''): + branchtorestore, keep=False, activebook='', interactive=False): info = { "name": name, "originalwctx": nodemod.hex(originalwctx.node()), @@ -267,6 +269,8 @@ "keep": cls._keep if keep else cls._nokeep, "activebook": activebook or cls._noactivebook } + if interactive: + info['interactive'] = cls._interactive scmutil.simplekeyvaluefile( repo.vfs, cls._filename).write(info, firstline=("%d" % cls._version)) @@ -694,11 +698,12 @@ if shfile.exists(): shfile.movetobackup() cleanupoldbackups(repo) -def unshelvecontinue(ui, repo, state, opts, basename=None): +def unshelvecontinue(ui, repo, state, opts): """subcommand to continue an in-progress unshelve""" # We're finishing off a merge. First parent is our original # parent, second is the temporary "fake" commit we're unshelving. - interactive = opts.get('interactive') + interactive = state.interactive + basename = state.name with repo.lock(): checkparents(repo, state) ms = merge.mergestate.read(repo) @@ -721,15 +726,8 @@ with repo.ui.configoverride(overrides, 'unshelve'): with repo.dirstate.parentchange(): repo.setparents(state.parents[0], nodemod.nullid) - if not interactive: - ispartialunshelve = False - newnode = repo.commit(text=shelvectx.description(), - extra=shelvectx.extra(), - user=shelvectx.user(), - date=shelvectx.date()) - else: - newnode, ispartialunshelve = _dounshelveinteractive(ui, - repo, shelvectx, basename, opts) + newnode, ispartialunshelve = _createunshelvectx(ui, + repo, shelvectx, basename, interactive, opts) if newnode is None: # If it ended up being a no-op commit, then the normal @@ -749,11 +747,11 @@ mergefiles(ui, repo, state.wctx, shelvectx) restorebranch(ui, repo, state.branchtorestore) + if not phases.supportinternal(repo): + repair.strip(ui, repo, state.nodestoremove, backup=False, + topic='shelve') + shelvedstate.clear(repo) if not ispartialunshelve: - if not phases.supportinternal(repo): - repair.strip(ui, repo, state.nodestoremove, backup=False, - topic='shelve') - shelvedstate.clear(repo) unshelvecleanup(ui, repo, state.name, opts) _restoreactivebookmark(repo, state.activebookmark) ui.status(_("unshelve of '%s' complete\n") % state.name) @@ -804,14 +802,37 @@ return repo, shelvectx -def _dounshelveinteractive(ui, repo, shelvectx, basename, opts): - """The user might want to unshelve certain changes only from the stored - shelve. So, we would create two commits. One with requested changes to - unshelve at that time and the latter is shelved for future. +def _createunshelvectx(ui, repo, shelvectx, basename, interactive, opts): + """Handles the creation of unshelve commit and updates the shelve if it + was partially unshelved. + + If interactive is: + + * False: Commits all the changes in the working directory. + * True: Prompts the user to select changes to unshelve and commit them. + Update the shelve with remaining changes. + + Returns the node of the new commit formed and a bool indicating whether + the shelve was partially unshelved.Creates a commit ctx to unshelve + interactively or non-interactively. + + The user might want to unshelve certain changes only from the stored + shelve in interactive. So, we would create two commits. One with requested + changes to unshelve at that time and the latter is shelved for future. + + Here, we return both the newnode which is created interactively and a + bool to know whether the shelve is partly done or completely done. """ opts['message'] = shelvectx.description() opts['interactive-unshelve'] = True pats = [] + if not interactive: + newnode = repo.commit(text=shelvectx.description(), + extra=shelvectx.extra(), + user=shelvectx.user(), + date=shelvectx.date()) + return newnode, False + commitfunc = getcommitfunc(shelvectx.extra(), interactive=True, editor=True) newnode = cmdutil.dorecord(ui, repo, commitfunc, None, False, @@ -819,10 +840,9 @@ **pycompat.strkwargs(opts)) snode = repo.commit(text=shelvectx.description(), extra=shelvectx.extra(), - user=shelvectx.user(), - date=shelvectx.date()) - m = scmutil.matchfiles(repo, repo[snode].files()) + user=shelvectx.user()) if snode: + m = scmutil.matchfiles(repo, repo[snode].files()) _shelvecreatedcommit(repo, snode, basename, m) return newnode, bool(snode) @@ -854,22 +874,16 @@ nodestoremove = [repo.changelog.node(rev) for rev in pycompat.xrange(oldtiprev, len(repo))] shelvedstate.save(repo, basename, pctx, tmpwctx, nodestoremove, - branchtorestore, opts.get('keep'), activebookmark) + branchtorestore, opts.get('keep'), activebookmark, + interactive) raise error.InterventionRequired( _("unresolved conflicts (see 'hg resolve', then " "'hg unshelve --continue')")) with repo.dirstate.parentchange(): repo.setparents(tmpwctx.node(), nodemod.nullid) - if not interactive: - ispartialunshelve = False - newnode = repo.commit(text=shelvectx.description(), - extra=shelvectx.extra(), - user=shelvectx.user(), - date=shelvectx.date()) - else: - newnode, ispartialunshelve = _dounshelveinteractive(ui, repo, - shelvectx, basename, opts) + newnode, ispartialunshelve = _createunshelvectx(ui, repo, + shelvectx, basename, interactive, opts) if newnode is None: # If it ended up being a no-op commit, then the normal @@ -928,7 +942,9 @@ if opts.get("name"): shelved.append(opts["name"]) - if abortf or continuef and not interactive: + if interactive and opts.get('keep'): + raise error.Abort(_('--keep on --interactive is not yet supported')) + if abortf or continuef: if abortf and continuef: raise error.Abort(_('cannot use both abort and continue')) if shelved: @@ -940,6 +956,8 @@ state = _loadshelvedstate(ui, repo, opts) if abortf: return unshelveabort(ui, repo, state) + elif continuef and interactive: + raise error.Abort(_('cannot use both continue and interactive')) elif continuef: return unshelvecontinue(ui, repo, state, opts) elif len(shelved) > 1: @@ -950,11 +968,8 @@ raise error.Abort(_('no shelved changes to apply!')) basename = util.split(shelved[0][1])[1] ui.status(_("unshelving change '%s'\n") % basename) - elif shelved: + else: basename = shelved[0] - if continuef and interactive: - state = _loadshelvedstate(ui, repo, opts) - return unshelvecontinue(ui, repo, state, opts, basename) if not shelvedfile(repo, basename, patchextension).exists(): raise error.Abort(_("shelved change '%s' not found") % basename) @@ -990,11 +1005,10 @@ with ui.configoverride(overrides, 'unshelve'): mergefiles(ui, repo, pctx, shelvectx) restorebranch(ui, repo, branchtorestore) + shelvedstate.clear(repo) + _finishunshelve(repo, oldtiprev, tr, activebookmark) + _forgetunknownfiles(repo, shelvectx, addedbefore) if not ispartialunshelve: - _forgetunknownfiles(repo, shelvectx, addedbefore) - - shelvedstate.clear(repo) - _finishunshelve(repo, oldtiprev, tr, activebookmark) unshelvecleanup(ui, repo, basename, opts) finally: if tr:
--- a/mercurial/stack.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/stack.py Fri Aug 23 17:03:42 2019 -0400 @@ -22,7 +22,7 @@ if rev is None: rev = '.' - revspec = 'reverse(only(%s) and not public() and not ::merge())' + revspec = 'only(%s) and not public() and not ::merge()' revset = revsetlang.formatspec(revspec, rev) revisions = scmutil.revrange(repo, [revset]) revisions.sort()
--- a/mercurial/testing/storage.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/testing/storage.py Fri Aug 23 17:03:42 2019 -0400 @@ -421,7 +421,7 @@ f.size(i) self.assertEqual(f.revision(nullid), b'') - self.assertEqual(f.revision(nullid, raw=True), b'') + self.assertEqual(f.rawdata(nullid), b'') with self.assertRaises(error.LookupError): f.revision(b'\x01' * 20) @@ -473,7 +473,7 @@ f.size(1) self.assertEqual(f.revision(node), fulltext) - self.assertEqual(f.revision(node, raw=True), fulltext) + self.assertEqual(f.rawdata(node), fulltext) self.assertEqual(f.read(node), fulltext) @@ -545,11 +545,11 @@ f.size(3) self.assertEqual(f.revision(node0), fulltext0) - self.assertEqual(f.revision(node0, raw=True), fulltext0) + self.assertEqual(f.rawdata(node0), fulltext0) self.assertEqual(f.revision(node1), fulltext1) - self.assertEqual(f.revision(node1, raw=True), fulltext1) + self.assertEqual(f.rawdata(node1), fulltext1) self.assertEqual(f.revision(node2), fulltext2) - self.assertEqual(f.revision(node2, raw=True), fulltext2) + self.assertEqual(f.rawdata(node2), fulltext2) with self.assertRaises(error.LookupError): f.revision(b'\x01' * 20) @@ -819,9 +819,9 @@ self.assertEqual(f.size(2), len(fulltext2)) self.assertEqual(f.revision(node1), stored1) - self.assertEqual(f.revision(node1, raw=True), stored1) + self.assertEqual(f.rawdata(node1), stored1) self.assertEqual(f.revision(node2), stored2) - self.assertEqual(f.revision(node2, raw=True), stored2) + self.assertEqual(f.rawdata(node2), stored2) self.assertEqual(f.read(node1), fulltext1) self.assertEqual(f.read(node2), fulltext2) @@ -862,10 +862,10 @@ self.assertEqual(f.size(1), len(fulltext1)) self.assertEqual(f.revision(node0), stored0) - self.assertEqual(f.revision(node0, raw=True), stored0) + self.assertEqual(f.rawdata(node0), stored0) self.assertEqual(f.revision(node1), stored1) - self.assertEqual(f.revision(node1, raw=True), stored1) + self.assertEqual(f.rawdata(node1), stored1) self.assertEqual(f.read(node0), fulltext0) self.assertEqual(f.read(node1), fulltext1) @@ -896,10 +896,10 @@ with self.assertRaises(error.StorageError): f.revision(node1) - # raw=True still verifies because there are no special storage + # rawdata() still verifies because there are no special storage # settings. with self.assertRaises(error.StorageError): - f.revision(node1, raw=True) + f.rawdata(node1) # read() behaves like revision(). with self.assertRaises(error.StorageError): @@ -909,7 +909,7 @@ # reading/validating the fulltext to return rename metadata. def testbadnoderevisionraw(self): - # Like above except we test revision(raw=True) first to isolate + # Like above except we test rawdata() first to isolate # revision caching behavior. f = self._makefilefn() @@ -924,10 +924,10 @@ rawtext=fulltext1) with self.assertRaises(error.StorageError): - f.revision(node1, raw=True) + f.rawdata(node1) with self.assertRaises(error.StorageError): - f.revision(node1, raw=True) + f.rawdata(node1) def testbadnoderevisionraw(self): # Like above except we test read() first to isolate revision caching @@ -1002,13 +1002,13 @@ f.revision(1) with self.assertRaises(error.CensoredNodeError): - f.revision(1, raw=True) + f.rawdata(1) with self.assertRaises(error.CensoredNodeError): f.read(1) def testcensoredrawrevision(self): - # Like above, except we do the revision(raw=True) request first to + # Like above, except we do the rawdata() request first to # isolate revision caching behavior. f = self._makefilefn() @@ -1027,7 +1027,7 @@ censored=True) with self.assertRaises(error.CensoredNodeError): - f.revision(1, raw=True) + f.rawdata(1) class ifilemutationtests(basetestcase): """Generic tests for the ifilemutation interface.
--- a/mercurial/ui.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/ui.py Fri Aug 23 17:03:42 2019 -0400 @@ -783,6 +783,17 @@ return None return default + def configdefault(self, section, name): + """returns the default value of the config item""" + item = self._knownconfig.get(section, {}).get(name) + itemdefault = None + if item is not None: + if callable(item.default): + itemdefault = item.default() + else: + itemdefault = item.default + return itemdefault + def hasconfig(self, section, name, untrusted=False): return self._data(untrusted).hasitem(section, name)
--- a/mercurial/unionrepo.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/unionrepo.py Fri Aug 23 17:03:42 2019 -0400 @@ -116,6 +116,9 @@ # already cached return text + def rawdata(self, nodeorrev, _df=None): + return self.revision(nodeorrev, _df=_df, raw=True) + def baserevision(self, nodeorrev): # Revlog subclasses may override 'revision' method to modify format of # content retrieved from revlog. To use unionrevlog with such class one
--- a/mercurial/upgrade.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/upgrade.py Fri Aug 23 17:03:42 2019 -0400 @@ -533,7 +533,55 @@ #reverse of "/".join(("data", path + ".i")) return filelog.filelog(repo.svfs, path[5:-2]) -def _copyrevlogs(ui, srcrepo, dstrepo, tr, deltareuse, forcedeltabothparents): +def _copyrevlog(tr, destrepo, oldrl, unencodedname): + """copy all relevant files for `oldrl` into `destrepo` store + + Files are copied "as is" without any transformation. The copy is performed + without extra checks. Callers are responsible for making sure the copied + content is compatible with format of the destination repository. + """ + oldrl = getattr(oldrl, '_revlog', oldrl) + newrl = _revlogfrompath(destrepo, unencodedname) + newrl = getattr(newrl, '_revlog', newrl) + + oldvfs = oldrl.opener + newvfs = newrl.opener + oldindex = oldvfs.join(oldrl.indexfile) + newindex = newvfs.join(newrl.indexfile) + olddata = oldvfs.join(oldrl.datafile) + newdata = newvfs.join(newrl.datafile) + + newdir = newvfs.dirname(newrl.indexfile) + newvfs.makedirs(newdir) + + util.copyfile(oldindex, newindex) + if oldrl.opener.exists(olddata): + util.copyfile(olddata, newdata) + + if not (unencodedname.endswith('00changelog.i') + or unencodedname.endswith('00manifest.i')): + destrepo.svfs.fncache.add(unencodedname) + +UPGRADE_CHANGELOG = object() +UPGRADE_MANIFEST = object() +UPGRADE_FILELOG = object() + +UPGRADE_ALL_REVLOGS = frozenset([UPGRADE_CHANGELOG, + UPGRADE_MANIFEST, + UPGRADE_FILELOG]) + +def matchrevlog(revlogfilter, entry): + """check is a revlog is selected for cloning + + The store entry is checked against the passed filter""" + if entry.endswith('00changelog.i'): + return UPGRADE_CHANGELOG in revlogfilter + elif entry.endswith('00manifest.i'): + return UPGRADE_MANIFEST in revlogfilter + return UPGRADE_FILELOG in revlogfilter + +def _clonerevlogs(ui, srcrepo, dstrepo, tr, deltareuse, forcedeltabothparents, + revlogs=UPGRADE_ALL_REVLOGS): """Copy revlogs between 2 repos.""" revcount = 0 srcsize = 0 @@ -554,9 +602,11 @@ crawsize = 0 cdstsize = 0 + alldatafiles = list(srcrepo.store.walk()) + # Perform a pass to collect metadata. This validates we can open all # source files and allows a unified progress bar to be displayed. - for unencoded, encoded, size in srcrepo.store.walk(): + for unencoded, encoded, size in alldatafiles: if unencoded.endswith('.d'): continue @@ -607,12 +657,11 @@ # Do the actual copying. # FUTURE this operation can be farmed off to worker processes. seen = set() - for unencoded, encoded, size in srcrepo.store.walk(): + for unencoded, encoded, size in alldatafiles: if unencoded.endswith('.d'): continue oldrl = _revlogfrompath(srcrepo, unencoded) - newrl = _revlogfrompath(dstrepo, unencoded) if isinstance(oldrl, changelog.changelog) and 'c' not in seen: ui.write(_('finished migrating %d manifest revisions across %d ' @@ -651,11 +700,19 @@ progress = srcrepo.ui.makeprogress(_('file revisions'), total=frevcount) + if matchrevlog(revlogs, unencoded): + ui.note(_('cloning %d revisions from %s\n') + % (len(oldrl), unencoded)) + newrl = _revlogfrompath(dstrepo, unencoded) + oldrl.clone(tr, newrl, addrevisioncb=oncopiedrevision, + deltareuse=deltareuse, + forcedeltabothparents=forcedeltabothparents) + else: + msg = _('blindly copying %s containing %i revisions\n') + ui.note(msg % (unencoded, len(oldrl))) + _copyrevlog(tr, dstrepo, oldrl, unencoded) - ui.note(_('cloning %d revisions from %s\n') % (len(oldrl), unencoded)) - oldrl.clone(tr, newrl, addrevisioncb=oncopiedrevision, - deltareuse=deltareuse, - forcedeltabothparents=forcedeltabothparents) + newrl = _revlogfrompath(dstrepo, unencoded) info = newrl.storageinfo(storedsize=True) datasize = info['storedsize'] or 0 @@ -715,7 +772,8 @@ before the new store is swapped into the original location. """ -def _upgraderepo(ui, srcrepo, dstrepo, requirements, actions): +def _upgraderepo(ui, srcrepo, dstrepo, requirements, actions, + revlogs=UPGRADE_ALL_REVLOGS): """Do the low-level work of upgrading a repository. The upgrade is effectively performed as a copy between a source @@ -743,8 +801,8 @@ deltareuse = revlog.revlog.DELTAREUSEALWAYS with dstrepo.transaction('upgrade') as tr: - _copyrevlogs(ui, srcrepo, dstrepo, tr, deltareuse, - 're-delta-multibase' in actions) + _clonerevlogs(ui, srcrepo, dstrepo, tr, deltareuse, + 're-delta-multibase' in actions, revlogs=revlogs) # Now copy other files in the store directory. # The sorted() makes execution deterministic.
--- a/mercurial/util.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/util.py Fri Aug 23 17:03:42 2019 -0400 @@ -53,7 +53,7 @@ stringutil, ) -rustdirs = policy.importrust('dirstate', 'Dirs') +rustdirs = policy.importrust(r'dirstate', r'Dirs') base85 = policy.importmod(r'base85') osutil = policy.importmod(r'osutil')
--- a/mercurial/utils/storageutil.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/utils/storageutil.py Fri Aug 23 17:03:42 2019 -0400 @@ -304,9 +304,9 @@ ``rawsizefn`` (optional) Callable receiving a revision number and returning the length of the - ``store.revision(rev, raw=True)``. + ``store.rawdata(rev)``. - If not defined, ``len(store.revision(rev, raw=True))`` will be called. + If not defined, ``len(store.rawdata(rev))`` will be called. ``revdifffn`` (optional) Callable receiving a pair of revision numbers that returns a delta @@ -422,7 +422,7 @@ if revisiondata: if store.iscensored(baserev) or store.iscensored(rev): try: - revision = store.revision(node, raw=True) + revision = store.rawdata(node) except error.CensoredNodeError as e: revision = e.tombstone @@ -430,19 +430,18 @@ if rawsizefn: baserevisionsize = rawsizefn(baserev) else: - baserevisionsize = len(store.revision(baserev, - raw=True)) + baserevisionsize = len(store.rawdata(baserev)) elif (baserev == nullrev and deltamode != repository.CG_DELTAMODE_PREV): - revision = store.revision(node, raw=True) + revision = store.rawdata(node) available.add(rev) else: if revdifffn: delta = revdifffn(baserev, rev) else: - delta = mdiff.textdiff(store.revision(baserev, raw=True), - store.revision(rev, raw=True)) + delta = mdiff.textdiff(store.rawdata(baserev), + store.rawdata(rev)) available.add(rev)
--- a/mercurial/wireprotov2server.py Wed Aug 21 17:56:50 2019 +0200 +++ b/mercurial/wireprotov2server.py Fri Aug 23 17:03:42 2019 -0400 @@ -937,7 +937,7 @@ followingdata = [] if b'revision' in fields: - revisiondata = cl.revision(node, raw=True) + revisiondata = cl.rawdata(node) followingmeta.append((b'revision', len(revisiondata))) followingdata.append(revisiondata)
--- a/rust/hg-core/Cargo.toml Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/Cargo.toml Fri Aug 23 17:03:42 2019 -0400 @@ -8,12 +8,10 @@ [lib] name = "hg" -[dev-dependencies] -rand = "*" -rand_pcg = "*" - [dependencies] byteorder = "1.3.1" lazy_static = "1.3.0" memchr = "2.2.0" +rand = "> 0.6.4" +rand_pcg = "> 0.1.0" regex = "^1.1"
--- a/rust/hg-core/src/dirstate.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/src/dirstate.rs Fri Aug 23 17:03:42 2019 -0400 @@ -1,36 +1,73 @@ +// dirstate module +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +use crate::DirstateParseError; +use std::collections::HashMap; +use std::convert::TryFrom; + pub mod dirs_multiset; +pub mod dirstate_map; pub mod parsers; -#[derive(Debug, PartialEq, Copy, Clone)] -pub struct DirstateParents<'a> { - pub p1: &'a [u8], - pub p2: &'a [u8], +#[derive(Debug, PartialEq, Clone)] +pub struct DirstateParents { + pub p1: [u8; 20], + pub p2: [u8; 20], } /// The C implementation uses all signed types. This will be an issue /// either when 4GB+ source files are commonplace or in 2038, whichever /// comes first. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Copy, Clone)] pub struct DirstateEntry { - pub state: i8, + pub state: EntryState, pub mode: i32, pub mtime: i32, pub size: i32, } -pub type DirstateVec = Vec<(Vec<u8>, DirstateEntry)>; +pub type StateMap = HashMap<Vec<u8>, DirstateEntry>; +pub type CopyMap = HashMap<Vec<u8>, Vec<u8>>; -#[derive(Debug, PartialEq)] -pub struct CopyVecEntry<'a> { - pub path: &'a [u8], - pub copy_path: &'a [u8], +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum EntryState { + Normal, + Added, + Removed, + Merged, + Unknown, } -pub type CopyVec<'a> = Vec<CopyVecEntry<'a>>; +impl TryFrom<u8> for EntryState { + type Error = DirstateParseError; -/// The Python implementation passes either a mapping (dirstate) or a flat -/// iterable (manifest) -pub enum DirsIterable { - Dirstate(DirstateVec), - Manifest(Vec<Vec<u8>>), + fn try_from(value: u8) -> Result<Self, Self::Error> { + match value { + b'n' => Ok(EntryState::Normal), + b'a' => Ok(EntryState::Added), + b'r' => Ok(EntryState::Removed), + b'm' => Ok(EntryState::Merged), + b'?' => Ok(EntryState::Unknown), + _ => Err(DirstateParseError::CorruptedEntry(format!( + "Incorrect entry state {}", + value + ))), + } + } } + +impl Into<u8> for EntryState { + fn into(self) -> u8 { + match self { + EntryState::Normal => b'n', + EntryState::Added => b'a', + EntryState::Removed => b'r', + EntryState::Merged => b'm', + EntryState::Unknown => b'?', + } + } +}
--- a/rust/hg-core/src/dirstate/dirs_multiset.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/src/dirstate/dirs_multiset.rs Fri Aug 23 17:03:42 2019 -0400 @@ -8,8 +8,10 @@ //! A multiset of directory names. //! //! Used to counts the references to directories in a manifest or dirstate. -use crate::{utils::files, DirsIterable, DirstateEntry, DirstateMapError}; -use std::collections::hash_map::{Entry, Iter}; +use crate::{ + dirstate::EntryState, utils::files, DirstateEntry, DirstateMapError, +}; +use std::collections::hash_map::Entry; use std::collections::HashMap; #[derive(PartialEq, Debug)] @@ -18,37 +20,44 @@ } impl DirsMultiset { - /// Initializes the multiset from a dirstate or a manifest. + /// Initializes the multiset from a dirstate. /// /// If `skip_state` is provided, skips dirstate entries with equal state. - pub fn new(iterable: DirsIterable, skip_state: Option<i8>) -> Self { + pub fn from_dirstate( + vec: &HashMap<Vec<u8>, DirstateEntry>, + skip_state: Option<EntryState>, + ) -> Self { let mut multiset = DirsMultiset { inner: HashMap::new(), }; - match iterable { - DirsIterable::Dirstate(vec) => { - for (ref filename, DirstateEntry { state, .. }) in vec { - // This `if` is optimized out of the loop - if let Some(skip) = skip_state { - if skip != state { - multiset.add_path(filename); - } - } else { - multiset.add_path(filename); - } - } - } - DirsIterable::Manifest(vec) => { - for ref filename in vec { + for (filename, DirstateEntry { state, .. }) in vec { + // This `if` is optimized out of the loop + if let Some(skip) = skip_state { + if skip != *state { multiset.add_path(filename); } + } else { + multiset.add_path(filename); } } multiset } + /// Initializes the multiset from a manifest. + pub fn from_manifest(vec: &Vec<Vec<u8>>) -> Self { + let mut multiset = DirsMultiset { + inner: HashMap::new(), + }; + + for filename in vec { + multiset.add_path(filename); + } + + multiset + } + /// Increases the count of deepest directory contained in the path. /// /// If the directory is not yet in the map, adds its parents. @@ -92,12 +101,12 @@ Ok(()) } - pub fn contains_key(&self, key: &[u8]) -> bool { + pub fn contains(&self, key: &[u8]) -> bool { self.inner.contains_key(key) } - pub fn iter(&self) -> Iter<Vec<u8>, u32> { - self.inner.iter() + pub fn iter(&self) -> impl Iterator<Item = &Vec<u8>> { + self.inner.keys() } pub fn len(&self) -> usize { @@ -108,10 +117,11 @@ #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; #[test] fn test_delete_path_path_not_found() { - let mut map = DirsMultiset::new(DirsIterable::Manifest(vec![]), None); + let mut map = DirsMultiset::from_manifest(&vec![]); let path = b"doesnotexist/"; assert_eq!( Err(DirstateMapError::PathNotFound(path.to_vec())), @@ -121,8 +131,7 @@ #[test] fn test_delete_path_empty_path() { - let mut map = - DirsMultiset::new(DirsIterable::Manifest(vec![vec![]]), None); + let mut map = DirsMultiset::from_manifest(&vec![vec![]]); let path = b""; assert_eq!(Ok(()), map.delete_path(path)); assert_eq!( @@ -162,7 +171,7 @@ #[test] fn test_add_path_empty_path() { - let mut map = DirsMultiset::new(DirsIterable::Manifest(vec![]), None); + let mut map = DirsMultiset::from_manifest(&vec![]); let path = b""; map.add_path(path); @@ -171,7 +180,7 @@ #[test] fn test_add_path_successful() { - let mut map = DirsMultiset::new(DirsIterable::Manifest(vec![]), None); + let mut map = DirsMultiset::from_manifest(&vec![]); map.add_path(b"a/"); assert_eq!(1, *map.inner.get(&b"a".to_vec()).unwrap()); @@ -216,15 +225,13 @@ #[test] fn test_dirsmultiset_new_empty() { - use DirsIterable::{Dirstate, Manifest}; - - let new = DirsMultiset::new(Manifest(vec![]), None); + let new = DirsMultiset::from_manifest(&vec![]); let expected = DirsMultiset { inner: HashMap::new(), }; assert_eq!(expected, new); - let new = DirsMultiset::new(Dirstate(vec![]), None); + let new = DirsMultiset::from_dirstate(&HashMap::new(), None); let expected = DirsMultiset { inner: HashMap::new(), }; @@ -233,8 +240,6 @@ #[test] fn test_dirsmultiset_new_no_skip() { - use DirsIterable::{Dirstate, Manifest}; - let input_vec = ["a/", "b/", "a/c", "a/d/"] .iter() .map(|e| e.as_bytes().to_vec()) @@ -244,7 +249,7 @@ .map(|(k, v)| (k.as_bytes().to_vec(), *v)) .collect(); - let new = DirsMultiset::new(Manifest(input_vec), None); + let new = DirsMultiset::from_manifest(&input_vec); let expected = DirsMultiset { inner: expected_inner, }; @@ -256,7 +261,7 @@ ( f.as_bytes().to_vec(), DirstateEntry { - state: 0, + state: EntryState::Normal, mode: 0, mtime: 0, size: 0, @@ -269,7 +274,7 @@ .map(|(k, v)| (k.as_bytes().to_vec(), *v)) .collect(); - let new = DirsMultiset::new(Dirstate(input_map), None); + let new = DirsMultiset::from_dirstate(&input_map, None); let expected = DirsMultiset { inner: expected_inner, }; @@ -278,39 +283,25 @@ #[test] fn test_dirsmultiset_new_skip() { - use DirsIterable::{Dirstate, Manifest}; - - let input_vec = ["a/", "b/", "a/c", "a/d/"] - .iter() - .map(|e| e.as_bytes().to_vec()) - .collect(); - let expected_inner = [("", 2), ("a", 3), ("b", 1), ("a/d", 1)] - .iter() - .map(|(k, v)| (k.as_bytes().to_vec(), *v)) - .collect(); - - let new = DirsMultiset::new(Manifest(input_vec), Some('n' as i8)); - let expected = DirsMultiset { - inner: expected_inner, - }; - // Skip does not affect a manifest - assert_eq!(expected, new); - - let input_map = - [("a/", 'n'), ("a/b/", 'n'), ("a/c", 'r'), ("a/d/", 'm')] - .iter() - .map(|(f, state)| { - ( - f.as_bytes().to_vec(), - DirstateEntry { - state: *state as i8, - mode: 0, - mtime: 0, - size: 0, - }, - ) - }) - .collect(); + let input_map = [ + ("a/", EntryState::Normal), + ("a/b/", EntryState::Normal), + ("a/c", EntryState::Removed), + ("a/d/", EntryState::Merged), + ] + .iter() + .map(|(f, state)| { + ( + f.as_bytes().to_vec(), + DirstateEntry { + state: *state, + mode: 0, + mtime: 0, + size: 0, + }, + ) + }) + .collect(); // "a" incremented with "a/c" and "a/d/" let expected_inner = [("", 1), ("a", 2), ("a/d", 1)] @@ -318,7 +309,8 @@ .map(|(k, v)| (k.as_bytes().to_vec(), *v)) .collect(); - let new = DirsMultiset::new(Dirstate(input_map), Some('n' as i8)); + let new = + DirsMultiset::from_dirstate(&input_map, Some(EntryState::Normal)); let expected = DirsMultiset { inner: expected_inner, };
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rust/hg-core/src/dirstate/dirstate_map.rs Fri Aug 23 17:03:42 2019 -0400 @@ -0,0 +1,424 @@ +// dirstate_map.rs +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +use crate::{ + dirstate::{parsers::PARENT_SIZE, EntryState}, + pack_dirstate, parse_dirstate, CopyMap, DirsMultiset, DirstateEntry, + DirstateError, DirstateMapError, DirstateParents, DirstateParseError, + StateMap, +}; +use core::borrow::Borrow; +use std::collections::{HashMap, HashSet}; +use std::convert::TryInto; +use std::iter::FromIterator; +use std::ops::Deref; +use std::time::Duration; + +pub type FileFoldMap = HashMap<Vec<u8>, Vec<u8>>; + +const NULL_ID: [u8; 20] = [0; 20]; +const MTIME_UNSET: i32 = -1; +const SIZE_DIRTY: i32 = -2; + +#[derive(Default)] +pub struct DirstateMap { + state_map: StateMap, + pub copy_map: CopyMap, + file_fold_map: Option<FileFoldMap>, + pub dirs: Option<DirsMultiset>, + pub all_dirs: Option<DirsMultiset>, + non_normal_set: HashSet<Vec<u8>>, + other_parent_set: HashSet<Vec<u8>>, + parents: Option<DirstateParents>, + dirty_parents: bool, +} + +/// Should only really be used in python interface code, for clarity +impl Deref for DirstateMap { + type Target = StateMap; + + fn deref(&self) -> &Self::Target { + &self.state_map + } +} + +impl FromIterator<(Vec<u8>, DirstateEntry)> for DirstateMap { + fn from_iter<I: IntoIterator<Item = (Vec<u8>, DirstateEntry)>>( + iter: I, + ) -> Self { + Self { + state_map: iter.into_iter().collect(), + ..Self::default() + } + } +} + +impl DirstateMap { + pub fn new() -> Self { + Self::default() + } + + pub fn clear(&mut self) { + self.state_map.clear(); + self.copy_map.clear(); + self.file_fold_map = None; + self.non_normal_set.clear(); + self.other_parent_set.clear(); + self.set_parents(&DirstateParents { + p1: NULL_ID, + p2: NULL_ID, + }) + } + + /// Add a tracked file to the dirstate + pub fn add_file( + &mut self, + filename: &[u8], + old_state: EntryState, + entry: DirstateEntry, + ) { + if old_state == EntryState::Unknown || old_state == EntryState::Removed + { + if let Some(ref mut dirs) = self.dirs { + dirs.add_path(filename) + } + } + if old_state == EntryState::Unknown { + if let Some(ref mut all_dirs) = self.all_dirs { + all_dirs.add_path(filename) + } + } + self.state_map.insert(filename.to_owned(), entry.to_owned()); + + if entry.state != EntryState::Normal || entry.mtime == MTIME_UNSET { + self.non_normal_set.insert(filename.to_owned()); + } + + if entry.size == SIZE_DIRTY { + self.other_parent_set.insert(filename.to_owned()); + } + } + + /// Mark a file as removed in the dirstate. + /// + /// The `size` parameter is used to store sentinel values that indicate + /// the file's previous state. In the future, we should refactor this + /// to be more explicit about what that state is. + pub fn remove_file( + &mut self, + filename: &[u8], + old_state: EntryState, + size: i32, + ) -> Result<(), DirstateMapError> { + if old_state != EntryState::Unknown && old_state != EntryState::Removed + { + if let Some(ref mut dirs) = self.dirs { + dirs.delete_path(filename)?; + } + } + if old_state == EntryState::Unknown { + if let Some(ref mut all_dirs) = self.all_dirs { + all_dirs.add_path(filename); + } + } + + if let Some(ref mut file_fold_map) = self.file_fold_map { + file_fold_map.remove(&filename.to_ascii_uppercase()); + } + self.state_map.insert( + filename.to_owned(), + DirstateEntry { + state: EntryState::Removed, + mode: 0, + size, + mtime: 0, + }, + ); + self.non_normal_set.insert(filename.to_owned()); + Ok(()) + } + + /// Remove a file from the dirstate. + /// Returns `true` if the file was previously recorded. + pub fn drop_file( + &mut self, + filename: &[u8], + old_state: EntryState, + ) -> Result<bool, DirstateMapError> { + let exists = self.state_map.remove(filename).is_some(); + + if exists { + if old_state != EntryState::Removed { + if let Some(ref mut dirs) = self.dirs { + dirs.delete_path(filename)?; + } + } + if let Some(ref mut all_dirs) = self.all_dirs { + all_dirs.delete_path(filename)?; + } + } + if let Some(ref mut file_fold_map) = self.file_fold_map { + file_fold_map.remove(&filename.to_ascii_uppercase()); + } + self.non_normal_set.remove(filename); + + Ok(exists) + } + + pub fn clear_ambiguous_times( + &mut self, + filenames: Vec<Vec<u8>>, + now: i32, + ) { + for filename in filenames { + let mut changed = false; + self.state_map + .entry(filename.to_owned()) + .and_modify(|entry| { + if entry.state == EntryState::Normal && entry.mtime == now + { + changed = true; + *entry = DirstateEntry { + mtime: MTIME_UNSET, + ..*entry + }; + } + }); + if changed { + self.non_normal_set.insert(filename.to_owned()); + } + } + } + + pub fn non_normal_other_parent_entries( + &self, + ) -> (HashSet<Vec<u8>>, HashSet<Vec<u8>>) { + let mut non_normal = HashSet::new(); + let mut other_parent = HashSet::new(); + + for ( + filename, + DirstateEntry { + state, size, mtime, .. + }, + ) in self.state_map.iter() + { + if *state != EntryState::Normal || *mtime == MTIME_UNSET { + non_normal.insert(filename.to_owned()); + } + if *state == EntryState::Normal && *size == SIZE_DIRTY { + other_parent.insert(filename.to_owned()); + } + } + + (non_normal, other_parent) + } + + /// Both of these setters and their uses appear to be the simplest way to + /// emulate a Python lazy property, but it is ugly and unidiomatic. + /// TODO One day, rewriting this struct using the typestate might be a + /// good idea. + pub fn set_all_dirs(&mut self) { + if self.all_dirs.is_none() { + self.all_dirs = + Some(DirsMultiset::from_dirstate(&self.state_map, None)); + } + } + + pub fn set_dirs(&mut self) { + if self.dirs.is_none() { + self.dirs = Some(DirsMultiset::from_dirstate( + &self.state_map, + Some(EntryState::Removed), + )); + } + } + + pub fn has_tracked_dir(&mut self, directory: &[u8]) -> bool { + self.set_dirs(); + self.dirs.as_ref().unwrap().contains(directory) + } + + pub fn has_dir(&mut self, directory: &[u8]) -> bool { + self.set_all_dirs(); + self.all_dirs.as_ref().unwrap().contains(directory) + } + + pub fn parents( + &mut self, + file_contents: &[u8], + ) -> Result<&DirstateParents, DirstateError> { + if let Some(ref parents) = self.parents { + return Ok(parents); + } + let parents; + if file_contents.len() == PARENT_SIZE * 2 { + parents = DirstateParents { + p1: file_contents[..PARENT_SIZE].try_into().unwrap(), + p2: file_contents[PARENT_SIZE..PARENT_SIZE * 2] + .try_into() + .unwrap(), + }; + } else if file_contents.is_empty() { + parents = DirstateParents { + p1: NULL_ID, + p2: NULL_ID, + }; + } else { + return Err(DirstateError::Parse(DirstateParseError::Damaged)); + } + + self.parents = Some(parents); + Ok(self.parents.as_ref().unwrap()) + } + + pub fn set_parents(&mut self, parents: &DirstateParents) { + self.parents = Some(parents.clone()); + self.dirty_parents = true; + } + + pub fn read( + &mut self, + file_contents: &[u8], + ) -> Result<Option<DirstateParents>, DirstateError> { + if file_contents.is_empty() { + return Ok(None); + } + + let parents = parse_dirstate( + &mut self.state_map, + &mut self.copy_map, + file_contents, + )?; + + if !self.dirty_parents { + self.set_parents(&parents); + } + + Ok(Some(parents)) + } + + pub fn pack( + &mut self, + parents: DirstateParents, + now: Duration, + ) -> Result<Vec<u8>, DirstateError> { + let packed = + pack_dirstate(&mut self.state_map, &self.copy_map, parents, now)?; + + self.dirty_parents = false; + + let result = self.non_normal_other_parent_entries(); + self.non_normal_set = result.0; + self.other_parent_set = result.1; + Ok(packed) + } + + pub fn build_file_fold_map(&mut self) -> &FileFoldMap { + if let Some(ref file_fold_map) = self.file_fold_map { + return file_fold_map; + } + let mut new_file_fold_map = FileFoldMap::new(); + for (filename, DirstateEntry { state, .. }) in self.state_map.borrow() + { + if *state == EntryState::Removed { + new_file_fold_map.insert( + filename.to_ascii_uppercase().to_owned(), + filename.to_owned(), + ); + } + } + self.file_fold_map = Some(new_file_fold_map); + self.file_fold_map.as_ref().unwrap() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dirs_multiset() { + let mut map = DirstateMap::new(); + assert!(map.dirs.is_none()); + assert!(map.all_dirs.is_none()); + + assert_eq!(false, map.has_dir(b"nope")); + assert!(map.all_dirs.is_some()); + assert!(map.dirs.is_none()); + + assert_eq!(false, map.has_tracked_dir(b"nope")); + assert!(map.dirs.is_some()); + } + + #[test] + fn test_add_file() { + let mut map = DirstateMap::new(); + + assert_eq!(0, map.len()); + + map.add_file( + b"meh", + EntryState::Normal, + DirstateEntry { + state: EntryState::Normal, + mode: 1337, + mtime: 1337, + size: 1337, + }, + ); + + assert_eq!(1, map.len()); + assert_eq!(0, map.non_normal_set.len()); + assert_eq!(0, map.other_parent_set.len()); + } + + #[test] + fn test_non_normal_other_parent_entries() { + let map: DirstateMap = [ + (b"f1", (EntryState::Removed, 1337, 1337, 1337)), + (b"f2", (EntryState::Normal, 1337, 1337, -1)), + (b"f3", (EntryState::Normal, 1337, 1337, 1337)), + (b"f4", (EntryState::Normal, 1337, -2, 1337)), + (b"f5", (EntryState::Added, 1337, 1337, 1337)), + (b"f6", (EntryState::Added, 1337, 1337, -1)), + (b"f7", (EntryState::Merged, 1337, 1337, -1)), + (b"f8", (EntryState::Merged, 1337, 1337, 1337)), + (b"f9", (EntryState::Merged, 1337, -2, 1337)), + (b"fa", (EntryState::Added, 1337, -2, 1337)), + (b"fb", (EntryState::Removed, 1337, -2, 1337)), + ] + .iter() + .map(|(fname, (state, mode, size, mtime))| { + ( + fname.to_vec(), + DirstateEntry { + state: *state, + mode: *mode, + size: *size, + mtime: *mtime, + }, + ) + }) + .collect(); + + let non_normal = [ + b"f1", b"f2", b"f5", b"f6", b"f7", b"f8", b"f9", b"fa", b"fb", + ] + .iter() + .map(|x| x.to_vec()) + .collect(); + + let mut other_parent = HashSet::new(); + other_parent.insert(b"f4".to_vec()); + + assert_eq!( + (non_normal, other_parent), + map.non_normal_other_parent_entries() + ); + } +}
--- a/rust/hg-core/src/dirstate/parsers.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/src/dirstate/parsers.rs Fri Aug 23 17:03:42 2019 -0400 @@ -4,31 +4,34 @@ // GNU General Public License version 2 or any later version. use crate::{ - CopyVec, CopyVecEntry, DirstateEntry, DirstatePackError, DirstateParents, - DirstateParseError, DirstateVec, + dirstate::{CopyMap, EntryState, StateMap}, + DirstateEntry, DirstatePackError, DirstateParents, DirstateParseError, }; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; -use std::collections::HashMap; +use std::convert::{TryFrom, TryInto}; use std::io::Cursor; +use std::time::Duration; /// Parents are stored in the dirstate as byte hashes. -const PARENT_SIZE: usize = 20; +pub const PARENT_SIZE: usize = 20; /// Dirstate entries have a static part of 8 + 32 + 32 + 32 + 32 bits. const MIN_ENTRY_SIZE: usize = 17; +// TODO parse/pack: is mutate-on-loop better for performance? + pub fn parse_dirstate( + state_map: &mut StateMap, + copy_map: &mut CopyMap, contents: &[u8], -) -> Result<(DirstateParents, DirstateVec, CopyVec), DirstateParseError> { +) -> Result<DirstateParents, DirstateParseError> { if contents.len() < PARENT_SIZE * 2 { return Err(DirstateParseError::TooLittleData); } - let mut dirstate_vec = vec![]; - let mut copies = vec![]; let mut curr_pos = PARENT_SIZE * 2; let parents = DirstateParents { - p1: &contents[..PARENT_SIZE], - p2: &contents[PARENT_SIZE..curr_pos], + p1: contents[..PARENT_SIZE].try_into().unwrap(), + p2: contents[PARENT_SIZE..curr_pos].try_into().unwrap(), }; while curr_pos < contents.len() { @@ -38,7 +41,7 @@ let entry_bytes = &contents[curr_pos..]; let mut cursor = Cursor::new(entry_bytes); - let state = cursor.read_i8()?; + let state = EntryState::try_from(cursor.read_u8()?)?; let mode = cursor.read_i32::<BigEndian>()?; let size = cursor.read_i32::<BigEndian>()?; let mtime = cursor.read_i32::<BigEndian>()?; @@ -57,9 +60,9 @@ }; if let Some(copy_path) = copy { - copies.push(CopyVecEntry { path, copy_path }); + copy_map.insert(path.to_owned(), copy_path.to_owned()); }; - dirstate_vec.push(( + state_map.insert( path.to_owned(), DirstateEntry { state, @@ -67,28 +70,28 @@ size, mtime, }, - )); + ); curr_pos = curr_pos + MIN_ENTRY_SIZE + (path_len); } - Ok((parents, dirstate_vec, copies)) + Ok(parents) } +/// `now` is the duration in seconds since the Unix epoch pub fn pack_dirstate( - dirstate_vec: &DirstateVec, - copymap: &HashMap<Vec<u8>, Vec<u8>>, + state_map: &mut StateMap, + copy_map: &CopyMap, parents: DirstateParents, - now: i32, -) -> Result<(Vec<u8>, DirstateVec), DirstatePackError> { - if parents.p1.len() != PARENT_SIZE || parents.p2.len() != PARENT_SIZE { - return Err(DirstatePackError::CorruptedParent); - } + now: Duration, +) -> Result<Vec<u8>, DirstatePackError> { + // TODO move away from i32 before 2038. + let now: i32 = now.as_secs().try_into().expect("time overflow"); - let expected_size: usize = dirstate_vec + let expected_size: usize = state_map .iter() - .map(|(ref filename, _)| { + .map(|(filename, _)| { let mut length = MIN_ENTRY_SIZE + filename.len(); - if let Some(ref copy) = copymap.get(filename) { + if let Some(copy) = copy_map.get(filename) { length += copy.len() + 1; } length @@ -97,15 +100,15 @@ let expected_size = expected_size + PARENT_SIZE * 2; let mut packed = Vec::with_capacity(expected_size); - let mut new_dirstate_vec = vec![]; + let mut new_state_map = vec![]; - packed.extend(parents.p1); - packed.extend(parents.p2); + packed.extend(&parents.p1); + packed.extend(&parents.p2); - for (ref filename, entry) in dirstate_vec { + for (filename, entry) in state_map.iter() { let mut new_filename: Vec<u8> = filename.to_owned(); let mut new_mtime: i32 = entry.mtime; - if entry.state == 'n' as i8 && entry.mtime == now.into() { + if entry.state == EntryState::Normal && entry.mtime == now { // The file was last modified "simultaneously" with the current // write to dirstate (i.e. within the same second for file- // systems with a granularity of 1 sec). This commonly happens @@ -116,7 +119,7 @@ // contents of the file if the size is the same. This prevents // mistakenly treating such files as clean. new_mtime = -1; - new_dirstate_vec.push(( + new_state_map.push(( filename.to_owned(), DirstateEntry { mtime: new_mtime, @@ -125,12 +128,12 @@ )); } - if let Some(copy) = copymap.get(filename) { + if let Some(copy) = copy_map.get(filename) { new_filename.push('\0' as u8); new_filename.extend(copy); } - packed.write_i8(entry.state)?; + packed.write_u8(entry.state.into())?; packed.write_i32::<BigEndian>(entry.mode)?; packed.write_i32::<BigEndian>(entry.size)?; packed.write_i32::<BigEndian>(new_mtime)?; @@ -142,143 +145,155 @@ return Err(DirstatePackError::BadSize(expected_size, packed.len())); } - Ok((packed, new_dirstate_vec)) + state_map.extend(new_state_map); + + Ok(packed) } #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; #[test] fn test_pack_dirstate_empty() { - let dirstate_vec: DirstateVec = vec![]; + let mut state_map: StateMap = HashMap::new(); let copymap = HashMap::new(); let parents = DirstateParents { - p1: b"12345678910111213141", - p2: b"00000000000000000000", + p1: *b"12345678910111213141", + p2: *b"00000000000000000000", }; - let now: i32 = 15000000; - let expected = - (b"1234567891011121314100000000000000000000".to_vec(), vec![]); + let now = Duration::new(15000000, 0); + let expected = b"1234567891011121314100000000000000000000".to_vec(); assert_eq!( expected, - pack_dirstate(&dirstate_vec, ©map, parents, now).unwrap() + pack_dirstate(&mut state_map, ©map, parents, now).unwrap() ); + + assert!(state_map.is_empty()) } #[test] fn test_pack_dirstate_one_entry() { - let dirstate_vec: DirstateVec = vec![( - vec!['f' as u8, '1' as u8], - DirstateEntry { - state: 'n' as i8, - mode: 0o644, - size: 0, - mtime: 791231220, - }, - )]; - let copymap = HashMap::new(); - let parents = DirstateParents { - p1: b"12345678910111213141", - p2: b"00000000000000000000", - }; - let now: i32 = 15000000; - let expected = ( - [ - 49, 50, 51, 52, 53, 54, 55, 56, 57, 49, 48, 49, 49, 49, 50, - 49, 51, 49, 52, 49, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 110, 0, 0, 1, 164, 0, - 0, 0, 0, 47, 41, 58, 244, 0, 0, 0, 2, 102, 49, - ] - .to_vec(), - vec![], - ); - - assert_eq!( - expected, - pack_dirstate(&dirstate_vec, ©map, parents, now).unwrap() - ); - } - #[test] - fn test_pack_dirstate_one_entry_with_copy() { - let dirstate_vec: DirstateVec = vec![( + let expected_state_map: StateMap = [( b"f1".to_vec(), DirstateEntry { - state: 'n' as i8, + state: EntryState::Normal, mode: 0o644, size: 0, mtime: 791231220, }, - )]; + )] + .iter() + .cloned() + .collect(); + let mut state_map = expected_state_map.clone(); + + let copymap = HashMap::new(); + let parents = DirstateParents { + p1: *b"12345678910111213141", + p2: *b"00000000000000000000", + }; + let now = Duration::new(15000000, 0); + let expected = [ + 49, 50, 51, 52, 53, 54, 55, 56, 57, 49, 48, 49, 49, 49, 50, 49, + 51, 49, 52, 49, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 110, 0, 0, 1, 164, 0, 0, 0, 0, 47, + 41, 58, 244, 0, 0, 0, 2, 102, 49, + ] + .to_vec(); + + assert_eq!( + expected, + pack_dirstate(&mut state_map, ©map, parents, now).unwrap() + ); + + assert_eq!(expected_state_map, state_map); + } + #[test] + fn test_pack_dirstate_one_entry_with_copy() { + let expected_state_map: StateMap = [( + b"f1".to_vec(), + DirstateEntry { + state: EntryState::Normal, + mode: 0o644, + size: 0, + mtime: 791231220, + }, + )] + .iter() + .cloned() + .collect(); + let mut state_map = expected_state_map.clone(); let mut copymap = HashMap::new(); copymap.insert(b"f1".to_vec(), b"copyname".to_vec()); let parents = DirstateParents { - p1: b"12345678910111213141", - p2: b"00000000000000000000", + p1: *b"12345678910111213141", + p2: *b"00000000000000000000", }; - let now: i32 = 15000000; - let expected = ( - [ - 49, 50, 51, 52, 53, 54, 55, 56, 57, 49, 48, 49, 49, 49, 50, - 49, 51, 49, 52, 49, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 110, 0, 0, 1, 164, 0, - 0, 0, 0, 47, 41, 58, 244, 0, 0, 0, 11, 102, 49, 0, 99, 111, - 112, 121, 110, 97, 109, 101, - ] - .to_vec(), - vec![], - ); + let now = Duration::new(15000000, 0); + let expected = [ + 49, 50, 51, 52, 53, 54, 55, 56, 57, 49, 48, 49, 49, 49, 50, 49, + 51, 49, 52, 49, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 110, 0, 0, 1, 164, 0, 0, 0, 0, 47, + 41, 58, 244, 0, 0, 0, 11, 102, 49, 0, 99, 111, 112, 121, 110, 97, + 109, 101, + ] + .to_vec(); assert_eq!( expected, - pack_dirstate(&dirstate_vec, ©map, parents, now).unwrap() + pack_dirstate(&mut state_map, ©map, parents, now).unwrap() ); + assert_eq!(expected_state_map, state_map); } #[test] fn test_parse_pack_one_entry_with_copy() { - let dirstate_vec: DirstateVec = vec![( + let mut state_map: StateMap = [( b"f1".to_vec(), DirstateEntry { - state: 'n' as i8, + state: EntryState::Normal, mode: 0o644, size: 0, mtime: 791231220, }, - )]; + )] + .iter() + .cloned() + .collect(); let mut copymap = HashMap::new(); copymap.insert(b"f1".to_vec(), b"copyname".to_vec()); let parents = DirstateParents { - p1: b"12345678910111213141", - p2: b"00000000000000000000", + p1: *b"12345678910111213141", + p2: *b"00000000000000000000", }; - let now: i32 = 15000000; + let now = Duration::new(15000000, 0); let result = - pack_dirstate(&dirstate_vec, ©map, parents, now).unwrap(); + pack_dirstate(&mut state_map, ©map, parents.clone(), now) + .unwrap(); + let mut new_state_map: StateMap = HashMap::new(); + let mut new_copy_map: CopyMap = HashMap::new(); + let new_parents = parse_dirstate( + &mut new_state_map, + &mut new_copy_map, + result.as_slice(), + ) + .unwrap(); assert_eq!( - ( - parents, - dirstate_vec, - copymap - .iter() - .map(|(k, v)| CopyVecEntry { - path: k.as_slice(), - copy_path: v.as_slice() - }) - .collect() - ), - parse_dirstate(result.0.as_slice()).unwrap() + (parents, state_map, copymap), + (new_parents, new_state_map, new_copy_map) ) } #[test] fn test_parse_pack_multiple_entries_with_copy() { - let dirstate_vec: DirstateVec = vec![ + let mut state_map: StateMap = [ ( b"f1".to_vec(), DirstateEntry { - state: 'n' as i8, + state: EntryState::Normal, mode: 0o644, size: 0, mtime: 791231220, @@ -287,7 +302,7 @@ ( b"f2".to_vec(), DirstateEntry { - state: 'm' as i8, + state: EntryState::Merged, mode: 0o777, size: 1000, mtime: 791231220, @@ -296,7 +311,7 @@ ( b"f3".to_vec(), DirstateEntry { - state: 'r' as i8, + state: EntryState::Removed, mode: 0o644, size: 234553, mtime: 791231220, @@ -305,84 +320,95 @@ ( b"f4\xF6".to_vec(), DirstateEntry { - state: 'a' as i8, + state: EntryState::Added, mode: 0o644, size: -1, mtime: -1, }, ), - ]; + ] + .iter() + .cloned() + .collect(); let mut copymap = HashMap::new(); copymap.insert(b"f1".to_vec(), b"copyname".to_vec()); copymap.insert(b"f4\xF6".to_vec(), b"copyname2".to_vec()); let parents = DirstateParents { - p1: b"12345678910111213141", - p2: b"00000000000000000000", + p1: *b"12345678910111213141", + p2: *b"00000000000000000000", }; - let now: i32 = 15000000; + let now = Duration::new(15000000, 0); let result = - pack_dirstate(&dirstate_vec, ©map, parents, now).unwrap(); + pack_dirstate(&mut state_map, ©map, parents.clone(), now) + .unwrap(); + let mut new_state_map: StateMap = HashMap::new(); + let mut new_copy_map: CopyMap = HashMap::new(); + let new_parents = parse_dirstate( + &mut new_state_map, + &mut new_copy_map, + result.as_slice(), + ) + .unwrap(); assert_eq!( - (parents, dirstate_vec, copymap), - parse_dirstate(result.0.as_slice()) - .and_then(|(p, dvec, cvec)| Ok(( - p, - dvec, - cvec.iter() - .map(|entry| ( - entry.path.to_vec(), - entry.copy_path.to_vec() - )) - .collect() - ))) - .unwrap() + (parents, state_map, copymap), + (new_parents, new_state_map, new_copy_map) ) } #[test] /// https://www.mercurial-scm.org/repo/hg/rev/af3f26b6bba4 fn test_parse_pack_one_entry_with_copy_and_time_conflict() { - let dirstate_vec: DirstateVec = vec![( + let mut state_map: StateMap = [( b"f1".to_vec(), DirstateEntry { - state: 'n' as i8, + state: EntryState::Normal, mode: 0o644, size: 0, mtime: 15000000, }, - )]; + )] + .iter() + .cloned() + .collect(); let mut copymap = HashMap::new(); copymap.insert(b"f1".to_vec(), b"copyname".to_vec()); let parents = DirstateParents { - p1: b"12345678910111213141", - p2: b"00000000000000000000", + p1: *b"12345678910111213141", + p2: *b"00000000000000000000", }; - let now: i32 = 15000000; + let now = Duration::new(15000000, 0); let result = - pack_dirstate(&dirstate_vec, ©map, parents, now).unwrap(); + pack_dirstate(&mut state_map, ©map, parents.clone(), now) + .unwrap(); + + let mut new_state_map: StateMap = HashMap::new(); + let mut new_copy_map: CopyMap = HashMap::new(); + let new_parents = parse_dirstate( + &mut new_state_map, + &mut new_copy_map, + result.as_slice(), + ) + .unwrap(); assert_eq!( ( parents, - vec![( + [( b"f1".to_vec(), DirstateEntry { - state: 'n' as i8, + state: EntryState::Normal, mode: 0o644, size: 0, mtime: -1 } - )], - copymap - .iter() - .map(|(k, v)| CopyVecEntry { - path: k.as_slice(), - copy_path: v.as_slice() - }) - .collect() + )] + .iter() + .cloned() + .collect::<StateMap>(), + copymap, ), - parse_dirstate(result.0.as_slice()).unwrap() + (new_parents, new_state_map, new_copy_map) ) } }
--- a/rust/hg-core/src/discovery.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/src/discovery.rs Fri Aug 23 17:03:42 2019 -0400 @@ -10,23 +10,124 @@ //! This is a Rust counterpart to the `partialdiscovery` class of //! `mercurial.setdiscovery` -use super::{Graph, GraphError, Revision}; +use super::{Graph, GraphError, Revision, NULL_REVISION}; use crate::ancestors::MissingAncestors; use crate::dagops; -use std::collections::HashSet; +use rand::seq::SliceRandom; +use rand::{thread_rng, RngCore, SeedableRng}; +use std::cmp::{max, min}; +use std::collections::{HashMap, HashSet, VecDeque}; + +type Rng = rand_pcg::Pcg32; pub struct PartialDiscovery<G: Graph + Clone> { target_heads: Option<Vec<Revision>>, graph: G, // plays the role of self._repo common: MissingAncestors<G>, undecided: Option<HashSet<Revision>>, + children_cache: Option<HashMap<Revision, Vec<Revision>>>, missing: HashSet<Revision>, + rng: Rng, + respect_size: bool, + randomize: bool, } pub struct DiscoveryStats { pub undecided: Option<usize>, } +/// Update an existing sample to match the expected size +/// +/// The sample is updated with revisions exponentially distant from each +/// element of `heads`. +/// +/// If a target size is specified, the sampling will stop once this size is +/// reached. Otherwise sampling will happen until roots of the <revs> set are +/// reached. +/// +/// - `revs`: set of revs we want to discover (if None, `assume` the whole dag +/// represented by `parentfn` +/// - `heads`: set of DAG head revs +/// - `sample`: a sample to update +/// - `parentfn`: a callable to resolve parents for a revision +/// - `quicksamplesize`: optional target size of the sample +fn update_sample<I>( + revs: Option<&HashSet<Revision>>, + heads: impl IntoIterator<Item = Revision>, + sample: &mut HashSet<Revision>, + parentsfn: impl Fn(Revision) -> Result<I, GraphError>, + quicksamplesize: Option<usize>, +) -> Result<(), GraphError> +where + I: Iterator<Item = Revision>, +{ + let mut distances: HashMap<Revision, u32> = HashMap::new(); + let mut visit: VecDeque<Revision> = heads.into_iter().collect(); + let mut factor: u32 = 1; + let mut seen: HashSet<Revision> = HashSet::new(); + while let Some(current) = visit.pop_front() { + if !seen.insert(current) { + continue; + } + + let d = *distances.entry(current).or_insert(1); + if d > factor { + factor *= 2; + } + if d == factor { + sample.insert(current); + if let Some(sz) = quicksamplesize { + if sample.len() >= sz { + return Ok(()); + } + } + } + for p in parentsfn(current)? { + if let Some(revs) = revs { + if !revs.contains(&p) { + continue; + } + } + distances.entry(p).or_insert(d + 1); + visit.push_back(p); + } + } + Ok(()) +} + +struct ParentsIterator { + parents: [Revision; 2], + cur: usize, +} + +impl ParentsIterator { + fn graph_parents( + graph: &impl Graph, + r: Revision, + ) -> Result<ParentsIterator, GraphError> { + Ok(ParentsIterator { + parents: graph.parents(r)?, + cur: 0, + }) + } +} + +impl Iterator for ParentsIterator { + type Item = Revision; + + fn next(&mut self) -> Option<Revision> { + if self.cur > 1 { + return None; + } + let rev = self.parents[self.cur]; + self.cur += 1; + if rev == NULL_REVISION { + return self.next(); + } + Some(rev) + } +} + impl<G: Graph + Clone> PartialDiscovery<G> { /// Create a PartialDiscovery object, with the intent /// of comparing our `::<target_heads>` revset to the contents of another @@ -38,22 +139,89 @@ /// If we want to make the signature more flexible, /// we'll have to make it a type argument of `PartialDiscovery` or a trait /// object since we'll keep it in the meanwhile - pub fn new(graph: G, target_heads: Vec<Revision>) -> Self { + /// + /// The `respect_size` boolean controls how the sampling methods + /// will interpret the size argument requested by the caller. If it's + /// `false`, they are allowed to produce a sample whose size is more + /// appropriate to the situation (typically bigger). + /// + /// The `randomize` boolean affects sampling, and specifically how + /// limiting or last-minute expanding is been done: + /// + /// If `true`, both will perform random picking from `self.undecided`. + /// This is currently the best for actual discoveries. + /// + /// If `false`, a reproductible picking strategy is performed. This is + /// useful for integration tests. + pub fn new( + graph: G, + target_heads: Vec<Revision>, + respect_size: bool, + randomize: bool, + ) -> Self { + let mut seed: [u8; 16] = [0; 16]; + if randomize { + thread_rng().fill_bytes(&mut seed); + } + Self::new_with_seed(graph, target_heads, seed, respect_size, randomize) + } + + pub fn new_with_seed( + graph: G, + target_heads: Vec<Revision>, + seed: [u8; 16], + respect_size: bool, + randomize: bool, + ) -> Self { PartialDiscovery { undecided: None, + children_cache: None, target_heads: Some(target_heads), graph: graph.clone(), common: MissingAncestors::new(graph, vec![]), missing: HashSet::new(), + rng: Rng::from_seed(seed), + respect_size: respect_size, + randomize: randomize, } } + /// Extract at most `size` random elements from sample and return them + /// as a vector + fn limit_sample( + &mut self, + mut sample: Vec<Revision>, + size: usize, + ) -> Vec<Revision> { + if !self.randomize { + sample.sort(); + sample.truncate(size); + return sample; + } + let sample_len = sample.len(); + if sample_len <= size { + return sample; + } + let rng = &mut self.rng; + let dropped_size = sample_len - size; + let limited_slice = if size < dropped_size { + sample.partial_shuffle(rng, size).0 + } else { + sample.partial_shuffle(rng, dropped_size).1 + }; + limited_slice.to_owned() + } + /// Register revisions known as being common pub fn add_common_revisions( &mut self, common: impl IntoIterator<Item = Revision>, ) -> Result<(), GraphError> { + let before_len = self.common.get_bases().len(); self.common.add_bases(common); + if self.common.get_bases().len() == before_len { + return Ok(()); + } if let Some(ref mut undecided) = self.undecided { self.common.remove_ancestors_from(undecided)?; } @@ -61,20 +229,50 @@ } /// Register revisions known as being missing + /// + /// # Performance note + /// + /// Except in the most trivial case, the first call of this method has + /// the side effect of computing `self.undecided` set for the first time, + /// and the related caches it might need for efficiency of its internal + /// computation. This is typically faster if more information is + /// available in `self.common`. Therefore, for good performance, the + /// caller should avoid calling this too early. pub fn add_missing_revisions( &mut self, missing: impl IntoIterator<Item = Revision>, ) -> Result<(), GraphError> { - self.ensure_undecided()?; - let range = dagops::range( - &self.graph, - missing, - self.undecided.as_ref().unwrap().iter().cloned(), - )?; + let mut tovisit: VecDeque<Revision> = missing.into_iter().collect(); + if tovisit.is_empty() { + return Ok(()); + } + self.ensure_children_cache()?; + self.ensure_undecided()?; // for safety of possible future refactors + let children = self.children_cache.as_ref().unwrap(); + let mut seen: HashSet<Revision> = HashSet::new(); let undecided_mut = self.undecided.as_mut().unwrap(); - for missrev in range { - self.missing.insert(missrev); - undecided_mut.remove(&missrev); + while let Some(rev) = tovisit.pop_front() { + if !self.missing.insert(rev) { + // either it's known to be missing from a previous + // invocation, and there's no need to iterate on its + // children (we now they are all missing) + // or it's from a previous iteration of this loop + // and its children have already been queued + continue; + } + undecided_mut.remove(&rev); + match children.get(&rev) { + None => { + continue; + } + Some(this_children) => { + for child in this_children.iter().cloned() { + if seen.insert(child) { + tovisit.push_back(child); + } + } + } + } } Ok(()) } @@ -124,12 +322,157 @@ Ok(()) } + fn ensure_children_cache(&mut self) -> Result<(), GraphError> { + if self.children_cache.is_some() { + return Ok(()); + } + self.ensure_undecided()?; + + let mut children: HashMap<Revision, Vec<Revision>> = HashMap::new(); + for &rev in self.undecided.as_ref().unwrap() { + for p in ParentsIterator::graph_parents(&self.graph, rev)? { + children.entry(p).or_insert_with(|| Vec::new()).push(rev); + } + } + self.children_cache = Some(children); + Ok(()) + } + /// Provide statistics about the current state of the discovery process pub fn stats(&self) -> DiscoveryStats { DiscoveryStats { undecided: self.undecided.as_ref().map(|s| s.len()), } } + + pub fn take_quick_sample( + &mut self, + headrevs: impl IntoIterator<Item = Revision>, + size: usize, + ) -> Result<Vec<Revision>, GraphError> { + self.ensure_undecided()?; + let mut sample = { + let undecided = self.undecided.as_ref().unwrap(); + if undecided.len() <= size { + return Ok(undecided.iter().cloned().collect()); + } + dagops::heads(&self.graph, undecided.iter())? + }; + if sample.len() >= size { + return Ok(self.limit_sample(sample.into_iter().collect(), size)); + } + update_sample( + None, + headrevs, + &mut sample, + |r| ParentsIterator::graph_parents(&self.graph, r), + Some(size), + )?; + Ok(sample.into_iter().collect()) + } + + /// Extract a sample from `self.undecided`, going from its heads and roots. + /// + /// The `size` parameter is used to avoid useless computations if + /// it turns out to be bigger than the whole set of undecided Revisions. + /// + /// The sample is taken by using `update_sample` from the heads, then + /// from the roots, working on the reverse DAG, + /// expressed by `self.children_cache`. + /// + /// No effort is being made to complete or limit the sample to `size` + /// but this method returns another interesting size that it derives + /// from its knowledge of the structure of the various sets, leaving + /// to the caller the decision to use it or not. + fn bidirectional_sample( + &mut self, + size: usize, + ) -> Result<(HashSet<Revision>, usize), GraphError> { + self.ensure_undecided()?; + { + // we don't want to compute children_cache before this + // but doing it after extracting self.undecided takes a mutable + // ref to self while a shareable one is still active. + let undecided = self.undecided.as_ref().unwrap(); + if undecided.len() <= size { + return Ok((undecided.clone(), size)); + } + } + + self.ensure_children_cache()?; + let revs = self.undecided.as_ref().unwrap(); + let mut sample: HashSet<Revision> = revs.clone(); + + // it's possible that leveraging the children cache would be more + // efficient here + dagops::retain_heads(&self.graph, &mut sample)?; + let revsheads = sample.clone(); // was again heads(revs) in python + + // update from heads + update_sample( + Some(revs), + revsheads.iter().cloned(), + &mut sample, + |r| ParentsIterator::graph_parents(&self.graph, r), + None, + )?; + + // update from roots + let revroots: HashSet<Revision> = + dagops::roots(&self.graph, revs)?.into_iter().collect(); + let prescribed_size = max(size, min(revroots.len(), revsheads.len())); + + let children = self.children_cache.as_ref().unwrap(); + let empty_vec: Vec<Revision> = Vec::new(); + update_sample( + Some(revs), + revroots, + &mut sample, + |r| Ok(children.get(&r).unwrap_or(&empty_vec).iter().cloned()), + None, + )?; + Ok((sample, prescribed_size)) + } + + /// Fill up sample up to the wished size with random undecided Revisions. + /// + /// This is intended to be used as a last resort completion if the + /// regular sampling algorithm returns too few elements. + fn random_complete_sample( + &mut self, + sample: &mut Vec<Revision>, + size: usize, + ) { + let sample_len = sample.len(); + if size <= sample_len { + return; + } + let take_from: Vec<Revision> = self + .undecided + .as_ref() + .unwrap() + .iter() + .filter(|&r| !sample.contains(r)) + .cloned() + .collect(); + sample.extend(self.limit_sample(take_from, size - sample_len)); + } + + pub fn take_full_sample( + &mut self, + size: usize, + ) -> Result<Vec<Revision>, GraphError> { + let (sample_set, prescribed_size) = self.bidirectional_sample(size)?; + let size = if self.respect_size { + size + } else { + prescribed_size + }; + let mut sample = + self.limit_sample(sample_set.into_iter().collect(), size); + self.random_complete_sample(&mut sample, size); + Ok(sample) + } } #[cfg(test)] @@ -138,8 +481,30 @@ use crate::testing::SampleGraph; /// A PartialDiscovery as for pushing all the heads of `SampleGraph` + /// + /// To avoid actual randomness in these tests, we give it a fixed + /// random seed, but by default we'll test the random version. fn full_disco() -> PartialDiscovery<SampleGraph> { - PartialDiscovery::new(SampleGraph, vec![10, 11, 12, 13]) + PartialDiscovery::new_with_seed( + SampleGraph, + vec![10, 11, 12, 13], + [0; 16], + true, + true, + ) + } + + /// A PartialDiscovery as for pushing the 12 head of `SampleGraph` + /// + /// To avoid actual randomness in tests, we give it a fixed random seed. + fn disco12() -> PartialDiscovery<SampleGraph> { + PartialDiscovery::new_with_seed( + SampleGraph, + vec![12], + [0; 16], + true, + true, + ) } fn sorted_undecided( @@ -206,4 +571,125 @@ assert_eq!(sorted_common_heads(&disco)?, vec![5, 11, 12]); Ok(()) } + + #[test] + fn test_add_missing_early_continue() -> Result<(), GraphError> { + eprintln!("test_add_missing_early_stop"); + let mut disco = full_disco(); + disco.add_common_revisions(vec![13, 3, 4])?; + disco.ensure_children_cache()?; + // 12 is grand-child of 6 through 9 + // passing them in this order maximizes the chances of the + // early continue to do the wrong thing + disco.add_missing_revisions(vec![6, 9, 12])?; + assert_eq!(sorted_undecided(&disco), vec![5, 7, 10, 11]); + assert_eq!(sorted_missing(&disco), vec![6, 9, 12]); + assert!(!disco.is_complete()); + Ok(()) + } + + #[test] + fn test_limit_sample_no_need_to() { + let sample = vec![1, 2, 3, 4]; + assert_eq!(full_disco().limit_sample(sample, 10), vec![1, 2, 3, 4]); + } + + #[test] + fn test_limit_sample_less_than_half() { + assert_eq!(full_disco().limit_sample((1..6).collect(), 2), vec![4, 2]); + } + + #[test] + fn test_limit_sample_more_than_half() { + assert_eq!(full_disco().limit_sample((1..4).collect(), 2), vec![3, 2]); + } + + #[test] + fn test_limit_sample_no_random() { + let mut disco = full_disco(); + disco.randomize = false; + assert_eq!( + disco.limit_sample(vec![1, 8, 13, 5, 7, 3], 4), + vec![1, 3, 5, 7] + ); + } + + #[test] + fn test_quick_sample_enough_undecided_heads() -> Result<(), GraphError> { + let mut disco = full_disco(); + disco.undecided = Some((1..=13).collect()); + + let mut sample_vec = disco.take_quick_sample(vec![], 4)?; + sample_vec.sort(); + assert_eq!(sample_vec, vec![10, 11, 12, 13]); + Ok(()) + } + + #[test] + fn test_quick_sample_climbing_from_12() -> Result<(), GraphError> { + let mut disco = disco12(); + disco.ensure_undecided()?; + + let mut sample_vec = disco.take_quick_sample(vec![12], 4)?; + sample_vec.sort(); + // r12's only parent is r9, whose unique grand-parent through the + // diamond shape is r4. This ends there because the distance from r4 + // to the root is only 3. + assert_eq!(sample_vec, vec![4, 9, 12]); + Ok(()) + } + + #[test] + fn test_children_cache() -> Result<(), GraphError> { + let mut disco = full_disco(); + disco.ensure_children_cache()?; + + let cache = disco.children_cache.unwrap(); + assert_eq!(cache.get(&2).cloned(), Some(vec![4])); + assert_eq!(cache.get(&10).cloned(), None); + + let mut children_4 = cache.get(&4).cloned().unwrap(); + children_4.sort(); + assert_eq!(children_4, vec![5, 6, 7]); + + let mut children_7 = cache.get(&7).cloned().unwrap(); + children_7.sort(); + assert_eq!(children_7, vec![9, 11]); + + Ok(()) + } + + #[test] + fn test_complete_sample() { + let mut disco = full_disco(); + let undecided: HashSet<Revision> = + [4, 7, 9, 2, 3].iter().cloned().collect(); + disco.undecided = Some(undecided); + + let mut sample = vec![0]; + disco.random_complete_sample(&mut sample, 3); + assert_eq!(sample.len(), 3); + + let mut sample = vec![2, 4, 7]; + disco.random_complete_sample(&mut sample, 1); + assert_eq!(sample.len(), 3); + } + + #[test] + fn test_bidirectional_sample() -> Result<(), GraphError> { + let mut disco = full_disco(); + disco.undecided = Some((0..=13).into_iter().collect()); + + let (sample_set, size) = disco.bidirectional_sample(7)?; + assert_eq!(size, 7); + let mut sample: Vec<Revision> = sample_set.into_iter().collect(); + sample.sort(); + // our DAG is a bit too small for the results to be really interesting + // at least it shows that + // - we went both ways + // - we didn't take all Revisions (6 is not in the sample) + assert_eq!(sample, vec![0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13]); + Ok(()) + } + }
--- a/rust/hg-core/src/filepatterns.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/src/filepatterns.rs Fri Aug 23 17:03:42 2019 -0400 @@ -1,3 +1,12 @@ +// filepatterns.rs +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Handling of Mercurial-specific patterns. + use crate::{ utils::{files::get_path_from_bytes, SliceExt}, LineNumber, PatternError, PatternFileError,
--- a/rust/hg-core/src/lib.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/src/lib.rs Fri Aug 23 17:03:42 2019 -0400 @@ -10,9 +10,9 @@ pub mod testing; // unconditionally built, for use from integration tests pub use dirstate::{ dirs_multiset::DirsMultiset, - parsers::{pack_dirstate, parse_dirstate}, - CopyVec, CopyVecEntry, DirsIterable, DirstateEntry, DirstateParents, - DirstateVec, + dirstate_map::DirstateMap, + parsers::{pack_dirstate, parse_dirstate, PARENT_SIZE}, + CopyMap, DirstateEntry, DirstateParents, EntryState, StateMap, }; mod filepatterns; pub mod utils; @@ -60,6 +60,25 @@ TooLittleData, Overflow, CorruptedEntry(String), + Damaged, +} + +impl From<std::io::Error> for DirstateParseError { + fn from(e: std::io::Error) -> Self { + DirstateParseError::CorruptedEntry(e.to_string()) + } +} + +impl ToString for DirstateParseError { + fn to_string(&self) -> String { + use crate::DirstateParseError::*; + match self { + TooLittleData => "Too little data for dirstate.".to_string(), + Overflow => "Overflow in dirstate.".to_string(), + CorruptedEntry(e) => format!("Corrupted entry: {:?}.", e), + Damaged => "Dirstate appears to be damaged.".to_string(), + } + } } #[derive(Debug, PartialEq)] @@ -69,21 +88,33 @@ BadSize(usize, usize), } +impl From<std::io::Error> for DirstatePackError { + fn from(e: std::io::Error) -> Self { + DirstatePackError::CorruptedEntry(e.to_string()) + } +} #[derive(Debug, PartialEq)] pub enum DirstateMapError { PathNotFound(Vec<u8>), EmptyPath, } -impl From<std::io::Error> for DirstatePackError { - fn from(e: std::io::Error) -> Self { - DirstatePackError::CorruptedEntry(e.to_string()) +pub enum DirstateError { + Parse(DirstateParseError), + Pack(DirstatePackError), + Map(DirstateMapError), + IO(std::io::Error), +} + +impl From<DirstateParseError> for DirstateError { + fn from(e: DirstateParseError) -> Self { + DirstateError::Parse(e) } } -impl From<std::io::Error> for DirstateParseError { - fn from(e: std::io::Error) -> Self { - DirstateParseError::CorruptedEntry(e.to_string()) +impl From<DirstatePackError> for DirstateError { + fn from(e: DirstatePackError) -> Self { + DirstateError::Pack(e) } } @@ -103,3 +134,15 @@ PatternFileError::IO(e) } } + +impl From<DirstateMapError> for DirstateError { + fn from(e: DirstateMapError) -> Self { + DirstateError::Map(e) + } +} + +impl From<std::io::Error> for DirstateError { + fn from(e: std::io::Error) -> Self { + DirstateError::IO(e) + } +}
--- a/rust/hg-core/src/utils.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/src/utils.rs Fri Aug 23 17:03:42 2019 -0400 @@ -1,3 +1,12 @@ +// utils module +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Contains useful functions, traits, structs, etc. for use in core. + pub mod files; /// Replaces the `from` slice with the `to` slice inside the `buf` slice.
--- a/rust/hg-core/src/utils/files.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-core/src/utils/files.rs Fri Aug 23 17:03:42 2019 -0400 @@ -1,3 +1,14 @@ +// files.rs +// +// Copyright 2019 +// Raphaël Gomès <rgomes@octobus.net>, +// Yuya Nishihara <yuya@tcha.org> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Functions for fiddling with files. + use std::iter::FusedIterator; use std::path::Path;
--- a/rust/hg-cpython/src/dirstate.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-cpython/src/dirstate.rs Fri Aug 23 17:03:42 2019 -0400 @@ -9,23 +9,21 @@ //! `hg-core` package. //! //! From Python, this will be seen as `mercurial.rustext.dirstate` - +mod copymap; +mod dirs_multiset; +mod dirstate_map; +use crate::dirstate::{dirs_multiset::Dirs, dirstate_map::DirstateMap}; use cpython::{ - exc, ObjectProtocol, PyBytes, PyDict, PyErr, PyInt, PyModule, PyObject, - PyResult, PySequence, PyTuple, Python, PythonObject, ToPyObject, + exc, PyBytes, PyDict, PyErr, PyModule, PyObject, PyResult, PySequence, + Python, }; -use hg::{ - pack_dirstate, parse_dirstate, CopyVecEntry, DirsIterable, DirsMultiset, - DirstateEntry, DirstateMapError, DirstatePackError, DirstateParents, - DirstateParseError, DirstateVec, -}; +use hg::{DirstateEntry, DirstateParseError, EntryState, StateMap}; use libc::{c_char, c_int}; #[cfg(feature = "python27")] use python27_sys::PyCapsule_Import; #[cfg(feature = "python3")] use python3_sys::PyCapsule_Import; -use std::cell::RefCell; -use std::collections::HashMap; +use std::convert::TryFrom; use std::ffi::CStr; use std::mem::transmute; @@ -45,7 +43,9 @@ /// This is largely a copy/paste from cindex.rs, pending the merge of a /// `py_capsule_fn!` macro in the rust-cpython project: /// https://github.com/dgrunwald/rust-cpython/pull/169 -fn decapsule_make_dirstate_tuple(py: Python) -> PyResult<MakeDirstateTupleFn> { +pub fn decapsule_make_dirstate_tuple( + py: Python, +) -> PyResult<MakeDirstateTupleFn> { unsafe { let caps_name = CStr::from_bytes_with_nul_unchecked( b"mercurial.cext.parsers.make_dirstate_tuple_CAPI\0", @@ -58,61 +58,17 @@ } } -fn parse_dirstate_wrapper( - py: Python, - dmap: PyDict, - copymap: PyDict, - st: PyBytes, -) -> PyResult<PyTuple> { - match parse_dirstate(st.data(py)) { - Ok((parents, dirstate_vec, copies)) => { - for (filename, entry) in dirstate_vec { - dmap.set_item( - py, - PyBytes::new(py, &filename[..]), - decapsule_make_dirstate_tuple(py)?( - entry.state as c_char, - entry.mode, - entry.size, - entry.mtime, - ), - )?; - } - for CopyVecEntry { path, copy_path } in copies { - copymap.set_item( - py, - PyBytes::new(py, path), - PyBytes::new(py, copy_path), - )?; - } - Ok((PyBytes::new(py, parents.p1), PyBytes::new(py, parents.p2)) - .to_py_object(py)) - } - Err(e) => Err(PyErr::new::<exc::ValueError, _>( - py, - match e { - DirstateParseError::TooLittleData => { - "too little data for parents".to_string() - } - DirstateParseError::Overflow => { - "overflow in dirstate".to_string() - } - DirstateParseError::CorruptedEntry(e) => e, - }, - )), - } -} - -fn extract_dirstate_vec( - py: Python, - dmap: &PyDict, -) -> Result<DirstateVec, PyErr> { +pub fn extract_dirstate(py: Python, dmap: &PyDict) -> Result<StateMap, PyErr> { dmap.items(py) .iter() .map(|(filename, stats)| { let stats = stats.extract::<PySequence>(py)?; let state = stats.get_item(py, 0)?.extract::<PyBytes>(py)?; - let state = state.data(py)[0] as i8; + let state = EntryState::try_from(state.data(py)[0]).map_err( + |e: DirstateParseError| { + PyErr::new::<exc::ValueError, _>(py, e.to_string()) + }, + )?; let mode = stats.get_item(py, 1)?.extract(py)?; let size = stats.get_item(py, 2)?.extract(py)?; let mtime = stats.get_item(py, 3)?.extract(py)?; @@ -131,167 +87,6 @@ .collect() } -fn pack_dirstate_wrapper( - py: Python, - dmap: PyDict, - copymap: PyDict, - pl: PyTuple, - now: PyInt, -) -> PyResult<PyBytes> { - let p1 = pl.get_item(py, 0).extract::<PyBytes>(py)?; - let p1: &[u8] = p1.data(py); - let p2 = pl.get_item(py, 1).extract::<PyBytes>(py)?; - let p2: &[u8] = p2.data(py); - - let dirstate_vec = extract_dirstate_vec(py, &dmap)?; - - let copies: Result<HashMap<Vec<u8>, Vec<u8>>, PyErr> = copymap - .items(py) - .iter() - .map(|(key, value)| { - Ok(( - key.extract::<PyBytes>(py)?.data(py).to_owned(), - value.extract::<PyBytes>(py)?.data(py).to_owned(), - )) - }) - .collect(); - - match pack_dirstate( - &dirstate_vec, - &copies?, - DirstateParents { p1, p2 }, - now.as_object().extract::<i32>(py)?, - ) { - Ok((packed, new_dirstate_vec)) => { - for ( - filename, - DirstateEntry { - state, - mode, - size, - mtime, - }, - ) in new_dirstate_vec - { - dmap.set_item( - py, - PyBytes::new(py, &filename[..]), - decapsule_make_dirstate_tuple(py)?( - state as c_char, - mode, - size, - mtime, - ), - )?; - } - Ok(PyBytes::new(py, &packed)) - } - Err(error) => Err(PyErr::new::<exc::ValueError, _>( - py, - match error { - DirstatePackError::CorruptedParent => { - "expected a 20-byte hash".to_string() - } - DirstatePackError::CorruptedEntry(e) => e, - DirstatePackError::BadSize(expected, actual) => { - format!("bad dirstate size: {} != {}", actual, expected) - } - }, - )), - } -} - -py_class!(pub class Dirs |py| { - data dirs_map: RefCell<DirsMultiset>; - - // `map` is either a `dict` or a flat iterator (usually a `set`, sometimes - // a `list`) - def __new__( - _cls, - map: PyObject, - skip: Option<PyObject> = None - ) -> PyResult<Self> { - let mut skip_state: Option<i8> = None; - if let Some(skip) = skip { - skip_state = Some(skip.extract::<PyBytes>(py)?.data(py)[0] as i8); - } - let dirs_map; - - if let Ok(map) = map.cast_as::<PyDict>(py) { - let dirstate_vec = extract_dirstate_vec(py, &map)?; - dirs_map = DirsMultiset::new( - DirsIterable::Dirstate(dirstate_vec), - skip_state, - ) - } else { - let map: Result<Vec<Vec<u8>>, PyErr> = map - .iter(py)? - .map(|o| Ok(o?.extract::<PyBytes>(py)?.data(py).to_owned())) - .collect(); - dirs_map = DirsMultiset::new( - DirsIterable::Manifest(map?), - skip_state, - ) - } - - Self::create_instance(py, RefCell::new(dirs_map)) - } - - def addpath(&self, path: PyObject) -> PyResult<PyObject> { - self.dirs_map(py).borrow_mut().add_path( - path.extract::<PyBytes>(py)?.data(py), - ); - Ok(py.None()) - } - - def delpath(&self, path: PyObject) -> PyResult<PyObject> { - self.dirs_map(py).borrow_mut().delete_path( - path.extract::<PyBytes>(py)?.data(py), - ) - .and(Ok(py.None())) - .or_else(|e| { - match e { - DirstateMapError::PathNotFound(_p) => { - Err(PyErr::new::<exc::ValueError, _>( - py, - "expected a value, found none".to_string(), - )) - } - DirstateMapError::EmptyPath => { - Ok(py.None()) - } - } - }) - } - - // This is really inefficient on top of being ugly, but it's an easy way - // of having it work to continue working on the rest of the module - // hopefully bypassing Python entirely pretty soon. - def __iter__(&self) -> PyResult<PyObject> { - let dict = PyDict::new(py); - - for (key, value) in self.dirs_map(py).borrow().iter() { - dict.set_item( - py, - PyBytes::new(py, &key[..]), - value.to_py_object(py), - )?; - } - - let locals = PyDict::new(py); - locals.set_item(py, "obj", dict)?; - - py.eval("iter(obj)", None, Some(&locals)) - } - - def __contains__(&self, item: PyObject) -> PyResult<bool> { - Ok(self - .dirs_map(py) - .borrow() - .contains_key(item.extract::<PyBytes>(py)?.data(py).as_ref())) - } -}); - /// Create the module, with `__package__` given from parent pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> { let dotted_name = &format!("{}.dirstate", package); @@ -299,29 +94,9 @@ m.add(py, "__package__", package)?; m.add(py, "__doc__", "Dirstate - Rust implementation")?; - m.add( - py, - "parse_dirstate", - py_fn!( - py, - parse_dirstate_wrapper(dmap: PyDict, copymap: PyDict, st: PyBytes) - ), - )?; - m.add( - py, - "pack_dirstate", - py_fn!( - py, - pack_dirstate_wrapper( - dmap: PyDict, - copymap: PyDict, - pl: PyTuple, - now: PyInt - ) - ), - )?; m.add_class::<Dirs>(py)?; + m.add_class::<DirstateMap>(py)?; let sys = PyModule::import(py, "sys")?; let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rust/hg-cpython/src/dirstate/copymap.rs Fri Aug 23 17:03:42 2019 -0400 @@ -0,0 +1,116 @@ +// copymap.rs +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Bindings for `hg::dirstate::dirstate_map::CopyMap` provided by the +//! `hg-core` package. + +use cpython::{PyBytes, PyClone, PyDict, PyObject, PyResult, Python}; +use std::cell::RefCell; + +use crate::dirstate::dirstate_map::{DirstateMap, DirstateMapLeakedRef}; + +py_class!(pub class CopyMap |py| { + data dirstate_map: DirstateMap; + + def __getitem__(&self, key: PyObject) -> PyResult<PyBytes> { + (*self.dirstate_map(py)).copymapgetitem(py, key) + } + + def __len__(&self) -> PyResult<usize> { + self.dirstate_map(py).copymaplen(py) + } + + def __contains__(&self, key: PyObject) -> PyResult<bool> { + self.dirstate_map(py).copymapcontains(py, key) + } + + def get( + &self, + key: PyObject, + default: Option<PyObject> = None + ) -> PyResult<Option<PyObject>> { + self.dirstate_map(py).copymapget(py, key, default) + } + + def pop( + &self, + key: PyObject, + default: Option<PyObject> = None + ) -> PyResult<Option<PyObject>> { + self.dirstate_map(py).copymappop(py, key, default) + } + + def __iter__(&self) -> PyResult<CopyMapKeysIterator> { + self.dirstate_map(py).copymapiter(py) + } + + // Python's `dict()` builtin works with either a subclass of dict + // or an abstract mapping. Said mapping needs to implement `__getitem__` + // and `keys`. + def keys(&self) -> PyResult<CopyMapKeysIterator> { + self.dirstate_map(py).copymapiter(py) + } + + def items(&self) -> PyResult<CopyMapItemsIterator> { + self.dirstate_map(py).copymapitemsiter(py) + } + + def iteritems(&self) -> PyResult<CopyMapItemsIterator> { + self.dirstate_map(py).copymapitemsiter(py) + } + + def __setitem__( + &self, + key: PyObject, + item: PyObject + ) -> PyResult<()> { + self.dirstate_map(py).copymapsetitem(py, key, item)?; + Ok(()) + } + + def copy(&self) -> PyResult<PyDict> { + self.dirstate_map(py).copymapcopy(py) + } + +}); + +impl CopyMap { + pub fn from_inner(py: Python, dm: DirstateMap) -> PyResult<Self> { + Self::create_instance(py, dm) + } + fn translate_key( + py: Python, + res: (&Vec<u8>, &Vec<u8>), + ) -> PyResult<Option<PyBytes>> { + Ok(Some(PyBytes::new(py, res.0))) + } + fn translate_key_value( + py: Python, + res: (&Vec<u8>, &Vec<u8>), + ) -> PyResult<Option<(PyBytes, PyBytes)>> { + let (k, v) = res; + Ok(Some((PyBytes::new(py, k), PyBytes::new(py, v)))) + } +} + +py_shared_mapping_iterator!( + CopyMapKeysIterator, + DirstateMapLeakedRef, + Vec<u8>, + Vec<u8>, + CopyMap::translate_key, + Option<PyBytes> +); + +py_shared_mapping_iterator!( + CopyMapItemsIterator, + DirstateMapLeakedRef, + Vec<u8>, + Vec<u8>, + CopyMap::translate_key_value, + Option<(PyBytes, PyBytes)> +);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rust/hg-cpython/src/dirstate/dirs_multiset.rs Fri Aug 23 17:03:42 2019 -0400 @@ -0,0 +1,121 @@ +// dirs_multiset.rs +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Bindings for the `hg::dirstate::dirs_multiset` file provided by the +//! `hg-core` package. + +use std::cell::RefCell; +use std::convert::TryInto; + +use cpython::{ + exc, ObjectProtocol, PyBytes, PyClone, PyDict, PyErr, PyObject, PyResult, + Python, +}; + +use crate::{dirstate::extract_dirstate, ref_sharing::PySharedState}; +use hg::{DirsMultiset, DirstateMapError, DirstateParseError, EntryState}; + +py_class!(pub class Dirs |py| { + data inner: RefCell<DirsMultiset>; + data py_shared_state: PySharedState; + + // `map` is either a `dict` or a flat iterator (usually a `set`, sometimes + // a `list`) + def __new__( + _cls, + map: PyObject, + skip: Option<PyObject> = None + ) -> PyResult<Self> { + let mut skip_state: Option<EntryState> = None; + if let Some(skip) = skip { + skip_state = Some( + skip.extract::<PyBytes>(py)?.data(py)[0] + .try_into() + .map_err(|e: DirstateParseError| { + PyErr::new::<exc::ValueError, _>(py, e.to_string()) + })?, + ); + } + let inner = if let Ok(map) = map.cast_as::<PyDict>(py) { + let dirstate = extract_dirstate(py, &map)?; + DirsMultiset::from_dirstate(&dirstate, skip_state) + } else { + let map: Result<Vec<Vec<u8>>, PyErr> = map + .iter(py)? + .map(|o| Ok(o?.extract::<PyBytes>(py)?.data(py).to_owned())) + .collect(); + DirsMultiset::from_manifest(&map?) + }; + + Self::create_instance( + py, + RefCell::new(inner), + PySharedState::default() + ) + } + + def addpath(&self, path: PyObject) -> PyResult<PyObject> { + self.borrow_mut(py)?.add_path( + path.extract::<PyBytes>(py)?.data(py), + ); + Ok(py.None()) + } + + def delpath(&self, path: PyObject) -> PyResult<PyObject> { + self.borrow_mut(py)?.delete_path( + path.extract::<PyBytes>(py)?.data(py), + ) + .and(Ok(py.None())) + .or_else(|e| { + match e { + DirstateMapError::PathNotFound(_p) => { + Err(PyErr::new::<exc::ValueError, _>( + py, + "expected a value, found none".to_string(), + )) + } + DirstateMapError::EmptyPath => { + Ok(py.None()) + } + } + }) + } + def __iter__(&self) -> PyResult<DirsMultisetKeysIterator> { + DirsMultisetKeysIterator::create_instance( + py, + RefCell::new(Some(DirsMultisetLeakedRef::new(py, &self))), + RefCell::new(Box::new(self.leak_immutable(py)?.iter())), + ) + } + + def __contains__(&self, item: PyObject) -> PyResult<bool> { + Ok(self + .inner(py) + .borrow() + .contains(item.extract::<PyBytes>(py)?.data(py).as_ref())) + } +}); + +py_shared_ref!(Dirs, DirsMultiset, inner, DirsMultisetLeakedRef,); + +impl Dirs { + pub fn from_inner(py: Python, d: DirsMultiset) -> PyResult<Self> { + Self::create_instance(py, RefCell::new(d), PySharedState::default()) + } + + fn translate_key(py: Python, res: &Vec<u8>) -> PyResult<Option<PyBytes>> { + Ok(Some(PyBytes::new(py, res))) + } +} + +py_shared_sequence_iterator!( + DirsMultisetKeysIterator, + DirsMultisetLeakedRef, + Vec<u8>, + Dirs::translate_key, + Option<PyBytes> +);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rust/hg-cpython/src/dirstate/dirstate_map.rs Fri Aug 23 17:03:42 2019 -0400 @@ -0,0 +1,515 @@ +// dirstate_map.rs +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Bindings for the `hg::dirstate::dirstate_map` file provided by the +//! `hg-core` package. + +use std::cell::RefCell; +use std::convert::TryInto; +use std::time::Duration; + +use cpython::{ + exc, ObjectProtocol, PyBool, PyBytes, PyClone, PyDict, PyErr, PyObject, + PyResult, PyTuple, Python, PythonObject, ToPyObject, +}; +use libc::c_char; + +use crate::{ + dirstate::copymap::{CopyMap, CopyMapItemsIterator, CopyMapKeysIterator}, + dirstate::{decapsule_make_dirstate_tuple, dirs_multiset::Dirs}, + ref_sharing::PySharedState, +}; +use hg::{ + DirsMultiset, DirstateEntry, DirstateMap as RustDirstateMap, + DirstateParents, DirstateParseError, EntryState, PARENT_SIZE, +}; + +// TODO +// This object needs to share references to multiple members of its Rust +// inner struct, namely `copy_map`, `dirs` and `all_dirs`. +// Right now `CopyMap` is done, but it needs to have an explicit reference +// to `RustDirstateMap` which itself needs to have an encapsulation for +// every method in `CopyMap` (copymapcopy, etc.). +// This is ugly and hard to maintain. +// The same logic applies to `dirs` and `all_dirs`, however the `Dirs` +// `py_class!` is already implemented and does not mention +// `RustDirstateMap`, rightfully so. +// All attributes also have to have a separate refcount data attribute for +// leaks, with all methods that go along for reference sharing. +py_class!(pub class DirstateMap |py| { + data inner: RefCell<RustDirstateMap>; + data py_shared_state: PySharedState; + + def __new__(_cls, _root: PyObject) -> PyResult<Self> { + let inner = RustDirstateMap::default(); + Self::create_instance( + py, + RefCell::new(inner), + PySharedState::default() + ) + } + + def clear(&self) -> PyResult<PyObject> { + self.borrow_mut(py)?.clear(); + Ok(py.None()) + } + + def get( + &self, + key: PyObject, + default: Option<PyObject> = None + ) -> PyResult<Option<PyObject>> { + let key = key.extract::<PyBytes>(py)?; + match self.inner(py).borrow().get(key.data(py)) { + Some(entry) => { + // Explicitly go through u8 first, then cast to + // platform-specific `c_char`. + let state: u8 = entry.state.into(); + Ok(Some(decapsule_make_dirstate_tuple(py)?( + state as c_char, + entry.mode, + entry.size, + entry.mtime, + ))) + }, + None => Ok(default) + } + } + + def addfile( + &self, + f: PyObject, + oldstate: PyObject, + state: PyObject, + mode: PyObject, + size: PyObject, + mtime: PyObject + ) -> PyResult<PyObject> { + self.borrow_mut(py)?.add_file( + f.extract::<PyBytes>(py)?.data(py), + oldstate.extract::<PyBytes>(py)?.data(py)[0] + .try_into() + .map_err(|e: DirstateParseError| { + PyErr::new::<exc::ValueError, _>(py, e.to_string()) + })?, + DirstateEntry { + state: state.extract::<PyBytes>(py)?.data(py)[0] + .try_into() + .map_err(|e: DirstateParseError| { + PyErr::new::<exc::ValueError, _>(py, e.to_string()) + })?, + mode: mode.extract(py)?, + size: size.extract(py)?, + mtime: mtime.extract(py)?, + }, + ); + Ok(py.None()) + } + + def removefile( + &self, + f: PyObject, + oldstate: PyObject, + size: PyObject + ) -> PyResult<PyObject> { + self.borrow_mut(py)? + .remove_file( + f.extract::<PyBytes>(py)?.data(py), + oldstate.extract::<PyBytes>(py)?.data(py)[0] + .try_into() + .map_err(|e: DirstateParseError| { + PyErr::new::<exc::ValueError, _>(py, e.to_string()) + })?, + size.extract(py)?, + ) + .or_else(|_| { + Err(PyErr::new::<exc::OSError, _>( + py, + "Dirstate error".to_string(), + )) + })?; + Ok(py.None()) + } + + def dropfile( + &self, + f: PyObject, + oldstate: PyObject + ) -> PyResult<PyBool> { + self.borrow_mut(py)? + .drop_file( + f.extract::<PyBytes>(py)?.data(py), + oldstate.extract::<PyBytes>(py)?.data(py)[0] + .try_into() + .map_err(|e: DirstateParseError| { + PyErr::new::<exc::ValueError, _>(py, e.to_string()) + })?, + ) + .and_then(|b| Ok(b.to_py_object(py))) + .or_else(|_| { + Err(PyErr::new::<exc::OSError, _>( + py, + "Dirstate error".to_string(), + )) + }) + } + + def clearambiguoustimes( + &self, + files: PyObject, + now: PyObject + ) -> PyResult<PyObject> { + let files: PyResult<Vec<Vec<u8>>> = files + .iter(py)? + .map(|filename| { + Ok(filename?.extract::<PyBytes>(py)?.data(py).to_owned()) + }) + .collect(); + self.inner(py) + .borrow_mut() + .clear_ambiguous_times(files?, now.extract(py)?); + Ok(py.None()) + } + + // TODO share the reference + def nonnormalentries(&self) -> PyResult<PyObject> { + let (non_normal, other_parent) = + self.inner(py).borrow().non_normal_other_parent_entries(); + + let locals = PyDict::new(py); + locals.set_item( + py, + "non_normal", + non_normal + .iter() + .map(|v| PyBytes::new(py, &v)) + .collect::<Vec<PyBytes>>() + .to_py_object(py), + )?; + locals.set_item( + py, + "other_parent", + other_parent + .iter() + .map(|v| PyBytes::new(py, &v)) + .collect::<Vec<PyBytes>>() + .to_py_object(py), + )?; + + py.eval("set(non_normal), set(other_parent)", None, Some(&locals)) + } + + def hastrackeddir(&self, d: PyObject) -> PyResult<PyBool> { + let d = d.extract::<PyBytes>(py)?; + Ok(self + .inner(py) + .borrow_mut() + .has_tracked_dir(d.data(py)) + .to_py_object(py)) + } + + def hasdir(&self, d: PyObject) -> PyResult<PyBool> { + let d = d.extract::<PyBytes>(py)?; + Ok(self + .inner(py) + .borrow_mut() + .has_dir(d.data(py)) + .to_py_object(py)) + } + + def parents(&self, st: PyObject) -> PyResult<PyTuple> { + self.inner(py) + .borrow_mut() + .parents(st.extract::<PyBytes>(py)?.data(py)) + .and_then(|d| { + Ok((PyBytes::new(py, &d.p1), PyBytes::new(py, &d.p2)) + .to_py_object(py)) + }) + .or_else(|_| { + Err(PyErr::new::<exc::OSError, _>( + py, + "Dirstate error".to_string(), + )) + }) + } + + def setparents(&self, p1: PyObject, p2: PyObject) -> PyResult<PyObject> { + let p1 = extract_node_id(py, &p1)?; + let p2 = extract_node_id(py, &p2)?; + + self.inner(py) + .borrow_mut() + .set_parents(&DirstateParents { p1, p2 }); + Ok(py.None()) + } + + def read(&self, st: PyObject) -> PyResult<Option<PyObject>> { + match self + .inner(py) + .borrow_mut() + .read(st.extract::<PyBytes>(py)?.data(py)) + { + Ok(Some(parents)) => Ok(Some( + (PyBytes::new(py, &parents.p1), PyBytes::new(py, &parents.p2)) + .to_py_object(py) + .into_object(), + )), + Ok(None) => Ok(Some(py.None())), + Err(_) => Err(PyErr::new::<exc::OSError, _>( + py, + "Dirstate error".to_string(), + )), + } + } + def write( + &self, + p1: PyObject, + p2: PyObject, + now: PyObject + ) -> PyResult<PyBytes> { + let now = Duration::new(now.extract(py)?, 0); + let parents = DirstateParents { + p1: extract_node_id(py, &p1)?, + p2: extract_node_id(py, &p2)?, + }; + + match self.borrow_mut(py)?.pack(parents, now) { + Ok(packed) => Ok(PyBytes::new(py, &packed)), + Err(_) => Err(PyErr::new::<exc::OSError, _>( + py, + "Dirstate error".to_string(), + )), + } + } + + def filefoldmapasdict(&self) -> PyResult<PyDict> { + let dict = PyDict::new(py); + for (key, value) in + self.borrow_mut(py)?.build_file_fold_map().iter() + { + dict.set_item(py, key, value)?; + } + Ok(dict) + } + + def __len__(&self) -> PyResult<usize> { + Ok(self.inner(py).borrow().len()) + } + + def __contains__(&self, key: PyObject) -> PyResult<bool> { + let key = key.extract::<PyBytes>(py)?; + Ok(self.inner(py).borrow().contains_key(key.data(py))) + } + + def __getitem__(&self, key: PyObject) -> PyResult<PyObject> { + let key = key.extract::<PyBytes>(py)?; + let key = key.data(py); + match self.inner(py).borrow().get(key) { + Some(entry) => { + // Explicitly go through u8 first, then cast to + // platform-specific `c_char`. + let state: u8 = entry.state.into(); + Ok(decapsule_make_dirstate_tuple(py)?( + state as c_char, + entry.mode, + entry.size, + entry.mtime, + )) + }, + None => Err(PyErr::new::<exc::KeyError, _>( + py, + String::from_utf8_lossy(key), + )), + } + } + + def keys(&self) -> PyResult<DirstateMapKeysIterator> { + DirstateMapKeysIterator::from_inner( + py, + Some(DirstateMapLeakedRef::new(py, &self)), + Box::new(self.leak_immutable(py)?.iter()), + ) + } + + def items(&self) -> PyResult<DirstateMapItemsIterator> { + DirstateMapItemsIterator::from_inner( + py, + Some(DirstateMapLeakedRef::new(py, &self)), + Box::new(self.leak_immutable(py)?.iter()), + ) + } + + def __iter__(&self) -> PyResult<DirstateMapKeysIterator> { + DirstateMapKeysIterator::from_inner( + py, + Some(DirstateMapLeakedRef::new(py, &self)), + Box::new(self.leak_immutable(py)?.iter()), + ) + } + + def getdirs(&self) -> PyResult<Dirs> { + // TODO don't copy, share the reference + self.inner(py).borrow_mut().set_dirs(); + Dirs::from_inner( + py, + DirsMultiset::from_dirstate( + &self.inner(py).borrow(), + Some(EntryState::Removed), + ), + ) + } + def getalldirs(&self) -> PyResult<Dirs> { + // TODO don't copy, share the reference + self.inner(py).borrow_mut().set_all_dirs(); + Dirs::from_inner( + py, + DirsMultiset::from_dirstate( + &self.inner(py).borrow(), + None, + ), + ) + } + + // TODO all copymap* methods, see docstring above + def copymapcopy(&self) -> PyResult<PyDict> { + let dict = PyDict::new(py); + for (key, value) in self.inner(py).borrow().copy_map.iter() { + dict.set_item(py, PyBytes::new(py, key), PyBytes::new(py, value))?; + } + Ok(dict) + } + + def copymapgetitem(&self, key: PyObject) -> PyResult<PyBytes> { + let key = key.extract::<PyBytes>(py)?; + match self.inner(py).borrow().copy_map.get(key.data(py)) { + Some(copy) => Ok(PyBytes::new(py, copy)), + None => Err(PyErr::new::<exc::KeyError, _>( + py, + String::from_utf8_lossy(key.data(py)), + )), + } + } + def copymap(&self) -> PyResult<CopyMap> { + CopyMap::from_inner(py, self.clone_ref(py)) + } + + def copymaplen(&self) -> PyResult<usize> { + Ok(self.inner(py).borrow().copy_map.len()) + } + def copymapcontains(&self, key: PyObject) -> PyResult<bool> { + let key = key.extract::<PyBytes>(py)?; + Ok(self.inner(py).borrow().copy_map.contains_key(key.data(py))) + } + def copymapget( + &self, + key: PyObject, + default: Option<PyObject> + ) -> PyResult<Option<PyObject>> { + let key = key.extract::<PyBytes>(py)?; + match self.inner(py).borrow().copy_map.get(key.data(py)) { + Some(copy) => Ok(Some(PyBytes::new(py, copy).into_object())), + None => Ok(default), + } + } + def copymapsetitem( + &self, + key: PyObject, + value: PyObject + ) -> PyResult<PyObject> { + let key = key.extract::<PyBytes>(py)?; + let value = value.extract::<PyBytes>(py)?; + self.inner(py) + .borrow_mut() + .copy_map + .insert(key.data(py).to_vec(), value.data(py).to_vec()); + Ok(py.None()) + } + def copymappop( + &self, + key: PyObject, + default: Option<PyObject> + ) -> PyResult<Option<PyObject>> { + let key = key.extract::<PyBytes>(py)?; + match self.inner(py).borrow_mut().copy_map.remove(key.data(py)) { + Some(_) => Ok(None), + None => Ok(default), + } + } + + def copymapiter(&self) -> PyResult<CopyMapKeysIterator> { + CopyMapKeysIterator::from_inner( + py, + Some(DirstateMapLeakedRef::new(py, &self)), + Box::new(self.leak_immutable(py)?.copy_map.iter()), + ) + } + + def copymapitemsiter(&self) -> PyResult<CopyMapItemsIterator> { + CopyMapItemsIterator::from_inner( + py, + Some(DirstateMapLeakedRef::new(py, &self)), + Box::new(self.leak_immutable(py)?.copy_map.iter()), + ) + } + +}); + +impl DirstateMap { + fn translate_key( + py: Python, + res: (&Vec<u8>, &DirstateEntry), + ) -> PyResult<Option<PyBytes>> { + Ok(Some(PyBytes::new(py, res.0))) + } + fn translate_key_value( + py: Python, + res: (&Vec<u8>, &DirstateEntry), + ) -> PyResult<Option<(PyBytes, PyObject)>> { + let (f, entry) = res; + + // Explicitly go through u8 first, then cast to + // platform-specific `c_char`. + let state: u8 = entry.state.into(); + Ok(Some(( + PyBytes::new(py, f), + decapsule_make_dirstate_tuple(py)?( + state as c_char, + entry.mode, + entry.size, + entry.mtime, + ), + ))) + } +} + +py_shared_ref!(DirstateMap, RustDirstateMap, inner, DirstateMapLeakedRef,); + +py_shared_mapping_iterator!( + DirstateMapKeysIterator, + DirstateMapLeakedRef, + Vec<u8>, + DirstateEntry, + DirstateMap::translate_key, + Option<PyBytes> +); + +py_shared_mapping_iterator!( + DirstateMapItemsIterator, + DirstateMapLeakedRef, + Vec<u8>, + DirstateEntry, + DirstateMap::translate_key_value, + Option<(PyBytes, PyObject)> +); + +fn extract_node_id(py: Python, obj: &PyObject) -> PyResult<[u8; PARENT_SIZE]> { + let bytes = obj.extract::<PyBytes>(py)?; + match bytes.data(py).try_into() { + Ok(s) => Ok(s), + Err(e) => Err(PyErr::new::<exc::ValueError, _>(py, e.to_string())), + } +}
--- a/rust/hg-cpython/src/discovery.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-cpython/src/discovery.rs Fri Aug 23 17:03:42 2019 -0400 @@ -18,7 +18,7 @@ exceptions::GraphError, }; use cpython::{ - ObjectProtocol, PyDict, PyModule, PyObject, PyResult, Python, + ObjectProtocol, PyDict, PyModule, PyObject, PyResult, PyTuple, Python, PythonObject, ToPyObject, }; use hg::discovery::PartialDiscovery as CorePartialDiscovery; @@ -29,16 +29,24 @@ py_class!(pub class PartialDiscovery |py| { data inner: RefCell<Box<CorePartialDiscovery<Index>>>; + // `_respectsize` is currently only here to replicate the Python API and + // will be used in future patches inside methods that are yet to be + // implemented. def __new__( _cls, - index: PyObject, - targetheads: PyObject + repo: PyObject, + targetheads: PyObject, + respectsize: bool, + randomize: bool = true ) -> PyResult<PartialDiscovery> { + let index = repo.getattr(py, "changelog")?.getattr(py, "index")?; Self::create_instance( py, RefCell::new(Box::new(CorePartialDiscovery::new( Index::new(py, index)?, rev_pyiter_collect(py, &targetheads)?, + respectsize, + randomize, ))) ) } @@ -105,6 +113,32 @@ .map_err(|e| GraphError::pynew(py, e))? ) } + + def takefullsample(&self, _headrevs: PyObject, + size: usize) -> PyResult<PyObject> { + let mut inner = self.inner(py).borrow_mut(); + let sample = inner.take_full_sample(size) + .map_err(|e| GraphError::pynew(py, e))?; + let as_vec: Vec<PyObject> = sample + .iter() + .map(|rev| rev.to_py_object(py).into_object()) + .collect(); + Ok(PyTuple::new(py, as_vec.as_slice()).into_object()) + } + + def takequicksample(&self, headrevs: PyObject, + size: usize) -> PyResult<PyObject> { + let mut inner = self.inner(py).borrow_mut(); + let revsvec: Vec<Revision> = rev_pyiter_collect(py, &headrevs)?; + let sample = inner.take_quick_sample(revsvec, size) + .map_err(|e| GraphError::pynew(py, e))?; + let as_vec: Vec<PyObject> = sample + .iter() + .map(|rev| rev.to_py_object(py).into_object()) + .collect(); + Ok(PyTuple::new(py, as_vec.as_slice()).into_object()) + } + }); /// Create the module, with __package__ given from parent
--- a/rust/hg-cpython/src/exceptions.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-cpython/src/exceptions.rs Fri Aug 23 17:03:42 2019 -0400 @@ -67,3 +67,5 @@ } } } + +py_exception!(shared_ref, AlreadyBorrowed, RuntimeError);
--- a/rust/hg-cpython/src/lib.rs Wed Aug 21 17:56:50 2019 +0200 +++ b/rust/hg-cpython/src/lib.rs Fri Aug 23 17:03:42 2019 -0400 @@ -27,8 +27,11 @@ pub mod ancestors; mod cindex; mod conversion; +#[macro_use] +pub mod ref_sharing; pub mod dagops; pub mod dirstate; +pub mod parsers; pub mod discovery; pub mod exceptions; pub mod filepatterns; @@ -50,6 +53,11 @@ "filepatterns", filepatterns::init_module(py, &dotted_name)?, )?; + m.add( + py, + "parsers", + parsers::init_parsers_module(py, &dotted_name)?, + )?; m.add(py, "GraphError", py.get_type::<exceptions::GraphError>())?; m.add( py,
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rust/hg-cpython/src/parsers.rs Fri Aug 23 17:03:42 2019 -0400 @@ -0,0 +1,208 @@ +// parsers.rs +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Bindings for the `hg::dirstate::parsers` module provided by the +//! `hg-core` package. +//! +//! From Python, this will be seen as `mercurial.rustext.parsers` +//! +use cpython::{ + exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyResult, PyTuple, Python, + PythonObject, ToPyObject, +}; +use hg::{ + pack_dirstate, parse_dirstate, DirstateEntry, DirstatePackError, + DirstateParents, DirstateParseError, PARENT_SIZE, +}; +use std::collections::HashMap; +use std::convert::TryInto; + +use libc::c_char; + +use crate::dirstate::{decapsule_make_dirstate_tuple, extract_dirstate}; +use std::time::Duration; + +fn parse_dirstate_wrapper( + py: Python, + dmap: PyDict, + copymap: PyDict, + st: PyBytes, +) -> PyResult<PyTuple> { + let mut dirstate_map = HashMap::new(); + let mut copies = HashMap::new(); + + match parse_dirstate(&mut dirstate_map, &mut copies, st.data(py)) { + Ok(parents) => { + for (filename, entry) in dirstate_map { + // Explicitly go through u8 first, then cast to + // platform-specific `c_char` because Into<u8> has a specific + // implementation while `as c_char` would just do a naive enum + // cast. + let state: u8 = entry.state.into(); + + dmap.set_item( + py, + PyBytes::new(py, &filename), + decapsule_make_dirstate_tuple(py)?( + state as c_char, + entry.mode, + entry.size, + entry.mtime, + ), + )?; + } + for (path, copy_path) in copies { + copymap.set_item( + py, + PyBytes::new(py, &path), + PyBytes::new(py, ©_path), + )?; + } + Ok( + (PyBytes::new(py, &parents.p1), PyBytes::new(py, &parents.p2)) + .to_py_object(py), + ) + } + Err(e) => Err(PyErr::new::<exc::ValueError, _>( + py, + match e { + DirstateParseError::TooLittleData => { + "too little data for parents".to_string() + } + DirstateParseError::Overflow => { + "overflow in dirstate".to_string() + } + DirstateParseError::CorruptedEntry(e) => e, + DirstateParseError::Damaged => { + "dirstate appears to be damaged".to_string() + } + }, + )), + } +} + +fn pack_dirstate_wrapper( + py: Python, + dmap: PyDict, + copymap: PyDict, + pl: PyTuple, + now: PyInt, +) -> PyResult<PyBytes> { + let p1 = pl.get_item(py, 0).extract::<PyBytes>(py)?; + let p1: &[u8] = p1.data(py); + let p2 = pl.get_item(py, 1).extract::<PyBytes>(py)?; + let p2: &[u8] = p2.data(py); + + let mut dirstate_map = extract_dirstate(py, &dmap)?; + + let copies: Result<HashMap<Vec<u8>, Vec<u8>>, PyErr> = copymap + .items(py) + .iter() + .map(|(key, value)| { + Ok(( + key.extract::<PyBytes>(py)?.data(py).to_owned(), + value.extract::<PyBytes>(py)?.data(py).to_owned(), + )) + }) + .collect(); + + if p1.len() != PARENT_SIZE || p2.len() != PARENT_SIZE { + return Err(PyErr::new::<exc::ValueError, _>( + py, + "expected a 20-byte hash".to_string(), + )); + } + + match pack_dirstate( + &mut dirstate_map, + &copies?, + DirstateParents { + p1: p1.try_into().unwrap(), + p2: p2.try_into().unwrap(), + }, + Duration::from_secs(now.as_object().extract::<u64>(py)?), + ) { + Ok(packed) => { + for ( + filename, + DirstateEntry { + state, + mode, + size, + mtime, + }, + ) in dirstate_map + { + // Explicitly go through u8 first, then cast to + // platform-specific `c_char` because Into<u8> has a specific + // implementation while `as c_char` would just do a naive enum + // cast. + let state: u8 = state.into(); + dmap.set_item( + py, + PyBytes::new(py, &filename[..]), + decapsule_make_dirstate_tuple(py)?( + state as c_char, + mode, + size, + mtime, + ), + )?; + } + Ok(PyBytes::new(py, &packed)) + } + Err(error) => Err(PyErr::new::<exc::ValueError, _>( + py, + match error { + DirstatePackError::CorruptedParent => { + "expected a 20-byte hash".to_string() + } + DirstatePackError::CorruptedEntry(e) => e, + DirstatePackError::BadSize(expected, actual) => { + format!("bad dirstate size: {} != {}", actual, expected) + } + }, + )), + } +} + +/// Create the module, with `__package__` given from parent +pub fn init_parsers_module(py: Python, package: &str) -> PyResult<PyModule> { + let dotted_name = &format!("{}.parsers", package); + let m = PyModule::new(py, dotted_name)?; + + m.add(py, "__package__", package)?; + m.add(py, "__doc__", "Parsers - Rust implementation")?; + + m.add( + py, + "parse_dirstate", + py_fn!( + py, + parse_dirstate_wrapper(dmap: PyDict, copymap: PyDict, st: PyBytes) + ), + )?; + m.add( + py, + "pack_dirstate", + py_fn!( + py, + pack_dirstate_wrapper( + dmap: PyDict, + copymap: PyDict, + pl: PyTuple, + now: PyInt + ) + ), + )?; + + let sys = PyModule::import(py, "sys")?; + let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?; + sys_modules.set_item(py, dotted_name, &m)?; + + Ok(m) +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rust/hg-cpython/src/ref_sharing.rs Fri Aug 23 17:03:42 2019 -0400 @@ -0,0 +1,375 @@ +// macros.rs +// +// Copyright 2019 Raphaël Gomès <rgomes@octobus.net> +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +//! Macros for use in the `hg-cpython` bridge library. + +use crate::exceptions::AlreadyBorrowed; +use cpython::{PyResult, Python}; +use std::cell::{Cell, RefCell, RefMut}; + +/// Manages the shared state between Python and Rust +#[derive(Default)] +pub struct PySharedState { + leak_count: Cell<usize>, + mutably_borrowed: Cell<bool>, +} + +impl PySharedState { + pub fn borrow_mut<'a, T>( + &'a self, + py: Python<'a>, + pyrefmut: RefMut<'a, T>, + ) -> PyResult<PyRefMut<'a, T>> { + if self.mutably_borrowed.get() { + return Err(AlreadyBorrowed::new( + py, + "Cannot borrow mutably while there exists another \ + mutable reference in a Python object", + )); + } + match self.leak_count.get() { + 0 => { + self.mutably_borrowed.replace(true); + Ok(PyRefMut::new(py, pyrefmut, self)) + } + // TODO + // For now, this works differently than Python references + // in the case of iterators. + // Python does not complain when the data an iterator + // points to is modified if the iterator is never used + // afterwards. + // Here, we are stricter than this by refusing to give a + // mutable reference if it is already borrowed. + // While the additional safety might be argued for, it + // breaks valid programming patterns in Python and we need + // to fix this issue down the line. + _ => Err(AlreadyBorrowed::new( + py, + "Cannot borrow mutably while there are \ + immutable references in Python objects", + )), + } + } + + /// Return a reference to the wrapped data with an artificial static + /// lifetime. + /// We need to be protected by the GIL for thread-safety. + pub fn leak_immutable<T>( + &self, + py: Python, + data: &RefCell<T>, + ) -> PyResult<&'static T> { + if self.mutably_borrowed.get() { + return Err(AlreadyBorrowed::new( + py, + "Cannot borrow immutably while there is a \ + mutable reference in Python objects", + )); + } + let ptr = data.as_ptr(); + self.leak_count.replace(self.leak_count.get() + 1); + unsafe { Ok(&*ptr) } + } + + pub fn decrease_leak_count(&self, _py: Python, mutable: bool) { + self.leak_count + .replace(self.leak_count.get().saturating_sub(1)); + if mutable { + self.mutably_borrowed.replace(false); + } + } +} + +/// Holds a mutable reference to data shared between Python and Rust. +pub struct PyRefMut<'a, T> { + inner: RefMut<'a, T>, + py_shared_state: &'a PySharedState, +} + +impl<'a, T> PyRefMut<'a, T> { + fn new( + _py: Python<'a>, + inner: RefMut<'a, T>, + py_shared_state: &'a PySharedState, + ) -> Self { + Self { + inner, + py_shared_state, + } + } +} + +impl<'a, T> std::ops::Deref for PyRefMut<'a, T> { + type Target = RefMut<'a, T>; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} +impl<'a, T> std::ops::DerefMut for PyRefMut<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl<'a, T> Drop for PyRefMut<'a, T> { + fn drop(&mut self) { + let gil = Python::acquire_gil(); + let py = gil.python(); + self.py_shared_state.decrease_leak_count(py, true); + } +} + +/// Allows a `py_class!` generated struct to share references to one of its +/// data members with Python. +/// +/// # Warning +/// +/// The targeted `py_class!` needs to have the +/// `data py_shared_state: PySharedState;` data attribute to compile. +/// A better, more complicated macro is needed to automatically insert it, +/// but this one is not yet really battle tested (what happens when +/// multiple references are needed?). See the example below. +/// +/// TODO allow Python container types: for now, integration with the garbage +/// collector does not extend to Rust structs holding references to Python +/// objects. Should the need surface, `__traverse__` and `__clear__` will +/// need to be written as per the `rust-cpython` docs on GC integration. +/// +/// # Parameters +/// +/// * `$name` is the same identifier used in for `py_class!` macro call. +/// * `$inner_struct` is the identifier of the underlying Rust struct +/// * `$data_member` is the identifier of the data member of `$inner_struct` +/// that will be shared. +/// * `$leaked` is the identifier to give to the struct that will manage +/// references to `$name`, to be used for example in other macros like +/// `py_shared_mapping_iterator`. +/// +/// # Example +/// +/// ``` +/// struct MyStruct { +/// inner: Vec<u32>; +/// } +/// +/// py_class!(pub class MyType |py| { +/// data inner: RefCell<MyStruct>; +/// data py_shared_state: PySharedState; +/// }); +/// +/// py_shared_ref!(MyType, MyStruct, inner, MyTypeLeakedRef); +/// ``` +macro_rules! py_shared_ref { + ( + $name: ident, + $inner_struct: ident, + $data_member: ident, + $leaked: ident, + ) => { + impl $name { + fn borrow_mut<'a>( + &'a self, + py: Python<'a>, + ) -> PyResult<crate::ref_sharing::PyRefMut<'a, $inner_struct>> + { + self.py_shared_state(py) + .borrow_mut(py, self.$data_member(py).borrow_mut()) + } + + fn leak_immutable<'a>( + &'a self, + py: Python<'a>, + ) -> PyResult<&'static $inner_struct> { + self.py_shared_state(py) + .leak_immutable(py, self.$data_member(py)) + } + } + + /// Manage immutable references to `$name` leaked into Python + /// iterators. + /// + /// In truth, this does not represent leaked references themselves; + /// it is instead useful alongside them to manage them. + pub struct $leaked { + inner: $name, + } + + impl $leaked { + fn new(py: Python, inner: &$name) -> Self { + Self { + inner: inner.clone_ref(py), + } + } + } + + impl Drop for $leaked { + fn drop(&mut self) { + let gil = Python::acquire_gil(); + let py = gil.python(); + self.inner + .py_shared_state(py) + .decrease_leak_count(py, false); + } + } + }; +} + +/// Defines a `py_class!` that acts as a Python iterator over a Rust iterator. +macro_rules! py_shared_iterator_impl { + ( + $name: ident, + $leaked: ident, + $iterator_type: ty, + $success_func: expr, + $success_type: ty + ) => { + py_class!(pub class $name |py| { + data inner: RefCell<Option<$leaked>>; + data it: RefCell<$iterator_type>; + + def __next__(&self) -> PyResult<$success_type> { + let mut inner_opt = self.inner(py).borrow_mut(); + if inner_opt.is_some() { + match self.it(py).borrow_mut().next() { + None => { + // replace Some(inner) by None, drop $leaked + inner_opt.take(); + Ok(None) + } + Some(res) => { + $success_func(py, res) + } + } + } else { + Ok(None) + } + } + + def __iter__(&self) -> PyResult<Self> { + Ok(self.clone_ref(py)) + } + }); + + impl $name { + pub fn from_inner( + py: Python, + leaked: Option<$leaked>, + it: $iterator_type + ) -> PyResult<Self> { + Self::create_instance( + py, + RefCell::new(leaked), + RefCell::new(it) + ) + } + } + }; +} + +/// Defines a `py_class!` that acts as a Python mapping iterator over a Rust +/// iterator. +/// +/// TODO: this is a bit awkward to use, and a better (more complicated) +/// procedural macro would simplify the interface a lot. +/// +/// # Parameters +/// +/// * `$name` is the identifier to give to the resulting Rust struct. +/// * `$leaked` corresponds to `$leaked` in the matching `py_shared_ref!` call. +/// * `$key_type` is the type of the key in the mapping +/// * `$value_type` is the type of the value in the mapping +/// * `$success_func` is a function for processing the Rust `(key, value)` +/// tuple on iteration success, turning it into something Python understands. +/// * `$success_func` is the return type of `$success_func` +/// +/// # Example +/// +/// ``` +/// struct MyStruct { +/// inner: HashMap<Vec<u8>, Vec<u8>>; +/// } +/// +/// py_class!(pub class MyType |py| { +/// data inner: RefCell<MyStruct>; +/// data py_shared_state: PySharedState; +/// +/// def __iter__(&self) -> PyResult<MyTypeItemsIterator> { +/// MyTypeItemsIterator::create_instance( +/// py, +/// RefCell::new(Some(MyTypeLeakedRef::new(py, &self))), +/// RefCell::new(self.leak_immutable(py).iter()), +/// ) +/// } +/// }); +/// +/// impl MyType { +/// fn translate_key_value( +/// py: Python, +/// res: (&Vec<u8>, &Vec<u8>), +/// ) -> PyResult<Option<(PyBytes, PyBytes)>> { +/// let (f, entry) = res; +/// Ok(Some(( +/// PyBytes::new(py, f), +/// PyBytes::new(py, entry), +/// ))) +/// } +/// } +/// +/// py_shared_ref!(MyType, MyStruct, inner, MyTypeLeakedRef); +/// +/// py_shared_mapping_iterator!( +/// MyTypeItemsIterator, +/// MyTypeLeakedRef, +/// Vec<u8>, +/// Vec<u8>, +/// MyType::translate_key_value, +/// Option<(PyBytes, PyBytes)> +/// ); +/// ``` +#[allow(unused)] // Removed in a future patch +macro_rules! py_shared_mapping_iterator { + ( + $name:ident, + $leaked:ident, + $key_type: ty, + $value_type: ty, + $success_func: path, + $success_type: ty + ) => { + py_shared_iterator_impl!( + $name, + $leaked, + Box< + Iterator<Item = (&'static $key_type, &'static $value_type)> + + Send, + >, + $success_func, + $success_type + ); + }; +} + +/// Works basically the same as `py_shared_mapping_iterator`, but with only a +/// key. +macro_rules! py_shared_sequence_iterator { + ( + $name:ident, + $leaked:ident, + $key_type: ty, + $success_func: path, + $success_type: ty + ) => { + py_shared_iterator_impl!( + $name, + $leaked, + Box<Iterator<Item = &'static $key_type> + Send>, + $success_func, + $success_type + ); + }; +}
--- a/setup.py Wed Aug 21 17:56:50 2019 +0200 +++ b/setup.py Fri Aug 23 17:03:42 2019 -0400 @@ -1078,8 +1078,8 @@ 'hgext', 'hgext.convert', 'hgext.fsmonitor', 'hgext.fastannotate', 'hgext.fsmonitor.pywatchman', + 'hgext.highlight', 'hgext.infinitepush', - 'hgext.highlight', 'hgext.largefiles', 'hgext.lfs', 'hgext.narrow', 'hgext.remotefilelog', 'hgext.zeroconf', 'hgext3rd',
--- a/tests/fakedirstatewritetime.py Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/fakedirstatewritetime.py Fri Aug 23 17:03:42 2019 -0400 @@ -30,6 +30,7 @@ ) parsers = policy.importmod(r'parsers') +rustmod = policy.importrust(r'parsers') def pack_dirstate(fakenow, orig, dmap, copymap, pl, now): # execute what original parsers.pack_dirstate should do actually @@ -57,16 +58,21 @@ # 'fakenow' value and 'touch -t YYYYmmddHHMM' argument easy fakenow = dateutil.parsedate(fakenow, [b'%Y%m%d%H%M'])[0] - if rustext is not None: - orig_module = rustext.dirstate - orig_pack_dirstate = rustext.dirstate.pack_dirstate - else: - orig_module = parsers - orig_pack_dirstate = parsers.pack_dirstate + if rustmod is not None: + # The Rust implementation does not use public parse/pack dirstate + # to prevent conversion round-trips + orig_dirstatemap_write = dirstate.dirstatemap.write + wrapper = lambda self, st, now: orig_dirstatemap_write(self, + st, + fakenow) + dirstate.dirstatemap.write = wrapper orig_dirstate_getfsnow = dirstate._getfsnow wrapper = lambda *args: pack_dirstate(fakenow, orig_pack_dirstate, *args) + orig_module = parsers + orig_pack_dirstate = parsers.pack_dirstate + orig_module.pack_dirstate = wrapper dirstate._getfsnow = lambda *args: fakenow try: @@ -74,6 +80,8 @@ finally: orig_module.pack_dirstate = orig_pack_dirstate dirstate._getfsnow = orig_dirstate_getfsnow + if rustmod is not None: + dirstate.dirstatemap.write = orig_dirstatemap_write def _poststatusfixup(orig, workingctx, status, fixup): ui = workingctx.repo().ui
--- a/tests/flagprocessorext.py Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/flagprocessorext.py Fri Aug 23 17:03:42 2019 -0400 @@ -12,6 +12,9 @@ revlog, util, ) +from mercurial.revlogutils import ( + flagutil, +) # Test only: These flags are defined here only in the context of testing the # behavior of the flag processor. The canonical way to add flags is to get in @@ -58,7 +61,7 @@ class wrappedfile(obj.__class__): def addrevision(self, text, transaction, link, p1, p2, cachedelta=None, node=None, - flags=revlog.REVIDX_DEFAULT_FLAGS): + flags=flagutil.REVIDX_DEFAULT_FLAGS): if b'[NOOP]' in text: flags |= REVIDX_NOOP @@ -102,7 +105,7 @@ # Teach revlog about our test flags flags = [REVIDX_NOOP, REVIDX_BASE64, REVIDX_GZIP, REVIDX_FAIL] - revlog.REVIDX_KNOWN_FLAGS |= util.bitsfrom(flags) + flagutil.REVIDX_KNOWN_FLAGS |= util.bitsfrom(flags) revlog.REVIDX_FLAGS_ORDER.extend(flags) # Teach exchange to use changegroup 3 @@ -110,7 +113,7 @@ exchange._bundlespeccontentopts[k][b"cg.version"] = b"03" # Register flag processors for each extension - revlog.addflagprocessor( + flagutil.addflagprocessor( REVIDX_NOOP, ( noopdonothing, @@ -118,7 +121,7 @@ validatehash, ) ) - revlog.addflagprocessor( + flagutil.addflagprocessor( REVIDX_BASE64, ( b64decode, @@ -126,7 +129,7 @@ bypass, ), ) - revlog.addflagprocessor( + flagutil.addflagprocessor( REVIDX_GZIP, ( gzipdecompress,
--- a/tests/simplestorerepo.py Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/simplestorerepo.py Fri Aug 23 17:03:42 2019 -0400 @@ -42,6 +42,9 @@ interfaceutil, storageutil, ) +from mercurial.revlogutils import ( + flagutil, +) # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should @@ -262,9 +265,9 @@ if flags == 0: return text, True - if flags & ~revlog.REVIDX_KNOWN_FLAGS: + if flags & ~flagutil.REVIDX_KNOWN_FLAGS: raise simplestoreerror(_("incompatible revision flag '%#x'") % - (flags & ~revlog.REVIDX_KNOWN_FLAGS)) + (flags & ~flagutil.REVIDX_KNOWN_FLAGS)) validatehash = True # Depending on the operation (read or write), the order might be @@ -326,6 +329,9 @@ return text + def rawdata(self, nodeorrev): + return self.revision(raw=True) + def read(self, node): validatenode(node)
--- a/tests/test-bookmarks-corner-case.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-bookmarks-corner-case.t Fri Aug 23 17:03:42 2019 -0400 @@ -119,7 +119,7 @@ > import atexit > import os > import time - > from mercurial import error, extensions, bookmarks + > from mercurial import bookmarks, error, extensions > > def wait(repo): > if not os.path.exists('push-A-started'):
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test-byteify-strings.t Fri Aug 23 17:03:42 2019 -0400 @@ -0,0 +1,266 @@ +#require py3 + + $ byteify_strings () { + > $PYTHON "$TESTDIR/../contrib/byteify-strings.py" "$@" + > } + +Test version + + $ byteify_strings --version + Byteify strings * (glob) + +Test in-place + + $ cat > testfile.py <<EOF + > obj['test'] = b"1234" + > mydict.iteritems() + > EOF + $ byteify_strings testfile.py -i + $ cat testfile.py + obj[b'test'] = b"1234" + mydict.iteritems() + +Test with dictiter + + $ cat > testfile.py <<EOF + > obj['test'] = b"1234" + > mydict.iteritems() + > EOF + $ byteify_strings testfile.py --dictiter + obj[b'test'] = b"1234" + mydict.items() + +Test kwargs-like objects + + $ cat > testfile.py <<EOF + > kwargs['test'] = "123" + > kwargs[test['testing']] + > kwargs[test[[['testing']]]] + > kwargs[kwargs['testing']] + > kwargs.get('test') + > kwargs.pop('test') + > kwargs.get('test', 'testing') + > kwargs.pop('test', 'testing') + > kwargs.setdefault('test', 'testing') + > + > opts['test'] = "123" + > opts[test['testing']] + > opts[test[[['testing']]]] + > opts[opts['testing']] + > opts.get('test') + > opts.pop('test') + > opts.get('test', 'testing') + > opts.pop('test', 'testing') + > opts.setdefault('test', 'testing') + > + > commitopts['test'] = "123" + > commitopts[test['testing']] + > commitopts[test[[['testing']]]] + > commitopts[commitopts['testing']] + > commitopts.get('test') + > commitopts.pop('test') + > commitopts.get('test', 'testing') + > commitopts.pop('test', 'testing') + > commitopts.setdefault('test', 'testing') + > EOF + $ byteify_strings testfile.py --treat-as-kwargs kwargs opts commitopts + kwargs['test'] = b"123" + kwargs[test[b'testing']] + kwargs[test[[[b'testing']]]] + kwargs[kwargs['testing']] + kwargs.get('test') + kwargs.pop('test') + kwargs.get('test', b'testing') + kwargs.pop('test', b'testing') + kwargs.setdefault('test', b'testing') + + opts['test'] = b"123" + opts[test[b'testing']] + opts[test[[[b'testing']]]] + opts[opts['testing']] + opts.get('test') + opts.pop('test') + opts.get('test', b'testing') + opts.pop('test', b'testing') + opts.setdefault('test', b'testing') + + commitopts['test'] = b"123" + commitopts[test[b'testing']] + commitopts[test[[[b'testing']]]] + commitopts[commitopts['testing']] + commitopts.get('test') + commitopts.pop('test') + commitopts.get('test', b'testing') + commitopts.pop('test', b'testing') + commitopts.setdefault('test', b'testing') + +Test attr*() as methods + + $ cat > testfile.py <<EOF + > setattr(o, 'a', 1) + > util.setattr(o, 'ae', 1) + > util.getattr(o, 'alksjdf', 'default') + > util.addattr(o, 'asdf') + > util.hasattr(o, 'lksjdf', 'default') + > util.safehasattr(o, 'lksjdf', 'default') + > @eh.wrapfunction(func, 'lksjdf') + > def f(): + > pass + > @eh.wrapclass(klass, 'lksjdf') + > def f(): + > pass + > EOF + $ byteify_strings testfile.py --allow-attr-methods + setattr(o, 'a', 1) + util.setattr(o, 'ae', 1) + util.getattr(o, 'alksjdf', b'default') + util.addattr(o, 'asdf') + util.hasattr(o, 'lksjdf', b'default') + util.safehasattr(o, 'lksjdf', b'default') + @eh.wrapfunction(func, 'lksjdf') + def f(): + pass + @eh.wrapclass(klass, 'lksjdf') + def f(): + pass + +Test without attr*() as methods + + $ cat > testfile.py <<EOF + > setattr(o, 'a', 1) + > util.setattr(o, 'ae', 1) + > util.getattr(o, 'alksjdf', 'default') + > util.addattr(o, 'asdf') + > util.hasattr(o, 'lksjdf', 'default') + > util.safehasattr(o, 'lksjdf', 'default') + > @eh.wrapfunction(func, 'lksjdf') + > def f(): + > pass + > @eh.wrapclass(klass, 'lksjdf') + > def f(): + > pass + > EOF + $ byteify_strings testfile.py + setattr(o, 'a', 1) + util.setattr(o, b'ae', 1) + util.getattr(o, b'alksjdf', b'default') + util.addattr(o, b'asdf') + util.hasattr(o, b'lksjdf', b'default') + util.safehasattr(o, b'lksjdf', b'default') + @eh.wrapfunction(func, b'lksjdf') + def f(): + pass + @eh.wrapclass(klass, b'lksjdf') + def f(): + pass + +Test ignore comments + + $ cat > testfile.py <<EOF + > # py3-transform: off + > "none" + > "of" + > 'these' + > s = """should""" + > d = '''be''' + > # py3-transform: on + > "this should" + > 'and this also' + > + > # no-py3-transform + > l = "this should be ignored" + > l2 = "this shouldn't" + > + > EOF + $ byteify_strings testfile.py + # py3-transform: off + "none" + "of" + 'these' + s = """should""" + d = '''be''' + # py3-transform: on + b"this should" + b'and this also' + + # no-py3-transform + l = "this should be ignored" + l2 = b"this shouldn't" + +Test triple-quoted strings + + $ cat > testfile.py <<EOF + > """This is ignored + > """ + > + > line = """ + > This should not be + > """ + > line = ''' + > Neither should this + > ''' + > EOF + $ byteify_strings testfile.py + """This is ignored + """ + + line = b""" + This should not be + """ + line = b''' + Neither should this + ''' + +Test prefixed strings + + $ cat > testfile.py <<EOF + > obj['test'] = b"1234" + > obj[r'test'] = u"1234" + > EOF + $ byteify_strings testfile.py + obj[b'test'] = b"1234" + obj[r'test'] = u"1234" + +Test multi-line alignment + + $ cat > testfile.py <<'EOF' + > def foo(): + > error.Abort(_("foo" + > "bar" + > "%s") + > % parameter) + > { + > 'test': dict, + > 'test2': dict, + > } + > [ + > "thing", + > "thing2" + > ] + > ( + > "tuple", + > "tuple2", + > ) + > {"thing", + > } + > EOF + $ byteify_strings testfile.py + def foo(): + error.Abort(_(b"foo" + b"bar" + b"%s") + % parameter) + { + b'test': dict, + b'test2': dict, + } + [ + b"thing", + b"thing2" + ] + ( + b"tuple", + b"tuple2", + ) + {b"thing", + }
--- a/tests/test-config.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-config.t Fri Aug 23 17:03:42 2019 -0400 @@ -57,11 +57,13 @@ $ hg showconfig Section -Tjson [ { + "defaultvalue": null, "name": "Section.KeY", "source": "*.hgrc:*", (glob) "value": "Case Sensitive" }, { + "defaultvalue": null, "name": "Section.key", "source": "*.hgrc:*", (glob) "value": "lower case" @@ -70,14 +72,15 @@ $ hg showconfig Section.KeY -Tjson [ { + "defaultvalue": null, "name": "Section.KeY", "source": "*.hgrc:*", (glob) "value": "Case Sensitive" } ] $ hg showconfig -Tjson | tail -7 - }, { + "defaultvalue": null, "name": "*", (glob) "source": "*", (glob) "value": "*" (glob) @@ -102,6 +105,7 @@ $ hg config empty.source -Tjson [ { + "defaultvalue": null, "name": "empty.source", "source": "", "value": "value"
--- a/tests/test-debugcommands.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-debugcommands.t Fri Aug 23 17:03:42 2019 -0400 @@ -546,7 +546,12 @@ .hg/cache/rbc-revs-v1 .hg/cache/rbc-names-v1 .hg/cache/hgtagsfnodes1 + .hg/cache/branch2-visible-hidden + .hg/cache/branch2-visible + .hg/cache/branch2-served.hidden .hg/cache/branch2-served + .hg/cache/branch2-immutable + .hg/cache/branch2-base Test debugcolor
--- a/tests/test-fix-metadata.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-fix-metadata.t Fri Aug 23 17:03:42 2019 -0400 @@ -43,6 +43,9 @@ > [extensions] > fix = > [fix] + > metadatafalse:command=cat $TESTTMP/missing + > metadatafalse:pattern=metadatafalse + > metadatafalse:metadata=false > missing:command=cat $TESTTMP/missing > missing:pattern=missing > missing:metadata=true @@ -65,6 +68,7 @@ $ hg init repo $ cd repo + $ printf "old content\n" > metadatafalse $ printf "old content\n" > invalid $ printf "old content\n" > missing $ printf "old content\n" > valid @@ -72,15 +76,20 @@ $ hg fix -w ignored invalid output from fixer tool: invalid + fixed metadatafalse in revision 2147483647 using metadatafalse ignored invalid output from fixer tool: missing fixed valid in revision 2147483647 using valid saw "key" 1 times fixed 1 files with valid fixed the working copy - $ cat missing invalid valid + $ cat metadatafalse + new content + $ cat missing old content + $ cat invalid old content + $ cat valid new content $ cd ..
--- a/tests/test-fix.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-fix.t Fri Aug 23 17:03:42 2019 -0400 @@ -147,6 +147,15 @@ {first} The 1-based line number of the first line in the modified range {last} The 1-based line number of the last line in the modified range + Deleted sections of a file will be ignored by :linerange, because there is no + corresponding line range in the version being fixed. + + By default, tools that set :linerange will only be executed if there is at + least one changed line range. This is meant to prevent accidents like running + a code formatter in such a way that it unexpectedly reformats the whole file. + If such a tool needs to operate on unchanged files, it should set the + :skipclean suboption to false. + The :pattern suboption determines which files will be passed through each configured tool. See 'hg help patterns' for possible values. If there are file arguments to 'hg fix', the intersection of these patterns is used. @@ -215,6 +224,13 @@ executions that modified a file. This aggregates the same metadata previously passed to the "postfixfile" hook. + Fixer tools are run the in repository's root directory. This allows them to + read configuration files from the working copy, or even write to the working + copy. The working copy is not updated to match the revision being fixed. In + fact, several revisions may be fixed in parallel. Writes to the working copy + are not amended into the revision being fixed; fixer tools should always write + fixed file content back to stdout as documented above. + list of commands: fix rewrite file content in changesets or working directory @@ -439,6 +455,18 @@ $ printf "a\nb\nc\nd\ne\nf\ng\n" > foo.changed $ hg commit -Aqm "foo" $ printf "zz\na\nc\ndd\nee\nff\nf\ngg\n" > foo.changed + + $ hg fix --working-dir + $ cat foo.changed + ZZ + a + c + DD + EE + FF + f + GG + $ hg fix --working-dir --whole $ cat foo.changed ZZ @@ -526,6 +554,21 @@ $ cd .. +If we try to fix a missing file, we still fix other files. + + $ hg init fixmissingfile + $ cd fixmissingfile + + $ printf "fix me!\n" > foo.whole + $ hg add + adding foo.whole + $ hg fix --working-dir foo.whole bar.whole + bar.whole: $ENOENT$ + $ cat *.whole + FIX ME! + + $ cd .. + Specifying a directory name should fix all its files and subdirectories. $ hg init fixdirectory @@ -1161,28 +1204,6 @@ $ cd .. -The :fileset subconfig was a misnomer, so we renamed it to :pattern. We will -still accept :fileset by itself as if it were :pattern, but this will issue a -warning. - - $ hg init filesetispattern - $ cd filesetispattern - - $ printf "foo\n" > foo.whole - $ printf "first\nsecond\n" > bar.txt - $ hg add -q - $ hg fix -w --config fix.sometool:fileset=bar.txt \ - > --config fix.sometool:command="sort -r" - the fix.tool:fileset config name is deprecated; please rename it to fix.tool:pattern - - $ cat foo.whole - FOO - $ cat bar.txt - second - first - - $ cd .. - The execution order of tools can be controlled. This example doesn't work if you sort after truncating, but the config defines the correct order while the definitions are out of order (which might imply the incorrect order given the @@ -1264,3 +1285,114 @@ $ cd .. +We run fixer tools in the repo root so they can look for config files or other +important things in the working directory. This does NOT mean we are +reconstructing a working copy of every revision being fixed; we're just giving +the tool knowledge of the repo's location in case it can do something +reasonable with that. + + $ hg init subprocesscwd + $ cd subprocesscwd + + $ cat >> .hg/hgrc <<EOF + > [fix] + > printcwd:command = pwd + > printcwd:pattern = path:foo/bar + > EOF + + $ mkdir foo + $ printf "bar\n" > foo/bar + $ hg commit -Aqm blah + + $ hg fix -w -r . foo/bar + $ hg cat -r tip foo/bar + $TESTTMP/subprocesscwd + $ cat foo/bar + $TESTTMP/subprocesscwd + + $ cd foo + + $ hg fix -w -r . bar + $ hg cat -r tip bar + $TESTTMP/subprocesscwd + $ cat bar + $TESTTMP/subprocesscwd + + $ cd ../.. + +Tools configured without a pattern are ignored. It would be too dangerous to +run them on all files, because this might happen while testing a configuration +that also deletes all of the file content. There is no reasonable subset of the +files to use as a default. Users should be explicit about what files are +affected by a tool. This test also confirms that we don't crash when the +pattern config is missing, and that we only warn about it once. + + $ hg init nopatternconfigured + $ cd nopatternconfigured + + $ printf "foo" > foo + $ printf "bar" > bar + $ hg add -q + $ hg fix --debug --working-dir --config "fix.nopattern:command=echo fixed" + fixer tool has no pattern configuration: nopattern + $ cat foo bar + foobar (no-eol) + + $ cd .. + +Test that we can configure a fixer to affect all files regardless of the cwd. +The way we invoke matching must not prohibit this. + + $ hg init affectallfiles + $ cd affectallfiles + + $ mkdir foo bar + $ printf "foo" > foo/file + $ printf "bar" > bar/file + $ printf "baz" > baz_file + $ hg add -q + + $ cd bar + $ hg fix --working-dir --config "fix.cooltool:command=echo fixed" \ + > --config "fix.cooltool:pattern=rootglob:**" + $ cd .. + + $ cat foo/file + fixed + $ cat bar/file + fixed + $ cat baz_file + fixed + + $ cd .. + +Tools should be able to run on unchanged files, even if they set :linerange. +This includes a corner case where deleted chunks of a file are not considered +changes. + + $ hg init skipclean + $ cd skipclean + + $ printf "a\nb\nc\n" > foo + $ printf "a\nb\nc\n" > bar + $ printf "a\nb\nc\n" > baz + $ hg commit -Aqm "base" + + $ printf "a\nc\n" > foo + $ printf "a\nx\nc\n" > baz + + $ hg fix --working-dir foo bar baz \ + > --config 'fix.changedlines:command=printf "Line ranges:\n"; ' \ + > --config 'fix.changedlines:linerange=printf "{first} through {last}\n"; ' \ + > --config 'fix.changedlines:pattern=rootglob:**' \ + > --config 'fix.changedlines:skipclean=false' + + $ cat foo + Line ranges: + $ cat bar + Line ranges: + $ cat baz + Line ranges: + 2 through 2 + + $ cd ..
--- a/tests/test-flagprocessor.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-flagprocessor.t Fri Aug 23 17:03:42 2019 -0400 @@ -205,9 +205,9 @@ extsetup(ui) File "*/tests/flagprocessorext.py", line *, in extsetup (glob) validatehash, - File "*/mercurial/revlog.py", line *, in addflagprocessor (glob) - _insertflagprocessor(flag, processor, _flagprocessors) - File "*/mercurial/revlog.py", line *, in _insertflagprocessor (glob) + File "*/mercurial/revlogutils/flagutil.py", line *, in addflagprocessor (glob) + insertflagprocessor(flag, processor, flagprocessors) + File "*/mercurial/revlogutils/flagutil.py", line *, in insertflagprocessor (glob) raise error.Abort(msg) mercurial.error.Abort: b"cannot register multiple processors on flag '0x8'." (py3 !) Abort: cannot register multiple processors on flag '0x8'. (no-py3 !)
--- a/tests/test-install.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-install.t Fri Aug 23 17:03:42 2019 -0400 @@ -153,6 +153,16 @@ 1 problems detected, please check your install! [1] +debuginstall extension support + $ hg debuginstall --config extensions.fsmonitor= --config fsmonitor.watchman_exe=false | grep atchman + fsmonitor checking for watchman binary... (false) + watchman binary missing or broken: warning: Watchman unavailable: watchman exited with code 1 +Verify the json works too: + $ hg debuginstall --config extensions.fsmonitor= --config fsmonitor.watchman_exe=false -Tjson | grep atchman + "fsmonitor-watchman": "false", + "fsmonitor-watchman-error": "warning: Watchman unavailable: watchman exited with code 1", + + #if test-repo $ . "$TESTDIR/helpers-testrepo.sh"
--- a/tests/test-lfs.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-lfs.t Fri Aug 23 17:03:42 2019 -0400 @@ -701,7 +701,7 @@ > if len(fl) == 0: > continue > sizes = [fl._revlog.rawsize(i) for i in fl] - > texts = [fl.revision(i, raw=True) for i in fl] + > texts = [fl.rawdata(i) for i in fl] > flags = [int(fl._revlog.flags(i)) for i in fl] > hashes = [hash(t) for t in texts] > pycompat.stdout.write(b' %s: rawsizes=%r flags=%r hashes=%s\n'
--- a/tests/test-rebase-inmemory.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-rebase-inmemory.t Fri Aug 23 17:03:42 2019 -0400 @@ -506,6 +506,7 @@ $ hg rebase -s 2 -d 7 rebasing 2:177f92b77385 "c" abort: outstanding merge conflicts + (use 'hg resolve' to resolve) [255] $ hg resolve -l U e
--- a/tests/test-resolve.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-resolve.t Fri Aug 23 17:03:42 2019 -0400 @@ -210,12 +210,15 @@ [1] $ hg up 0 abort: outstanding merge conflicts + (use 'hg resolve' to resolve) [255] $ hg merge 2 abort: outstanding merge conflicts + (use 'hg resolve' to resolve) [255] $ hg merge --force 2 abort: outstanding merge conflicts + (use 'hg resolve' to resolve) [255] set up conflict-free merge
--- a/tests/test-revlog-raw.py Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-revlog-raw.py Fri Aug 23 17:03:42 2019 -0400 @@ -16,6 +16,7 @@ from mercurial.revlogutils import ( deltas, + flagutil, ) # TESTTMP is optional. This makes it convenient to run without run-tests.py @@ -56,7 +57,7 @@ # can be used to verify hash. return False -revlog.addflagprocessor(revlog.REVIDX_EXTSTORED, +flagutil.addflagprocessor(revlog.REVIDX_EXTSTORED, (readprocessor, writeprocessor, rawprocessor)) # Utilities about reading and appending revlog @@ -161,7 +162,7 @@ p1 = rlog.node(r - 1) p2 = node.nullid if r == 0 or (rlog.flags(r) & revlog.REVIDX_EXTSTORED): - text = rlog.revision(r, raw=True) + text = rlog.rawdata(r) cachedelta = None else: # deltaparent cannot have EXTSTORED flag. @@ -268,7 +269,7 @@ abort('rev %d: wrong rawsize' % rev) if rlog.revision(rev, raw=False) != text: abort('rev %d: wrong text' % rev) - if rlog.revision(rev, raw=True) != rawtext: + if rlog.rawdata(rev) != rawtext: abort('rev %d: wrong rawtext' % rev) result.append((text, rawtext)) @@ -293,7 +294,10 @@ nlog = newrevlog() for rev in revorder: for raw in raworder: - t = nlog.revision(rev, raw=raw) + if raw: + t = nlog.rawdata(rev) + else: + t = nlog.revision(rev) if t != expected[rev][int(raw)]: abort('rev %d: corrupted %stext' % (rev, raw and 'raw' or ''))
--- a/tests/test-rust-discovery.py Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-rust-discovery.py Fri Aug 23 17:03:42 2019 -0400 @@ -1,16 +1,9 @@ from __future__ import absolute_import import unittest -try: - from mercurial import rustext - rustext.__name__ # trigger immediate actual import -except ImportError: - rustext = None -else: - # this would fail already without appropriate ancestor.__package__ - from mercurial.rustext.discovery import ( - PartialDiscovery, - ) +from mercurial import policy + +PartialDiscovery = policy.importrust('discovery', member='PartialDiscovery') try: from mercurial.cext import parsers as cparsers @@ -38,8 +31,16 @@ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00' ) +class fakechangelog(object): + def __init__(self, idx): + self.index = idx -@unittest.skipIf(rustext is None or cparsers is None, +class fakerepo(object): + def __init__(self, idx): + """Just make so that self.changelog.index is the given idx.""" + self.changelog = fakechangelog(idx) + +@unittest.skipIf(PartialDiscovery is None or cparsers is None, "rustext or the C Extension parsers module " "discovery relies on is not available") class rustdiscoverytest(unittest.TestCase): @@ -57,6 +58,9 @@ def parseindex(self): return cparsers.parse_index2(data_non_inlined, False)[0] + def repo(self): + return fakerepo(self.parseindex()) + def testindex(self): idx = self.parseindex() # checking our assumptions about the index binary data: @@ -67,8 +71,7 @@ 3: (2, -1)}) def testaddcommonsmissings(self): - idx = self.parseindex() - disco = PartialDiscovery(idx, [3]) + disco = PartialDiscovery(self.repo(), [3], True) self.assertFalse(disco.hasinfo()) self.assertFalse(disco.iscomplete()) @@ -83,29 +86,29 @@ self.assertEqual(disco.commonheads(), {1}) def testaddmissingsstats(self): - idx = self.parseindex() - disco = PartialDiscovery(idx, [3]) + disco = PartialDiscovery(self.repo(), [3], True) self.assertIsNone(disco.stats()['undecided'], None) disco.addmissings([2]) self.assertEqual(disco.stats()['undecided'], 2) def testaddinfocommonfirst(self): - idx = self.parseindex() - disco = PartialDiscovery(idx, [3]) + disco = PartialDiscovery(self.repo(), [3], True) disco.addinfo([(1, True), (2, False)]) self.assertTrue(disco.hasinfo()) self.assertTrue(disco.iscomplete()) self.assertEqual(disco.commonheads(), {1}) def testaddinfomissingfirst(self): - idx = self.parseindex() - disco = PartialDiscovery(idx, [3]) + disco = PartialDiscovery(self.repo(), [3], True) disco.addinfo([(2, False), (1, True)]) self.assertTrue(disco.hasinfo()) self.assertTrue(disco.iscomplete()) self.assertEqual(disco.commonheads(), {1}) + def testinitnorandom(self): + PartialDiscovery(self.repo(), [3], True, randomize=False) + if __name__ == '__main__': import silenttestrunner silenttestrunner.main(__name__)
--- a/tests/test-server-view.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-server-view.t Fri Aug 23 17:03:42 2019 -0400 @@ -50,7 +50,12 @@ $ hg -R test --config experimental.extra-filter-revs='not public()' debugupdatecache $ ls -1 test/.hg/cache/ branch2-base%89c45d2fa07e + branch2-immutable%89c45d2fa07e branch2-served + branch2-served%89c45d2fa07e + branch2-served.hidden%89c45d2fa07e + branch2-visible%89c45d2fa07e + branch2-visible-hidden%89c45d2fa07e hgtagsfnodes1 rbc-names-v1 rbc-revs-v1
--- a/tests/test-setdiscovery.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-setdiscovery.t Fri Aug 23 17:03:42 2019 -0400 @@ -968,7 +968,7 @@ updating to branch b 0 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ hg -R a debugdiscovery b --debug --verbose --config progress.debug=true + $ hg -R a debugdiscovery b --debug --verbose --config progress.debug=true --config devel.discovery.randomize=false comparing with b query 1; heads searching for changes @@ -980,13 +980,14 @@ query 3; still undecided: 980, sample size is: 200 sampling from both directions searching: 4 queries - query 4; still undecided: 435, sample size is: 210 (no-py3 !) - query 4; still undecided: 430, sample size is: 210 (py3 !) + query 4; still undecided: 497, sample size is: 210 sampling from both directions searching: 5 queries - query 5; still undecided: 185, sample size is: 185 (no-py3 !) - query 5; still undecided: 187, sample size is: 187 (py3 !) - 5 total queries in *.????s (glob) + query 5; still undecided: 285, sample size is: 220 + sampling from both directions + searching: 6 queries + query 6; still undecided: 63, sample size is: 63 + 6 total queries in *.????s (glob) elapsed time: * seconds (glob) heads summary: total common heads: 1 @@ -1095,16 +1096,9 @@ give 'all remote heads known locally' without checking the remaining heads - fixed in 86c35b7ae300: - $ cat >> $TESTTMP/unrandomsample.py << EOF - > import random - > def sample(population, k): - > return sorted(population)[:k] - > random.sample = sample - > EOF - $ cat >> r1/.hg/hgrc << EOF - > [extensions] - > unrandomsample = $TESTTMP/unrandomsample.py + > [devel] + > discovery.randomize = False > EOF $ hg -R r1 outgoing r2 -T'{rev} ' --config extensions.blackbox= \
--- a/tests/test-shelve.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-shelve.t Fri Aug 23 17:03:42 2019 -0400 @@ -1239,6 +1239,7 @@ > y > EOF unshelving change 'default' + temporarily committing pending changes (restore with 'hg unshelve --abort') rebasing shelved changes diff --git a/d b/d new file mode 100644 @@ -1250,6 +1251,10 @@ record this change to 'd'? (enter ? for help) [Ynesfdaq?] y + + $ hg status -v + A c + A d $ ls b c @@ -1267,15 +1272,21 @@ > B > C > EOF - $ hg shelve + $ echo > garbage + $ hg st + M foo + ? garbage + $ hg shelve --unknown shelved as default - 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + 1 files updated, 0 files merged, 1 files removed, 0 files unresolved $ cat foo B $ hg unshelve -i <<EOF > y > y > n + > y + > y > EOF unshelving change 'default' rebasing shelved changes @@ -1287,15 +1298,28 @@ @@ -1,1 +1,2 @@ +A B - record change 1/2 to 'foo'? + record change 1/3 to 'foo'? (enter ? for help) [Ynesfdaq?] y @@ -1,1 +2,2 @@ B +C - record change 2/2 to 'foo'? + record change 2/3 to 'foo'? (enter ? for help) [Ynesfdaq?] n + diff --git a/garbage b/garbage + new file mode 100644 + examine changes to 'garbage'? + (enter ? for help) [Ynesfdaq?] y + + @@ -0,0 +1,1 @@ + + + record change 3/3 to 'garbage'? + (enter ? for help) [Ynesfdaq?] y + + $ hg st + M foo + ? garbage $ cat foo A B @@ -1347,17 +1371,44 @@ $ hg resolve -m bar1 bar2 (no more unresolved files) continue: hg unshelve --continue + +-- using --continue with --interactive should throw an error + $ hg unshelve --continue -i + abort: cannot use both continue and interactive + [255] + $ cat bar1 A B C - $ hg unshelve --continue -i <<EOF + +#if stripbased + $ hg log -r 3:: -G + @ changeset: 5:f1d5f53e397b + | tag: tip + | parent: 3:e28fd7fa7938 + | user: shelve@localhost + | date: Thu Jan 01 00:00:00 1970 +0000 + | summary: changes to: add A to bars + | + | @ changeset: 4:fe451a778c81 + |/ user: test + | date: Thu Jan 01 00:00:00 1970 +0000 + | summary: add C to bars + | + o changeset: 3:e28fd7fa7938 + | user: test + ~ date: Thu Jan 01 00:00:00 1970 +0000 + summary: add A to bars + +#endif + + $ hg unshelve --continue <<EOF > y > y > y - > y + > n > EOF - unshelving change 'default-01' diff --git a/bar1 b/bar1 1 hunks, 1 lines changed examine changes to 'bar1'? @@ -1380,6 +1431,51 @@ +B C record change 2/2 to 'bar2'? + (enter ? for help) [Ynesfdaq?] n + + unshelve of 'default-01' complete + +#if stripbased + $ hg log -r 3:: -G + @ changeset: 4:fe451a778c81 + | tag: tip + | user: test + | date: Thu Jan 01 00:00:00 1970 +0000 + | summary: add C to bars + | + o changeset: 3:e28fd7fa7938 + | user: test + ~ date: Thu Jan 01 00:00:00 1970 +0000 + summary: add A to bars + +#endif + + $ hg unshelve --continue + abort: no unshelve in progress + [255] + + $ hg shelve --list + default-01 (*)* changes to: add A to bars (glob) + default (*)* changes to: add B to foo (glob) + $ hg unshelve -n default-01 -i <<EOF + > y + > y + > EOF + temporarily committing pending changes (restore with 'hg unshelve --abort') + rebasing shelved changes + diff --git a/bar2 b/bar2 + 1 hunks, 1 lines changed + examine changes to 'bar2'? (enter ? for help) [Ynesfdaq?] y - unshelve of 'default-01' complete + @@ -1,2 +1,3 @@ + A + +B + C + record this change to 'bar2'? + (enter ? for help) [Ynesfdaq?] y + +-- test for --interactive --keep + $ hg unshelve -i --keep + abort: --keep on --interactive is not yet supported + [255]
--- a/tests/test-transplant.t Wed Aug 21 17:56:50 2019 +0200 +++ b/tests/test-transplant.t Fri Aug 23 17:03:42 2019 -0400 @@ -1,8 +1,17 @@ +#testcases commandmode continueflag $ cat <<EOF >> $HGRCPATH > [extensions] > transplant= + > graphlog= > EOF +#if continueflag + $ cat >> $HGRCPATH <<EOF + > [alias] + > continue = transplant --continue + > EOF +#endif + $ hg init t $ cd t $ hg transplant @@ -11,6 +20,9 @@ $ hg transplant --continue --all abort: --continue is incompatible with --branch, --all and --merge [255] + $ hg transplant --stop --all + abort: --stop is incompatible with --branch, --all and --merge + [255] $ hg transplant --all tip abort: --all requires a branch revision [255] @@ -368,7 +380,8 @@ applying 722f4667af76 722f4667af76 transplanted to 76e321915884 -transplant --continue + +transplant --continue and --stop behaviour $ hg init ../tc $ cd ../tc @@ -408,6 +421,36 @@ $ echo foobar > foo $ hg ci -mfoobar created new head + +Repo log before transplant + $ hg glog + @ changeset: 4:e8643552fde5 + | tag: tip + | parent: 0:493149fa1541 + | user: test + | date: Thu Jan 01 00:00:00 1970 +0000 + | summary: foobar + | + | o changeset: 3:1dab759070cf + | | user: test + | | date: Thu Jan 01 00:00:00 1970 +0000 + | | summary: bar2 + | | + | o changeset: 2:9d6d6b5a8275 + | | user: test + | | date: Thu Jan 01 00:00:00 1970 +0000 + | | summary: bar + | | + | o changeset: 1:46ae92138f3c + |/ user: test + | date: Thu Jan 01 00:00:00 1970 +0000 + | summary: foo2 + | + o changeset: 0:493149fa1541 + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + summary: foo + $ hg transplant 1:3 applying 46ae92138f3c patching file foo @@ -417,6 +460,49 @@ abort: fix up the working directory and run hg transplant --continue [255] + $ hg transplant --stop + stopped the interrupted transplant + working directory is now at e8643552fde5 +Repo log after abort + $ hg glog + @ changeset: 4:e8643552fde5 + | tag: tip + | parent: 0:493149fa1541 + | user: test + | date: Thu Jan 01 00:00:00 1970 +0000 + | summary: foobar + | + | o changeset: 3:1dab759070cf + | | user: test + | | date: Thu Jan 01 00:00:00 1970 +0000 + | | summary: bar2 + | | + | o changeset: 2:9d6d6b5a8275 + | | user: test + | | date: Thu Jan 01 00:00:00 1970 +0000 + | | summary: bar + | | + | o changeset: 1:46ae92138f3c + |/ user: test + | date: Thu Jan 01 00:00:00 1970 +0000 + | summary: foo2 + | + o changeset: 0:493149fa1541 + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + summary: foo + + $ hg transplant 1:3 + applying 46ae92138f3c + file added already exists + 1 out of 1 hunks FAILED -- saving rejects to file added.rej + patching file foo + Hunk #1 FAILED at 0 + 1 out of 1 hunks FAILED -- saving rejects to file foo.rej + patch failed to apply + abort: fix up the working directory and run hg transplant --continue + [255] + transplant -c shouldn't use an old changeset $ hg up -C @@ -424,8 +510,12 @@ updated to "e8643552fde5: foobar" 1 other heads for branch "default" $ rm added - $ hg transplant --continue - abort: no transplant to continue + $ hg continue + abort: no transplant to continue (continueflag !) + abort: no operation in progress (no-continueflag !) + [255] + $ hg transplant --stop + abort: no interrupted transplant found [255] $ hg transplant 1 applying 46ae92138f3c @@ -480,23 +570,23 @@ [255] $ hg transplant 1:3 abort: transplant in progress - (use 'hg transplant --continue' or 'hg update' to abort) + (use 'hg transplant --continue' or 'hg transplant --stop') [255] $ hg status -v A bar + ? added.rej ? baz.rej ? foo.rej # The repository is in an unfinished *transplant* state. # To continue: hg transplant --continue - # To abort: hg update + # To stop: hg transplant --stop $ echo fixed > baz - $ hg transplant --continue + $ hg continue 9d6d6b5a8275 transplanted as d80c49962290 applying 1dab759070cf 1dab759070cf transplanted to aa0ffe6bd5ae - $ cd .. Issue1111: Test transplant --merge @@ -881,7 +971,7 @@ [255] $ hg status ? b.rej - $ hg transplant --continue + $ hg continue 645035761929 skipped due to empty diff $ cd ..