tests/testlib/exchange-obsmarker-util.sh

streamclone: also stream caches to the client

When stream clone is used over bundle2, relevant cache files are also streamed.
This is expected to be a massive performance win for clone since no important
cache will have to be recomputed.

Some performance numbers:

(All times are wall-clock times in seconds, 2 attempts per case.)

# Mozilla-Central

## Clone over ssh over lan
V1 streaming: 234.3 239.6
V2 streaming: 248.4 243.7

## Clone over ssh over Internet
V1 streaming: 175.5 110.9
V2 streaming: 109.1 111.0

## Clone over HTTP over lan
V1 streaming: 105.3 105.6
V2 streaming: 112.7 111.4

## Clone over HTTP over internet
V1 streaming: 105.6 114.6
V2 streaming: 226.7 225.9

## Hg tags
V1 streaming (no cache): 1.084 1.071
V2 streaming (cache):    0.312 0.325

## Hg branches
V1 streaming (no cache):   14.047 14.148
V2 streaming (with cache):  0.312  0.333

# Pypy

## Clone over ssh over internet
V1 streaming: 29.4 30.1
V2 streaming: 31.2 30.1

## Clone over http over internet
V1 streaming: 29.7 29.7
V2 streaming: 75.2 72.9

(since ssh and lan are not affected, there seems to be an issue with how we
read/write the http stream on connection with latency, unrelated to the format)

## Hg tags
V1 streaming (no cache):   1.752 1.664
V2 streaming (with cache): 0.274 0.260

## Hg branches
V1 streaming (no cache):   4.469 4.728
V2 streaming (with cache): 0.318 0.321

# Private repository:
    * 500K revision revisions
    * 11K topological heads
    * 28K branch heads

## hg tags
no cache:   1543.332
with cache:    4.900

## hg branches
no cache:   91.828
with cache:  2.955

#!/bin/sh
# setup config and various utility to test obsolescence marker exchanges tests

cat >> $TESTTMP/prune.sh << EOF
rev=\`hg log --hidden --template '{node}\n' --rev "\$3"\`

hg debugobsolete --record-parents \$1 "\$2" \$rev \
   && hg up --quiet 'max((::.) - obsolete())'
EOF

cat >> $HGRCPATH <<EOF
[web]
# We test http pull and push, drop authentication requirement
push_ssl = false
allow_push = *

[ui]
# simpler log output
logtemplate ="{node|short} ({phase}): {desc}\n"

[phases]
# non publishing server
publish=False

[experimental]
# reduce output changes
bundle2-output-capture=True
# enable evolution
evolution=true

[extensions]
# we need to strip some changeset for some test cases
hgext.strip=

[devel]
strip-obsmarkers = no

[alias]
# fix date used to create obsolete markers.
debugobsolete=debugobsolete -d '0 0'
# poor man substiture to the evolve 'hg prune'. using prune makes the test clearer and 
prune = !sh $TESTTMP/prune.sh \$1 "\$2" "\$3"
EOF

mkcommit() {
   echo "$1" > "$1"
   hg add "$1"
   hg ci -m "$1"
}
getid() {
   hg log --hidden --template '{node}\n' --rev "$1"
}

setuprepos() {
    echo creating test repo for test case $1
    mkdir $1
    cd $1
    echo - pulldest
    hg init pushdest
    cd pushdest
    mkcommit O
    hg phase --public .
    cd ..
    echo - main
    hg clone -q pushdest main
    echo - pushdest
    hg clone -q main pulldest
    echo 'cd into `main` and proceed with env setup'
}

inspect_obsmarkers (){
    # This exist as its own function to help the evolve extension reuse the tests as is.
    # The evolve extensions version will includes more advances query (eg:
    # related to obsmarkers discovery) to this.
    echo 'obsstore content'
    echo '================'
    hg debugobsolete
}

dotest() {
    # dotest TESTNAME [TARGETNODE] [PUSHFLAGS+]
    #
    # test exchange for the given test case.
    #
    # This function performs push and pull in all directions through all
    # protocols and display the resulting obsolescence markers on all sides.

    testcase=$1
    shift
    target="$1"
    if [ $# -gt 0 ]; then
        shift
    fi
    targetnode=""
    desccall=""
    cd $testcase
    echo "## Running testcase $testcase"
    if [ -n "$target" ]; then
        desccall="desc("\'"$target"\'")"
        targetnode="`hg -R main id -qr \"$desccall\"`"
        echo "# testing echange of \"$target\" ($targetnode)"
    fi
    echo "## initial state"
    echo "# obstore: main"
    hg -R main     debugobsolete | sort
    echo "# obstore: pushdest"
    hg -R pushdest debugobsolete | sort
    echo "# obstore: pulldest"
    hg -R pulldest debugobsolete | sort

    if [ -n "$target" ]; then
        echo "## pushing \"$target\"" from main to pushdest
        hg -R main push -r "$desccall" $@ pushdest
    else
        echo "## pushing from main to pushdest"
        hg -R main push pushdest $@
    fi
    echo "## post push state"
    echo "# obstore: main"
    hg -R main     debugobsolete | sort
    echo "# obstore: pushdest"
    hg -R pushdest debugobsolete | sort
    echo "# obstore: pulldest"
    hg -R pulldest debugobsolete | sort
    if [ -n "$target" ]; then
        echo "## pulling \"$targetnode\"" from main into pulldest
        hg -R pulldest pull -r $targetnode $@ main
    else
        echo "## pulling from main into pulldest"
        hg -R pulldest pull main $@
    fi
    echo "## post pull state"
    echo "# obstore: main"
    hg -R main     debugobsolete | sort
    echo "# obstore: pushdest"
    hg -R pushdest debugobsolete | sort
    echo "# obstore: pulldest"
    hg -R pulldest debugobsolete | sort

    cd ..

}