comparison mercurial/localrepo.py @ 39700:b10d145837bc

localrepo: extract resolving of opener options to standalone functions

Requirements and config options are converted into a dict which is
available to the store vfs to consult. This is how storage options
are communicated from the repo layer to the storage layer.

Currently, we do that option resolution in a private method on the
repo instance. And there is a single method doing that resolution.

Opener options are logically specific to the storage backend they
apply to. And, opener options may wish to influence how the repo
object/type is constructed. So it makes sense to have more granular
storage option resolution that occurs before the repo object is
instantiated.

This commit extracts the code for resolving opener options into new
module-level functions. These functions are run before the repo
instance is constructed.

As part of the code move, we split the option resolution into
generic and revlog-specific options. After this commit, we no longer
add revlog-specific options to repos that don't have a revlog
requirement.

Some of these opener options and associated config options might make
sense on alternate storage backends. We can always reuse config
options and opener option names for other backends. But we shouldn't
be passing opener options to storage backends that won't recognize
them. I haven't done it here, but after this commit it should be
possible for store backends to validate the set of opener options
it receives.

Because localrepository.openerreqs is no longer used after this commit,
it has been removed.

I'm not super thrilled about the code outside of localrepo that is
adding requirements and updating opener options. We'll probably want
to create a more formal API for that use case that constructs a new
repo instance and poisons the old repo object. But this was a
pre-existing issue and can be dealt with later. I have little doubt
it will cause me troubles as I continue to refactor how repository
objects are instantiated.

.. api::

   ``localrepository.openerreqs`` has been removed. Override
   ``localrepo.resolvestorevfsoptions()`` to add custom opener options.

.. api::

   ``localrepository._applyopenerreqs()`` has been removed. Use
   ``localrepo.resolvestorevfsoptions()`` to add custom opener options.

Differential Revision: https://phab.mercurial-scm.org/D4576

39699:6f26417b71bb

    # The store has changed over time and the exact layout is dictated by
    # requirements. The store interface abstracts differences across all
    # of them.
    store = makestore(requirements, storebasepath,
                      lambda base: vfsmod.vfs(base, cacheaudited=True))

    hgvfs.createmode = store.createmode

    # The cache vfs is used to manage cache files.
    cachevfs = vfsmod.vfs(cachepath, cacheaudited=True)
    cachevfs.createmode = store.createmode

                                         b'dotencode' in requirements)

        return storemod.encodedstore(path, vfstype)

    return storemod.basicstore(path, vfstype)

@interfaceutil.implementer(repository.completelocalrepository)
class localrepository(object):

    # obsolete experimental requirements:

        'relshared',
        'dotencode',
        'exp-sparse',
        'internal-phase'
    }
    openerreqs = {
        'revlogv1',
        'generaldelta',
        'treemanifest',
    }

    # list of prefix for file which can be written without 'wlock'
    # Extensions should extend this list when needed
    _wlockfreeprefix = {
        # We migh consider requiring 'wlock' for the next

            self.ui.configbool('devel', 'check-locks')):
            if util.safehasattr(self.svfs, 'vfs'): # this is filtervfs
                self.svfs.vfs.audit = self._getsvfsward(self.svfs.vfs.audit)
            else: # standard vfs
                self.svfs.audit = self._getsvfsward(self.svfs.audit)
        self._applyopenerreqs()

        self._dirstatevalidatewarned = False

        self._branchcaches = {}
        self._revbranchcache = None

            caps = set(caps)
            capsblob = bundle2.encodecaps(bundle2.getrepocaps(self,
                                                              role='client'))
            caps.add('bundle2=' + urlreq.quote(capsblob))
        return caps

    def _applyopenerreqs(self):
        self.svfs.options = {r: True for r in self.requirements
                             if r in self.openerreqs}
        # experimental config: format.chunkcachesize
        chunkcachesize = self.ui.configint('format', 'chunkcachesize')
        if chunkcachesize is not None:
            self.svfs.options['chunkcachesize'] = chunkcachesize
        # experimental config: format.manifestcachesize
        manifestcachesize = self.ui.configint('format', 'manifestcachesize')
        if manifestcachesize is not None:
            self.svfs.options['manifestcachesize'] = manifestcachesize
        deltabothparents = self.ui.configbool('storage',
            'revlog.optimize-delta-parent-choice')
        self.svfs.options['deltabothparents'] = deltabothparents
        self.svfs.options['lazydeltabase'] = not scmutil.gddeltaconfig(self.ui)
        chainspan = self.ui.configbytes('experimental', 'maxdeltachainspan')
        if 0 <= chainspan:
            self.svfs.options['maxdeltachainspan'] = chainspan
        mmapindexthreshold = self.ui.configbytes('experimental',
                                                 'mmapindexthreshold')
        if mmapindexthreshold is not None:
            self.svfs.options['mmapindexthreshold'] = mmapindexthreshold
        withsparseread = self.ui.configbool('experimental', 'sparse-read')
        srdensitythres = float(self.ui.config('experimental',
                                              'sparse-read.density-threshold'))
        srmingapsize = self.ui.configbytes('experimental',
                                           'sparse-read.min-gap-size')
        self.svfs.options['with-sparse-read'] = withsparseread
        self.svfs.options['sparse-read-density-threshold'] = srdensitythres
        self.svfs.options['sparse-read-min-gap-size'] = srmingapsize
        sparserevlog = SPARSEREVLOG_REQUIREMENT in self.requirements
        self.svfs.options['sparse-revlog'] = sparserevlog
        if sparserevlog:
            self.svfs.options['generaldelta'] = True
        maxchainlen = None
        if sparserevlog:
            maxchainlen = revlogconst.SPARSE_REVLOG_MAX_CHAIN_LENGTH
        # experimental config: format.maxchainlen
        maxchainlen = self.ui.configint('format', 'maxchainlen', maxchainlen)
        if maxchainlen is not None:
            self.svfs.options['maxchainlen'] = maxchainlen

        for r in self.requirements:
            if r.startswith('exp-compression-'):
                self.svfs.options['compengine'] = r[len('exp-compression-'):]

        # TODO move "revlogv2" to openerreqs once finalized.
        if REVLOGV2_REQUIREMENT in self.requirements:
            self.svfs.options['revlogv2'] = True

    def _writerequirements(self):
        scmutil.writerequires(self.vfs, self.requirements)

    # Don't cache auditor/nofsauditor, or you'll end up with reference cycle:

39700:b10d145837bc

    # The store has changed over time and the exact layout is dictated by
    # requirements. The store interface abstracts differences across all
    # of them.
    store = makestore(requirements, storebasepath,
                      lambda base: vfsmod.vfs(base, cacheaudited=True))
    hgvfs.createmode = store.createmode

    storevfs = store.vfs
    storevfs.options = resolvestorevfsoptions(ui, requirements)

    # The cache vfs is used to manage cache files.
    cachevfs = vfsmod.vfs(cachepath, cacheaudited=True)
    cachevfs.createmode = store.createmode

                                         b'dotencode' in requirements)

        return storemod.encodedstore(path, vfstype)

    return storemod.basicstore(path, vfstype)

def resolvestorevfsoptions(ui, requirements):
    """Resolve the options to pass to the store vfs opener.

    The returned dict is used to influence behavior of the storage layer.
    """
    options = {}

    if b'treemanifest' in requirements:
        options[b'treemanifest'] = True

    # experimental config: format.manifestcachesize
    manifestcachesize = ui.configint(b'format', b'manifestcachesize')
    if manifestcachesize is not None:
        options[b'manifestcachesize'] = manifestcachesize

    # In the absence of another requirement superseding a revlog-related
    # requirement, we have to assume the repo is using revlog version 0.
    # This revlog format is super old and we don't bother trying to parse
    # opener options for it because those options wouldn't do anything
    # meaningful on such old repos.
    if b'revlogv1' in requirements or REVLOGV2_REQUIREMENT in requirements:
        options.update(resolverevlogstorevfsoptions(ui, requirements))

    return options

def resolverevlogstorevfsoptions(ui, requirements):
    """Resolve opener options specific to revlogs."""

    options = {}

    if b'revlogv1' in requirements:
        options[b'revlogv1'] = True
    if REVLOGV2_REQUIREMENT in requirements:
        options[b'revlogv2'] = True

    if b'generaldelta' in requirements:
        options[b'generaldelta'] = True

    # experimental config: format.chunkcachesize
    chunkcachesize = ui.configint(b'format', b'chunkcachesize')
    if chunkcachesize is not None:
        options[b'chunkcachesize'] = chunkcachesize

    deltabothparents = ui.configbool(b'storage',
                                     b'revlog.optimize-delta-parent-choice')
    options[b'deltabothparents'] = deltabothparents

    options[b'lazydeltabase'] = not scmutil.gddeltaconfig(ui)

    chainspan = ui.configbytes(b'experimental', b'maxdeltachainspan')
    if 0 <= chainspan:
        options[b'maxdeltachainspan'] = chainspan

    mmapindexthreshold = ui.configbytes(b'experimental',
                                        b'mmapindexthreshold')
    if mmapindexthreshold is not None:
        options[b'mmapindexthreshold'] = mmapindexthreshold

    withsparseread = ui.configbool(b'experimental', b'sparse-read')
    srdensitythres = float(ui.config(b'experimental',
                                     b'sparse-read.density-threshold'))
    srmingapsize = ui.configbytes(b'experimental',
                                  b'sparse-read.min-gap-size')
    options[b'with-sparse-read'] = withsparseread
    options[b'sparse-read-density-threshold'] = srdensitythres
    options[b'sparse-read-min-gap-size'] = srmingapsize

    sparserevlog = SPARSEREVLOG_REQUIREMENT in requirements
    options[b'sparse-revlog'] = sparserevlog
    if sparserevlog:
        options[b'generaldelta'] = True

    maxchainlen = None
    if sparserevlog:
        maxchainlen = revlogconst.SPARSE_REVLOG_MAX_CHAIN_LENGTH
    # experimental config: format.maxchainlen
    maxchainlen = ui.configint(b'format', b'maxchainlen', maxchainlen)
    if maxchainlen is not None:
        options[b'maxchainlen'] = maxchainlen

    for r in requirements:
        if r.startswith(b'exp-compression-'):
            options[b'compengine'] = r[len(b'exp-compression-'):]

    return options

@interfaceutil.implementer(repository.completelocalrepository)
class localrepository(object):

    # obsolete experimental requirements:

        'relshared',
        'dotencode',
        'exp-sparse',
        'internal-phase'
    }

    # list of prefix for file which can be written without 'wlock'
    # Extensions should extend this list when needed
    _wlockfreeprefix = {
        # We migh consider requiring 'wlock' for the next

            self.ui.configbool('devel', 'check-locks')):
            if util.safehasattr(self.svfs, 'vfs'): # this is filtervfs
                self.svfs.vfs.audit = self._getsvfsward(self.svfs.vfs.audit)
            else: # standard vfs
                self.svfs.audit = self._getsvfsward(self.svfs.audit)

        self._dirstatevalidatewarned = False

        self._branchcaches = {}
        self._revbranchcache = None

            caps = set(caps)
            capsblob = bundle2.encodecaps(bundle2.getrepocaps(self,
                                                              role='client'))
            caps.add('bundle2=' + urlreq.quote(capsblob))
        return caps

    def _writerequirements(self):
        scmutil.writerequires(self.vfs, self.requirements)

    # Don't cache auditor/nofsauditor, or you'll end up with reference cycle: