diff mercurial/metadata.py @ 45634:9a6b409b8ebc

changing-files: rework the way we store changed files in side-data

We need to store new data so this is a good opportunity to rework this fully.

1) We directly store the list of affected file in the side data:

* This avoid having to fetch and parse the `files` list in the revision in
  addition to the sidedata. Making the data more self sufficient.

* This work around situation where that `files` field contains wrong
  information, and open the way to other bug fixing (eg: issue6219)

* The format (fixed initial index, sorted files) allow for fast lookup of
  filename within the structure.

* This unify the storage of affected files and copies sources and destination,
  limiting the number filename stored redundantly.

* This prepare for the fact we should drop the `files` as soon as we do any
  change affecting the revision schema.

* This rely on compression to avoid a significant increase of the changelog.d.
  More testing on this will be done before we freeze the final format.


2) We can store additional data:

* The new "merged" field,

* A future "salvaged" set recording files that might have been deleted but have
  were still present in the final result.

Differential Revision: https://phab.mercurial-scm.org/D9090

--- a/mercurial/metadata.py	Mon Oct 05 10:33:52 2020 +0200
+++ b/mercurial/metadata.py	Tue Sep 15 10:55:17 2020 +0200
@@ -8,6 +8,7 @@
 from __future__ import absolute_import, print_function
 
 import multiprocessing
+import struct
 
 from . import (
     error,
@@ -373,54 +374,112 @@
         return None
 
 
+# see mercurial/helptext/internals/revlogs.txt for details about the format
+
+ACTION_MASK = int("111" "00", 2)
+# note: untouched file used as copy source will as `000` for this mask.
+ADDED_FLAG = int("001" "00", 2)
+MERGED_FLAG = int("010" "00", 2)
+REMOVED_FLAG = int("011" "00", 2)
+# `100` is reserved for future use
+TOUCHED_FLAG = int("101" "00", 2)
+
+COPIED_MASK = int("11", 2)
+COPIED_FROM_P1_FLAG = int("10", 2)
+COPIED_FROM_P2_FLAG = int("11", 2)
+
+# structure is <flag><filename-end><copy-source>
+INDEX_HEADER = struct.Struct(">L")
+INDEX_ENTRY = struct.Struct(">bLL")
+
+
 def encode_files_sidedata(files):
-    sortedfiles = sorted(files.touched)
-    sidedata = {}
-    p1copies = files.copied_from_p1
-    if p1copies:
-        p1copies = encodecopies(sortedfiles, p1copies)
-        sidedata[sidedatamod.SD_P1COPIES] = p1copies
-    p2copies = files.copied_from_p2
-    if p2copies:
-        p2copies = encodecopies(sortedfiles, p2copies)
-        sidedata[sidedatamod.SD_P2COPIES] = p2copies
-    filesadded = files.added
-    if filesadded:
-        filesadded = encodefileindices(sortedfiles, filesadded)
-        sidedata[sidedatamod.SD_FILESADDED] = filesadded
-    filesremoved = files.removed
-    if filesremoved:
-        filesremoved = encodefileindices(sortedfiles, filesremoved)
-        sidedata[sidedatamod.SD_FILESREMOVED] = filesremoved
-    if not sidedata:
-        sidedata = None
-    return sidedata
+    all_files = set(files.touched)
+    all_files.update(files.copied_from_p1.values())
+    all_files.update(files.copied_from_p2.values())
+    all_files = sorted(all_files)
+    file_idx = {f: i for (i, f) in enumerate(all_files)}
+    file_idx[None] = 0
+
+    chunks = [INDEX_HEADER.pack(len(all_files))]
+
+    filename_length = 0
+    for f in all_files:
+        filename_size = len(f)
+        filename_length += filename_size
+        flag = 0
+        if f in files.added:
+            flag |= ADDED_FLAG
+        elif f in files.merged:
+            flag |= MERGED_FLAG
+        elif f in files.removed:
+            flag |= REMOVED_FLAG
+        elif f in files.touched:
+            flag |= TOUCHED_FLAG
+
+        copy = None
+        if f in files.copied_from_p1:
+            flag |= COPIED_FROM_P1_FLAG
+            copy = files.copied_from_p1.get(f)
+        elif f in files.copied_from_p2:
+            copy = files.copied_from_p2.get(f)
+            flag |= COPIED_FROM_P2_FLAG
+        copy_idx = file_idx[copy]
+        chunks.append(INDEX_ENTRY.pack(flag, filename_length, copy_idx))
+    chunks.extend(all_files)
+    return {sidedatamod.SD_FILES: b''.join(chunks)}
 
 
 def decode_files_sidedata(changelogrevision, sidedata):
-    """Return a ChangingFiles instance from a changelogrevision using sidata
-    """
-    touched = changelogrevision.files
+    md = ChangingFiles()
+    raw = sidedata.get(sidedatamod.SD_FILES)
+
+    if raw is None:
+        return md
+
+    copies = []
+    all_files = []
 
-    rawindices = sidedata.get(sidedatamod.SD_FILESADDED)
-    added = decodefileindices(touched, rawindices)
+    assert len(raw) >= INDEX_HEADER.size
+    total_files = INDEX_HEADER.unpack_from(raw, 0)[0]
 
-    rawindices = sidedata.get(sidedatamod.SD_FILESREMOVED)
-    removed = decodefileindices(touched, rawindices)
+    offset = INDEX_HEADER.size
+    file_offset_base = offset + (INDEX_ENTRY.size * total_files)
+    file_offset_last = file_offset_base
+
+    assert len(raw) >= file_offset_base
 
-    rawcopies = sidedata.get(sidedatamod.SD_P1COPIES)
-    p1_copies = decodecopies(touched, rawcopies)
-
-    rawcopies = sidedata.get(sidedatamod.SD_P2COPIES)
-    p2_copies = decodecopies(touched, rawcopies)
+    for idx in range(total_files):
+        flag, file_end, copy_idx = INDEX_ENTRY.unpack_from(raw, offset)
+        file_end += file_offset_base
+        filename = raw[file_offset_last:file_end]
+        filesize = file_end - file_offset_last
+        assert len(filename) == filesize
+        offset += INDEX_ENTRY.size
+        file_offset_last = file_end
+        all_files.append(filename)
+        if flag & ACTION_MASK == ADDED_FLAG:
+            md.mark_added(filename)
+        elif flag & ACTION_MASK == MERGED_FLAG:
+            md.mark_merged(filename)
+        elif flag & ACTION_MASK == REMOVED_FLAG:
+            md.mark_removed(filename)
+        elif flag & ACTION_MASK == TOUCHED_FLAG:
+            md.mark_touched(filename)
 
-    return ChangingFiles(
-        touched=touched,
-        added=added,
-        removed=removed,
-        p1_copies=p1_copies,
-        p2_copies=p2_copies,
-    )
+        copied = None
+        if flag & COPIED_MASK == COPIED_FROM_P1_FLAG:
+            copied = md.mark_copied_from_p1
+        elif flag & COPIED_MASK == COPIED_FROM_P2_FLAG:
+            copied = md.mark_copied_from_p2
+
+        if copied is not None:
+            copies.append((copied, filename, copy_idx))
+
+    for copied, filename, copy_idx in copies:
+        copied(all_files[copy_idx], filename)
+
+    return md
 
 
 def _getsidedata(srcrepo, rev):
@@ -428,23 +487,15 @@
     filescopies = computechangesetcopies(ctx)
     filesadded = computechangesetfilesadded(ctx)
     filesremoved = computechangesetfilesremoved(ctx)
-    sidedata = {}
-    if any([filescopies, filesadded, filesremoved]):
-        sortedfiles = sorted(ctx.files())
-        p1copies, p2copies = filescopies
-        p1copies = encodecopies(sortedfiles, p1copies)
-        p2copies = encodecopies(sortedfiles, p2copies)
-        filesadded = encodefileindices(sortedfiles, filesadded)
-        filesremoved = encodefileindices(sortedfiles, filesremoved)
-        if p1copies:
-            sidedata[sidedatamod.SD_P1COPIES] = p1copies
-        if p2copies:
-            sidedata[sidedatamod.SD_P2COPIES] = p2copies
-        if filesadded:
-            sidedata[sidedatamod.SD_FILESADDED] = filesadded
-        if filesremoved:
-            sidedata[sidedatamod.SD_FILESREMOVED] = filesremoved
-    return sidedata
+    filesmerged = computechangesetfilesmerged(ctx)
+    files = ChangingFiles()
+    files.update_touched(ctx.files())
+    files.update_added(filesadded)
+    files.update_removed(filesremoved)
+    files.update_merged(filesmerged)
+    files.update_copies_from_p1(filescopies[0])
+    files.update_copies_from_p2(filescopies[1])
+    return encode_files_sidedata(files)
 
 
 def getsidedataadder(srcrepo, destrepo):