comparison mercurial/cext/revlog.c @ 45811:4404f129341e

revlog: don't cache parsed tuples in the C module

A cached entry creates ~8 Python objects per cached changeset, which
comes to around 200 Bytes per cached changeset on AMD64. Especially for
operations that touch a lot of changesets, that can easily sum up to
more than a 100MB of memory. Simple tests on large repositories show
<2% runtime penalty for ripping out the cache, even for cache heavy
operations like "hg log" for all revisions.

Differential Revision: https://phab.mercurial-scm.org/D9155

45810:09914d934cf4

struct indexObjectStruct {
	PyObject_HEAD
	    /* Type-specific fields go here. */
	    PyObject *data;     /* raw bytes of index */
	Py_buffer buf;          /* buffer of data */
	PyObject **cache;       /* cached tuples */
	const char **offsets;   /* populated on demand */
	Py_ssize_t raw_length;  /* original number of elements */
	Py_ssize_t length;      /* current number of elements */
	PyObject *added;        /* populated on demand */
	PyObject *headrevs;     /* cache, invalidated on changes */

	uint64_t offset_flags;
	int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2;
	const char *c_node_id;
	const char *data;
	Py_ssize_t length = index_length(self);
	PyObject *entry;

	if (pos == nullrev) {
		Py_INCREF(nullentry);
		return nullentry;
	}

	if (pos >= self->length) {
		PyObject *obj;
		obj = PyList_GET_ITEM(self->added, pos - self->length);
		Py_INCREF(obj);
		return obj;
	}

	if (self->cache) {
		if (self->cache[pos]) {
			Py_INCREF(self->cache[pos]);
			return self->cache[pos];
		}
	} else {
		self->cache = calloc(self->raw_length, sizeof(PyObject *));
		if (self->cache == NULL)
			return PyErr_NoMemory();
	}

	data = index_deref(self, pos);
	if (data == NULL)
		return NULL;

	link_rev = getbe32(data + 20);
	parent_1 = getbe32(data + 24);
	parent_2 = getbe32(data + 28);
	c_node_id = data + 32;

	entry = Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
	                      base_rev, link_rev, parent_1, parent_2, c_node_id,
	                      (Py_ssize_t)20);

	if (entry) {
		PyObject_GC_UnTrack(entry);
		Py_INCREF(entry);
	}

	self->cache[pos] = entry;

	return entry;
}

/*
 * Return the 20-byte SHA of the node corresponding to the given rev.
 */

		} else if (self->added) {
			Py_CLEAR(self->added);
		}

		self->length = start;
		if (start < self->raw_length) {
			if (self->cache) {
				Py_ssize_t i;
				for (i = start; i < self->raw_length; i++)
					Py_CLEAR(self->cache[i]);
			}
			self->raw_length = start;
		}
		goto done;
	}

	if (self->ntinitialized) {
		index_invalidate_added(self, start - self->length);

	/* Initialize before argument-checking to avoid index_dealloc() crash.
	 */
	self->raw_length = 0;
	self->added = NULL;
	self->cache = NULL;
	self->data = NULL;
	memset(&self->buf, 0, sizeof(self->buf));
	self->headrevs = NULL;
	self->filteredrevs = Py_None;
	Py_INCREF(Py_None);

	return (PyObject *)self;
}

static void _index_clearcaches(indexObject *self)
{
	if (self->cache) {
		Py_ssize_t i;

		for (i = 0; i < self->raw_length; i++)
			Py_CLEAR(self->cache[i]);
		free(self->cache);
		self->cache = NULL;
	}
	if (self->offsets) {
		PyMem_Free((void *)self->offsets);
		self->offsets = NULL;
	}
	if (self->ntinitialized) {

45811:4404f129341e

struct indexObjectStruct {
	PyObject_HEAD
	    /* Type-specific fields go here. */
	    PyObject *data;     /* raw bytes of index */
	Py_buffer buf;          /* buffer of data */
	const char **offsets;   /* populated on demand */
	Py_ssize_t raw_length;  /* original number of elements */
	Py_ssize_t length;      /* current number of elements */
	PyObject *added;        /* populated on demand */
	PyObject *headrevs;     /* cache, invalidated on changes */

	uint64_t offset_flags;
	int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2;
	const char *c_node_id;
	const char *data;
	Py_ssize_t length = index_length(self);

	if (pos == nullrev) {
		Py_INCREF(nullentry);
		return nullentry;
	}

	if (pos >= self->length) {
		PyObject *obj;
		obj = PyList_GET_ITEM(self->added, pos - self->length);
		Py_INCREF(obj);
		return obj;
	}

	data = index_deref(self, pos);
	if (data == NULL)
		return NULL;

	link_rev = getbe32(data + 20);
	parent_1 = getbe32(data + 24);
	parent_2 = getbe32(data + 28);
	c_node_id = data + 32;

	return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
	                     base_rev, link_rev, parent_1, parent_2, c_node_id,
	                     (Py_ssize_t)20);
}

/*
 * Return the 20-byte SHA of the node corresponding to the given rev.
 */

		} else if (self->added) {
			Py_CLEAR(self->added);
		}

		self->length = start;
		if (start < self->raw_length)
			self->raw_length = start;
		goto done;
	}

	if (self->ntinitialized) {
		index_invalidate_added(self, start - self->length);

	/* Initialize before argument-checking to avoid index_dealloc() crash.
	 */
	self->raw_length = 0;
	self->added = NULL;
	self->data = NULL;
	memset(&self->buf, 0, sizeof(self->buf));
	self->headrevs = NULL;
	self->filteredrevs = Py_None;
	Py_INCREF(Py_None);

	return (PyObject *)self;
}

static void _index_clearcaches(indexObject *self)
{
	if (self->offsets) {
		PyMem_Free((void *)self->offsets);
		self->offsets = NULL;
	}
	if (self->ntinitialized) {