Mercurial > hg
comparison mercurial/osutil.c @ 24461:05ccfe6763f1
osutil: use getdirentriesattr on OS X if possible
This is a significant win for large repositories on OS X, especially with a
cold cache. Unfortunately we need to keep the lstat-based implementation around
for two reasons:
- Not all filesystems support this call.
- There's an edge case in which it's best to fall back to avoid a retry loop.
More about this in the comments.
The below tests are all performed on a Mac with an SSD running OS X 10.9, on a
repository with over 200k files. The results are best of 5 with simulated
best-effort conditions.
The gains with a hot cache are pretty impressive: 'hg status' goes from 5.18
seconds to 3.79 seconds.
However, a repository that large will probably already be using something like
hgwatchman [1], which helps much more (for this repo, 'hg status' with
hgwatchman is approximately 1 second). Where this really helps is when the
cache is cold [2]: hg status goes from 31.0 seconds to 9.66.
See http://lists.apple.com/archives/filesystem-dev/2014/Dec/msg00002.html for
some more discussion about this function.
This is based on a patch by Sean Farley <sean@farley.io>.
[1] https://bitbucket.org/facebook/hgwatchman
[2] There appears to be no easy way to clear the file cache (aka "vnodes") on
OS X short of rebooting. purge(8) purportedly does that but in my testing had
little effect. The workaround I came up with was to assume that vnode eviction
was LRU, make sure the kern.maxvnodes sysctl is smaller than the size of the
repository, then make sure we'd always miss the cache by running 'hg status' in
another clone of the repository before running it in the test repository.
author | Siddharth Agarwal <sid0@fb.com> |
---|---|
date | Wed, 25 Mar 2015 15:55:31 -0700 |
parents | 73477e755cd2 |
children | 40b05303ac32 |
comparison
equal
deleted
inserted
replaced
24460:73477e755cd2 | 24461:05ccfe6763f1 |
---|---|
20 #else | 20 #else |
21 #include <dirent.h> | 21 #include <dirent.h> |
22 #include <sys/stat.h> | 22 #include <sys/stat.h> |
23 #include <sys/types.h> | 23 #include <sys/types.h> |
24 #include <unistd.h> | 24 #include <unistd.h> |
25 #endif | |
26 | |
27 #ifdef __APPLE__ | |
28 #include <sys/attr.h> | |
29 #include <sys/vnode.h> | |
25 #endif | 30 #endif |
26 | 31 |
27 #include "util.h" | 32 #include "util.h" |
28 | 33 |
29 /* some platforms lack the PATH_MAX definition (eg. GNU/Hurd) */ | 34 /* some platforms lack the PATH_MAX definition (eg. GNU/Hurd) */ |
390 #endif | 395 #endif |
391 error_value: | 396 error_value: |
392 return ret; | 397 return ret; |
393 } | 398 } |
394 | 399 |
400 #ifdef __APPLE__ | |
401 | |
402 typedef struct { | |
403 u_int32_t length; | |
404 attrreference_t name; | |
405 fsobj_type_t obj_type; | |
406 struct timespec mtime; | |
407 #if __LITTLE_ENDIAN__ | |
408 mode_t access_mask; | |
409 uint16_t padding; | |
410 #else | |
411 uint16_t padding; | |
412 mode_t access_mask; | |
413 #endif | |
414 off_t size; | |
415 } __attribute__((packed)) attrbuf_entry; | |
416 | |
417 int attrkind(attrbuf_entry *entry) | |
418 { | |
419 switch (entry->obj_type) { | |
420 case VREG: return S_IFREG; | |
421 case VDIR: return S_IFDIR; | |
422 case VLNK: return S_IFLNK; | |
423 case VBLK: return S_IFBLK; | |
424 case VCHR: return S_IFCHR; | |
425 case VFIFO: return S_IFIFO; | |
426 case VSOCK: return S_IFSOCK; | |
427 } | |
428 return -1; | |
429 } | |
430 | |
431 /* get these many entries at a time */ | |
432 #define LISTDIR_BATCH_SIZE 50 | |
433 | |
434 static PyObject *_listdir_batch(char *path, int pathlen, int keepstat, | |
435 char *skip, bool *fallback) | |
436 { | |
437 PyObject *list, *elem, *stat = NULL, *ret = NULL; | |
438 int kind, err; | |
439 unsigned long index; | |
440 unsigned int count, old_state, new_state; | |
441 bool state_seen = false; | |
442 attrbuf_entry *entry; | |
443 /* from the getattrlist(2) man page: a path can be no longer than | |
444 (NAME_MAX * 3 + 1) bytes. Also, "The getattrlist() function will | |
445 silently truncate attribute data if attrBufSize is too small." So | |
446 pass in a buffer big enough for the worst case. */ | |
447 char attrbuf[LISTDIR_BATCH_SIZE * (sizeof(attrbuf_entry) + NAME_MAX * 3 + 1)]; | |
448 unsigned int basep_unused; | |
449 | |
450 struct stat st; | |
451 int dfd = -1; | |
452 | |
453 /* these must match the attrbuf_entry struct, otherwise you'll end up | |
454 with garbage */ | |
455 struct attrlist requested_attr = {0}; | |
456 requested_attr.bitmapcount = ATTR_BIT_MAP_COUNT; | |
457 requested_attr.commonattr = (ATTR_CMN_NAME | ATTR_CMN_OBJTYPE | | |
458 ATTR_CMN_MODTIME | ATTR_CMN_ACCESSMASK); | |
459 requested_attr.fileattr = ATTR_FILE_TOTALSIZE; | |
460 | |
461 *fallback = false; | |
462 | |
463 if (pathlen >= PATH_MAX) { | |
464 errno = ENAMETOOLONG; | |
465 PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); | |
466 goto error_value; | |
467 } | |
468 | |
469 dfd = open(path, O_RDONLY); | |
470 if (dfd == -1) { | |
471 PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); | |
472 goto error_value; | |
473 } | |
474 | |
475 list = PyList_New(0); | |
476 if (!list) | |
477 goto error_dir; | |
478 | |
479 do { | |
480 count = LISTDIR_BATCH_SIZE; | |
481 err = getdirentriesattr(dfd, &requested_attr, &attrbuf, | |
482 sizeof(attrbuf), &count, &basep_unused, | |
483 &new_state, 0); | |
484 if (err < 0) { | |
485 if (errno == ENOTSUP) { | |
486 /* We're on a filesystem that doesn't support | |
487 getdirentriesattr. Fall back to the | |
488 stat-based implementation. */ | |
489 *fallback = true; | |
490 } else | |
491 PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); | |
492 goto error; | |
493 } | |
494 | |
495 if (!state_seen) { | |
496 old_state = new_state; | |
497 state_seen = true; | |
498 } else if (old_state != new_state) { | |
499 /* There's an edge case with getdirentriesattr. Consider | |
500 the following initial list of files: | |
501 | |
502 a | |
503 b | |
504 <-- | |
505 c | |
506 d | |
507 | |
508 If the iteration is paused at the arrow, and b is | |
509 deleted before it is resumed, getdirentriesattr will | |
510 not return d at all! Ordinarily we're expected to | |
511 restart the iteration from the beginning. To avoid | |
512 getting stuck in a retry loop here, fall back to | |
513 stat. */ | |
514 *fallback = true; | |
515 goto error; | |
516 } | |
517 | |
518 entry = (attrbuf_entry *)attrbuf; | |
519 | |
520 for (index = 0; index < count; index++) { | |
521 char *filename = ((char *)&entry->name) + | |
522 entry->name.attr_dataoffset; | |
523 | |
524 if (!strcmp(filename, ".") || !strcmp(filename, "..")) | |
525 continue; | |
526 | |
527 kind = attrkind(entry); | |
528 if (kind == -1) { | |
529 PyErr_Format(PyExc_OSError, | |
530 "unknown object type %u for file " | |
531 "%s%s!", | |
532 entry->obj_type, path, filename); | |
533 goto error; | |
534 } | |
535 | |
536 /* quit early? */ | |
537 if (skip && kind == S_IFDIR && !strcmp(filename, skip)) { | |
538 ret = PyList_New(0); | |
539 goto error; | |
540 } | |
541 | |
542 if (keepstat) { | |
543 /* from the getattrlist(2) man page: "Only the | |
544 permission bits ... are valid". */ | |
545 st.st_mode = (entry->access_mask & ~S_IFMT) | kind; | |
546 st.st_mtime = entry->mtime.tv_sec; | |
547 st.st_size = entry->size; | |
548 stat = makestat(&st); | |
549 if (!stat) | |
550 goto error; | |
551 elem = Py_BuildValue("siN", filename, kind, stat); | |
552 } else | |
553 elem = Py_BuildValue("si", filename, kind); | |
554 if (!elem) | |
555 goto error; | |
556 stat = NULL; | |
557 | |
558 PyList_Append(list, elem); | |
559 Py_DECREF(elem); | |
560 | |
561 entry = (attrbuf_entry *)((char *)entry + entry->length); | |
562 } | |
563 } while (err == 0); | |
564 | |
565 ret = list; | |
566 Py_INCREF(ret); | |
567 | |
568 error: | |
569 Py_DECREF(list); | |
570 Py_XDECREF(stat); | |
571 error_dir: | |
572 close(dfd); | |
573 error_value: | |
574 return ret; | |
575 } | |
576 | |
577 #endif /* __APPLE__ */ | |
578 | |
395 static PyObject *_listdir(char *path, int pathlen, int keepstat, char *skip) | 579 static PyObject *_listdir(char *path, int pathlen, int keepstat, char *skip) |
396 { | 580 { |
581 #ifdef __APPLE__ | |
582 PyObject *ret; | |
583 bool fallback = false; | |
584 | |
585 ret = _listdir_batch(path, pathlen, keepstat, skip, &fallback); | |
586 if (ret != NULL || !fallback) | |
587 return ret; | |
588 #endif | |
397 return _listdir_stat(path, pathlen, keepstat, skip); | 589 return _listdir_stat(path, pathlen, keepstat, skip); |
398 } | 590 } |
399 | 591 |
400 static PyObject *statfiles(PyObject *self, PyObject *args) | 592 static PyObject *statfiles(PyObject *self, PyObject *args) |
401 { | 593 { |