Mercurial > hg
annotate tests/list-tree.py @ 50400:95acba2c29f6
encoding: avoid quadratic time complexity when json-encoding non-UTF8 strings
Apparently the code uses "+=" with a bytes object, which is linear-time, so the
whole encoding is quadratic-time. This patch makes us use a bytearray object,
instead, which has a(n amortized-)constant-time append operation.
The encoding is still not particularly fast, but at least a 10MB file
takes tens of seconds, not many hours to encode.
author | Arseniy Alekseyev <aalekseyev@janestreet.com> |
---|---|
date | Mon, 06 Mar 2023 11:27:57 +0000 |
parents | 6000f5b25c9b |
children |
rev | line source |
---|---|
35217
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
1 import argparse |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
2 import os |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
3 |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
4 ap = argparse.ArgumentParser() |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
5 ap.add_argument('path', nargs='+') |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
6 opts = ap.parse_args() |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
7 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
35380
diff
changeset
|
8 |
35217
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
9 def gather(): |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
10 for p in opts.path: |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
11 if not os.path.exists(p): |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
12 return |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
13 if os.path.isdir(p): |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
14 yield p + os.path.sep |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
15 for dirpath, dirs, files in os.walk(p): |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
16 for d in dirs: |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
17 yield os.path.join(dirpath, d) + os.path.sep |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
18 for f in files: |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
19 yield os.path.join(dirpath, f) |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
20 else: |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
21 yield p |
aa905f9cdcda
tests: write and use a custom helper script to avoid find's -printf
Augie Fackler <augie@google.com>
parents:
diff
changeset
|
22 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
35380
diff
changeset
|
23 |
35380
acff41957b34
tests: stabilize the sorted output of list-tree.py on Windows
Matt Harbison <matt_harbison@yahoo.com>
parents:
35217
diff
changeset
|
24 print('\n'.join(sorted(gather(), key=lambda x: x.replace(os.path.sep, '/')))) |