Lucas WintherandClaude Opus 5.5
e28fbeb0b3
Single-file, stdlib-only CLI that writes the CSV format WinDirStat's CsvLoader accepts (parent-before-child rows, subtree totals, localized headers for all 25 UI languages). Includes install.sh, pyproject entry point, unit tests and a generator for the language table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
289 lines
14 KiB
Python
Executable File
289 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""wdsexport - scan a Linux/macOS directory tree and write a WinDirStat 2.x
|
||
"saved results" CSV, to be opened on Windows via File > Open saved results.
|
||
|
||
Format notes (from windirstat/CsvLoader.cpp):
|
||
* Header names must match WinDirStat's UI language exactly.
|
||
* Rows must be ordered parent-before-child; parents are found by the path
|
||
prefix before the last backslash.
|
||
* Container rows carry subtree totals (the loader does not sum children).
|
||
|
||
Rows are written in post-order (when subtree totals are known) to a temp file,
|
||
which is then copied out in reverse: reversed post-order is parent-first.
|
||
"""
|
||
import argparse
|
||
import os
|
||
import socket
|
||
import stat
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import zipfile
|
||
|
||
__version__ = "1.0.0"
|
||
|
||
IT_DIRECTORY = 1 << 2
|
||
IT_FILE = 1 << 3
|
||
ITF_SCANERROR = 1 << 14
|
||
ITF_ROOTITEM = 1 << 28
|
||
|
||
# Localized column names from windirstat/res/langs/lang_*.txt, in the order:
|
||
# Name, Files, Folders, Logical Size, Physical Size, Attributes, Last Change, Index.
|
||
# Regenerate with tools/gen_langs.py.
|
||
# @@LANGS-BEGIN@@
|
||
LANGS = {
|
||
'ar': ['الاسم', 'الملفات', 'المجلدات الفرعية', 'الحجم المنطقي', 'الحجم الفعلي على القرص', 'السمات', 'آخر تعديل', 'الفهرس'],
|
||
'cs': ['Název', 'Soubory', 'Složky', 'Logická velikost', 'Fyzická velikost', 'Atributy', 'Poslední změna', 'Index'],
|
||
'da': ['Navn', 'Filer', 'Mapper', 'Logisk størrelse', 'Fysisk størrelse', 'Attributter', 'Sidste ændring', 'Indeks'],
|
||
'de': ['Name', 'Dateien', 'Verzeichnisse', 'Logische Größe', 'Physische Größe', 'Attribute', 'Letzte Änderung', 'Index'],
|
||
'en': ['Name', 'Files', 'Folders', 'Logical Size', 'Physical Size', 'Attributes', 'Last Change', 'Index'],
|
||
'es': ['Nombre', 'Archivos', 'Subdirectorios', 'Tamaño Lógico', 'Tamaño Físico', 'Atributos', 'Último Cambio', 'Índice'],
|
||
'et': ['Nimi', 'Failid', 'Alamkataloogid', 'Loogiline suurus', 'Füüsiline suurus', 'Atribuudid', 'Viimane muudatus', 'Indeks'],
|
||
'fi': ['Nimi', 'Tiedostoja', 'Alikansioita', 'Looginen koko', 'Fyysinen koko', 'Määritteet', 'Edellinen muutos', 'Indeksi'],
|
||
'fr': ['Nom', 'Fichiers', 'Sous-dossiers', 'Taille Logique', 'Taille Physique', 'Attributs', 'Dernier changement', 'Index'],
|
||
'hi': ['नाम', 'फ़ाइलें', 'फ़ोल्डर', 'तार्किक आकार', 'भौतिक आकार', 'विशेषताएँ', 'अंतिम बदलाव', 'सूचकांक'],
|
||
'hu': ['Név', 'Fájlok', 'Alkönyvtárak', 'Logikai méret', 'Fizikai méret', 'Attribútumok', 'Utoljára módosítva', 'Index'],
|
||
'it': ['Nome', 'File', 'Sottocartella', 'Dimensione logica', 'Dimensione fisica', 'Attributi', 'Ultima modifica', 'Indice'],
|
||
'ja': ['名前', 'ファイル', 'フォルダー', '論理サイズ', '物理サイズ', '属性', '最終更新日時', 'インデックス'],
|
||
'ko': ['이름', '파일', '폴더', '논리적 크기', '물리적 크기', '속성', '최근 변경', '인덱스'],
|
||
'nb': ['Navn', 'Filer', 'Mapper', 'Størrelse', 'Diskbruk', 'Attributter', 'Endringsdato', 'Indeks'],
|
||
'nl': ['Naam', 'Bestanden', 'Onderliggende mappen', 'Logische Grootte', 'Fysieke Grootte', 'Kenmerken', 'Laatste wijziging', 'Index'],
|
||
'pl': ['Nazwa', 'Plików', 'Podkatalogi', 'Rozmiar logiczny', 'Rozmiar fizyczny', 'Atrybuty', 'Ostatnia modyfikacja', 'Indeks'],
|
||
'pt': ['Nome', 'Arquivos', 'Subdiretórios', 'Tamanho Lógico', 'Tamanho Físico', 'Atributos', 'Última Alteração', 'Índice'],
|
||
'ru': ['Имя', 'Файлы', 'Подкаталоги', 'Логический размер', 'Физический размер', 'Атрибуты', 'Дата изменения', 'Индекс'],
|
||
'sl': ['Ime', 'Datoteke', 'Mape', 'Logična velikost', 'Fizična velikost', 'Atributi', 'Zadnja sprememba', 'Indeks'],
|
||
'sv': ['Namn', 'Filer', 'Mappar', 'Logisk storlek', 'Fysisk storlek', 'Attribut', 'Senast ändrad', 'Index'],
|
||
'tr': ['Ad', 'Dosyalar', 'Dizinler', 'Mantıksal Boyut', 'Fiziksel Boyut', 'Özellikler', 'Son Değişiklik', 'Dizin'],
|
||
'uk': ['Ім’я', 'Файли', 'Підкаталоги', 'Логічний розмір', 'Фізичний розмір', 'Атрибути', 'Дата зміни', 'Індекс'],
|
||
'zh-HK': ['名稱', '檔案數目', '資料夾數目', '邏輯大小', '實際大小', '檔案屬性', '最後修改時間', '索引'],
|
||
'zh': ['名称', '文件数', '文件夹数', '逻辑大小', '物理大小', '属性', '修改时间', '索引'],
|
||
}
|
||
# @@LANGS-END@@
|
||
|
||
|
||
def header(lang):
|
||
name, files, folders, logical, physical, attrs, last_change, index = LANGS[lang]
|
||
return [name, files, folders, logical, physical, attrs, last_change,
|
||
"WinDirStat " + attrs, index, "ScanOutcome", "ScanError"]
|
||
|
||
|
||
def clean(name):
|
||
"""Make a path component safe for WinDirStat's minimal CSV parser."""
|
||
s = os.fsdecode(name).encode("utf-8", "replace").decode("utf-8")
|
||
for bad, good in (("\\", "_"), ('"', "'"), ("\n", " "), ("\r", " ")):
|
||
s = s.replace(bad, good)
|
||
return s
|
||
|
||
|
||
def timestamp(mtime):
|
||
try:
|
||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(mtime))
|
||
except (OverflowError, OSError, ValueError):
|
||
return ""
|
||
|
||
|
||
class Scanner:
|
||
def __init__(self, out, cross_fs=False, exclude=(), skip_inodes=(), progress=None):
|
||
self.out = out
|
||
self.cross_fs = cross_fs
|
||
self.exclude = {os.path.realpath(p) for p in exclude}
|
||
self.skip_inodes = set(skip_inodes)
|
||
self.seen_links = set()
|
||
self.progress = progress
|
||
self.entries = 0
|
||
self.errors = 0
|
||
self.root_dev = None
|
||
|
||
def row(self, path, files, folders, logical, physical, attrs, mtime, itype,
|
||
outcome="complete", err=""):
|
||
self.out.write(
|
||
f'"{path}",{files},{folders},{logical},{physical},"{attrs}",'
|
||
f'{timestamp(mtime)},0x{itype:08X},0x{0:016X},"{outcome}","{err}"\n')
|
||
|
||
def scan(self, root, root_name):
|
||
st = os.stat(root)
|
||
self.root_dev = st.st_dev
|
||
return self._walk(root, clean(root_name), st.st_mtime, "", True)
|
||
|
||
def _walk(self, dirpath, wpath, mtime, attrs, is_root=False):
|
||
# One stack frame per directory level; main() raises the recursion limit.
|
||
files = folders = logical = physical = 0
|
||
outcome, err = "complete", ""
|
||
try:
|
||
it = os.scandir(dirpath)
|
||
except OSError as e:
|
||
it, outcome, err = None, "partial", clean(e.strerror or type(e).__name__)
|
||
self.errors += 1
|
||
if it is not None:
|
||
with it:
|
||
for entry in it:
|
||
self.entries += 1
|
||
if self.progress and self.entries % 100000 == 0:
|
||
self.progress(self.entries)
|
||
try:
|
||
st = entry.stat(follow_symlinks=False)
|
||
except OSError:
|
||
self.errors += 1
|
||
continue
|
||
name = clean(entry.name)
|
||
child = wpath + "\\" + name
|
||
a = "H" if name.startswith(".") else ""
|
||
if stat.S_ISDIR(st.st_mode):
|
||
full = os.path.join(dirpath, entry.name)
|
||
if (not self.cross_fs and st.st_dev != self.root_dev) or \
|
||
os.path.realpath(full) in self.exclude:
|
||
continue
|
||
r = self._walk(full, child, st.st_mtime, a)
|
||
files += r[0]
|
||
folders += r[1] + 1
|
||
logical += r[2]
|
||
physical += r[3]
|
||
continue
|
||
key = (st.st_dev, st.st_ino)
|
||
if key in self.skip_inodes:
|
||
continue # our own output/temp files
|
||
if st.st_nlink > 1:
|
||
if key in self.seen_links:
|
||
continue # count each hard-linked inode once
|
||
self.seen_links.add(key)
|
||
lsz, psz = st.st_size, st.st_blocks * 512
|
||
self.row(child, 0, 0, lsz, psz, a, st.st_mtime, IT_FILE)
|
||
files += 1
|
||
logical += lsz
|
||
physical += psz
|
||
itype = IT_DIRECTORY | (ITF_ROOTITEM if is_root else 0) | \
|
||
(ITF_SCANERROR if outcome != "complete" else 0)
|
||
self.row(wpath, files, folders, logical, physical, attrs, mtime, itype, outcome, err)
|
||
return files, folders, logical, physical
|
||
|
||
|
||
def copy_reversed_lines(src, dst, block=1 << 20):
|
||
"""Write the lines of binary file `src` to `dst` in reverse order."""
|
||
src.seek(0, os.SEEK_END)
|
||
pos = src.tell()
|
||
tail = b""
|
||
while pos > 0:
|
||
step = min(block, pos)
|
||
pos -= step
|
||
src.seek(pos)
|
||
buf = src.read(step) + tail
|
||
lines = buf.split(b"\n")
|
||
tail = lines[0] # may be incomplete; joined with the previous block
|
||
for line in reversed(lines[1:]):
|
||
if line:
|
||
dst.write(line + b"\n")
|
||
if tail:
|
||
dst.write(tail + b"\n")
|
||
|
||
|
||
def default_name(path):
|
||
host = socket.gethostname().split(".")[0] or "host"
|
||
real = os.path.realpath(path)
|
||
base = os.path.basename(real.rstrip(os.sep)) or "root"
|
||
return f"{host}-{base}"
|
||
|
||
|
||
def human(n):
|
||
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
|
||
if n < 1024 or unit == "TiB":
|
||
return f"{n:.1f} {unit}" if unit != "B" else f"{n} B"
|
||
n /= 1024
|
||
|
||
|
||
def parse_args(argv):
|
||
p = argparse.ArgumentParser(
|
||
prog="wdsexport",
|
||
description="Scan a directory tree and write a WinDirStat 2.x saved-results CSV "
|
||
"(open it in WinDirStat via File > Open saved results).")
|
||
p.add_argument("path", nargs="?", default="/", help="directory to scan (default: /)")
|
||
p.add_argument("-o", "--output",
|
||
help="output CSV path, or - for stdout (default: ./<name>.csv)")
|
||
p.add_argument("--name", help="root folder name shown in WinDirStat "
|
||
"(default: <hostname>-<dirname>, e.g. myhost-root)")
|
||
p.add_argument("--exclude", action="append", default=[], metavar="DIR",
|
||
help="skip this directory (repeatable)")
|
||
p.add_argument("--zip", action="store_true",
|
||
help="also write <output>.zip (CSVs compress ~15x)")
|
||
p.add_argument("--lang", default="en", choices=sorted(LANGS), metavar="CODE",
|
||
help="column-name language; must match WinDirStat's UI language "
|
||
f"(default: en; choices: {', '.join(sorted(LANGS))})")
|
||
p.add_argument("--cross-fs", action="store_true",
|
||
help="descend into other mounted filesystems (default: stay on one, like du -x)")
|
||
p.add_argument("-q", "--quiet", action="store_true", help="no progress output")
|
||
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
||
args = p.parse_args(argv)
|
||
if not os.path.isdir(args.path):
|
||
p.error(f"not a directory: {args.path}")
|
||
if args.zip and args.output == "-":
|
||
p.error("--zip cannot be used with --output -")
|
||
return args
|
||
|
||
|
||
def main(argv=None):
|
||
args = parse_args(argv)
|
||
sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000))
|
||
name = args.name or default_name(args.path)
|
||
to_stdout = args.output == "-"
|
||
output = None if to_stdout else os.path.abspath(args.output or f"{clean(name)}.csv")
|
||
log = (lambda *a: None) if args.quiet else (lambda msg: print(msg, file=sys.stderr, flush=True))
|
||
tty = sys.stderr.isatty() and not args.quiet
|
||
t0 = time.monotonic()
|
||
|
||
def progress(n):
|
||
if tty:
|
||
print(f"\r{n:,} entries, {time.monotonic() - t0:.0f}s", end="", file=sys.stderr, flush=True)
|
||
|
||
tmp_dir = tempfile.gettempdir() if to_stdout else os.path.dirname(output)
|
||
tmp = tempfile.NamedTemporaryFile("w+b", dir=tmp_dir, prefix=".wdsexport-", suffix=".tmp", delete=False)
|
||
skip = set()
|
||
for f in [tmp.name] + ([output] if output and os.path.exists(output) else []):
|
||
st = os.stat(f)
|
||
skip.add((st.st_dev, st.st_ino))
|
||
try:
|
||
text = open(tmp.fileno(), "w", encoding="utf-8", newline="\n", buffering=1 << 20, closefd=False)
|
||
scanner = Scanner(text, cross_fs=args.cross_fs, exclude=args.exclude,
|
||
skip_inodes=skip, progress=progress)
|
||
files, folders, logical, physical = scanner.scan(args.path, name)
|
||
text.flush()
|
||
if tty:
|
||
print(file=sys.stderr)
|
||
log(f"Scanned {scanner.entries:,} entries in {time.monotonic() - t0:.0f}s: "
|
||
f"{files:,} files, {folders:,} folders, {human(physical)} on disk")
|
||
if scanner.errors:
|
||
log(f"{scanner.errors:,} entries could not be read (marked as partial); "
|
||
"run with sudo to include them")
|
||
|
||
head = ("" + ",".join(f'"{h}"' for h in header(args.lang)) + "\n").encode("utf-8")
|
||
if to_stdout:
|
||
dst = sys.stdout.buffer
|
||
dst.write(head)
|
||
copy_reversed_lines(tmp, dst)
|
||
dst.flush()
|
||
else:
|
||
# Write next to the target, then rename, so a failed run never
|
||
# leaves a truncated CSV behind under the final name.
|
||
part = output + ".part"
|
||
with open(part, "wb", buffering=1 << 20) as dst:
|
||
dst.write(head)
|
||
copy_reversed_lines(tmp, dst)
|
||
os.replace(part, output)
|
||
log(f"Wrote {output} ({human(os.path.getsize(output))})")
|
||
if args.zip:
|
||
zpath = os.path.splitext(output)[0] + ".zip"
|
||
with zipfile.ZipFile(zpath, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as z:
|
||
z.write(output, os.path.basename(output))
|
||
log(f"Wrote {zpath} ({human(os.path.getsize(zpath))})")
|
||
except KeyboardInterrupt:
|
||
print("\ninterrupted", file=sys.stderr)
|
||
return 130
|
||
finally:
|
||
tmp.close()
|
||
os.unlink(tmp.name)
|
||
if output and os.path.exists(output + ".part"):
|
||
os.unlink(output + ".part")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|