software/hoardy-web/./tool/development/describe-dir.py

Passively capture, archive, and hoard your web browsing history, including the contents of the pages you visit, for later offline viewing, mirroring, and/or indexing. Your own personal private Wayback Machine that can also archive HTTP POST requests and responses, as well as most other HTTP-level data.

Files

Raw Source

Contents

#!/usr/bin/env python3

import argparse
import hashlib
import os
import os.path
import stat
import sys
import time
import typing as _t

def any_to_bytes(s : _t.Any) -> bytes:
    if isinstance(s, bytes):
        return s
    elif isinstance(s, str):
        return os.fsencode(s)
    else:
        try:
            return os.fsencode(str(s))
        except Exception:
            return os.fsencode(repr(s))

binstdout = os.fdopen(sys.stdout.fileno(), "wb")
def printbin(*ls : _t.Any) -> None:
    res = b" ".join(map(any_to_bytes, ls)).replace(b"\\", b"\\\\").replace(b"\n", b"\\n")
    binstdout.write(res + b"\n")
    binstdout.flush()

BUFFER_SIZE = 4 * 1024 * 1024
def hex_sha256_of(fpath : bytes) -> str:
    with open(fpath, "rb") as f:
        fhash=hashlib.sha256()
        while True:
            data = f.read(BUFFER_SIZE)
            if len(data) == 0: break
            fhash.update(data)
        return fhash.hexdigest()

def main() -> None:
    parser = argparse.ArgumentParser(prog="describe-dir", description="generate a recursive deterministic textual textual description of given input directories")
    parser.add_argument("--no-mtime", action="store_true", help="ignore mtimes")
    parser.add_argument("path", metavar="PATH", nargs="*", type=str, help="input directories")
    args = parser.parse_args(sys.argv[1:])

    argvb = [os.fsencode(a) for a in args.path]
    argvb.sort()

    seen : dict[tuple[int, int], bytes] = {}
    for dirpath in argvb:
        for root, dirs, files in os.walk(dirpath):
            dirs.sort()
            files.sort()
            everything = dirs + files
            everything.sort()
            for name in everything:
                fpath=os.path.join(root, name)
                opath=os.path.relpath(fpath, dirpath)

                fstat = os.lstat(fpath)
                ino = (fstat.st_dev, fstat.st_ino)
                try:
                    hardlink = seen[ino]
                except KeyError:
                    seen[ino] = fpath
                else:
                    printbin(opath, "=>", hardlink)
                    continue

                if args.no_mtime:
                    mtime = "no"
                else:
                    mtime = "[" + time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(fstat.st_mtime)) + "]"
                size = fstat.st_size
                mode = oct(stat.S_IMODE(fstat.st_mode))[2:]
                if stat.S_ISDIR(fstat.st_mode):
                    printbin(opath, "dir", "mode", mode, "mtime", mtime)
                elif stat.S_ISREG(fstat.st_mode):
                    sha256 = hex_sha256_of(fpath)
                    printbin(opath, "reg", "mode", mode, "mtime", mtime, "size", size, "sha256", sha256)
                elif stat.S_ISLNK(fstat.st_mode):
                    symlink = os.path.relpath(os.readlink(fpath), fpath)
                    printbin(opath, "lnk", "mode", mode, "mtime", mtime, "->", symlink)
                else:
                    printbin(opath, "...", "mode", mode, "mtime", mtime, "size", size)

if __name__ == "__main__":
    main()