# coding: utf-8
from __future__ import print_function, unicode_literals

import argparse
import base64
import hashlib
import json
import os
import re
import stat
import sys
import threading
import time
from datetime import datetime

from .__init__ import ANYWIN, MACOS, PY2, TYPE_CHECKING, WINDOWS, E
from .bos import bos
from .cfg import flagdescs, permdescs, vf_bmap, vf_cmap, vf_vmap
from .pwhash import PWHash
from .util import (
    DEF_MTE,
    DEF_MTH,
    EXTS,
    FAVICON_MIMES,
    FN_EMB,
    HAVE_SQLITE3,
    IMPLICATIONS,
    META_NOBOTS,
    MIMES,
    SQLITE_VER,
    UNPLICATIONS,
    UTC,
    ODict,
    Pebkac,
    absreal,
    afsenc,
    get_df,
    humansize,
    json_hesc,
    min_ex,
    odfusion,
    read_utf8,
    relchk,
    statdir,
    ub64enc,
    uncyg,
    undot,
    unhumanize,
    vjoin,
    vsplit,
)

if HAVE_SQLITE3:
    import sqlite3

if TYPE_CHECKING:
    from .broker_mp import BrokerMp
    from .broker_thr import BrokerThr
    from .broker_util import BrokerCli


if PY2:
    range = xrange


LEELOO_DALLAS = "leeloo_dallas"



UP_MTE_MAP = {
    "w": "substr(w,1,16)",
    "up_ip": "ip",
    ".up_at": "at",
    "up_by": "un",
}

SEE_LOG = "see log for details"
SEESLOG = " (see fileserver log for details)"
SSEELOG = " ({})".format(SEE_LOG)
BAD_CFG = "invalid config; {}".format(SEE_LOG)
SBADCFG = " ({})".format(BAD_CFG)

PTN_U_GRP = re.compile(r"\$\{u(%[+-][^}]+)\}")
PTN_G_GRP = re.compile(r"\$\{g(%[+-][^}]+)\}")
PTN_U_ANY = re.compile(r"(\${[u][}%])")
PTN_G_ANY = re.compile(r"(\${[g][}%])")
PTN_SIGIL = re.compile(r"(\${[ug][}%])")


class CfgEx(Exception):
    pass


class AXS(object):
    def __init__(
        self,
        uread   = None,
        uwrite   = None,
        umove   = None,
        udel   = None,
        uget   = None,
        upget   = None,
        uhtml   = None,
        uadmin   = None,
        udot   = None,
    )  :
        self.uread  = set(uread or [])
        self.uwrite  = set(uwrite or [])
        self.umove  = set(umove or [])
        self.udel  = set(udel or [])
        self.uget  = set(uget or [])
        self.upget  = set(upget or [])
        self.uhtml  = set(uhtml or [])
        self.uadmin  = set(uadmin or [])
        self.udot  = set(udot or [])

    def __repr__(self)  :
        ks = "uread uwrite umove udel uget upget uhtml uadmin udot".split()
        return "AXS(%s)" % (", ".join("%s=%r" % (k, self.__dict__[k]) for k in ks),)


class Lim(object):
    def __init__(self, log_func )  :
        self.log_func = log_func

        self.reg    = None

        self.chmod_d = 0o755
        self.uid = self.gid = -1
        self.chown = False

        self.nups   = {}
        self.bups    = {}
        self.bupc   = {}

        self.nosub = False

        self.dfl = 0
        self.dft = 0
        self.dfv = 0
        self.vbmax = 0
        self.vnmax = 0

        self.c_vb_v = 0
        self.c_vb_r = 0

        self.smin = 0
        self.smax = 0

        self.bwin = 0
        self.bmax = 0
        self.nwin = 0
        self.nmax = 0

        self.rotn = 0
        self.rotl = 0
        self.rotf = ""
        self.rotf_tz = UTC
        self.rot_re = re.compile("")

    def log(self, msg , c   = 0)  :
        if self.log_func:
            self.log_func("up-lim", msg, c)

    def set_rotf(self, fmt , tz )  :
        self.rotf = fmt
        if tz != "UTC":
            from zoneinfo import ZoneInfo

            self.rotf_tz = ZoneInfo(tz)
        r = re.escape(fmt).replace("%Y", "[0-9]{4}").replace("%j", "[0-9]{3}")
        r = re.sub("%[mdHMSWU]", "[0-9]{2}", r)
        self.rot_re = re.compile("(^|/)" + r + "$")

    def all(
        self,
        ip ,
        rem ,
        sz ,
        ptop ,
        abspath ,
        broker    = None,
        reg    = None,
        volgetter  = "up2k.get_volsize",
    )   :
        if reg is not None and self.reg is None:
            self.reg = reg
            self.dft = 0

        self.chk_nup(ip)
        self.chk_bup(ip)
        self.chk_rem(rem)
        if sz != -1:
            self.chk_sz(sz)
        else:
            sz = 0

        self.chk_vsz(broker, ptop, sz, volgetter)
        self.chk_df(abspath, sz)

        ap2, vp2 = self.rot(abspath)
        if abspath == ap2:
            return ap2, rem

        return ap2, ("{}/{}".format(rem, vp2) if rem else vp2)

    def chk_sz(self, sz )  :
        if sz < self.smin:
            raise Pebkac(400, "file too small")

        if self.smax and sz > self.smax:
            raise Pebkac(400, "file too big")

    def chk_vsz(
        self,
        broker   ,
        ptop ,
        sz ,
        volgetter  = "up2k.get_volsize",
    )  :
        if not broker or not self.vbmax + self.vnmax:
            return

        x = broker.ask(volgetter, ptop)
        nbytes, nfiles = x.get()

        if self.vbmax and self.vbmax < nbytes + sz:
            raise Pebkac(400, "volume has exceeded max size")

        if self.vnmax and self.vnmax < nfiles + 1:
            raise Pebkac(400, "volume has exceeded max num.files")

    def chk_df(self, abspath , sz , already_written  = False)  :
        if not self.dfl:
            return

        if self.dft < time.time():
            self.dft = int(time.time()) + 300

            df, du, err = get_df(abspath, True)
            if err:
                t = "failed to read disk space usage for %r: %s"
                self.log(t % (abspath, err), 3)
                self.dfv = 0xAAAAAAAAA
            else:
                self.dfv = df or 0

            for j in list(self.reg.values()) if self.reg else []:
                self.dfv -= int(j["size"] / (len(j["hash"]) or 999) * len(j["need"]))

            if already_written:
                sz = 0

        if self.dfv - sz < self.dfl:
            self.dft = min(self.dft, int(time.time()) + 10)
            t = "server HDD is full; {} free, need {}"
            raise Pebkac(500, t.format(humansize(self.dfv - self.dfl), humansize(sz)))

        self.dfv -= int(sz)

    def chk_rem(self, rem )  :
        if self.nosub and rem:
            raise Pebkac(500, "no subdirectories allowed")

    def rot(self, path )   :
        if not self.rotf and not self.rotn:
            return path, ""

        if self.rotf:
            path = path.rstrip("/\\")
            if self.rot_re.search(path.replace("\\", "/")):
                return path, ""

            suf = datetime.now(self.rotf_tz).strftime(self.rotf)
            if path:
                path += "/"

            return path + suf, suf

        ret = self.dive(path, self.rotl)
        if not ret:
            raise Pebkac(500, "no available slots in volume")

        d = ret[len(path) :].strip("/\\").replace("\\", "/")
        return ret, d

    def dive(self, path , lvs )  :
        items = bos.listdir(path)

        if not lvs:

            return None if len(items) >= self.rotn else ""

        dirs = [int(x) for x in items if x and all(y in "1234567890" for y in x)]
        dirs.sort()

        if not dirs:

            sub = os.path.join(path, "0")
            bos.mkdir(sub, self.chmod_d)
            if self.chown:
                os.chown(sub, self.uid, self.gid)
        else:

            sub = os.path.join(path, str(dirs[-1]))

        ret = self.dive(sub, lvs - 1)
        if ret is not None:
            return os.path.join(sub, ret)

        if len(dirs) >= self.rotn:

            return None

        sub = os.path.join(path, str(dirs[-1] + 1))
        bos.mkdir(sub, self.chmod_d)
        if self.chown:
            os.chown(sub, self.uid, self.gid)
        ret = self.dive(sub, lvs - 1)
        if ret is None:
            raise Pebkac(500, "rotation bug")

        return os.path.join(sub, ret)

    def nup(self, ip )  :
        try:
            self.nups[ip].append(time.time())
        except:
            self.nups[ip] = [time.time()]

    def bup(self, ip , nbytes )  :
        v = (time.time(), nbytes)
        try:
            self.bups[ip].append(v)
            self.bupc[ip] += nbytes
        except:
            self.bups[ip] = [v]
            self.bupc[ip] = nbytes

    def chk_nup(self, ip )  :
        if not self.nmax or ip not in self.nups:
            return

        nups = self.nups[ip]
        cutoff = time.time() - self.nwin
        while nups and nups[0] < cutoff:
            nups.pop(0)

        if len(nups) >= self.nmax:
            raise Pebkac(429, "too many uploads")

    def chk_bup(self, ip )  :
        if not self.bmax or ip not in self.bups:
            return

        bups = self.bups[ip]
        cutoff = time.time() - self.bwin
        mark = self.bupc[ip]
        while bups and bups[0][0] < cutoff:
            mark -= bups.pop(0)[1]

        self.bupc[ip] = mark
        if mark >= self.bmax:
            raise Pebkac(429, "upload size limit exceeded")


class VFS(object):
    "a"

    def __init__(
        self,
        log ,
        realpath ,
        vpath ,
        vpath0 ,
        axs ,
        flags  ,
    )  :
        self.log = log
        self.realpath = realpath
        self.vpath = vpath
        self.vpath0 = vpath0
        self.axs = axs
        self.uaxs                               = {}


        self.flags = flags
        self.root = self
        self.dev = 0
        self.nodes   = {}
        self.histtab   = {}
        self.dbpaths   = {}
        self.dbv  = None
        self.lim  = None
        self.shr_src   = None
        self.shr_files  = set()
        self.shr_owner  = ""
        self.shr_all_aps   = []
        self.aread   = {}
        self.awrite   = {}
        self.amove   = {}
        self.adel   = {}
        self.aget   = {}
        self.apget   = {}
        self.ahtml   = {}
        self.aadmin   = {}
        self.adot   = {}
        self.js_ls = {}
        self.js_htm = ""
        self.all_vols   = {}
        self.all_nodes   = {}
        self.all_fvols   = {}

        if realpath:
            rp = realpath + ("" if realpath.endswith(os.sep) else os.sep)
            vp = vpath + ("/" if vpath else "")
            self.histpath = os.path.join(realpath, ".hist")
            self.dbpath = self.histpath
            self.all_vols[vpath] = self
            self.all_nodes[vpath] = self
            self.all_aps = [(rp, [self])]
            self.all_vps = [(vp, self)]
            self.canonical = self._canonical
            self.dcanonical = self._dcanonical
        else:
            self.histpath = self.dbpath = ""
            self.all_aps = []
            self.all_vps = []
            self.canonical = self._canonical_null
            self.dcanonical = self._dcanonical_null

        self.get_dbv = self._get_dbv
        self.ls = self._ls

    def __repr__(self)  :
        return "VFS(%s)" % (
            ", ".join(
                "%s=%r" % (k, self.__dict__[k])
                for k in "realpath vpath axs flags".split()
            )
        )

    def get_all_vols(
        self,
        vols  ,
        nodes  ,
        aps  ,
        vps  ,
    )  :
        nodes[self.vpath] = self
        if self.realpath:
            vols[self.vpath] = self
            rp = self.realpath
            rp += "" if rp.endswith(os.sep) else os.sep
            vp = self.vpath + ("/" if self.vpath else "")
            hit = next((x[1] for x in aps if x[0] == rp), None)
            if hit:
                hit.append(self)
            else:
                aps.append((rp, [self]))
            vps.append((vp, self))

        for v in self.nodes.values():
            v.get_all_vols(vols, nodes, aps, vps)

    def add(self, src , dst , dst0 )  :
        "a"
        assert src == "/" or not src.endswith("/")
        assert not dst.endswith("/")

        if "/" in dst:

            name, dst = dst.split("/", 1)
            name0, dst0 = dst0.split("/", 1)
            if name in self.nodes:

                return self.nodes[name].add(src, dst, dst0)

            vn = VFS(
                self.log,
                os.path.join(self.realpath, name) if self.realpath else "",
                "{}/{}".format(self.vpath, name).lstrip("/"),
                "{}/{}".format(self.vpath0, name0).lstrip("/"),
                self.axs,
                self._copy_flags(name),
            )
            vn.dbv = self.dbv or self
            self.nodes[name] = vn
            return vn.add(src, dst, dst0)

        if dst in self.nodes:

            return self.nodes[dst]

        vp = "{}/{}".format(self.vpath, dst).lstrip("/")
        vp0 = "{}/{}".format(self.vpath0, dst0).lstrip("/")
        vn = VFS(self.log, src, vp, vp0, AXS(), {})
        vn.dbv = self.dbv or self
        self.nodes[dst] = vn
        return vn

    def _copy_flags(self, name )   :
        flags = {k: v for k, v in self.flags.items()}

        hist = flags.get("hist")
        if hist and hist != "-":
            zs = "{}/{}".format(hist.rstrip("/"), name)
            flags["hist"] = os.path.expandvars(os.path.expanduser(zs))

        dbp = flags.get("dbpath")
        if dbp and dbp != "-":
            zs = "{}/{}".format(dbp.rstrip("/"), name)
            flags["dbpath"] = os.path.expandvars(os.path.expanduser(zs))

        return flags

    def bubble_flags(self)  :
        if self.dbv:
            for k, v in self.dbv.flags.items():
                if k not in ("hist", "dbpath"):
                    self.flags[k] = v

        for n in self.nodes.values():
            n.bubble_flags()

    def _find(self, vpath )   :
        "a"
        if not vpath:
            return self, ""

        if "/" in vpath:
            name, rem = vpath.split("/", 1)
        else:
            name = vpath
            rem = ""

        if name in self.nodes:
            return self.nodes[name]._find(rem)

        return self, vpath

    def can_access(
        self, vpath , uname 
    )          :
        "a"

        if vpath:
            vn, _ = self._find(undot(vpath))
        else:
            vn = self

        return vn.uaxs[uname]

    def get_perms(self, vpath , uname )  :
        zbl = self.can_access(vpath, uname)
        ret = "".join(ch for ch, ok in zip("rwmdgGha.", zbl) if ok)
        if "rwmd" in ret and "a." in ret:
            ret += "A"
        return ret

    def get(
        self,
        vpath ,
        uname ,
        will_read ,
        will_write ,
        will_move  = False,
        will_del  = False,
        will_get  = False,
        err  = 403,
    )   :
        "a"
        if relchk(vpath):
            if self.log:
                self.log("vfs", "invalid relpath %r @%s" % (vpath, uname))
            raise Pebkac(422)

        cvpath = undot(vpath)
        vn, rem = self._find(cvpath)
        c  = vn.axs

        for req, d, msg in [
            (will_read, c.uread, "read"),
            (will_write, c.uwrite, "write"),
            (will_move, c.umove, "move"),
            (will_del, c.udel, "delete"),
            (will_get, c.uget, "get"),
        ]:
            if req and uname not in d and uname != LEELOO_DALLAS:
                if vpath != cvpath and vpath != "." and self.log:
                    ap = vn.canonical(rem)
                    t = "%s has no %s in %r => %r => %r"
                    self.log("vfs", t % (uname, msg, vpath, cvpath, ap), 6)

                t = "you don't have %s-access in %r or below %r"
                raise Pebkac(err, t % (msg, "/" + cvpath, "/" + vn.vpath))

        return vn, rem

    def _get_share_src(self, vrem )   :
        src = self.shr_src
        if not src:
            return self._get_dbv(vrem)

        shv, srem = src
        return shv._get_dbv(vjoin(srem, vrem))

    def _get_dbv(self, vrem )   :
        dbv = self.dbv
        if not dbv:
            return self, vrem

        vrem = vjoin(self.vpath[len(dbv.vpath) :].lstrip("/"), vrem)
        return dbv, vrem

    def casechk(self, rem , do_stat )  :
        ap = self.canonical(rem, False)
        if do_stat and not bos.path.exists(ap):
            return True
        dp, fn = os.path.split(ap)
        if not fn:
            return True
        try:
            fns = os.listdir(dp)
        except:
            return True
        if fn in fns:
            return True
        hit = "<?>"
        lfn = fn.lower()
        for zs in fns:
            if lfn == zs.lower():
                hit = zs
                break
        if not hit:
            return True
        if self.log:
            t = "returning 404 due to underlying case-insensitive filesystem:\n  http-req: %r\n  local-fs: %r"
            self.log("vfs", t % (fn, hit))
        return False

    def _canonical_null(self, rem , resolve  = True)  :
        return ""

    def _dcanonical_null(self, rem )  :
        return ""

    def _canonical(self, rem , resolve  = True)  :
        "a"
        ap = self.realpath
        if rem:
            ap += "/" + rem

        return absreal(ap) if resolve else ap

    def _dcanonical(self, rem )  :
        "a"
        ap = self.realpath
        if rem:
            ap += "/" + rem

        ad, fn = os.path.split(ap)
        return os.path.join(absreal(ad), fn)

    def _canonical_shr(self, rem , resolve  = True)  :
        "a"
        ap = self.realpath
        if rem:
            ap += "/" + rem

        rap = absreal(ap)
        if self.shr_files:
            vn, rem = self.shr_src
            chk = absreal(os.path.join(vn.realpath, rem))
            if chk != rap:

                ad, fn = os.path.split(rap)
                if chk != ad or fn not in self.shr_files:
                    return "\n\n"

        return rap if resolve else ap

    def _dcanonical_shr(self, rem )  :
        "a"
        ap = self.realpath
        if rem:
            ap += "/" + rem

        ad, fn = os.path.split(ap)
        ad = absreal(ad)
        if self.shr_files:
            vn, rem = self.shr_src
            chk = absreal(os.path.join(vn.realpath, rem))
            if chk != absreal(ap):

                if ad != chk or fn not in self.shr_files:
                    return "\n\n"

        return os.path.join(ad, fn)

    def _ls_nope(
        self, *a, **ka
    )      :
        raise Pebkac(500, "nope.avi")

    def _ls_shr(
        self,
        rem ,
        uname ,
        scandir ,
        permsets ,
        lstat  = False,
        throw  = False,
    )      :
        "a"
        vn, rem = self.shr_src
        abspath, real, _ = vn.ls(rem, "\n", scandir, permsets, lstat, throw)
        real = [x for x in real if os.path.basename(x[0]) in self.shr_files]
        return abspath, real, {}

    def _ls(
        self,
        rem ,
        uname ,
        scandir ,
        permsets ,
        lstat  = False,
        throw  = False,
    )      :
        "a"
        virt_vis = {}
        abspath = self.canonical(rem)
        if abspath:
            real = list(statdir(self.log, scandir, lstat, abspath, throw))
            real.sort()
        else:
            real = []

        if not rem:

            real = [x for x in real if x[0] not in self.nodes]

            dbv = self.dbv or self
            for name, vn2 in sorted(self.nodes.items()):
                if vn2.dbv == dbv and self.flags.get("dk"):
                    virt_vis[name] = vn2
                    continue

                u_has = vn2.uaxs.get(uname) or [False] * 9
                for pset in permsets:
                    ok = True
                    for req, zb in zip(pset, u_has):
                        if req and not zb:
                            ok = False
                            break
                    if ok:
                        virt_vis[name] = vn2
                        break

        if ".hist" in abspath:
            p = abspath.replace("\\", "/") if WINDOWS else abspath
            if p.endswith("/.hist"):
                real = [x for x in real if not x[0].startswith("up2k.")]
            elif "/.hist/th/" in p:
                real = [x for x in real if not x[0].endswith("dir.txt")]

        return abspath, real, virt_vis

    def walk(
        self,
        rel ,
        rem ,
        seen ,
        uname ,
        permsets ,
        wantdots ,
        scandir ,
        lstat ,
        subvols  = True,
    ):  
        
            
            
            
            
             
             
             
        
        
        
    
        "a"

        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, permsets, lstat=lstat)
        dbv, vrem = self.get_dbv(rem)

        if (
            seen
            and (not fsroot.startswith(seen[-1]) or fsroot == seen[-1])
            and fsroot in seen
        ):
            if self.log:
                t = "bailing from symlink loop,\n  prev: %r\n  curr: %r\n  from: %r / %r"
                self.log("vfs.walk", t % (seen[-1], fsroot, self.vpath, rem), 3)
            return

        if "xdev" in self.flags or "xvol" in self.flags:
            rm1 = []
            for le in vfs_ls:
                ap = absreal(os.path.join(fsroot, le[0]))
                vn2 = self.chk_ap(ap)
                if not vn2 or not vn2.get("", uname, True, False):
                    rm1.append(le)
            _ = [vfs_ls.remove(x) for x in rm1]

        dots_ok = wantdots and (wantdots == 2 or uname in dbv.axs.udot)
        if not dots_ok:
            vfs_ls = [x for x in vfs_ls if "/." not in "/" + x[0]]

        seen = seen[:] + [fsroot]
        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]


        rfiles.sort()
        rdirs.sort()

        yield dbv, vrem, rel, fsroot, rfiles, rdirs, vfs_virt

        for rdir, _ in rdirs:
            if not dots_ok and rdir.startswith("."):
                continue

            wrel = (rel + "/" + rdir).lstrip("/")
            wrem = (rem + "/" + rdir).lstrip("/")
            for x in self.walk(
                wrel, wrem, seen, uname, permsets, wantdots, scandir, lstat, subvols
            ):
                yield x

        if not subvols:
            return

        for n, vfs in sorted(vfs_virt.items()):
            if not dots_ok and n.startswith("."):
                continue

            wrel = (rel + "/" + n).lstrip("/")
            for x in vfs.walk(
                wrel, "", seen, uname, permsets, wantdots, scandir, lstat
            ):
                yield x

    def zipgen(
        self,
        vpath ,
        vrem ,
        flt ,
        uname ,
        dirs ,
        dots ,
        scandir ,
        wrap  = True,
    )     :

        folder = "" if flt or not wrap else (vpath.split("/")[-1].lstrip(".") or "top")

        g = self.walk(folder, vrem, [], uname, [[True, False]], dots, scandir, False)
        for _, _, vpath, apath, files, rd, vd in g:
            if flt:
                files = [x for x in files if x[0] in flt]

                rm1 = [x for x in rd if x[0] not in flt]
                _ = [rd.remove(x) for x in rm1]

                rm2 = [x for x in vd.keys() if x not in flt]
                _ = [vd.pop(x) for x in rm2]

                flt = set()

            fnames = [n[0] for n in files]
            vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
            apaths = [os.path.join(apath, n) for n in fnames]
            ret = list(zip(vpaths, apaths, files))

            for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in ret]:
                yield f

            if not dirs:
                continue

            ts = int(time.time())
            st = os.stat_result((16877, -1, -1, 1, 1000, 1000, 8, ts, ts, ts))
            dnames = [n[0] for n in rd]
            dstats = [n[1] for n in rd]
            dnames += list(vd.keys())
            dstats += [st] * len(vd)
            vpaths = [vpath + "/" + n for n in dnames] if vpath else dnames
            apaths = [os.path.join(apath, n) for n in dnames]
            ret2 = list(zip(vpaths, apaths, dstats))
            for d in [{"vp": v, "ap": a, "st": n} for v, a, n in ret2]:
                yield d

    def chk_ap(self, ap , st  = None)  :
        aps = ap + os.sep
        if "xdev" in self.flags and not ANYWIN:
            if not st:
                ap2 = ap.replace("\\", "/") if ANYWIN else ap
                while ap2:
                    try:
                        st = bos.stat(ap2)
                        break
                    except:
                        if "/" not in ap2:
                            raise
                        ap2 = ap2.rsplit("/", 1)[0]
                assert st

            vdev = self.dev
            if not vdev:
                vdev = self.dev = bos.stat(self.realpath).st_dev

            if vdev != st.st_dev:
                if self.log:
                    t = "xdev: %s[%r] => %s[%r]"
                    self.log("vfs", t % (vdev, self.realpath, st.st_dev, ap), 3)

                return None

        if "xvol" in self.flags:
            self_ap = self.realpath + os.sep
            if aps.startswith(self_ap):
                vp = aps[len(self_ap) :]
                if ANYWIN:
                    vp = vp.replace(os.sep, "/")
                vn2, _ = self._find(vp)
                if self == vn2:
                    return self

            all_aps = self.shr_all_aps or self.root.all_aps

            for vap, vns in all_aps:
                if aps.startswith(vap):
                    return self if self in vns else vns[0]

            if self.log:
                self.log("vfs", "xvol: %r" % (ap,), 3)

            return None

        return self

    def check_landmarks(self)  :
        if self.dbv:
            return True

        vps = self.flags.get("landmark") or []
        if not vps:
            return True

        failed = ""
        for vp in vps:
            if "^=" in vp:
                vp, zs = vp.split("^=", 1)
                expect = zs.encode("utf-8")
            else:
                expect = b""

            if self.log:
                t = "checking [/%s] landmark [%s]"
                self.log("vfs", t % (self.vpath, vp), 6)

            ap = "?"
            try:
                ap = self.canonical(vp)
                with open(ap, "rb") as f:
                    buf = f.read(4096)
                    if not buf.startswith(expect):
                        t = "file [%s] does not start with the expected bytes %s"
                        failed = t % (ap, expect)
                        break
            except Exception as ex:
                t = "%r while trying to read [%s] => [%s]"
                failed = t % (ex, vp, ap)
                break

        if not failed:
            return True

        if self.log:
            t = "WARNING: landmark verification failed; %s; will now disable up2k database for volume [/%s]"
            self.log("vfs", t % (failed, self.vpath), 3)

        for rm in "e2d e2t e2v".split():
            self.flags = {k: v for k, v in self.flags.items() if not k.startswith(rm)}
        self.flags["d2d"] = True
        self.flags["d2t"] = True
        return False


if WINDOWS:
    re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
else:
    re_vol = re.compile(r"^([^:]*):([^:]*):(.*)$")


class AuthSrv(object):
    "a"

    def __init__(
        self,
        args ,
        log_func ,
        warn_anonwrite  = True,
        dargs  = None,
    )  :
        self.ah = PWHash(args)
        self.args = args
        self.dargs = dargs or args
        self.log_func = log_func
        self.warn_anonwrite = warn_anonwrite
        self.line_ctr = 0
        self.indent = ""
        self.is_lxc = args.c == ["/z/initcfg"]

        self._vf0b = {
            "cachectl": self.args.cachectl,
            "tcolor": self.args.tcolor,
            "du_iwho": self.args.du_iwho,
            "shr_who": self.args.shr_who if self.args.shr else "no",
            "emb_all": FN_EMB,
            "ls_q_m": ("", ""),
        }
        self._vf0 = self._vf0b.copy()
        self._vf0["d2d"] = True

        self.vfs = VFS(log_func, "", "", "", AXS(), {})
        self.acct   = {}
        self.iacct   = {}
        self.ases   = {}
        self.sesa   = {}
        self.defpw   = {}
        self.grps   = {}
        self.re_pwd  = None
        self.cfg_files_loaded  = []
        self.badcfg1 = False

        self.idp_vols   = {}

        self.idp_accs   = {}
        self.idp_usr_gh   = {}

        self.hid_cache   = {}
        self.mutex = threading.Lock()
        self.reload()

    def log(self, msg , c   = 0)  :
        if self.log_func:
            self.log_func("auth", msg, c)

    def laggy_iter(self, iterable )    :
        "a"
        it = iter(iterable)
        prev = next(it)
        for x in it:
            yield prev, False
            prev = x

        yield prev, True

    def vf0(self):
        return self._vf0.copy()

    def vf0b(self):
        return self._vf0b.copy()

    def idp_checkin(
        self, broker , uname , gname 
    )  :
        if self.idp_usr_gh.get(uname) == gname:
            return False

        gnames = [x.strip() for x in self.args.idp_gsep.split(gname)]
        gnames.sort()

        with self.mutex:
            self.idp_usr_gh[uname] = gname
            if self.idp_accs.get(uname) == gnames:
                return False

            self.idp_accs[uname] = gnames
            try:
                self._update_idp_db(uname, gname)
            except:
                self.log("failed to update the --idp-db:\n%s" % (min_ex(),), 3)

            t = "reinitializing due to new user from IdP: [%r:%r]"
            self.log(t % (uname, gnames), 3)

            if not broker:

                self._reload()
                return True

        broker.ask("reload", False, True).get()
        return True

    def _update_idp_db(self, uname , gname )  :
        if not self.args.idp_store:
            return


        db = sqlite3.connect(self.args.idp_db)
        cur = db.cursor()

        cur.execute("delete from us where un = ?", (uname,))
        cur.execute("insert into us values (?,?)", (uname, gname))

        db.commit()
        cur.close()
        db.close()

    def _map_volume_idp(
        self,
        src ,
        dst ,
        mount   ,
        daxs  ,
        mflags   ,
        un_gns  ,
    )     :
        ret     = []
        visited = set()
        src0 = src
        dst0 = dst

        zsl = []
        for ptn, sigil in ((PTN_U_ANY, "${u}"), (PTN_G_ANY, "${g}")):
            if bool(ptn.search(src)) != bool(ptn.search(dst)):
                zsl.append(sigil)
        if zsl:
            t = "ERROR: if %s is mentioned in a volume definition, it must be included in both the filesystem-path [%s] and the volume-url [/%s]"
            t = "\n".join([t % (x, src, dst) for x in zsl])
            self.log(t, 1)
            raise Exception(t)

        un_gn = [(un, gn) for un, gns in un_gns.items() for gn in gns]
        if not un_gn:

            un_gn = [("", "")]

        for un, gn in un_gn:
            rejected = False
            for ptn in [PTN_U_GRP, PTN_G_GRP]:
                m = ptn.search(dst0)
                if not m:
                    continue
                zs = m.group(1)
                zs = zs.replace(",%+", "\n%+")
                zs = zs.replace(",%-", "\n%-")
                for rule in zs.split("\n"):
                    gnc = rule[2:]
                    if ptn == PTN_U_GRP:

                        hit = gnc in (un_gns.get(un) or [])
                    else:

                        hit = gn == gnc

                    if rule.startswith("%+") != hit:
                        rejected = True
            if rejected:
                continue

            if gn == self.args.grp_all:
                gn = ""


            src1 = src0.replace("${u}", un or "\n")
            dst1 = dst0.replace("${u}", un or "\n")
            src1 = PTN_U_GRP.sub(un or "\n", src1)
            dst1 = PTN_U_GRP.sub(un or "\n", dst1)
            if src0 == src1 and dst0 == dst1:
                un = ""

            src = src1.replace("${g}", gn or "\n")
            dst = dst1.replace("${g}", gn or "\n")
            src = PTN_G_GRP.sub(gn or "\n", src)
            dst = PTN_G_GRP.sub(gn or "\n", dst)
            if src == src1 and dst == dst1:
                gn = ""

            if "\n" in (src + dst):
                continue

            label = "%s\n%s" % (src, dst)
            if label in visited:
                continue
            visited.add(label)

            src, dst = self._map_volume(src, dst, dst0, mount, daxs, mflags)
            if src:
                ret.append((src, dst, un, gn))
                if un or gn:
                    self.idp_vols[dst] = src

        return ret

    def _map_volume(
        self,
        src ,
        dst ,
        dst0 ,
        mount   ,
        daxs  ,
        mflags   ,
    )   :
        src = os.path.expandvars(os.path.expanduser(src))
        src = absreal(src)
        dst = dst.strip("/")

        if dst in mount:
            t = "multiple filesystem-paths mounted at [/{}]:\n  [{}]\n  [{}]"
            self.log(t.format(dst, mount[dst][0], src), c=1)
            raise Exception(BAD_CFG)

        if src in mount.values():
            t = "filesystem-path [{}] mounted in multiple locations:"
            t = t.format(src)
            for v in [k for k, v in mount.items() if v[0] == src] + [dst]:
                t += "\n  /{}".format(v)

            self.log(t, c=3)
            raise Exception(BAD_CFG)

        if not bos.path.exists(src):
            self.log("warning: filesystem-path did not exist: %r" % (src,), 3)

        mount[dst] = (src, dst0)
        daxs[dst] = AXS()
        mflags[dst] = {}
        return (src, dst)

    def _e(self, desc  = None)  :
        if not self.args.vc or not self.line_ctr:
            return

        if not desc and not self.indent:
            self.log("")
            return

        desc = desc or ""
        desc = desc.replace("[", "[\033[0m").replace("]", "\033[90m]")
        self.log(" >>> {}{}".format(self.indent, desc), "90")

    def _l(self, ln , c , desc )  :
        if not self.args.vc or not self.line_ctr:
            return

        if c < 10:
            c += 30

        t = "\033[97m{:4} \033[{}m{}{}"
        if desc:
            t += "  \033[0;90m# {}\033[0m"
            desc = desc.replace("[", "[\033[0m").replace("]", "\033[90m]")

        self.log(t.format(self.line_ctr, c, self.indent, ln, desc))

    def _all_un_gn(
        self,
        acct  ,
        grps  ,
    )   :
        "a"
        self.load_idp_db(bool(self.idp_accs))
        ret = {un: gns[:] for un, gns in self.idp_accs.items()}
        ret.update({zs: [""] for zs in acct if zs not in ret})
        grps[self.args.grp_all] = list(ret.keys())
        for gn, uns in grps.items():
            for un in uns:
                try:
                    ret[un].append(gn)
                except:
                    ret[un] = [gn]

        return ret

    def _parse_config_file(
        self,
        fp ,
        cfg_lines ,
        acct  ,
        grps  ,
        daxs  ,
        mflags   ,
        mount   ,
    )  :
        self.line_ctr = 0

        expand_config_file(self.log, cfg_lines, fp, "")
        if self.args.vc:
            lns = ["{:4}: {}".format(n, s) for n, s in enumerate(cfg_lines, 1)]
            self.log("expanded config file (unprocessed):\n" + "\n".join(lns))

        cfg_lines = upgrade_cfg_fmt(self.log, self.args, cfg_lines, fp)

        for npass in range(1, 3):
            if self.args.vc:
                self.log("parsing config files; pass %d/%d" % (npass, 2))
            self._parse_config_file_2(cfg_lines, acct, grps, daxs, mflags, mount, npass)

    def _parse_config_file_2(
        self,
        cfg_lines ,
        acct  ,
        grps  ,
        daxs  ,
        mflags   ,
        mount   ,
        npass ,
    )  :
        self.line_ctr = 0
        all_un_gn = self._all_un_gn(acct, grps)

        cat = ""
        catg = "[global]"
        cata = "[accounts]"
        catgrp = "[groups]"
        catx = "accs:"
        catf = "flags:"
        ap  = None
        vp  = None
        vols     = []
        for ln in cfg_lines:
            self.line_ctr += 1
            ln = ln.split("  #")[0].strip()
            if not ln.split("#")[0].strip():
                continue

            if re.match(r"^\[.*\]:$", ln):
                ln = ln[:-1]

            subsection = ln in (catx, catf)
            if ln.startswith("[") or subsection:
                self._e()
                if npass > 1 and ap is None and vp is not None:
                    t = "the first line after [/{}] must be a filesystem path to share on that volume"
                    raise Exception(t.format(vp))

                cat = ln
                if not subsection:
                    ap = vp = None
                    self.indent = ""
                else:
                    self.indent = "  "

                if ln == catg:
                    t = "begin commandline-arguments (anything from --help; dashes are optional)"
                    self._l(ln, 6, t)
                elif ln == cata:
                    self._l(ln, 5, "begin user-accounts section")
                elif ln == catgrp:
                    self._l(ln, 5, "begin user-groups section")
                elif ln.startswith("[/"):
                    vp = ln[1:-1].strip("/")
                    self._l(ln, 2, "define volume at URL [/{}]".format(vp))
                elif subsection:
                    if ln == catx:
                        self._l(ln, 5, "volume access config:")
                    else:
                        t = "volume-specific config (anything from --help-flags)"
                        self._l(ln, 6, t)
                else:
                    raise Exception("invalid section header" + SBADCFG)

                self.indent = "    " if subsection else "  "
                continue

            if cat == catg:
                self._l(ln, 6, "")
                zt = split_cfg_ln(ln)
                for zs, za in zt.items():
                    zs = zs.lstrip("-")
                    if "=" in zs:
                        t = "WARNING: found an option named [%s] in your [global] config; did you mean to say [%s: %s] instead?"
                        zs1, zs2 = zs.split("=", 1)
                        self.log(t % (zs, zs1, zs2), 3)
                    if za is True:
                        self._e("└─argument [{}]".format(zs))
                    else:
                        self._e("└─argument [{}] with value [{}]".format(zs, za))
                continue

            if cat == cata:
                try:
                    u, p = [zs.strip() for zs in ln.split(":", 1)]
                    if "=" in u and not p:
                        t = "WARNING: found username [%s] in your [accounts] config; did you mean to say [%s: %s] instead?"
                        zs1, zs2 = u.split("=", 1)
                        self.log(t % (u, zs1, zs2), 3)
                    self._l(ln, 5, "account [{}], password [{}]".format(u, p))
                    acct[u] = p
                except:
                    t = 'lines inside the [accounts] section must be "username: password"'
                    raise Exception(t + SBADCFG)
                continue

            if cat == catgrp:
                try:
                    gn, zs1 = [zs.strip() for zs in ln.split(":", 1)]
                    uns = [zs.strip() for zs in zs1.split(",")]
                    t = "group [%s] = " % (gn,)
                    t += ", ".join("user [%s]" % (x,) for x in uns)
                    self._l(ln, 5, t)
                    if gn in grps:
                        grps[gn].extend(uns)
                    else:
                        grps[gn] = uns
                except:
                    t = 'lines inside the [groups] section must be "groupname: user1, user2, user..."'
                    raise Exception(t + SBADCFG)
                continue

            if vp is not None and ap is None:
                if npass != 2:
                    continue

                ap = ln
                self._l(ln, 2, "bound to filesystem-path [{}]".format(ap))
                vols = self._map_volume_idp(ap, vp, mount, daxs, mflags, all_un_gn)
                if not vols:
                    ap = vp = None
                    self._l(ln, 2, "└─no users/groups known; was not mapped")
                elif len(vols) > 1:
                    for vol in vols:
                        self._l(ln, 2, "└─mapping: [%s] => [%s]" % (vol[1], vol[0]))
                continue

            if cat == catx:
                if npass != 2 or not ap:

                    continue

                err = ""
                try:
                    self._l(ln, 5, "volume access config:")
                    sk, sv = ln.split(":")
                    if re.sub("[rwmdgGhaA.]", "", sk) or not sk:
                        err = "invalid accs permissions list; "
                        raise Exception(err)
                    if " " in re.sub(", *", "", sv).strip():
                        err = "list of users is not comma-separated; "
                        raise Exception(err)
                    sv = sv.replace(" ", "")
                    self._read_vol_str_idp(sk, sv, vols, all_un_gn, daxs, mflags)
                    continue
                except CfgEx:
                    raise
                except:
                    err += "accs entries must be 'rwmdgGhaA.: user1, user2, ...'"
                    raise CfgEx(err + SBADCFG)

            if cat == catf:
                if npass != 2 or not ap:

                    continue

                err = ""
                try:
                    self._l(ln, 6, "volume-specific config:")
                    zd = split_cfg_ln(ln)
                    fstr = ""
                    for sk, sv in zd.items():
                        if "=" in sk:
                            t = "WARNING: found a volflag named [%s] in your config; did you mean to say [%s: %s] instead?"
                            zs1, zs2 = sk.split("=", 1)
                            self.log(t % (sk, zs1, zs2), 3)
                        bad = re.sub(r"[a-z0-9_-]", "", sk).lstrip("-")
                        if bad:
                            err = "bad characters [{}] in volflag name [{}]; "
                            err = err.format(bad, sk)
                            raise Exception(err + SBADCFG)
                        if sv is True:
                            fstr += "," + sk
                        else:
                            fstr += ",{}={}".format(sk, sv)
                            assert vp is not None
                            self._read_vol_str_idp(
                                "c", fstr[1:], vols, all_un_gn, daxs, mflags
                            )
                            fstr = ""
                    if fstr:
                        self._read_vol_str_idp(
                            "c", fstr[1:], vols, all_un_gn, daxs, mflags
                        )
                    continue
                except:
                    err += "flags entries (volflags) must be one of the following:\n  'flag1, flag2, ...'\n  'key: value'\n  'flag1, flag2, key: value'"
                    raise Exception(err + SBADCFG)

            raise Exception("unprocessable line in config" + SBADCFG)

        self._e()
        self.line_ctr = 0

    def _read_vol_str_idp(
        self,
        lvl ,
        uname ,
        vols    ,
        un_gns  ,
        axs  ,
        flags   ,
    )  :
        if lvl.strip("crwmdgGhaA."):
            t = "%s,%s" % (lvl, uname) if uname else lvl
            raise CfgEx("invalid config value (volume or volflag): %s" % (t,))

        if lvl == "c":

            cval   = True
            try:

                uname, cval = uname.split("=", 1)
            except:

                pass

            while "," in uname:

                n1, uname = uname.split(",", 1)
                for _, vp, _, _ in vols:
                    self._read_volflag(vp, flags[vp], n1, True, False)

            for _, vp, _, _ in vols:
                self._read_volflag(vp, flags[vp], uname, cval, False)

            return

        if uname == "":
            uname = "*"

        unames = []
        for un in uname.replace(",", " ").strip().split():
            if un.startswith("@"):
                grp = un[1:]
                uns = [x[0] for x in un_gns.items() if grp in x[1]]
                if grp == "${g}":
                    unames.append(un)
                elif not uns and not self.args.idp_h_grp and grp != self.args.grp_all:
                    t = "group [%s] must be defined with --grp argument (or in a [groups] config section)"
                    raise CfgEx(t % (grp,))

                unames.extend(uns)
            else:
                unames.append(un)

        un_gn = [(un, gn) for un, gns in un_gns.items() for gn in gns]

        for src, dst, vu, vg in vols:
            unames2 = set(unames)

            if "${u}" in unames:
                if not vu:
                    t = "cannot use ${u} in accs of volume [%s] because the volume url does not contain ${u}"
                    raise CfgEx(t % (src,))
                unames2.add(vu)

            if "@${g}" in unames:
                if not vg:
                    t = "cannot use @${g} in accs of volume [%s] because the volume url does not contain @${g}"
                    raise CfgEx(t % (src,))
                unames2.update([un for un, gn in un_gn if gn == vg])

            if "${g}" in unames:
                t = 'the accs of volume [%s] contains "${g}" but the only supported way of specifying that is "@${g}"'
                raise CfgEx(t % (src,))

            unames2.discard("${u}")
            unames2.discard("@${g}")

            self._read_vol_str(lvl, list(unames2), axs[dst])

    def _read_vol_str(self, lvl , unames , axs )  :
        junkset = set()
        for un in unames:
            for alias, mapping in [
                ("h", "gh"),
                ("G", "gG"),
                ("A", "rwmda.A"),
            ]:
                expanded = ""
                for ch in mapping:
                    if ch not in lvl:
                        expanded += ch
                    lvl = lvl.replace(alias, expanded + alias)

            for ch, al in [
                ("r", axs.uread),
                ("w", axs.uwrite),
                ("m", axs.umove),
                ("d", axs.udel),
                (".", axs.udot),
                ("a", axs.uadmin),
                ("A", junkset),
                ("g", axs.uget),
                ("G", axs.upget),
                ("h", axs.uhtml),
            ]:
                if ch in lvl:
                    if un == "*":
                        t = "└─add permission [{0}] for [everyone] -- {2}"
                    else:
                        t = "└─add permission [{0}] for user [{1}] -- {2}"

                    desc = permdescs.get(ch, "?")
                    self._e(t.format(ch, un, desc))
                    al.add(un)

    def _read_volflag(
        self,
        vpath ,
        flags  ,
        name ,
        value   ,
        is_list ,
    )  :
        if name not in flagdescs:
            name = name.lower()

            stripped = name.lstrip("-")
            zi = len(name) - len(stripped)
            if zi > 1:
                t = "WARNING: the config for volume [/%s] specified a volflag with multiple leading hyphens (%s); use one hyphen to remove, or zero hyphens to add a flag. Will now enable flag [%s]"
                self.log(t % (vpath, name, stripped), 3)
                name = stripped
                zi = 0

            if stripped not in flagdescs and "-" in stripped:
                name = ("-" * zi) + stripped.replace("-", "_")

        desc = flagdescs.get(name.lstrip("-"), "?").replace("\n", " ")

        if not name:
            self._e("└─unreadable-line")
            t = "WARNING: the config for volume [/%s] indicated that a volflag was to be defined, but the volflag name was blank"
            self.log(t % (vpath,), 3)
            return

        if re.match("^-[^-]+$", name):
            t = "└─unset volflag [{}]  ({})"
            self._e(t.format(name[1:], desc))
            flags[name] = True
            return

        zs = "ext_th landmark mtp on403 on404 xbu xau xiu xbc xac xbr xar xbd xad xm xban"
        if name not in zs.split():
            if value is True:
                t = "└─add volflag [{}] = {}  ({})"
            else:
                t = "└─add volflag [{}] = [{}]  ({})"
            self._e(t.format(name, value, desc))
            flags[name] = value
            return

        vals = flags.get(name, [])
        if not value:
            return
        elif is_list:
            vals += value
        else:
            vals += [value]

        flags[name] = vals
        self._e("volflag [{}] += {}  ({})".format(name, vals, desc))

    def reload(self, verbosity  = 9)  :
        "a"
        with self.mutex:
            self._reload(verbosity)

    def _reload(self, verbosity  = 9)  :
        acct   = {}
        grps   = {}
        daxs   = {}
        mflags    = {}
        mount    = {}
        cfg_files_loaded  = []

        self.idp_vols = {}
        self.badcfg1 = False

        if self.args.a:

            for x in self.args.a:
                try:
                    u, p = x.split(":", 1)
                    acct[u] = p
                except:
                    t = '\n  invalid value "{}" for argument -a, must be username:password'
                    raise Exception(t.format(x))

        if self.args.grp:

            for x in self.args.grp:
                try:

                    zs1, zs2 = x.replace("=", ":").split(":", 1)
                    grps[zs1] = zs2.replace(":", ",").split(",")
                    grps[zs1] = [x.strip() for x in grps[zs1]]
                except:
                    t = '\n  invalid value "{}" for argument --grp, must be groupname:username1,username2,...'
                    raise Exception(t.format(x))

        if self.args.v:

            all_un_gn = self._all_un_gn(acct, grps)
            for v_str in self.args.v:
                m = re_vol.match(v_str)
                if not m:
                    raise Exception("invalid -v argument: [{}]".format(v_str))

                src, dst, perms = m.groups()
                if WINDOWS:
                    src = uncyg(src)

                vols = self._map_volume_idp(src, dst, mount, daxs, mflags, all_un_gn)

                for x in perms.split(":"):
                    lvl, uname = x.split(",", 1) if "," in x else [x, ""]
                    self._read_vol_str_idp(lvl, uname, vols, all_un_gn, daxs, mflags)

        if self.args.c:
            for cfg_fn in self.args.c:
                lns  = []
                try:
                    self._parse_config_file(
                        cfg_fn, lns, acct, grps, daxs, mflags, mount
                    )

                    zs = "#\033[36m cfg files in "
                    zst = [x[len(zs) :] for x in lns if x.startswith(zs)]
                    for zs in list(set(zst)):
                        self.log("discovered config files in " + zs, 6)

                    zs = "#\033[36m opening cfg file"
                    zstt = [x.split(" -> ") for x in lns if x.startswith(zs)]
                    zst = [(max(0, len(x) - 2) * " ") + "└" + x[-1] for x in zstt]
                    t = "loaded {} config files:\n{}"
                    self.log(t.format(len(zst), "\n".join(zst)))
                    cfg_files_loaded = zst

                except:
                    lns = lns[: self.line_ctr]
                    slns = ["{:4}: {}".format(n, s) for n, s in enumerate(lns, 1)]
                    t = "\033[1;31m\nerror @ line {}, included from {}\033[0m"
                    t = t.format(self.line_ctr, cfg_fn)
                    self.log("\n{0}\n{1}{0}".format(t, "\n".join(slns)))
                    raise

        derive_args(self.args)
        self.setup_auth_ord()

        if self.args.ipu and not self.args.have_idp_hdrs:

            zsl = [x.split("=", 1)[1] for x in self.args.ipu]
            zsl = [x for x in zsl if x and x not in acct]
            if zsl:
                t = "ERROR: unknown users in ipu: %s" % (zsl,)
                self.log(t, 1)
                raise Exception(t)

        self.setup_pwhash(acct)
        defpw = acct.copy()
        self.setup_chpw(acct)

        if WINDOWS:
            cased = {}
            for vp, (ap, vp0) in mount.items():
                cased[vp] = (absreal(ap), vp0)

            mount = cased

        if not mount and not self.args.have_idp_hdrs:

            axs = AXS(["*"], ["*"], None, None)
            ehint = ""
            if self.is_lxc:
                t = "Read-access has been disabled due to failsafe: Docker detected, but %s. This failsafe is to prevent unintended access if this is due to accidental loss of config. You can override this safeguard and allow read/write to all of /w/ by adding the following arguments to the docker container:  -v .::rw"
                if len(cfg_files_loaded) == 1:
                    self.log(t % ("no config-file was provided",), 1)
                    t = "it is strongly recommended to add a config-file instead, for example based on https://github.com/9001/copyparty/blob/hovudstraum/docs/examples/docker/basic-docker-compose/copyparty.conf"
                    self.log(t, 3)
                else:
                    self.log(t % ("the config does not define any volumes",), 1)
                axs = AXS()
                ehint = "; please try moving them up one level, into the parent folder:"
            elif self.args.c:
                t = "Read-access has been disabled due to failsafe: No volumes were defined by the config-file. This failsafe is to prevent unintended access if this is due to accidental loss of config. You can override this safeguard and allow read/write to the working-directory by adding the following arguments:  -v .::rw"
                self.log(t, 1)
                axs = AXS()
                ehint = ":"
            if ehint:
                try:
                    files = os.listdir(E.cfg)
                except:
                    files = []
                hits = [
                    x
                    for x in files
                    if x.lower().endswith(".conf") and not x.startswith(".")
                ]
                if hits:
                    t = "Hint: Found some config files in [%s], but these were not automatically loaded because they are in the wrong place%s %s\n"
                    self.log(t % (E.cfg, ehint, ", ".join(hits)), 3)
            vfs = VFS(self.log_func, absreal("."), "", "", axs, self.vf0b())
            if not axs.uread:
                self.badcfg1 = True
        elif "" not in mount:

            vfs = VFS(self.log_func, "", "", "", AXS(), self.vf0())

        maxdepth = 0
        for dst in sorted(mount.keys(), key=lambda x: (x.count("/"), len(x))):
            depth = dst.count("/")
            assert maxdepth <= depth
            maxdepth = depth
            src, dst0 = mount[dst]

            if dst == "":

                vfs = VFS(self.log_func, src, dst, dst0, daxs[dst], mflags[dst])
                continue

            assert vfs
            zv = vfs.add(src, dst, dst0)
            zv.axs = daxs[dst]
            zv.flags = mflags[dst]
            zv.dbv = None

        assert vfs
        vfs.all_vols = {}
        vfs.all_nodes = {}
        vfs.all_aps = []
        vfs.all_vps = []
        vfs.get_all_vols(vfs.all_vols, vfs.all_nodes, vfs.all_aps, vfs.all_vps)
        for vol in vfs.all_nodes.values():
            vol.all_aps.sort(key=lambda x: len(x[0]), reverse=True)
            vol.all_vps.sort(key=lambda x: len(x[0]), reverse=True)
            vol.root = vfs

        zs = "du_iwho ls_q_m neversymlink"
        k_ign = set(zs.split())
        for vol in vfs.all_vols.values():
            unknown_flags = set()
            for k, v in vol.flags.items():
                ks = k.lstrip("-")
                if ks not in flagdescs and ks not in k_ign:
                    unknown_flags.add(k)
            if unknown_flags:
                t = "WARNING: the config for volume [/%s] has unrecognized volflags; will ignore: '%s'"
                self.log(t % (vol.vpath, "', '".join(unknown_flags)), 3)

        enshare = self.args.shr
        shr = enshare[1:-1]
        shrs = enshare[1:]
        if enshare:

            shv = VFS(self.log_func, "", shr, shr, AXS(), self.vf0())

            db_path = self.args.shr_db
            db = sqlite3.connect(db_path)
            cur = db.cursor()
            cur2 = db.cursor()
            now = time.time()
            for row in cur.execute("select * from sh"):
                s_k, s_pw, s_vp, s_pr, s_nf, s_un, s_t0, s_t1 = row
                if s_t1 and s_t1 < now:
                    continue

                if self.args.shr_v:
                    t = "loading %s share %r by %r => %r"
                    self.log(t % (s_pr, s_k, s_un, s_vp))

                if s_pw:

                    zb = hashlib.sha512(s_pw.encode("utf-8")).digest()
                    sun = "s_%s" % (ub64enc(zb)[4:16].decode("ascii"),)
                    acct[sun] = s_pw
                else:
                    sun = "*"

                s_axs = AXS(
                    [sun] if "r" in s_pr else [],
                    [sun] if "w" in s_pr else [],
                    [sun] if "m" in s_pr else [],
                    [sun] if "d" in s_pr else [],
                    [sun] if "g" in s_pr else [],
                )

                vp = "%s/%s" % (shr, s_k)
                shv.nodes[s_k] = VFS(self.log_func, "", vp, vp, s_axs, shv.flags.copy())

            vfs.nodes[shr] = vfs.all_vols[shr] = shv
            for vol in shv.nodes.values():
                vfs.all_vols[vol.vpath] = vfs.all_nodes[vol.vpath] = vol
                vol.get_dbv = vol._get_share_src
                vol.ls = vol._ls_nope

        zss = set(acct)
        zss.update(self.idp_accs)
        zss.discard("*")
        unames = ["*"] + list(sorted(zss))

        for perm in "read write move del get pget html admin dot".split():
            axs_key = "u" + perm
            for vp, vol in vfs.all_vols.items():
                zx = getattr(vol.axs, axs_key)
                if "*" in zx and "-@acct" not in zx:
                    for usr in unames:
                        zx.add(usr)
                for zs in list(zx):
                    if zs.startswith("-"):
                        zx.discard(zs)
                        zs = zs[1:]
                        zx.discard(zs)
                        if zs.startswith("@"):
                            zs = zs[1:]
                            for zs in grps.get(zs) or []:
                                zx.discard(zs)

            umap   = {x: [] for x in unames}
            for usr in unames:
                for vp, vol in vfs.all_vols.items():
                    zx = getattr(vol.axs, axs_key)
                    if usr in zx and (not enshare or not vp.startswith(shrs)):
                        umap[usr].append(vp)
                umap[usr].sort()
            setattr(vfs, "a" + perm, umap)

        for vol in vfs.all_nodes.values():
            za = vol.axs
            vol.uaxs = {
                un: (
                    un in za.uread,
                    un in za.uwrite,
                    un in za.umove,
                    un in za.udel,
                    un in za.uget,
                    un in za.upget,
                    un in za.uhtml,
                    un in za.uadmin,
                    un in za.udot,
                )
                for un in unames
            }

        all_users = {}
        missing_users = {}
        associated_users = {}
        for axs in daxs.values():
            for d in [
                axs.uread,
                axs.uwrite,
                axs.umove,
                axs.udel,
                axs.uget,
                axs.upget,
                axs.uhtml,
                axs.uadmin,
                axs.udot,
            ]:
                for usr in d:
                    all_users[usr] = 1
                    if usr != "*" and usr not in acct and usr not in self.idp_accs:
                        missing_users[usr] = 1
                    if "*" not in d:
                        associated_users[usr] = 1

        if missing_users:
            zs = ", ".join(k for k in sorted(missing_users))
            if self.args.have_idp_hdrs:
                t = "the following users are unknown, and assumed to come from IdP: "
                self.log(t + zs, c=6)
            else:
                t = "you must -a the following users: "
                self.log(t + zs, c=1)
                raise Exception(BAD_CFG)

        if LEELOO_DALLAS in all_users:
            raise Exception("sorry, reserved username: " + LEELOO_DALLAS)

        zsl = []
        for usr in list(acct)[:]:
            zs = acct[usr].strip()
            if not zs:
                zs = ub64enc(os.urandom(48)).decode("ascii")
                zsl.append(usr)
            acct[usr] = zs
        if zsl:
            self.log("generated random passwords for users %r" % (zsl,), 6)

        seenpwds = {}
        for usr, pwd in acct.items():
            if pwd in seenpwds:
                t = "accounts [{}] and [{}] have the same password; this is not supported"
                self.log(t.format(seenpwds[pwd], usr), 1)
                raise Exception(BAD_CFG)
            seenpwds[pwd] = usr

        for usr in acct:
            if usr not in associated_users:
                if enshare and usr.startswith("s_"):
                    continue
                if len(vfs.all_vols) > 1:

                    t = "account [%s] is not mentioned in any volume definitions; see --help-accounts"
                    self.log(t % (usr,), 1)
                else:
                    t = "WARNING: the account [%s] is not mentioned in any volume definitions and thus has the same access-level and privileges that guests have; please see --help-accounts for details.  For example, if you intended to give that user full access to the current directory, you could do this:  -v .::A,%s"
                    self.log(t % (usr, usr), 1)

        dropvols = []
        errors = False
        for vol in vfs.all_vols.values():
            if (
                not vol.realpath
                or (
                    "assert_root" not in vol.flags
                    and "nospawn" not in vol.flags
                    and not self.args.vol_or_crash
                    and not self.args.vol_nospawn
                )
                or bos.path.exists(vol.realpath)
            ):
                pass
            elif "assert_root" in vol.flags or self.args.vol_or_crash:
                t = "ERROR: volume [/%s] root folder %r does not exist on server HDD; will now crash due to volflag 'assert_root'"
                self.log(t % (vol.vpath, vol.realpath), 1)
                errors = True
            else:
                t = "WARNING: volume [/%s] root folder %r does not exist on server HDD; volume will be unavailable due to volflag 'nospawn'"
                self.log(t % (vol.vpath, vol.realpath), 3)
                dropvols.append(vol)
        if errors:
            sys.exit(1)
        for vol in dropvols:
            vol.axs = AXS()
            vol.uaxs = {}
            vfs.all_vols.pop(vol.vpath, None)
            vfs.all_nodes.pop(vol.vpath, None)
            for zv in vfs.all_nodes.values():
                try:
                    zv.all_aps.remove(vol.realpath)
                    zv.all_vps.remove(vol.vpath)

                    zv.all_vols.pop(vol.vpath)
                    zv.all_nodes.pop(vol.vpath)
                except:
                    pass
                zs = next((x for x, y in zv.nodes.items() if y == vol), "")
                if zs:
                    zv.nodes.pop(zs)
            vol.realpath = ""

        promote = []
        demote = []
        for vol in vfs.all_vols.values():
            if not vol.realpath:
                continue
            hid = self.hid_cache.get(vol.realpath)
            if not hid:
                zb = hashlib.sha512(afsenc(vol.realpath)).digest()
                hid = base64.b32encode(zb).decode("ascii").lower()
                self.hid_cache[vol.realpath] = hid

            vflag = vol.flags.get("hist")
            if vflag == "-":
                pass
            elif vflag:
                vflag = os.path.expandvars(os.path.expanduser(vflag))
                vol.histpath = vol.dbpath = uncyg(vflag) if WINDOWS else vflag
            elif self.args.hist:
                for nch in range(len(hid)):
                    hpath = os.path.join(self.args.hist, hid[: nch + 1])
                    bos.makedirs(hpath)

                    powner = os.path.join(hpath, "owner.txt")
                    try:
                        with open(powner, "rb") as f:
                            owner = f.read().rstrip()
                    except:
                        owner = None

                    me = afsenc(vol.realpath).rstrip()
                    if owner not in [None, me]:
                        continue

                    if owner is None:
                        with open(powner, "wb") as f:
                            f.write(me)

                    vol.histpath = vol.dbpath = hpath
                    break

            vol.histpath = absreal(vol.histpath)

        for vol in vfs.all_vols.values():
            if not vol.realpath:
                continue
            hid = self.hid_cache[vol.realpath]
            vflag = vol.flags.get("dbpath")
            if vflag == "-":
                pass
            elif vflag:
                vflag = os.path.expandvars(os.path.expanduser(vflag))
                vol.dbpath = uncyg(vflag) if WINDOWS else vflag
            elif self.args.dbpath:
                for nch in range(len(hid)):
                    hpath = os.path.join(self.args.dbpath, hid[: nch + 1])
                    bos.makedirs(hpath)

                    powner = os.path.join(hpath, "owner.txt")
                    try:
                        with open(powner, "rb") as f:
                            owner = f.read().rstrip()
                    except:
                        owner = None

                    me = afsenc(vol.realpath).rstrip()
                    if owner not in [None, me]:
                        continue

                    if owner is None:
                        with open(powner, "wb") as f:
                            f.write(me)

                    vol.dbpath = hpath
                    break

            vol.dbpath = absreal(vol.dbpath)
            if vol.dbv:
                if bos.path.exists(os.path.join(vol.dbpath, "up2k.db")):
                    promote.append(vol)
                    vol.dbv = None
                else:
                    demote.append(vol)

        for zv in demote:
            vfs.all_vols.pop(zv.vpath)

        if promote:
            ta = [
                "\n  the following jump-volumes were generated to assist the vfs.\n  As they contain a database (probably from v0.11.11 or older),\n  they are promoted to full volumes:"
            ]
            for vol in promote:
                ta.append("  /%s  (%s)  (%s)" % (vol.vpath, vol.realpath, vol.dbpath))

            self.log("\n\n".join(ta) + "\n", c=3)

        rhisttab = {}
        vfs.histtab = {}
        for zv in vfs.all_vols.values():
            histp = zv.histpath
            is_shr = shr and zv.vpath.split("/")[0] == shr
            if histp and not is_shr and histp in rhisttab:
                zv2 = rhisttab[histp]
                t = "invalid config; multiple volumes share the same histpath (database+thumbnails location):\n  histpath: %s\n  volume 1: /%s  [%s]\n  volume 2: /%s  [%s]"
                t = t % (histp, zv2.vpath, zv2.realpath, zv.vpath, zv.realpath)
                self.log(t, 1)
                raise Exception(t)
            rhisttab[histp] = zv
            vfs.histtab[zv.realpath] = histp

        rdbpaths = {}
        vfs.dbpaths = {}
        for zv in vfs.all_vols.values():
            dbp = zv.dbpath
            is_shr = shr and zv.vpath.split("/")[0] == shr
            if dbp and not is_shr and dbp in rdbpaths:
                zv2 = rdbpaths[dbp]
                t = "invalid config; multiple volumes share the same dbpath (database location):\n  dbpath: %s\n  volume 1: /%s  [%s]\n  volume 2: /%s  [%s]"
                t = t % (dbp, zv2.vpath, zv2.realpath, zv.vpath, zv.realpath)
                self.log(t, 1)
                raise Exception(t)
            rdbpaths[dbp] = zv
            vfs.dbpaths[zv.realpath] = dbp

        for vol in vfs.all_vols.values():
            use = False
            for k in ["zipmaxn", "zipmaxs"]:
                try:
                    zs = vol.flags[k]
                except:
                    zs = getattr(self.args, k)
                if zs in ("", "0"):
                    vol.flags[k] = 0
                    continue

                zf = unhumanize(zs)
                vol.flags[k + "_v"] = zf
                if zf:
                    use = True
            if use:
                vol.flags["zipmax"] = True

        for vol in vfs.all_vols.values():
            lim = Lim(self.log_func)
            use = False

            if vol.flags.get("nosub"):
                use = True
                lim.nosub = True

            zs = vol.flags.get("df") or self.args.df or ""
            if zs not in ("", "0"):
                use = True
                try:
                    _ = float(zs)
                    zs = "%sg" % (zs,)
                except:
                    pass
                lim.dfl = unhumanize(zs)

            zs = vol.flags.get("sz")
            if zs:
                use = True
                lim.smin, lim.smax = [unhumanize(x) for x in zs.split("-")]

            zs = vol.flags.get("rotn")
            if zs:
                use = True
                lim.rotn, lim.rotl = [int(x) for x in zs.split(",")]

            zs = vol.flags.get("rotf")
            if zs:
                use = True
                lim.set_rotf(zs, vol.flags.get("rotf_tz") or "UTC")

            zs = vol.flags.get("maxn")
            if zs:
                use = True
                lim.nmax, lim.nwin = [int(x) for x in zs.split(",")]

            zs = vol.flags.get("maxb")
            if zs:
                use = True
                lim.bmax, lim.bwin = [unhumanize(x) for x in zs.split(",")]

            zs = vol.flags.get("vmaxb")
            if zs:
                use = True
                lim.vbmax = unhumanize(zs)

            zs = vol.flags.get("vmaxn")
            if zs:
                use = True
                lim.vnmax = unhumanize(zs)

            if use:
                vol.lim = lim

        if self.args.no_robots:
            for vol in vfs.all_nodes.values():

                if not vol.flags.get("robots"):
                    vol.flags["norobots"] = True

        for vol in vfs.all_nodes.values():
            if self.args.no_vthumb:
                vol.flags["dvthumb"] = True
            if self.args.no_athumb:
                vol.flags["dathumb"] = True
            if self.args.no_thumb or vol.flags.get("dthumb", False):
                vol.flags["dthumb"] = True
                vol.flags["dvthumb"] = True
                vol.flags["dathumb"] = True
                vol.flags["dithumb"] = True

        have_fk = False
        for vol in vfs.all_nodes.values():
            fk = vol.flags.get("fk")
            fka = vol.flags.get("fka")
            if fka and not fk:
                fk = fka
            if fk:
                fk = 8 if fk is True else int(fk)
                if fk > 72:
                    t = "max filekey-length is 72; volume /%s specified %d (anything higher than 16 is pointless btw)"
                    raise Exception(t % (vol.vpath, fk))
                vol.flags["fk"] = fk
                have_fk = True

            dk = vol.flags.get("dk")
            dks = vol.flags.get("dks")
            dky = vol.flags.get("dky")
            if dks is not None and dky is not None:
                t = "WARNING: volume /%s has both dks and dky enabled; this is too yolo and not permitted"
                raise Exception(t % (vol.vpath,))

            if dks and not dk:
                dk = dks
            if dky and not dk:
                dk = dky
            if dk:
                vol.flags["dk"] = int(dk) if dk is not True else 8

        if have_fk and re.match(r"^[0-9\.]+$", self.args.fk_salt):
            self.log("filekey salt: {}".format(self.args.fk_salt))

        fk_len = len(self.args.fk_salt)
        if have_fk and fk_len < 14:
            t = "WARNING: filekeys are enabled, but the salt is only %d chars long; %d or longer is recommended. Either specify a stronger salt using --fk-salt or delete this file and restart copyparty: %s"
            zs = os.path.join(E.cfg, "fk-salt.txt")
            self.log(t % (fk_len, 16, zs), 3)

        for vol in vfs.all_nodes.values():
            if "pk" in vol.flags and "gz" not in vol.flags and "xz" not in vol.flags:
                vol.flags["gz"] = False

            if "scan" in vol.flags:
                vol.flags["scan"] = int(vol.flags["scan"])
            elif self.args.re_maxage:
                vol.flags["scan"] = self.args.re_maxage

        self.args.have_unlistc = False

        all_mte = {}
        errors = False
        free_umask = False
        have_reflink = False
        for vol in vfs.all_nodes.values():
            if os.path.isfile(vol.realpath):
                vol.flags["is_file"] = True
                vol.flags["d2d"] = True

            if (self.args.e2ds and vol.axs.uwrite) or self.args.e2dsa:
                vol.flags["e2ds"] = True

            if self.args.e2d or "e2ds" in vol.flags:
                vol.flags["e2d"] = True

            for ga, vf in [
                ["no_hash", "nohash"],
                ["no_idx", "noidx"],
                ["og_ua", "og_ua"],
                ["srch_excl", "srch_excl"],
            ]:
                if vf in vol.flags:
                    ptn = re.compile(vol.flags.pop(vf))
                else:
                    ptn = getattr(self.args, ga)

                if ptn:
                    vol.flags[vf] = ptn

            for ga, vf in vf_bmap().items():
                if getattr(self.args, ga):
                    vol.flags[vf] = True

            for ve, vd in (
                ("nodotsrch", "dotsrch"),
                ("sb_lg", "no_sb_lg"),
                ("sb_md", "no_sb_md"),
            ):
                if ve in vol.flags:
                    vol.flags.pop(vd, None)

            for ga, vf in vf_vmap().items():
                if vf not in vol.flags:
                    vol.flags[vf] = getattr(self.args, ga)

            zs = "forget_ip gid nrand tail_who th_qv th_spec_p u2abort u2ow uid unp_who ups_who zip_who"
            for k in zs.split():
                if k in vol.flags:
                    vol.flags[k] = int(vol.flags[k])

            zs = "aconvt convt tail_fd tail_rate tail_tmax"
            for k in zs.split():
                if k in vol.flags:
                    vol.flags[k] = float(vol.flags[k])

            for k in ("mv_re", "rm_re"):
                try:
                    zs1, zs2 = vol.flags[k + "try"].split("/")
                    vol.flags[k + "_t"] = float(zs1)
                    vol.flags[k + "_r"] = float(zs2)
                except:
                    t = 'volume "/%s" has invalid %stry [%s]'
                    raise Exception(t % (vol.vpath, k, vol.flags.get(k + "try")))

            for k in ("chmod_d", "chmod_f"):
                is_d = k == "chmod_d"
                zs = vol.flags.get(k, "")
                if not zs and is_d:
                    zs = "755"
                if not zs:
                    vol.flags.pop(k, None)
                    continue
                if not re.match("^[0-7]{3}$", zs):
                    t = "config-option '%s' must be a three-digit octal value such as [755] or [644] but the value was [%s]"
                    t = t % (k, zs)
                    self.log(t, 1)
                    raise Exception(t)
                zi = int(zs, 8)
                vol.flags[k] = zi
                if (is_d and zi != 0o755) or not is_d:
                    free_umask = True

            vol.flags.pop("chown", None)
            if vol.flags["uid"] != -1 or vol.flags["gid"] != -1:
                vol.flags["chown"] = True
            vol.flags.pop("fperms", None)
            if "chown" in vol.flags or vol.flags.get("chmod_f"):
                vol.flags["fperms"] = True
            if vol.lim:
                vol.lim.chmod_d = vol.flags["chmod_d"]
                vol.lim.chown = "chown" in vol.flags
                vol.lim.uid = vol.flags["uid"]
                vol.lim.gid = vol.flags["gid"]

            vol.flags["du_iwho"] = n_du_who(vol.flags["du_who"])

            if not enshare:
                vol.flags["shr_who"] = self.args.shr_who = "no"

            if vol.flags.get("og"):
                self.args.uqe = True

            if "unlistcr" in vol.flags or "unlistcw" in vol.flags:
                self.args.have_unlistc = True

            if "reflink" in vol.flags:
                have_reflink = True

            zs = str(vol.flags.get("tcolor", "")).lstrip("#")
            if len(zs) == 3:
                vol.flags["tcolor"] = "".join([x * 2 for x in zs])

            zs = vol.flags["put_name"]
            vol.flags["put_name2"] = zs.replace("{now.", "{now:.")

            if vol.flags.get("neversymlink"):
                vol.flags["hardlinkonly"] = True
            if vol.flags.get("hardlinkonly"):
                vol.flags["hardlink"] = True

            for k1, k2 in IMPLICATIONS:
                if k1 in vol.flags:
                    vol.flags[k2] = True

            for k1, k2 in UNPLICATIONS:
                if k1 in vol.flags:
                    vol.flags[k2] = False

            dbds = "acid|swal|wal|yolo"
            vol.flags["dbd"] = dbd = vol.flags.get("dbd") or self.args.dbd
            if dbd not in dbds.split("|"):
                t = 'volume "/%s" has invalid dbd [%s]; must be one of [%s]'
                raise Exception(t % (vol.vpath, dbd, dbds))

            for k in ("mte", "mth", "exp_md", "exp_lg"):
                if k not in vol.flags:
                    vol.flags[k] = getattr(self.args, k).copy()
                else:
                    vol.flags[k] = odfusion(getattr(self.args, k), vol.flags[k])

            hooks = "xbu xau xiu xbc xac xbr xar xbd xad xm xban".split()
            for name in "ext_th mtp on404 on403".split() + hooks:
                self._read_volflag(
                    vol.vpath, vol.flags, name, getattr(self.args, name), True
                )

            for hn in hooks:
                cmds = vol.flags.get(hn)
                if not cmds:
                    continue

                ncmds = []
                for cmd in cmds:
                    hfs = []
                    ocmd = cmd
                    while "," in cmd[:6]:
                        zs, cmd = cmd.split(",", 1)
                        hfs.append(zs)

                    if "c" in hfs and "f" in hfs:
                        t = "cannot combine flags c and f; removing f from eventhook [{}]"
                        self.log(t.format(ocmd), 1)
                        hfs = [x for x in hfs if x != "f"]
                        ocmd = ",".join(hfs + [cmd])

                    if "c" not in hfs and "f" not in hfs and hn == "xban":
                        hfs = ["c"] + hfs
                        ocmd = ",".join(hfs + [cmd])

                    ncmds.append(ocmd)
                vol.flags[hn] = ncmds

            ext_th = vol.flags["ext_th_d"] = {}
            etv = "(?)"
            try:
                for etv in vol.flags.get("ext_th") or []:
                    k, v = etv.split("=")
                    ext_th[k] = v
            except:
                t = "WARNING: volume [/%s]: invalid value specified for ext-th: %s"
                self.log(t % (vol.vpath, etv), 3)

            emb_all = vol.flags["emb_all"] = set()

            zsl1 = [x for x in vol.flags["preadmes"].split(",") if x]
            zsl2 = [x for x in vol.flags["readmes"].split(",") if x]
            zsl3 = list(set([x.lower() for x in zsl1]))
            zsl4 = list(set([x.lower() for x in zsl2]))
            emb_all.update(zsl3)
            emb_all.update(zsl4)
            vol.flags["emb_mds"] = [[0, zsl1, zsl3], [1, zsl2, zsl4]]

            zsl1 = [x for x in vol.flags["prologues"].split(",") if x]
            zsl2 = [x for x in vol.flags["epilogues"].split(",") if x]
            zsl3 = list(set([x.lower() for x in zsl1]))
            zsl4 = list(set([x.lower() for x in zsl2]))
            emb_all.update(zsl3)
            emb_all.update(zsl4)
            vol.flags["emb_lgs"] = [[0, zsl1, zsl3], [1, zsl2, zsl4]]

            zs = str(vol.flags.get("html_head") or "")
            if zs and zs[:1] in "%@":
                vol.flags["html_head_d"] = zs
                head_s = str(vol.flags.get("html_head_s") or "")
            else:
                zs2 = str(vol.flags.get("html_head_s") or "")
                if zs2 and zs:
                    head_s = "%s\n%s\n" % (zs2.strip(), zs.strip())
                else:
                    head_s = zs2 or zs

            if head_s and not head_s.endswith("\n"):
                head_s += "\n"

            if "norobots" in vol.flags:
                head_s += META_NOBOTS

            ico_url = vol.flags.get("ufavico")
            if ico_url:
                ico_h = ""
                ico_ext = ico_url.split("?")[0].split(".")[-1].lower()
                if ico_ext in FAVICON_MIMES:
                    zs = '<link rel="icon" type="%s" href="%s">\n'
                    ico_h = zs % (FAVICON_MIMES[ico_ext], ico_url)
                elif ico_ext == "ico":
                    zs = '<link rel="shortcut icon" href="%s">\n'
                    ico_h = zs % (ico_url,)
                if ico_h:
                    vol.flags["ufavico_h"] = ico_h
                    head_s += ico_h

            if head_s:
                vol.flags["html_head_s"] = head_s
            else:
                vol.flags.pop("html_head_s", None)

            if not vol.flags.get("html_head_d"):
                vol.flags.pop("html_head_d", None)

            vol.check_landmarks()

            if vol.flags.get("db_xattr"):
                self.args.have_db_xattr = True
                zs = str(vol.flags["db_xattr"])
                neg = zs.startswith("~~")
                if neg:
                    zs = zs[2:]
                zsl = [x.strip() for x in zs.split(",")]
                zsl = [x for x in zsl if x]
                if neg:
                    vol.flags["db_xattr_no"] = set(zsl)
                else:
                    vol.flags["db_xattr_yes"] = zsl

            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"], ["d2d", "e2v"]]:
                if not vol.flags.get(grp, False):
                    continue

                vol.flags["d2t"] = True
                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}

            for grp, rm in [["d2ds", "e2ds"], ["d2ts", "e2ts"]]:
                if not vol.flags.get(grp, False):
                    continue

                vol.flags["d2ts"] = True
                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}

            for grp, rm in [["e2t", "mt"]]:
                if vol.flags.get(grp, False):
                    continue

                vol.flags = {
                    k: v
                    for k, v in vol.flags.items()
                    if not k.startswith(rm) or k == "mte"
                }

            for grp, rm in [["d2v", "e2v"]]:
                if not vol.flags.get(grp, False):
                    continue

                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}

            ints = ["lifetime"]
            for k in list(vol.flags):
                if k in ints:
                    vol.flags[k] = int(vol.flags[k])

            if "e2d" not in vol.flags:
                zs = "lifetime rss"
                drop = [x for x in zs.split() if x in vol.flags]

                zs = "xau xiu"
                drop += [x for x in zs.split() if vol.flags.get(x)]

                for k in drop:
                    t = 'cannot enable [%s] for volume "/%s" because this requires one of the following: e2d / e2ds / e2dsa  (either as volflag or global-option)'
                    self.log(t % (k, vol.vpath), 1)
                    vol.flags.pop(k)

            zi = vol.flags.get("lifetime") or 0
            zi2 = time.time() // (86400 * 365)
            zi3 = zi2 * 86400 * 365
            if zi < 0 or zi > zi3:
                t = "the lifetime of volume [/%s] (%d) exceeds max value (%d years; %d)"
                t = t % (vol.vpath, zi, zi2, zi3)
                self.log(t, 1)
                raise Exception(t)

            if (
                "dedup" in vol.flags
                and "reflink" not in vol.flags
                and vol.flags["apnd_who"] != "no"
            ):
                vol.flags["apnd_who"] = "ndd"

            local_mtp = {}
            local_only_mtp = {}
            tags = vol.flags.get("mtp", []) + vol.flags.get("mtm", [])
            tags = [x.split("=")[0] for x in tags]
            tags = [y for x in tags for y in x.split(",")]
            for a in tags:
                local_mtp[a] = True
                local = True
                for b in self.args.mtp or []:
                    b = b.split("=")[0]
                    if a == b:
                        local = False

                if local:
                    local_only_mtp[a] = True

            local_mte = ODict()
            for a in vol.flags.get("mte", {}).keys():
                local = True
                all_mte[a] = True
                local_mte[a] = True
                for b in self.args.mte.keys():
                    if not a or not b:
                        continue

                    if a == b:
                        local = False

            for mtp in local_only_mtp:
                if mtp not in local_mte:
                    t = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volflags)'
                    self.log(t.format(vol.vpath, mtp), 1)
                    errors = True

            mte = vol.flags.get("mte") or {}
            up_m = [x for x in UP_MTE_MAP if x in mte]
            up_q = [UP_MTE_MAP[x] for x in up_m]
            zs = "select %s from up where rd=? and fn=?" % (", ".join(up_q),)
            vol.flags["ls_q_m"] = (zs if up_m else "", up_m)

        vfs.all_fvols = {
            zs: vol for zs, vol in vfs.all_vols.items() if "is_file" in vol.flags
        }

        for vol in vfs.all_nodes.values():
            if not vol.flags.get("is_file"):
                continue
            zs = "og opds xlink"
            for zs in zs.split():
                vol.flags.pop(zs, None)

        for vol in vfs.all_vols.values():
            if not vol.realpath or vol.flags.get("is_file"):
                continue
            ccs = vol.flags["casechk"][:1].lower()
            if ccs in ("y", "n"):
                if ccs == "y":
                    vol.flags["bcasechk"] = True
                continue
            try:
                bos.makedirs(vol.realpath, vf=vol.flags)
                files = os.listdir(vol.realpath)
                for fn in files:
                    fn2 = fn.lower()
                    if fn == fn2:
                        fn2 = fn.upper()
                    if fn == fn2 or fn2 in files:
                        continue
                    is_ci = os.path.exists(os.path.join(vol.realpath, fn2))
                    ccs = "y" if is_ci else "n"
                    break
                if ccs not in ("y", "n"):
                    ap = os.path.join(vol.realpath, "casechk")
                    open(ap, "wb").close()
                    ccs = "y" if os.path.exists(ap[:-1] + "K") else "n"
                    os.unlink(ap)
            except Exception as ex:
                if ANYWIN:
                    zs = "Windows"
                    ccs = "y"
                elif MACOS:
                    zs = "Macos"
                    ccs = "y"
                else:
                    zs = "Linux"
                    ccs = "n"
                t = "unable to determine if filesystem at %r is case-insensitive due to %r; assuming casechk=%s due to %s"
                self.log(t % (vol.realpath, ex, ccs, zs), 3)
            vol.flags["casechk"] = ccs
            if ccs == "y":
                vol.flags["bcasechk"] = True

        tags = self.args.mtp or []
        tags = [x.split("=")[0] for x in tags]
        tags = [y for x in tags for y in x.split(",")]
        for mtp in tags:
            if mtp not in all_mte:
                t = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volflag)'
                self.log(t.format(mtp), 1)
                errors = True

        for vol in vfs.all_vols.values():
            re1  = vol.flags.get("srch_excl")
            excl = [re1.pattern] if re1 else []

            vpaths = []
            vtop = vol.vpath
            for vp2 in vfs.all_vols.keys():
                if vp2.startswith((vtop + "/").lstrip("/")) and vtop != vp2:
                    vpaths.append(re.escape(vp2[len(vtop) :].lstrip("/")))
            if vpaths:
                excl.append("^(%s)/" % ("|".join(vpaths),))

            vol.flags["srch_re_dots"] = re.compile("|".join(excl or ["^$"]))
            excl.extend([r"^\.", r"/\."])
            vol.flags["srch_re_nodot"] = re.compile("|".join(excl))

        have_daw = False
        for vol in vfs.all_nodes.values():
            daw = vol.flags.get("daw") or self.args.daw
            if daw:
                vol.flags["daw"] = True
                have_daw = True

        if have_daw and self.args.no_dav:
            t = 'volume "/{}" has volflag "daw" (webdav write-access), but --no-dav is set'
            self.log(t, 1)
            errors = True

        if self.args.smb and self.ah.on and acct:
            self.log("--smb can only be used when --ah-alg is none", 1)
            errors = True

        for vol in vfs.all_nodes.values():
            for k in list(vol.flags.keys()):
                if re.match("^-[^-]+$", k):
                    vol.flags.pop(k)
                    zs = k[1:]
                    if zs in vol.flags:
                        vol.flags.pop(k[1:])
                    else:
                        t = "WARNING: the config for volume [/%s] tried to remove volflag [%s] by specifying [%s] but that volflag was not already set"
                        self.log(t % (vol.vpath, zs, k), 3)

            if vol.flags.get("dots"):
                for name in vol.axs.uread:
                    vol.axs.udot.add(name)

        if errors:
            sys.exit(1)

        setattr(self.args, "free_umask", free_umask)
        if free_umask:
            os.umask(0)

        vfs.bubble_flags()

        have_e2d = False
        have_e2t = False
        have_dedup = False
        unsafe_dedup = []
        t = "volumes and permissions:\n"
        for zv in vfs.all_vols.values():
            if not self.warn_anonwrite or verbosity < 5:
                break

            if enshare and (zv.vpath == shr or zv.vpath.startswith(shrs)):
                continue

            t += '\n\033[36m"/{}"  \033[33m{}\033[0m'.format(zv.vpath, zv.realpath)
            for txt, attr in [
                ["  read", "uread"],
                [" write", "uwrite"],
                ["  move", "umove"],
                ["delete", "udel"],
                ["  dots", "udot"],
                ["   get", "uget"],
                [" upGet", "upget"],
                ["  html", "uhtml"],
                ["uadmin", "uadmin"],
            ]:
                u = list(sorted(getattr(zv.axs, attr)))
                if u == ["*"] and acct:
                    u = ["\033[35monly-anonymous\033[0m"]
                elif "*" in u:
                    u = ["\033[35meverybody\033[0m"]
                if not u:
                    u = ["\033[36m--none--\033[0m"]
                u = ", ".join(u)
                t += "\n|  {}:  {}".format(txt, u)

            if "e2d" in zv.flags:
                have_e2d = True

            if "e2t" in zv.flags:
                have_e2t = True

            if "dedup" in zv.flags:
                have_dedup = True
                if (
                    "e2d" not in zv.flags
                    and "hardlink" not in zv.flags
                    and "reflink" not in zv.flags
                ):
                    unsafe_dedup.append("/" + zv.vpath)

            t += "\n"

        if self.warn_anonwrite and verbosity > 4:
            if not self.args.no_voldump:
                self.log(t)

            if have_e2d or self.args.have_idp_hdrs:
                t = self.chk_sqlite_threadsafe()
                if t:
                    self.log("\n\033[{}\033[0m\n".format(t))
            if have_e2d:
                if not have_e2t:
                    t = "hint: enable multimedia indexing (artist/title/...) with argument -e2ts"
                    self.log(t, 6)
            else:
                t = "hint: enable searching and upload-undo with argument -e2dsa"
                self.log(t, 6)

            if unsafe_dedup:
                t = "WARNING: symlink-based deduplication is enabled for some volumes, but without indexing. Please enable -e2dsa and/or --hardlink to avoid problems when moving/renaming files. Affected volumes: %s"
                self.log(t % (", ".join(unsafe_dedup)), 3)
            elif not have_dedup:
                t = "hint: enable upload deduplication with --dedup (but see readme for consequences)"
                self.log(t, 6)

            zv, _ = vfs.get("/", "*", False, False)
            zs = zv.realpath.lower()
            if zs in ("/", "c:\\") or zs.startswith(r"c:\windows"):
                t = "you are sharing a system directory: {}\n"
                self.log(t.format(zv.realpath), c=1)

        try:
            zv, _ = vfs.get("", "*", False, True, err=999)
            if self.warn_anonwrite and verbosity > 4 and os.getcwd() == zv.realpath:
                t = "anyone can write to the current directory: {}\n"
                self.log(t.format(zv.realpath), c=1)

            self.warn_anonwrite = False
        except Pebkac:
            self.warn_anonwrite = True

        self.idp_warn = []
        self.idp_err = []
        for idp_vp in self.idp_vols:
            idp_vn, _ = vfs.get(idp_vp, "*", False, False)
            idp_vp0 = idp_vn.vpath0

            sigils = set(PTN_SIGIL.findall(idp_vp0))
            if len(sigils) > 1:
                t = '\nWARNING: IdP-volume "/%s" created by "/%s" has multiple IdP placeholders: %s'
                self.idp_warn.append(t % (idp_vp, idp_vp0, list(sigils)))
                continue

            sigil = sigils.pop()
            par_vp = idp_vp
            while par_vp:
                par_vp = vsplit(par_vp)[0]
                par_vn, _ = vfs.get(par_vp, "*", False, False)
                if sigil in par_vn.vpath0:
                    continue

                oth_read = []
                oth_write = []
                for usr in par_vn.axs.uread:
                    if usr not in idp_vn.axs.uread:
                        oth_read.append(usr)
                for usr in par_vn.axs.uwrite:
                    if usr not in idp_vn.axs.uwrite:
                        oth_write.append(usr)

                if "*" in oth_read:
                    taxs = "WORLD-READABLE"
                elif "*" in oth_write:
                    taxs = "WORLD-WRITABLE"
                elif oth_read:
                    taxs = "READABLE BY %r" % (oth_read,)
                elif oth_write:
                    taxs = "WRITABLE BY %r" % (oth_write,)
                else:
                    break

                t = '\nWARNING: IdP-volume "/%s" created by "/%s" has parent/grandparent "/%s" and would be %s'
                self.idp_err.append(t % (idp_vp, idp_vp0, par_vn.vpath, taxs))

        if self.idp_warn:
            t = "WARNING! Some IdP volumes include multiple IdP placeholders; this is too complex to automatically determine if safe or not. To ensure that no users gain unintended access, please use only a single placeholder for each IdP volume."
            self.log(t + "".join(self.idp_warn), 1)

        if self.idp_err:
            t = "WARNING! The following IdP volumes are mounted below another volume where other users can read and/or write files. This is a SECURITY HAZARD!! When copyparty is restarted, it will not know about these IdP volumes yet. These volumes will then be accessible by an unexpected set of permissions UNTIL one of the users associated with their volume sends a request to the server. RECOMMENDATION: You should create a restricted volume where nobody can read/write files, and make sure that all IdP volumes are configured to appear somewhere below that volume."
            self.log(t + "".join(self.idp_err), 1)

        if have_reflink:
            t = "WARNING: Reflink-based dedup was requested, but %s. This will not work; files will be full copies instead."
            if not sys.platform.startswith("linux"):
                self.log(t % "your OS is not Linux", 1)

        self.vfs = vfs
        self.acct = acct
        self.defpw = defpw
        self.grps = grps
        self.iacct = {v: k for k, v in acct.items()}
        self.cfg_files_loaded = cfg_files_loaded

        self.load_sessions()

        self.re_pwd = None
        pwds = [re.escape(x) for x in self.iacct.keys()]
        pwds.extend(list(self.sesa))
        if self.args.usernames:
            pwds.extend([x.split(":", 1)[1] for x in pwds if ":" in x])
        if pwds:
            if self.ah.on:
                zs = r"(\[H\] %s:.*|[?&]%s=)([^&]+)"
                zs = zs % (self.args.pw_hdr, self.args.pw_urlp)
            else:
                zs = r"(\[H\] %s:.*|=)(" % (self.args.pw_hdr,)
                zs += "|".join(pwds) + r")([]&; ]|$)"

            self.re_pwd = re.compile(zs)

        if self.args.mime:
            for zs in self.args.mime:
                ext, mime = zs.split("=", 1)
                MIMES[ext] = mime
            EXTS.update({v: k for k, v in MIMES.items()})

        if enshare:

            vfs.all_vols = {
                x: y
                for x, y in vfs.all_vols.items()
                if x != shr and not x.startswith(shrs)
            }

            assert db and cur and cur2 and shv
            for row in cur.execute("select * from sh"):
                s_k, s_pw, s_vp, s_pr, s_nf, s_un, s_t0, s_t1 = row
                shn = shv.nodes.get(s_k, None)
                if not shn:
                    continue

                try:
                    s_vfs, s_rem = vfs.get(
                        s_vp, s_un, "r" in s_pr, "w" in s_pr, "m" in s_pr, "d" in s_pr
                    )
                except Exception as ex:
                    t = "removing share [%s] by [%s] to [%s] due to %r"
                    self.log(t % (s_k, s_un, s_vp, ex), 3)
                    shv.nodes.pop(s_k)
                    continue

                fns = []
                if s_nf:
                    q = "select vp from sf where k = ?"
                    for (s_fn,) in cur2.execute(q, (s_k,)):
                        fns.append(s_fn)

                    shn.shr_files = set(fns)
                    shn.ls = shn._ls_shr
                    shn.canonical = shn._canonical_shr
                    shn.dcanonical = shn._dcanonical_shr
                else:
                    shn.ls = shn._ls
                    shn.canonical = shn._canonical
                    shn.dcanonical = shn._dcanonical

                shn.shr_owner = s_un
                shn.shr_src = (s_vfs, s_rem)
                shn.realpath = s_vfs.canonical(s_rem)

                o_vn, _ = shn._get_share_src("")
                shn.lim = o_vn.lim
                shn.flags = o_vn.flags.copy()
                shn.dbpath = o_vn.dbpath
                shn.histpath = o_vn.histpath

                ap = shn.realpath
                if not ap.endswith(os.sep):
                    ap += os.sep
                shn.shr_all_aps = [(x, y[:]) for x, y in vfs.all_aps]
                exact = False
                for ap2, vns in shn.shr_all_aps:
                    if ap == ap2:
                        exact = True
                    if ap2.startswith(ap):
                        try:
                            vp2 = vjoin(s_rem, ap2[len(ap) :])
                            vn2, _ = s_vfs.get(vp2, "*", False, False)
                            if vn2 == s_vfs or vn2.dbv == s_vfs:
                                vns.append(shn)
                        except:
                            pass
                if not exact:
                    shn.shr_all_aps.append((ap, [shn]))
                shn.shr_all_aps.sort(key=lambda x: len(x[0]), reverse=True)

                if self.args.shr_v:
                    t = "mapped %s share [%s] by [%s] => [%s] => [%s]"
                    self.log(t % (s_pr, s_k, s_un, s_vp, shn.realpath))

            for vn in shv.nodes.values():
                svn, srem = vn.shr_src
                if srem:
                    continue
                ap = svn.canonical(srem)
                if bos.path.isfile(ap):
                    continue
                for zs in svn.nodes.keys():

                    vn.nodes[zs] = VFS(self.log_func, "", "", "", AXS(), self.vf0())

            cur2.close()
            cur.close()
            db.close()

        self.js_ls = {}
        self.js_htm = {}
        for vp, vn in self.vfs.all_nodes.items():
            if enshare and vp.startswith(shrs):
                continue
            vf = vn.flags
            vn.js_ls = {
                "idx": "e2d" in vf,
                "itag": "e2t" in vf,
                "dlni": "dlni" in vf,
                "dgrid": "grid" in vf,
                "dnsort": "nsort" in vf,
                "dhsortn": vf["hsortn"],
                "dsort": vf["sort"],
                "dcrop": vf["crop"],
                "dth3x": vf["th3x"],
                "u2ts": vf["u2ts"],
                "shr_who": vf["shr_who"],
                "frand": bool(vf.get("rand")),
                "lifetime": vf.get("lifetime") or 0,
                "unlist": vf.get("unlist") or "",
                "sb_lg": "" if "no_sb_lg" in vf else (vf.get("lg_sbf") or "y"),
                "sb_md": "" if "no_sb_md" in vf else (vf.get("md_sbf") or "y"),
            }
            if "ufavico_h" in vf:
                vn.js_ls["ufavico"] = vf["ufavico_h"]
            js_htm = {
                "SPINNER": self.args.spinner,
                "s_name": self.args.bname,
                "idp_login": self.args.idp_login,
                "have_up2k_idx": "e2d" in vf,
                "have_acode": not self.args.no_acode,
                "have_c2flac": self.args.allow_flac,
                "have_c2wav": self.args.allow_wav,
                "have_shr": self.args.shr,
                "shr_who": vf["shr_who"],
                "have_zip": not self.args.no_zip,
                "have_zls": not self.args.no_zls,
                "have_mv": not self.args.no_mv,
                "have_del": not self.args.no_del,
                "have_unpost": int(self.args.unpost),
                "have_emp": int(self.args.emp),
                "md_no_br": int(vf.get("md_no_br") or 0),
                "ext_th": vf.get("ext_th_d") or {},
                "sb_lg": vn.js_ls["sb_lg"],
                "sb_md": vn.js_ls["sb_md"],
                "sba_md": vf.get("md_sba") or "",
                "sba_lg": vf.get("lg_sba") or "",
                "txt_ext": self.args.textfiles.replace(",", " "),
                "def_hcols": list(vf.get("mth") or []),
                "unlist0": vf.get("unlist") or "",
                "see_dots": self.args.see_dots,
                "dqdel": self.args.qdel,
                "dlni": vn.js_ls["dlni"],
                "dgrid": vn.js_ls["dgrid"],
                "dgsel": "gsel" in vf,
                "dnsort": "nsort" in vf,
                "dhsortn": vf["hsortn"],
                "dsort": vf["sort"],
                "dcrop": vf["crop"],
                "dth3x": vf["th3x"],
                "drcm": self.args.rcm,
                "dvol": self.args.au_vol,
                "idxh": int(self.args.ih),
                "dutc": not self.args.localtime,
                "dfszf": self.args.ui_filesz.strip("-"),
                "themes": self.args.themes,
                "turbolvl": self.args.turbo,
                "nosubtle": self.args.nosubtle,
                "u2j": self.args.u2j,
                "u2sz": self.args.u2sz,
                "u2ts": vf["u2ts"],
                "u2ow": vf["u2ow"],
                "frand": bool(vf.get("rand")),
                "lifetime": vn.js_ls["lifetime"],
                "u2sort": self.args.u2sort,
            }
            zs = "ui_noacci ui_nocpla ui_noctxb ui_nolbar ui_nombar ui_nonav ui_notree ui_norepl ui_nosrvi"
            for zs in zs.split():
                if vf.get(zs):
                    js_htm[zs] = 1
            zs = "notooltips"
            for zs in zs.split():
                if getattr(self.args, zs, False):
                    js_htm[zs] = 1
            zs = "up_site"
            for zs in zs.split():
                zs2 = getattr(self.args, zs, "")
                if zs2:
                    js_htm[zs] = zs2
            vn.js_htm = json_hesc(json.dumps(js_htm))

        vols = list(vfs.all_nodes.values())
        if enshare:
            for vol in shv.nodes.values():
                if vol.vpath not in vfs.all_nodes:
                    self.log("BUG: /%s not in all_nodes" % (vol.vpath,), 1)
                    vols.append(vol)
            if shr in vfs.all_nodes:
                t = "invalid config: a volume is overlapping with the --shr global-option (/%s)"
                t = t % (shr,)
                self.log(t, 1)
                raise Exception(t)

        for vol in vols:
            dbv = vol.get_dbv("")[0]
            vol.js_ls = vol.js_ls or dbv.js_ls or {}
            vol.js_htm = vol.js_htm or dbv.js_htm or "{}"

            zs = str(vol.flags.get("tcolor") or self.args.tcolor)
            vol.flags["tcolor"] = zs.lstrip("#")

    def setup_auth_ord(self)  :
        ao = [x.strip() for x in self.args.auth_ord.split(",")]
        if "idp" in ao:
            zi = ao.index("idp")
            ao = ao[:zi] + ["idp-hm", "idp-h"] + ao[zi:]
        zsl = "pw idp-h idp-hm ipu".split()
        pw, h, hm, ipu = [ao.index(x) if x in ao else 99 for x in zsl]
        self.args.ao_idp_before_pw = min(h, hm) < pw
        self.args.ao_h_before_hm = h < hm
        self.args.ao_ipu_wins = ipu == 0
        self.args.ao_have_pw = pw < 99 or not self.args.have_idp_hdrs

    def load_idp_db(self, quiet=False)  :

        level = self.args.idp_store
        if level < 2 or not self.args.have_idp_hdrs:
            return


        db = sqlite3.connect(self.args.idp_db)
        cur = db.cursor()
        from_cache = cur.execute("select un, gs from us").fetchall()
        cur.close()
        db.close()

        self.idp_accs.clear()
        self.idp_usr_gh.clear()

        gsep = self.args.idp_gsep
        n = []
        for uname, gname in from_cache:
            if level < 3:
                if uname in self.idp_accs:
                    continue
                gname = ""
            gnames = [x.strip() for x in gsep.split(gname)]
            gnames.sort()

            self.idp_accs[uname] = gnames
            n.append(uname)

        if n and not quiet:
            t = ", ".join(n[:9])
            if len(n) > 9:
                t += "..."
            self.log("found %d IdP users in db (%s)" % (len(n), t))

    def load_sessions(self, quiet=False)  :

        if self.args.no_ses:
            self.ases = {}
            self.sesa = {}
            return


        ases = {}
        blen = (self.args.ses_len // 4) * 4
        blen = (blen * 3) // 4

        db = sqlite3.connect(self.args.ses_db)
        cur = db.cursor()

        for uname, sid in cur.execute("select un, si from us"):
            if uname in self.acct:
                ases[uname] = sid

        n = []
        q = "insert into us values (?,?,?)"
        accs = list(self.acct)
        if self.args.have_idp_hdrs and self.args.idp_cookie:
            accs.extend(self.idp_accs.keys())
        for uname in accs:
            if uname not in ases:
                sid = ub64enc(os.urandom(blen)).decode("ascii")
                cur.execute(q, (uname, sid, int(time.time())))
                ases[uname] = sid
                n.append(uname)

        if n:
            db.commit()

        cur.close()
        db.close()

        self.ases = ases
        self.sesa = {v: k for k, v in ases.items()}
        if n and not quiet:
            t = ", ".join(n[:3])
            if len(n) > 3:
                t += "..."
            self.log("added %d new sessions (%s)" % (len(n), t))

    def forget_session(self, broker , uname )  :
        with self.mutex:
            self._forget_session(uname)

        if broker:
            broker.ask("_reload_sessions").get()

    def _forget_session(self, uname )  :
        if self.args.no_ses:
            return


        db = sqlite3.connect(self.args.ses_db)
        cur = db.cursor()
        cur.execute("delete from us where un = ?", (uname,))
        db.commit()
        cur.close()
        db.close()

        self.sesa.pop(self.ases.get(uname, ""), "")
        self.ases.pop(uname, "")

    def chpw(self, broker , uname, pw)   :
        if not self.args.chpw:
            return False, "feature disabled in server config"

        if uname == "*" or uname not in self.defpw:
            return False, "not logged in"

        if uname in self.args.chpw_no:
            return False, "not allowed for this account"

        if len(pw) < self.args.chpw_len:
            t = "minimum password length: %d characters"
            return False, t % (self.args.chpw_len,)

        if self.args.usernames:
            pw = "%s:%s" % (uname, pw)

        hpw = self.ah.hash(pw) if self.ah.on else pw

        if hpw == self.acct[uname]:
            return False, "that's already your password my dude"

        if hpw in self.iacct or hpw in self.sesa:
            return False, "password is taken"

        with self.mutex:
            ap = self.args.chpw_db
            if not bos.path.exists(ap):
                pwdb = {}
            else:
                jtxt = read_utf8(self.log, ap, True)
                pwdb = json.loads(jtxt) if jtxt.strip() else {}

            pwdb = [x for x in pwdb if x[0] != uname]
            pwdb.append((uname, self.defpw[uname], hpw))

            with open(ap, "w", encoding="utf-8") as f:
                json.dump(pwdb, f, separators=(",\n", ": "))

            self.log("reinitializing due to password-change for user [%s]" % (uname,))

            if not broker:

                self._reload()
                return True, "new password OK"

        broker.ask("reload", False, False).get()
        return True, "new password OK"

    def setup_chpw(self, acct  )  :
        ap = self.args.chpw_db
        if not self.args.chpw or not bos.path.exists(ap):
            return

        jtxt = read_utf8(self.log, ap, True)
        pwdb = json.loads(jtxt) if jtxt.strip() else {}

        useen = set()
        urst = set()
        uok = set()
        for usr, orig, mod in pwdb:
            useen.add(usr)
            if usr not in acct:

                continue
            if acct[usr] != orig:
                urst.add(usr)
                continue
            uok.add(usr)
            acct[usr] = mod

        if not self.args.chpw_v:
            return

        for usr in acct:
            if usr not in useen:
                urst.add(usr)

        for zs in uok:
            urst.discard(zs)

        if self.args.chpw_v == 1 or (self.args.chpw_v == 2 and not urst):
            t = "chpw: %d changed, %d unchanged"
            self.log(t % (len(uok), len(urst)))
            return

        elif self.args.chpw_v == 2:
            t = "chpw: %d changed" % (len(uok),)
            if urst:
                t += ", \033[0munchanged:\033[35m %s" % (", ".join(list(urst)))

            self.log(t, 6)
            return

        msg = ""
        if uok:
            t = "\033[0mchanged: \033[32m%s"
            msg += t % (", ".join(list(uok)),)
        if urst:
            t = "%s\033[0munchanged: \033[35m%s"
            msg += t % (
                ", " if msg else "",
                ", ".join(list(urst)),
            )

        self.log("chpw: " + msg, 6)

    def setup_pwhash(self, acct  )  :
        if self.args.usernames:
            for uname, pw in list(acct.items())[:]:
                if pw.startswith("+") and len(pw) == 33:
                    continue
                acct[uname] = "%s:%s" % (uname, pw)

        self.ah = PWHash(self.args)
        if not self.ah.on:
            if self.args.ah_cli or self.args.ah_gen:
                t = "\n  BAD CONFIG:\n    cannot --ah-cli or --ah-gen without --ah-alg"
                raise Exception(t)
            return

        if self.args.ah_cli:
            self.ah.cli()
            sys.exit()
        elif self.args.ah_gen == "-":
            self.ah.stdin()
            sys.exit()
        elif self.args.ah_gen:
            print(self.ah.hash(self.args.ah_gen))
            sys.exit()

        if not acct:
            return

        changed = False
        for uname, pw in list(acct.items())[:]:
            if pw.startswith("+") and len(pw) == 33:
                continue

            changed = True
            hpw = self.ah.hash(pw)
            acct[uname] = hpw
            t = "hashed password for account {}: {}"
            self.log(t.format(uname, hpw), 3)

        if not changed:
            return

        lns = []
        for uname, pw in acct.items():
            lns.append("  {}: {}".format(uname, pw))

        t = "please use the following hashed passwords in your config:\n{}"
        self.log(t.format("\n".join(lns)), 3)

    def chk_sqlite_threadsafe(self)  :
        v = SQLITE_VER[-1:]

        if v == "1":

            return ""

        if v == "2":

            return "33m  your sqlite3 was compiled with reduced thread-safety;\n   database features (-e2d, -e2t) SHOULD be fine\n    but MAY cause database-corruption and crashes"

        if v == "0":

            return "31m  your sqlite3 was compiled WITHOUT thread-safety!\n   database features (-e2d, -e2t) will PROBABLY cause crashes!"

        return "36m  cannot verify sqlite3 thread-safety; strange but probably fine"

    def dbg_ls(self)  :
        users = self.args.ls
        vol = "*"
        flags  = []

        try:
            users, vol = users.split(",", 1)
        except:
            pass

        try:
            vol, zf = vol.split(",", 1)
            flags = zf.split(",")
        except:
            pass

        if users == "**":
            users = list(self.acct.keys()) + ["*"]
        else:
            users = [users]

        for u in users:
            if u not in self.acct and u != "*":
                raise Exception("user not found: " + u)

        if vol == "*":
            vols = ["/" + x for x in self.vfs.all_vols]
        else:
            vols = [vol]

        for zs in vols:
            if not zs.startswith("/"):
                raise Exception("volumes must start with /")

            if zs[1:] not in self.vfs.all_vols:
                raise Exception("volume not found: " + zs)

        self.log(str({"users": users, "vols": vols, "flags": flags}))
        t = "/{}: read({}) write({}) move({}) del({}) dots({}) get({}) upGet({}) html({}) uadmin({})"
        for k, zv in self.vfs.all_vols.items():
            vc = zv.axs
            vs = [
                k,
                vc.uread,
                vc.uwrite,
                vc.umove,
                vc.udel,
                vc.udot,
                vc.uget,
                vc.upget,
                vc.uhtml,
                vc.uadmin,
            ]
            self.log(t.format(*vs))

        flag_v = "v" in flags
        flag_ln = "ln" in flags
        flag_p = "p" in flags
        flag_r = "r" in flags

        bads = []
        for v in vols:
            v = v[1:]
            vtop = "/{}/".format(v) if v else "/"
            for u in users:
                self.log("checking /{} as {}".format(v, u))
                try:
                    vn, _ = self.vfs.get(v, u, True, False, False, False, False)
                except:
                    continue

                atop = vn.realpath
                safeabs = atop + os.sep
                g = vn.walk(
                    vn.vpath,
                    "",
                    [],
                    u,
                    [[True, False]],
                    1,
                    not self.args.no_scandir,
                    False,
                    False,
                )
                for _, _, vpath, apath, files1, dirs, _ in g:
                    fnames = [n[0] for n in files1]
                    zsl = [vpath + "/" + n for n in fnames] if vpath else fnames
                    vpaths = [vtop + x for x in zsl]
                    apaths = [os.path.join(apath, n) for n in fnames]
                    files = [(vpath + "/", apath + os.sep)] + list(
                        [(zs1, zs2) for zs1, zs2 in zip(vpaths, apaths)]
                    )

                    if flag_ln:
                        files = [x for x in files if not x[1].startswith(safeabs)]
                        if files:
                            dirs[:] = []
                            bads.append(files[0][0])

                    if not files:
                        continue
                    elif flag_v:
                        ta = [""] + [
                            '# user "{}", vpath "{}"\n{}'.format(u, vp, ap)
                            for vp, ap in files
                        ]
                    else:
                        ta = ["user {}, vol {}: {} =>".format(u, vtop, files[0][0])]
                        ta += [x[1] for x in files]

                    self.log("\n".join(ta))

                if bads:
                    self.log("\n  ".join(["found symlinks leaving volume:"] + bads))

                if bads and flag_p:
                    raise Exception(
                        "\033[31m\n  [--ls] found a safety issue and prevented startup:\n    found symlinks leaving volume, and strict is set\n\033[0m"
                    )

        if not flag_r:
            sys.exit(0)

    def cgen(self)  :
        ret = [
            "## WARNING:",
            "##  there will probably be mistakes in",
            "##  commandline-args (and maybe volflags)",
            "",
        ]

        csv = set("i p th_covers zm_on zm_off zs_on zs_off".split())
        zs = "c ihead ohead mtm mtp on403 on404 xac xad xar xau xiu xban xbc xbd xbr xbu xm"
        lst = set(zs.split())
        askip = set("a v c vc cgen exp_lg exp_md theme".split())

        t = "exp_lg exp_md ext_th_d mv_re_r mv_re_t rm_re_r rm_re_t srch_re_dots srch_re_nodot"
        fskip = set(t.split())

        amap = vf_bmap()
        amap.update(vf_vmap())
        amap.update(vf_cmap())
        vmap = {v: k for k, v in amap.items()}

        args = {k: v for k, v in vars(self.args).items()}
        pops = []
        for k1, k2 in IMPLICATIONS:
            if args.get(k1):
                pops.append(k2)
        for pop in pops:
            args.pop(pop, None)

        if args:
            ret.append("[global]")
            for k, v in args.items():
                if k in askip:
                    continue

                try:
                    v = v.pattern
                    if k in ("idp_gsep", "tftp_lsf"):
                        v = v[1:-1]
                except:
                    pass

                skip = False
                for k2, defstr in (("mte", DEF_MTE), ("mth", DEF_MTH)):
                    if k != k2:
                        continue
                    s1 = list(sorted(list(v)))
                    s2 = list(sorted(defstr.split(",")))
                    if s1 == s2:
                        skip = True
                        break
                    v = ",".join(s1)

                if skip:
                    continue

                if k in csv:
                    v = ", ".join([str(za) for za in v])
                try:
                    v2 = getattr(self.dargs, k)
                    if k == "tcolor" and len(v2) == 3:
                        v2 = "".join([x * 2 for x in v2])
                    if v == v2 or v.replace(", ", ",") == v2:
                        continue
                except:
                    continue

                dk = "  " + k.replace("_", "-")
                if k in lst:
                    for ve in v:
                        ret.append("{}: {}".format(dk, ve))
                else:
                    if v is True:
                        ret.append(dk)
                    elif v not in (False, None, ""):
                        ret.append("{}: {}".format(dk, v))
            ret.append("")

        if self.acct:
            ret.append("[accounts]")
            for u, p in self.acct.items():
                ret.append("  {}: {}".format(u, p))
            ret.append("")

        if self.grps:
            ret.append("[groups]")
            for gn, uns in self.grps.items():
                ret.append("  %s: %s" % (gn, ", ".join(uns)))
            ret.append("")

        for vol in self.vfs.all_vols.values():
            ret.append("[/{}]".format(vol.vpath))
            ret.append("  " + vol.realpath)
            ret.append("  accs:")
            perms = {
                "r": "uread",
                "w": "uwrite",
                "m": "umove",
                "d": "udel",
                ".": "udot",
                "g": "uget",
                "G": "upget",
                "h": "uhtml",
                "a": "uadmin",
            }
            users = {}
            for pkey in perms.values():
                for uname in getattr(vol.axs, pkey):
                    try:
                        users[uname] += 1
                    except:
                        users[uname] = 1
            lusers = [(v, k) for k, v in users.items()]
            vperms = {}
            for _, uname in sorted(lusers):
                pstr = ""
                for pchar, pkey in perms.items():
                    if uname in getattr(vol.axs, pkey):
                        pstr += pchar
                if "g" in pstr and "G" in pstr:
                    pstr = pstr.replace("g", "")
                pstr = pstr.replace("rwmd.a", "A")
                try:
                    vperms[pstr].append(uname)
                except:
                    vperms[pstr] = [uname]
            for pstr, uname in vperms.items():
                ret.append("    {}: {}".format(pstr, ", ".join(uname)))
            trues = []
            vals = []
            for k, v in sorted(vol.flags.items()):
                if k in fskip:
                    continue

                try:
                    v = v.pattern
                except:
                    pass

                try:
                    ak = vmap[k]
                    v2 = getattr(self.args, ak)

                    try:
                        v2 = v2.pattern
                    except:
                        pass

                    if v2 is v:
                        continue
                except:
                    pass

                skip = False
                for k2, defstr in (("mte", DEF_MTE), ("mth", DEF_MTH)):
                    if k != k2:
                        continue
                    s1 = list(sorted(list(v)))
                    s2 = list(sorted(defstr.split(",")))
                    if s1 == s2:
                        skip = True
                        break
                    v = ",".join(s1)

                if skip:
                    continue

                if k in lst:
                    for ve in v:
                        vals.append("{}: {}".format(k, ve))
                elif v is True:
                    trues.append(k)
                elif v is not False:
                    vals.append("{}: {}".format(k, v))
            pops = []
            for k1, k2 in IMPLICATIONS:
                if k1 in trues:
                    pops.append(k2)
            trues = [x for x in trues if x not in pops]
            if trues:
                vals.append(", ".join(trues))
            if vals:
                ret.append("  flags:")
                for zs in vals:
                    ret.append("    " + zs)
            ret.append("")

        self.log("generated config:\n\n" + "\n".join(ret))


def derive_args(args )  :
    args.have_idp_hdrs = bool(args.idp_h_usr or args.idp_hm_usr)
    args.have_ipu_or_ipr = bool(args.ipu or args.ipr)


def n_du_who(s )  :
    if s == "all":
        return 9
    if s == "auth":
        return 7
    if s == "w":
        return 5
    if s == "rw":
        return 4
    if s == "a":
        return 3
    return 0


def n_ver_who(s )  :
    if s == "all":
        return 9
    if s == "auth":
        return 6
    if s == "a":
        return 3
    return 0


def split_cfg_ln(ln )   :

    ret = {}
    while True:
        ln = ln.strip()
        if not ln:
            break
        ofs_sep = ln.find(",") + 1
        ofs_var = ln.find(":") + 1
        if not ofs_sep and not ofs_var:
            ret[ln] = True
            break
        if ofs_sep and (ofs_sep < ofs_var or not ofs_var):
            k, ln = ln.split(",", 1)
            ret[k.strip()] = True
        else:
            k, ln = ln.split(":", 1)
            ret[k.strip()] = ln.strip()
            break
    return ret


def expand_config_file(
    log , ret , fp , ipath 
)  :
    "a"
    fp = absreal(fp)
    if len(ipath.split(" -> ")) > 64:
        raise Exception("hit max depth of 64 includes")

    if os.path.isdir(fp):
        names = list(sorted(os.listdir(fp)))
        cnames = [
            x for x in names if x.lower().endswith(".conf") and not x.startswith(".")
        ]
        if not cnames:
            t = "warning: tried to read config-files from folder '%s' but it does not contain any "
            if names:
                t += ".conf files; the following files/subfolders were ignored: %s"
                t = t % (fp, ", ".join(names[:8]))
            else:
                t += "files at all"
                t = t % (fp,)

            if log:
                log(t, 3)

            ret.append("#\033[33m %s\033[0m" % (t,))
        else:
            zs = "#\033[36m cfg files in %s => %s\033[0m" % (fp, cnames)
            ret.append(zs)

        for fn in cnames:
            fp2 = os.path.join(fp, fn)
            if fp2 in ipath:
                continue

            expand_config_file(log, ret, fp2, ipath)

        return

    if not os.path.exists(fp):
        t = "warning: tried to read config from '%s' but the file/folder does not exist"
        t = t % (fp,)
        if log:
            log(t, 3)

        ret.append("#\033[31m %s\033[0m" % (t,))
        return

    ipath += " -> " + fp
    ret.append("#\033[36m opening cfg file{}\033[0m".format(ipath))

    cfg_lines = read_utf8(log, fp, True).replace("\t", " ").split("\n")
    if True:
        for oln in [x.rstrip() for x in cfg_lines]:
            ln = oln.split("  #")[0].strip()
            if ln.startswith("% "):
                pad = " " * len(oln.split("%")[0])
                fp2 = ln[1:].strip()
                fp2 = os.path.join(os.path.dirname(fp), fp2)
                ofs = len(ret)
                expand_config_file(log, ret, fp2, ipath)
                for n in range(ofs, len(ret)):
                    ret[n] = pad + ret[n]
                continue

            ret.append(oln)

    ret.append("#\033[36m closed{}\033[0m".format(ipath))

    zsl = []
    for ln in ret:
        zs = ln.split("  #")[0]
        if " #" in zs and zs.split("#")[0].strip():
            zsl.append(ln)
    if zsl and "no-cfg-cmt-warn" not in "\n".join(ret):
        t = "\033[33mWARNING: there is less than two spaces before the # in the following config lines, so instead of assuming that this is a comment, the whole line will become part of the config value:\n\n>>> %s\n\nif you are familiar with this and would like to mute this warning, specify the global-option no-cfg-cmt-warn\n\033[0m"
        t = t % ("\n>>> ".join(zsl),)
        if log:
            log(t)
        else:
            print(t, file=sys.stderr)


def upgrade_cfg_fmt(
    log , args , orig , cfg_fp 
)  :
    "a"
    zst = [x.split("#")[0].strip() for x in orig]
    zst = [x for x in zst if x]
    if (
        "[global]" in zst
        or "[accounts]" in zst
        or "accs:" in zst
        or "flags:" in zst
        or [x for x in zst if x.startswith("[/")]
        or len(zst) == len([x for x in zst if x.startswith("%")])
    ):
        return orig

    zst = [x for x in orig if "#\033[36m opening cfg file" not in x]
    incl = len(zst) != len(orig) - 1

    t = "upgrading config file [{}] from v1 to v2"
    if not args.vc:
        t += ". Run with argument '--vc' to see the converted config if you want to upgrade"
    if incl:
        t += ". Please don't include v1 configs from v2 files or vice versa! Upgrade all of them at the same time."
    if log:
        log(t.format(cfg_fp), 3)

    ret = []
    vp = ""
    ap = ""
    cat = ""
    catg = "[global]"
    cata = "[accounts]"
    catx = "  accs:"
    catf = "  flags:"
    for ln in orig:
        sn = ln.strip()
        if not sn:
            cat = vp = ap = ""
        if not sn.split("#")[0]:
            ret.append(ln)
        elif sn.startswith("-") and cat in ("", catg):
            if cat != catg:
                cat = catg
                ret.append(cat)
            sn = sn.lstrip("-")
            zst = sn.split(" ", 1)
            if len(zst) > 1:
                sn = "{}: {}".format(zst[0], zst[1].strip())
            ret.append("  " + sn)
        elif sn.startswith("u ") and cat in ("", catg, cata):
            if cat != cata:
                cat = cata
                ret.append(cat)
            s1, s2 = sn[1:].split(":", 1)
            ret.append("  {}: {}".format(s1.strip(), s2.strip()))
        elif not ap:
            ap = sn
        elif not vp:
            vp = "/" + sn.strip("/")
            cat = "[{}]".format(vp)
            ret.append(cat)
            ret.append("  " + ap)
        elif sn.startswith("c "):
            if cat != catf:
                cat = catf
                ret.append(cat)
            sn = sn[1:].strip()
            if "=" in sn:
                zst = sn.split("=", 1)
                sn = zst[0].replace(",", ", ")
                sn += ": " + zst[1]
            else:
                sn = sn.replace(",", ", ")
            ret.append("    " + sn)
        elif sn[:1] in "rwmdgGhaA.":
            if cat != catx:
                cat = catx
                ret.append(cat)
            zst = sn.split(" ")
            zst = [x for x in zst if x]
            if len(zst) == 1:
                zst.append("*")
            ret.append("    {}: {}".format(zst[0], ", ".join(zst[1:])))
        else:
            t = "did not understand line {} in the config"
            t1 = t
            n = 0
            for ln in orig:
                n += 1
                t += "\n{:4} {}".format(n, ln)
            if log:
                log(t, 1)
            else:
                print("\033[31m" + t)
            raise Exception(t1)

    if args.vc and log:
        t = "new config syntax (copy/paste this to upgrade your config):\n"
        t += "\n# ======================[ begin upgraded config ]======================\n\n"
        for ln in ret:
            t += ln + "\n"
        t += "\n# ======================[ end of upgraded config ]======================\n"
        log(t)

    return ret
