diff --git a/src/main/kotlin/com/gitvantage/app/AllocationMeter.kt b/src/main/kotlin/com/gitvantage/app/AllocationMeter.kt index 6999545..de38e6e 100644 --- a/src/main/kotlin/com/gitvantage/app/AllocationMeter.kt +++ b/src/main/kotlin/com/gitvantage/app/AllocationMeter.kt @@ -20,7 +20,9 @@ import java.lang.management.ManagementFactory * cause exactly where it was. * * Measured from the per-thread allocation counters, which a native image may not support; when it - * doesn't, the line says so rather than reporting zero. A thread that ends between two samples takes + * doesn't, the line says so rather than reporting zero. That includes the native image's way of not + * supporting them, which is to say it does and then read 0 for every thread: a field report showed + * "allocated 0MB since start" beside 218 young collections. A thread that ends between two samples takes * whatever it allocated since the last one with it, so the figures are a slight undercount. */ object AllocationMeter { @@ -45,6 +47,12 @@ object AllocationMeter { /** Whether [start] has taken the baseline [recent] is measured from. */ private var baselined = false + /** Whether any thread's counter has ever read above zero — which a working one does at once. */ + private var counted = false + + private const val UNSUPPORTED = "alloc unavailable (this runtime has no per-thread allocation counters)" + private const val ALL_ZERO = "alloc unavailable (this runtime's per-thread counters all read 0)" + /** * Which group a thread belongs to, by its name — and by identity for the UI thread, whose name * is the platform's ("main" or an AppKit thread) and says nothing. @@ -69,6 +77,7 @@ object AllocationMeter { val name = infos[i]?.threadName if (now < 0 || name == null) return@forEachIndexed // ended since it was listed live += id + if (now > 0) counted = true val delta = now - (lastSeen.put(id, now) ?: 0L) if (delta <= 0) return@forEachIndexed val g = groupOf(name, id == ui).ordinal @@ -84,12 +93,13 @@ object AllocationMeter { */ @Synchronized fun interval(): String { - val b = bean ?: return "alloc unavailable (this runtime has no per-thread allocation counters)" + val b = bean ?: return UNSUPPORTED if (!baselined) { start() return "alloc measuring from now" } runCatching { sample(b) }.onFailure { return "alloc unavailable (${it.javaClass.simpleName})" } + if (!counted) return ALL_ZERO val now = System.nanoTime() val seconds = ((now - recentSince) / 1_000_000_000.0).coerceAtLeast(0.001) val line = "alloc %.1fMB/s over %ds".format(recent.sum() / seconds / (1024 * 1024), seconds.toLong()) + @@ -118,8 +128,9 @@ object AllocationMeter { /** Everything attributed since the process started, for the clipboard report. Doesn't move [interval]'s baseline. */ @Synchronized fun total(): String { - val b = bean ?: return "alloc unavailable (this runtime has no per-thread allocation counters)" + val b = bean ?: return UNSUPPORTED runCatching { sample(b) }.onFailure { return "alloc unavailable (${it.javaClass.simpleName})" } + if (!counted) return ALL_ZERO return "allocated ${mb(total.sum().toDouble())}MB since start" + breakdown(total) } diff --git a/src/main/kotlin/com/gitvantage/app/AppState.kt b/src/main/kotlin/com/gitvantage/app/AppState.kt index 3441f29..85f7b0e 100644 --- a/src/main/kotlin/com/gitvantage/app/AppState.kt +++ b/src/main/kotlin/com/gitvantage/app/AppState.kt @@ -18,6 +18,7 @@ import com.gitvantage.git.LogOps import com.gitvantage.git.RepoOps import com.gitvantage.git.RepoScanner import com.gitvantage.git.SubmoduleOps +import com.gitvantage.git.IgnoredDirs import com.gitvantage.git.WatchRoots import com.gitvantage.git.WorktreeOps import com.gitvantage.git.model.Branch @@ -649,6 +650,7 @@ class AppState(private val scope: CoroutineScope) { (watchRegs.keys - want).forEach { id -> runCatching { watchRegs.remove(id)?.forEach { it.close() } } watchRootById.remove(id) + ignoredDirsById.remove(id) watchBursts.remove(id) } } @@ -663,6 +665,34 @@ class AppState(private val scope: CoroutineScope) { watchRegs[id] = WatchRoots.forRepo(id).mapNotNull { root -> runCatching { w.watch(watchRootFor(root.path), root.recursive, id) }.getOrNull() } + refreshIgnoredDirs(id) + } + + /** + * Repo id -> its top-level directories git ignores, whose events [actionFor] drops. Absent until + * the first answer arrives, and nothing is dropped meanwhile. + */ + private val ignoredDirsById = ConcurrentHashMap() + + /** Repos with an [IgnoredDirs] query in flight, so a burst of stale events asks git once. */ + private val ignoredDirsPending: MutableSet = ConcurrentHashMap.newKeySet() + + /** + * Ask git again which of [id]'s top-level directories it ignores — at registration, and when an + * event says the answer may be out of date ([WatchPolicy.ignoresStale]). Off the watcher's + * thread: it is two `git` runs, and the thread delivering events must not wait on a process. + */ + private fun refreshIgnoredDirs(id: String) { + if (!ignoredDirsPending.add(id)) return + watchScope.launch { + try { + val dirs = WatchRoots.ignoredDirs(id) + // Still watched: a repo untracked while git was answering must not be put back. + if (watchRootById.containsKey(id)) ignoredDirsById[id] = dirs + } finally { + ignoredDirsPending.remove(id) + } + } } /** @@ -702,14 +732,27 @@ class AppState(private val scope: CoroutineScope) { * scheduled at all. * * Only worktrees a scan has already found are filtered, so one that has just appeared still gets - * through and triggers the scan that discovers it. - */ - private fun actionFor(ev: FsWatchEvent): WatchAction = WatchPolicy.actionFor( - id = ev.source?.name, - needsRescan = ev.needsRescan, - nested = ev.source?.name?.let(nestedByRepo::get).orEmpty(), - paths = { eventPaths(ev) }, - ) + * through and triggers the scan that discovers it. Ignored directories work the same way: a new + * one gets through until the git query its first event sets off has answered. + */ + private fun actionFor(ev: FsWatchEvent): WatchAction { + val id = ev.source?.name + val root = id?.let(watchRootById::get) + val known = id?.let(ignoredDirsById::get) + if (id != null && root != null && known != null && !ev.needsRescan && + WatchPolicy.ignoresStale(root, known.checked, eventPaths(ev)) + ) { + refreshIgnoredDirs(id) + } + return WatchPolicy.actionFor( + id = id, + needsRescan = ev.needsRescan, + nested = id?.let(nestedByRepo::get).orEmpty(), + root = root, + ignored = known?.ignored.orEmpty(), + paths = { eventPaths(ev) }, + ) + } /** The paths an [FsWatchEvent] concerns; empty for an Overflow, which names none. */ private fun eventPaths(ev: FsWatchEvent): List = when (ev) { diff --git a/src/main/kotlin/com/gitvantage/git/WatchRoots.kt b/src/main/kotlin/com/gitvantage/git/WatchRoots.kt index 2209aea..63b1c08 100644 --- a/src/main/kotlin/com/gitvantage/git/WatchRoots.kt +++ b/src/main/kotlin/com/gitvantage/git/WatchRoots.kt @@ -3,9 +3,21 @@ package com.gitvantage.git +import java.io.File + /** One directory to register with the filesystem watcher, and whether to descend into it. */ data class WatchRoot(val path: String, val recursive: Boolean) +/** + * A repo's top-level directories as they were when last asked about ([checked]), and which of them + * git ignores without tracking anything inside ([ignored]) — the ones whose events can be dropped. + */ +data class IgnoredDirs(val checked: Set, val ignored: Set) { + companion object { + val NONE = IgnoredDirs(emptySet(), emptySet()) + } +} + /** * Which directories of a repo are worth watching. * @@ -29,6 +41,11 @@ data class WatchRoot(val path: String, val recursive: Boolean) * That is a bad trade at any exchange rate: what pruning bought was about half a second of startup * on a background thread; what it cost was half the machine's capacity to watch files at all. * + * What is still possible without that is to drop the events: the build directory is walked and + * watched as before, but a write inside it no longer rescans the repo — see [ignoredDirs]. That + * saves the rescans, which a field report showed are what a build actually costs (one repo's + * `build/` was 97% of its events and half of all the app's rescans), and leaves the walk. + * * So this stays a single watch until the watcher can either share one inotify instance across * registrations or take an exclusion list directly — see NucleusFramework/Nucleus#571. That one is * fixed upstream (a shared native watcher) but only on the unreleased 2.6/3.0 line: it is not in @@ -42,4 +59,40 @@ object WatchRoots { * pruning experiment. See this file's header for why it is not several. */ fun forRepo(repoPath: String): List = listOf(WatchRoot(repoPath, recursive = true)) + + /** + * Which of the top-level directories of the repo at [repoPath] git ignores and holds nothing + * tracked in — the ones a write inside can't change what a scan finds. + * + * Asked of git rather than worked out here. `.gitignore` is a bigger specification than it looks + * — negation, directory-only patterns, `core.excludesFile`, `.git/info/exclude` — and getting any + * of it wrong would drop the events of a directory the user is working in. Names go as + * arguments, not on stdin, because [Git] never writes to a child's stdin. + * + * Then `ls-files` over the ignored ones, because ignoring a directory doesn't untrack what was + * committed there before, or force-added since: an `.idea/` with its shared run configurations + * checked in is ignored and still has files whose edits `git status` reports. + * + * Any failure reads as nothing ignored, which costs rescans and is never wrong. `.git` is never + * asked about: its events are how commits, checkouts and fetches are noticed at all. + */ + fun ignoredDirs(repoPath: String): IgnoredDirs { + val dir = File(repoPath) + val names = dir.listFiles()?.filter { it.isDirectory && it.name != ".git" }?.map { it.name }?.toSet() + ?: return IgnoredDirs.NONE + if (names.isEmpty()) return IgnoredDirs(names, emptySet()) + // check-ignore exits 1 when nothing matched, which [Git.read] reads as "". It takes `-z` only + // with `--stdin`, so a name git still quotes after quotePath (a quote, a newline) fails the + // `in names` filter and stays watched — the safe way to be wrong. + val ignored = Git.read(dir, "-c", "core.quotePath=false", "check-ignore", *names.toTypedArray()) + .lineSequence() + .map { it.trimEnd('/') } + .filter { it in names } + .toSet() + if (ignored.isEmpty()) return IgnoredDirs(names, ignored) + val tracked = Git.readOrNull(dir, "ls-files", "-z", "--", *ignored.map { "$it/" }.toTypedArray()) + ?.split('\u0000')?.map { it.substringBefore('/') }?.toSet() + ?: return IgnoredDirs(names, emptySet()) + return IgnoredDirs(names, ignored - tracked) + } } diff --git a/src/main/kotlin/com/gitvantage/model/WatchPolicy.kt b/src/main/kotlin/com/gitvantage/model/WatchPolicy.kt index f8164ca..847e448 100644 --- a/src/main/kotlin/com/gitvantage/model/WatchPolicy.kt +++ b/src/main/kotlin/com/gitvantage/model/WatchPolicy.kt @@ -32,10 +32,10 @@ sealed interface WatchAction { * it was when the churn started: changes made since missing, and with the counts stuck at zero so * was the Diff button. * - * `WatchRoots` has since stopped the ignored directories being watched at all, which removes most - * of that churn at the source. The ceiling stays, for the churn it does not remove: a worktree in a - * directory git isn't ignoring, a generated file that is tracked, or a build configured to write - * somewhere committed. + * Events in a nested checkout and in top-level directories git ignores are now dropped before they + * reach the debounce (see [actionFor]), which removes most of that churn. The ceiling stays, for the + * churn they don't remove: an ignored directory deeper down, a generated file that is tracked, or a + * build configured to write somewhere committed. * * [MAX_WAIT_MS] is the ceiling that makes starvation impossible — the scan runs that long after the * *first* event of a burst however busy the tree stays. Pure and clock-injected, so both halves can @@ -138,19 +138,65 @@ object WatchPolicy { * precisely the failure [concernsRepo]'s empty-paths case was written to prevent and could not, * because it was never reached. * - * [paths] is a lambda so an ordinary event on a repo with no nested checkouts — nearly every - * event — costs no allocation at all. This runs once per filesystem event. + * An event wholly inside directories git ignores — [ignored], the top-level names under [root] + * — is dropped too: see [inIgnoredDirs]. + * + * [paths] is a lambda so an ordinary event on a repo with neither nested checkouts nor ignored + * directories costs no allocation at all. This runs once per filesystem event. */ fun actionFor( id: String?, needsRescan: Boolean, nested: List, + root: Path? = null, + ignored: Set = emptySet(), paths: () -> List, - ): WatchAction = when { - id == null -> if (needsRescan) WatchAction.RescanAll else WatchAction.Ignore - needsRescan -> WatchAction.Rescan(id) - nested.isEmpty() -> WatchAction.Rescan(id) - concernsRepo(nested.map(Path::of), paths()) -> WatchAction.Rescan(id) - else -> WatchAction.Ignore + ): WatchAction { + if (id == null) return if (needsRescan) WatchAction.RescanAll else WatchAction.Ignore + if (needsRescan) return WatchAction.Rescan(id) + val filtersIgnored = root != null && ignored.isNotEmpty() + if (nested.isEmpty() && !filtersIgnored) return WatchAction.Rescan(id) + val ps = paths() + if (filtersIgnored && inIgnoredDirs(root, ignored, ps)) return WatchAction.Ignore + return if (concernsRepo(nested.map(Path::of), ps)) WatchAction.Rescan(id) else WatchAction.Ignore + } + + /** + * Whether every one of [paths] is inside one of the top-level directories [ignored] under + * [root] — a write git doesn't see, so a rescan would find nothing new. + * + * A build writes thousands of files into `build/`, and while it runs the debounce's ceiling + * rescans the repo every [MAX_WAIT_MS]: one field report had a single repo's `build/` behind 97% + * of its events and half of every rescan the app made. Not watching the directory at all would + * be better still and is blocked upstream — see `WatchRoots`. + * + * Inside, not at: an event *for* `build` itself, its creation or removal, still counts, as does + * anything at the top of the repo. Empty [paths] — an event naming nothing — is never inside. + */ + fun inIgnoredDirs(root: Path, ignored: Set, paths: List): Boolean = + paths.isNotEmpty() && paths.all { p -> topLevelDirOf(root, p)?.let { it in ignored } == true } + + /** + * Whether what is known of [root]'s ignored directories may be out of date, given that its + * top-level directories were [checked] when last asked: an event in a directory that wasn't + * there then, or to `.gitignore` or `.git/info/exclude`, which decide what is ignored. + * + * Neither means anything is wrong now — the event is not dropped, so the worst case is a rescan + * that wasn't needed — only that the next answer should be asked of git again. + */ + fun ignoresStale(root: Path, checked: Set, paths: List): Boolean = paths.any { p -> + if (!p.startsWith(root)) return@any false + val rel = root.relativize(p) + val spelled = rel.joinToString("/") + spelled == ".gitignore" || spelled == ".git/info/exclude" || + topLevelDirOf(root, p)?.let { it !in checked } == true + } + + /** The top-level directory under [root] that [path] is inside, or null for anything at the top, outside, or in `.git`. */ + private fun topLevelDirOf(root: Path, path: Path): String? { + if (!path.startsWith(root)) return null + val rel = root.relativize(path) + if (rel.nameCount < 2) return null + return rel.getName(0).toString().takeIf { it != ".git" } } } diff --git a/src/test/kotlin/com/gitvantage/git/WatchRootsTest.kt b/src/test/kotlin/com/gitvantage/git/WatchRootsTest.kt index 2486c4d..017fe81 100644 --- a/src/test/kotlin/com/gitvantage/git/WatchRootsTest.kt +++ b/src/test/kotlin/com/gitvantage/git/WatchRootsTest.kt @@ -53,3 +53,49 @@ val RepoWatchRoots by testSuite { } } } + +/** + * Which top-level directories a repo's events can be dropped for. + * + * The failure worth guarding against is dropping too much: a directory listed here stops refreshing + * the repo, so one the user edits tracked files in must never appear, however git's ignore rules + * read. Dropping too little only costs a rescan. + */ +val RepoIgnoredDirs by testSuite { + testFixture { Sandbox() } asContextForEach { + + test("an ignored build directory is listed, and source directories are not") { + val work = repo() + commit(work, ".gitignore", "build/\nnode_modules\n", "ignore output") + listOf("build/classes", "node_modules/x", "src/main").forEach { java.io.File(work, it).mkdirs() } + + val dirs = WatchRoots.ignoredDirs(work.path) + + assert(dirs.ignored == setOf("build", "node_modules")) { "ignored: ${dirs.ignored}" } + assert(dirs.checked == setOf("build", "node_modules", "src")) { "checked: ${dirs.checked}" } + } + + test("an ignored directory with tracked files in it is not listed — their edits still count") { + val work = repo() + commit(work, ".gitignore", ".idea/\n", "ignore the IDE") + java.io.File(work, ".idea").mkdirs() + java.io.File(work, ".idea/runConfig.xml").writeText("") + git(work, "add", "-f", ".idea/runConfig.xml") + git(work, "commit", "-q", "-m", "share a run configuration") + + assert(WatchRoots.ignoredDirs(work.path).ignored.isEmpty()) + } + + test(".git is never asked about, so its events always count") { + val work = repo() + commit(work, ".gitignore", ".git\n", "a rule that names .git") + + assert(".git" !in WatchRoots.ignoredDirs(work.path).checked) + } + + test("a folder that isn't a repo ignores nothing") { + val plain = java.io.File(root, "plain").apply { java.io.File(this, "build").mkdirs() } + assert(WatchRoots.ignoredDirs(plain.path).ignored.isEmpty()) + } + } +} diff --git a/src/test/kotlin/com/gitvantage/model/WatchPolicyTest.kt b/src/test/kotlin/com/gitvantage/model/WatchPolicyTest.kt index 82a1fd4..f269d0a 100644 --- a/src/test/kotlin/com/gitvantage/model/WatchPolicyTest.kt +++ b/src/test/kotlin/com/gitvantage/model/WatchPolicyTest.kt @@ -205,3 +205,74 @@ val WatchActionRouting by testSuite { assert(!called) { "the common path allocated an event's path list to reach a known answer" } } } + +/** + * Dropping events from directories git ignores. + * + * What matters is the edge: a build's writes inside `build/` are dropped, while the directory itself + * appearing, anything at the top, `.git`, and an event naming nothing all still rescan. + */ +val WatchIgnoredDirs by testSuite { + + val id = "/src/app" + val root = Path.of(id) + val ignored = setOf("build", ".gradle") + fun p(rel: String) = root.resolve(rel) + + test("a write inside an ignored directory is dropped") { + val action = WatchPolicy.actionFor(id, needsRescan = false, nested = emptyList(), root, ignored) { + listOf(p("build/classes/App.class")) + } + assert(action == WatchAction.Ignore) { "a build write rescanned the repo: $action" } + } + + test("a write anywhere else still rescans") { + val action = WatchPolicy.actionFor(id, needsRescan = false, nested = emptyList(), root, ignored) { + listOf(p("src/main/App.kt")) + } + assert(action == WatchAction.Rescan(id)) + } + + test("a move out of an ignored directory into the code still rescans") { + val action = WatchPolicy.actionFor(id, needsRescan = false, nested = emptyList(), root, ignored) { + listOf(p("build/gen/App.kt"), p("src/main/App.kt")) + } + assert(action == WatchAction.Rescan(id)) + } + + test("the ignored directory itself, a top-level file and an event naming nothing are not inside") { + assert(!WatchPolicy.inIgnoredDirs(root, ignored, listOf(p("build")))) + assert(!WatchPolicy.inIgnoredDirs(root, ignored, listOf(p("build.gradle.kts")))) + assert(!WatchPolicy.inIgnoredDirs(root, ignored, emptyList())) + } + + test("a lost-events signal is never dropped, whatever paths it carries") { + val action = WatchPolicy.actionFor(id, needsRescan = true, nested = emptyList(), root, ignored) { + listOf(p("build/x")) + } + assert(action == WatchAction.Rescan(id)) + } + + test("events in nested worktrees are still dropped alongside ignored ones") { + val nested = listOf("/src/app/.claude/worktrees/feat") + val action = WatchPolicy.actionFor(id, needsRescan = false, nested = nested, root, ignored) { + listOf(p(".claude/worktrees/feat/App.kt")) + } + assert(action == WatchAction.Ignore) + } + + test("the answer is stale after a new top-level directory or a change to the ignore rules") { + val checked = setOf("build", "src") + assert(WatchPolicy.ignoresStale(root, checked, listOf(p("dist/app.js")))) + assert(WatchPolicy.ignoresStale(root, checked, listOf(p(".gitignore")))) + assert(WatchPolicy.ignoresStale(root, checked, listOf(p(".git/info/exclude")))) + } + + test("and not after ordinary writes, git's own files, or a nested .gitignore") { + val checked = setOf("build", "src") + assert(!WatchPolicy.ignoresStale(root, checked, listOf(p("src/main/App.kt")))) + assert(!WatchPolicy.ignoresStale(root, checked, listOf(p(".git/index")))) + assert(!WatchPolicy.ignoresStale(root, checked, listOf(p("src/.gitignore")))) + assert(!WatchPolicy.ignoresStale(root, checked, listOf(p("README.md")))) + } +}