Don't rescan for writes in git-ignored folders

src/main/kotlin/com/gitvantage/app/AllocationMeter.kt

@@ -20,7 +20,9 @@ import java.lang.management.ManagementFactory
  * cause exactly where it was.
  *
  * Measured from the per-thread allocation counters, which a native image may not support; when it
- * doesn't, the line says so rather than reporting zero. A thread that ends between two samples takes
+ * doesn't, the line says so rather than reporting zero. That includes the native image's way of not
+ * supporting them, which is to say it does and then read 0 for every thread: a field report showed
+ * "allocated 0MB since start" beside 218 young collections. A thread that ends between two samples takes
  * whatever it allocated since the last one with it, so the figures are a slight undercount.
  */
 object AllocationMeter {
@@ -45,6 +47,12 @@ object AllocationMeter {
     /** Whether [start] has taken the baseline [recent] is measured from. */
     private var baselined = false
 
+    /** Whether any thread's counter has ever read above zero — which a working one does at once. */
+    private var counted = false
+
+    private const val UNSUPPORTED = "alloc unavailable (this runtime has no per-thread allocation counters)"
+    private const val ALL_ZERO = "alloc unavailable (this runtime's per-thread counters all read 0)"
+
     /**
      * Which group a thread belongs to, by its name — and by identity for the UI thread, whose name
      * is the platform's ("main" or an AppKit thread) and says nothing.
@@ -69,6 +77,7 @@ object AllocationMeter {
             val name = infos[i]?.threadName
             if (now < 0 || name == null) return@forEachIndexed // ended since it was listed
             live += id
+            if (now > 0) counted = true
             val delta = now - (lastSeen.put(id, now) ?: 0L)
             if (delta <= 0) return@forEachIndexed
             val g = groupOf(name, id == ui).ordinal
@@ -84,12 +93,13 @@ object AllocationMeter {
      */
     @Synchronized
     fun interval(): String {
-        val b = bean ?: return "alloc unavailable (this runtime has no per-thread allocation counters)"
+        val b = bean ?: return UNSUPPORTED
         if (!baselined) {
             start()
             return "alloc measuring from now"
         }
         runCatching { sample(b) }.onFailure { return "alloc unavailable (${it.javaClass.simpleName})" }
+        if (!counted) return ALL_ZERO
         val now = System.nanoTime()
         val seconds = ((now - recentSince) / 1_000_000_000.0).coerceAtLeast(0.001)
         val line = "alloc %.1fMB/s over %ds".format(recent.sum() / seconds / (1024 * 1024), seconds.toLong()) +
@@ -118,8 +128,9 @@ object AllocationMeter {
     /** Everything attributed since the process started, for the clipboard report. Doesn't move [interval]'s baseline. */
     @Synchronized
     fun total(): String {
-        val b = bean ?: return "alloc unavailable (this runtime has no per-thread allocation counters)"
+        val b = bean ?: return UNSUPPORTED
         runCatching { sample(b) }.onFailure { return "alloc unavailable (${it.javaClass.simpleName})" }
+        if (!counted) return ALL_ZERO
         return "allocated ${mb(total.sum().toDouble())}MB since start" + breakdown(total)
     }
 

src/main/kotlin/com/gitvantage/app/AppState.kt

@@ -18,6 +18,7 @@ import com.gitvantage.git.LogOps
 import com.gitvantage.git.RepoOps
 import com.gitvantage.git.RepoScanner
 import com.gitvantage.git.SubmoduleOps
+import com.gitvantage.git.IgnoredDirs
 import com.gitvantage.git.WatchRoots
 import com.gitvantage.git.WorktreeOps
 import com.gitvantage.git.model.Branch
@@ -649,6 +650,7 @@ class AppState(private val scope: CoroutineScope) {
                 (watchRegs.keys - want).forEach { id ->
                     runCatching { watchRegs.remove(id)?.forEach { it.close() } }
                     watchRootById.remove(id)
+                    ignoredDirsById.remove(id)
                     watchBursts.remove(id)
                 }
             }
@@ -663,6 +665,34 @@ class AppState(private val scope: CoroutineScope) {
         watchRegs[id] = WatchRoots.forRepo(id).mapNotNull { root ->
             runCatching { w.watch(watchRootFor(root.path), root.recursive, id) }.getOrNull()
         }
+        refreshIgnoredDirs(id)
+    }
+
+    /**
+     * Repo id -> its top-level directories git ignores, whose events [actionFor] drops. Absent until
+     * the first answer arrives, and nothing is dropped meanwhile.
+     */
+    private val ignoredDirsById = ConcurrentHashMap<String, IgnoredDirs>()
+
+    /** Repos with an [IgnoredDirs] query in flight, so a burst of stale events asks git once. */
+    private val ignoredDirsPending: MutableSet<String> = ConcurrentHashMap.newKeySet()
+
+    /**
+     * Ask git again which of [id]'s top-level directories it ignores — at registration, and when an
+     * event says the answer may be out of date ([WatchPolicy.ignoresStale]). Off the watcher's
+     * thread: it is two `git` runs, and the thread delivering events must not wait on a process.
+     */
+    private fun refreshIgnoredDirs(id: String) {
+        if (!ignoredDirsPending.add(id)) return
+        watchScope.launch {
+            try {
+                val dirs = WatchRoots.ignoredDirs(id)
+                // Still watched: a repo untracked while git was answering must not be put back.
+                if (watchRootById.containsKey(id)) ignoredDirsById[id] = dirs
+            } finally {
+                ignoredDirsPending.remove(id)
+            }
+        }
     }
 
     /**
@@ -702,14 +732,27 @@ class AppState(private val scope: CoroutineScope) {
      * scheduled at all.
      *
      * Only worktrees a scan has already found are filtered, so one that has just appeared still gets
-     * through and triggers the scan that discovers it.
-     */
-    private fun actionFor(ev: FsWatchEvent): WatchAction = WatchPolicy.actionFor(
-        id = ev.source?.name,
-        needsRescan = ev.needsRescan,
-        nested = ev.source?.name?.let(nestedByRepo::get).orEmpty(),
-        paths = { eventPaths(ev) },
-    )
+     * through and triggers the scan that discovers it. Ignored directories work the same way: a new
+     * one gets through until the git query its first event sets off has answered.
+     */
+    private fun actionFor(ev: FsWatchEvent): WatchAction {
+        val id = ev.source?.name
+        val root = id?.let(watchRootById::get)
+        val known = id?.let(ignoredDirsById::get)
+        if (id != null && root != null && known != null && !ev.needsRescan &&
+            WatchPolicy.ignoresStale(root, known.checked, eventPaths(ev))
+        ) {
+            refreshIgnoredDirs(id)
+        }
+        return WatchPolicy.actionFor(
+            id = id,
+            needsRescan = ev.needsRescan,
+            nested = id?.let(nestedByRepo::get).orEmpty(),
+            root = root,
+            ignored = known?.ignored.orEmpty(),
+            paths = { eventPaths(ev) },
+        )
+    }
 
     /** The paths an [FsWatchEvent] concerns; empty for an Overflow, which names none. */
     private fun eventPaths(ev: FsWatchEvent): List<Path> = when (ev) {

src/main/kotlin/com/gitvantage/git/WatchRoots.kt

@@ -3,9 +3,21 @@
 
 package com.gitvantage.git
 
+import java.io.File
+
 /** One directory to register with the filesystem watcher, and whether to descend into it. */
 data class WatchRoot(val path: String, val recursive: Boolean)
 
+/**
+ * A repo's top-level directories as they were when last asked about ([checked]), and which of them
+ * git ignores without tracking anything inside ([ignored]) — the ones whose events can be dropped.
+ */
+data class IgnoredDirs(val checked: Set<String>, val ignored: Set<String>) {
+    companion object {
+        val NONE = IgnoredDirs(emptySet(), emptySet())
+    }
+}
+
 /**
  * Which directories of a repo are worth watching.
  *
@@ -29,6 +41,11 @@ data class WatchRoot(val path: String, val recursive: Boolean)
  * That is a bad trade at any exchange rate: what pruning bought was about half a second of startup
  * on a background thread; what it cost was half the machine's capacity to watch files at all.
  *
+ * What is still possible without that is to drop the events: the build directory is walked and
+ * watched as before, but a write inside it no longer rescans the repo — see [ignoredDirs]. That
+ * saves the rescans, which a field report showed are what a build actually costs (one repo's
+ * `build/` was 97% of its events and half of all the app's rescans), and leaves the walk.
+ *
  * So this stays a single watch until the watcher can either share one inotify instance across
  * registrations or take an exclusion list directly — see NucleusFramework/Nucleus#571. That one is
  * fixed upstream (a shared native watcher) but only on the unreleased 2.6/3.0 line: it is not in
@@ -42,4 +59,40 @@ object WatchRoots {
      * pruning experiment. See this file's header for why it is not several.
      */
     fun forRepo(repoPath: String): List<WatchRoot> = listOf(WatchRoot(repoPath, recursive = true))
+
+    /**
+     * Which of the top-level directories of the repo at [repoPath] git ignores and holds nothing
+     * tracked in — the ones a write inside can't change what a scan finds.
+     *
+     * Asked of git rather than worked out here. `.gitignore` is a bigger specification than it looks
+     * — negation, directory-only patterns, `core.excludesFile`, `.git/info/exclude` — and getting any
+     * of it wrong would drop the events of a directory the user is working in. Names go as
+     * arguments, not on stdin, because [Git] never writes to a child's stdin.
+     *
+     * Then `ls-files` over the ignored ones, because ignoring a directory doesn't untrack what was
+     * committed there before, or force-added since: an `.idea/` with its shared run configurations
+     * checked in is ignored and still has files whose edits `git status` reports.
+     *
+     * Any failure reads as nothing ignored, which costs rescans and is never wrong. `.git` is never
+     * asked about: its events are how commits, checkouts and fetches are noticed at all.
+     */
+    fun ignoredDirs(repoPath: String): IgnoredDirs {
+        val dir = File(repoPath)
+        val names = dir.listFiles()?.filter { it.isDirectory && it.name != ".git" }?.map { it.name }?.toSet()
+            ?: return IgnoredDirs.NONE
+        if (names.isEmpty()) return IgnoredDirs(names, emptySet())
+        // check-ignore exits 1 when nothing matched, which [Git.read] reads as "". It takes `-z` only
+        // with `--stdin`, so a name git still quotes after quotePath (a quote, a newline) fails the
+        // `in names` filter and stays watched — the safe way to be wrong.
+        val ignored = Git.read(dir, "-c", "core.quotePath=false", "check-ignore", *names.toTypedArray())
+            .lineSequence()
+            .map { it.trimEnd('/') }
+            .filter { it in names }
+            .toSet()
+        if (ignored.isEmpty()) return IgnoredDirs(names, ignored)
+        val tracked = Git.readOrNull(dir, "ls-files", "-z", "--", *ignored.map { "$it/" }.toTypedArray())
+            ?.split('\u0000')?.map { it.substringBefore('/') }?.toSet()
+            ?: return IgnoredDirs(names, emptySet())
+        return IgnoredDirs(names, ignored - tracked)
+    }
 }

src/main/kotlin/com/gitvantage/model/WatchPolicy.kt

@@ -32,10 +32,10 @@ sealed interface WatchAction {
  * it was when the churn started: changes made since missing, and with the counts stuck at zero so
  * was the Diff button.
  *
- * `WatchRoots` has since stopped the ignored directories being watched at all, which removes most
- * of that churn at the source. The ceiling stays, for the churn it does not remove: a worktree in a
- * directory git isn't ignoring, a generated file that is tracked, or a build configured to write
- * somewhere committed.
+ * Events in a nested checkout and in top-level directories git ignores are now dropped before they
+ * reach the debounce (see [actionFor]), which removes most of that churn. The ceiling stays, for the
+ * churn they don't remove: an ignored directory deeper down, a generated file that is tracked, or a
+ * build configured to write somewhere committed.
  *
  * [MAX_WAIT_MS] is the ceiling that makes starvation impossible — the scan runs that long after the
  * *first* event of a burst however busy the tree stays. Pure and clock-injected, so both halves can
@@ -138,19 +138,65 @@ object WatchPolicy {
      * precisely the failure [concernsRepo]'s empty-paths case was written to prevent and could not,
      * because it was never reached.
      *
-     * [paths] is a lambda so an ordinary event on a repo with no nested checkouts — nearly every
-     * event — costs no allocation at all. This runs once per filesystem event.
+     * An event wholly inside directories git ignores — [ignored], the top-level names under [root]
+     * — is dropped too: see [inIgnoredDirs].
+     *
+     * [paths] is a lambda so an ordinary event on a repo with neither nested checkouts nor ignored
+     * directories costs no allocation at all. This runs once per filesystem event.
      */
     fun actionFor(
         id: String?,
         needsRescan: Boolean,
         nested: List<String>,
+        root: Path? = null,
+        ignored: Set<String> = emptySet(),
         paths: () -> List<Path>,
-    ): WatchAction = when {
-        id == null -> if (needsRescan) WatchAction.RescanAll else WatchAction.Ignore
-        needsRescan -> WatchAction.Rescan(id)
-        nested.isEmpty() -> WatchAction.Rescan(id)
-        concernsRepo(nested.map(Path::of), paths()) -> WatchAction.Rescan(id)
-        else -> WatchAction.Ignore
+    ): WatchAction {
+        if (id == null) return if (needsRescan) WatchAction.RescanAll else WatchAction.Ignore
+        if (needsRescan) return WatchAction.Rescan(id)
+        val filtersIgnored = root != null && ignored.isNotEmpty()
+        if (nested.isEmpty() && !filtersIgnored) return WatchAction.Rescan(id)
+        val ps = paths()
+        if (filtersIgnored && inIgnoredDirs(root, ignored, ps)) return WatchAction.Ignore
+        return if (concernsRepo(nested.map(Path::of), ps)) WatchAction.Rescan(id) else WatchAction.Ignore
+    }
+
+    /**
+     * Whether every one of [paths] is inside one of the top-level directories [ignored] under
+     * [root] — a write git doesn't see, so a rescan would find nothing new.
+     *
+     * A build writes thousands of files into `build/`, and while it runs the debounce's ceiling
+     * rescans the repo every [MAX_WAIT_MS]: one field report had a single repo's `build/` behind 97%
+     * of its events and half of every rescan the app made. Not watching the directory at all would
+     * be better still and is blocked upstream — see `WatchRoots`.
+     *
+     * Inside, not at: an event *for* `build` itself, its creation or removal, still counts, as does
+     * anything at the top of the repo. Empty [paths] — an event naming nothing — is never inside.
+     */
+    fun inIgnoredDirs(root: Path, ignored: Set<String>, paths: List<Path>): Boolean =
+        paths.isNotEmpty() && paths.all { p -> topLevelDirOf(root, p)?.let { it in ignored } == true }
+
+    /**
+     * Whether what is known of [root]'s ignored directories may be out of date, given that its
+     * top-level directories were [checked] when last asked: an event in a directory that wasn't
+     * there then, or to `.gitignore` or `.git/info/exclude`, which decide what is ignored.
+     *
+     * Neither means anything is wrong now — the event is not dropped, so the worst case is a rescan
+     * that wasn't needed — only that the next answer should be asked of git again.
+     */
+    fun ignoresStale(root: Path, checked: Set<String>, paths: List<Path>): Boolean = paths.any { p ->
+        if (!p.startsWith(root)) return@any false
+        val rel = root.relativize(p)
+        val spelled = rel.joinToString("/")
+        spelled == ".gitignore" || spelled == ".git/info/exclude" ||
+            topLevelDirOf(root, p)?.let { it !in checked } == true
+    }
+
+    /** The top-level directory under [root] that [path] is inside, or null for anything at the top, outside, or in `.git`. */
+    private fun topLevelDirOf(root: Path, path: Path): String? {
+        if (!path.startsWith(root)) return null
+        val rel = root.relativize(path)
+        if (rel.nameCount < 2) return null
+        return rel.getName(0).toString().takeIf { it != ".git" }
     }
 }

src/test/kotlin/com/gitvantage/git/WatchRootsTest.kt

@@ -53,3 +53,49 @@ val RepoWatchRoots by testSuite {
         }
     }
 }
+
+/**
+ * Which top-level directories a repo's events can be dropped for.
+ *
+ * The failure worth guarding against is dropping too much: a directory listed here stops refreshing
+ * the repo, so one the user edits tracked files in must never appear, however git's ignore rules
+ * read. Dropping too little only costs a rescan.
+ */
+val RepoIgnoredDirs by testSuite {
+    testFixture { Sandbox() } asContextForEach {
+
+        test("an ignored build directory is listed, and source directories are not") {
+            val work = repo()
+            commit(work, ".gitignore", "build/\nnode_modules\n", "ignore output")
+            listOf("build/classes", "node_modules/x", "src/main").forEach { java.io.File(work, it).mkdirs() }
+
+            val dirs = WatchRoots.ignoredDirs(work.path)
+
+            assert(dirs.ignored == setOf("build", "node_modules")) { "ignored: ${dirs.ignored}" }
+            assert(dirs.checked == setOf("build", "node_modules", "src")) { "checked: ${dirs.checked}" }
+        }
+
+        test("an ignored directory with tracked files in it is not listed — their edits still count") {
+            val work = repo()
+            commit(work, ".gitignore", ".idea/\n", "ignore the IDE")
+            java.io.File(work, ".idea").mkdirs()
+            java.io.File(work, ".idea/runConfig.xml").writeText("<shared/>")
+            git(work, "add", "-f", ".idea/runConfig.xml")
+            git(work, "commit", "-q", "-m", "share a run configuration")
+
+            assert(WatchRoots.ignoredDirs(work.path).ignored.isEmpty())
+        }
+
+        test(".git is never asked about, so its events always count") {
+            val work = repo()
+            commit(work, ".gitignore", ".git\n", "a rule that names .git")
+
+            assert(".git" !in WatchRoots.ignoredDirs(work.path).checked)
+        }
+
+        test("a folder that isn't a repo ignores nothing") {
+            val plain = java.io.File(root, "plain").apply { java.io.File(this, "build").mkdirs() }
+            assert(WatchRoots.ignoredDirs(plain.path).ignored.isEmpty())
+        }
+    }
+}