Fix tight-circle selection stealing neighboring text
Prefer a11y nodes centered in the circle, skip OCR merge when a11y is already useful, and smoke-test the How to circle heading case.
This commit is contained in:
@@ -256,8 +256,10 @@ class CrklAccessibilityService : AccessibilityService() {
|
||||
}
|
||||
}
|
||||
|
||||
// OCR when a11y is thin, or always merge for image-like sparse text.
|
||||
val needOcr = a11y.isEmpty || a11y.text.length < 40
|
||||
// OCR only when a11y found nothing useful. Short labels like
|
||||
// "How to circle" are real hits — OCR-merge used to re-pull the
|
||||
// neighboring Ship demo block from the selection bitmap.
|
||||
val needOcr = a11y.isEmpty || !ContentCapture.hasUsefulText(a11y.text)
|
||||
val merged = if (needOcr) {
|
||||
withContext(Dispatchers.Main) {
|
||||
val debug = IntegrationSettings(this@CrklAccessibilityService).showDebugMeta
|
||||
|
||||
@@ -44,7 +44,8 @@ object LocalAssistStub {
|
||||
title = "Circled text",
|
||||
meta = "",
|
||||
body = buildString {
|
||||
if (summary.isNotBlank()) {
|
||||
// Headline only when it shortens a longer blob — never a lone "1." marker.
|
||||
if (summary.length >= 12 && summary.length < preview.length) {
|
||||
appendLine(summary)
|
||||
appendLine()
|
||||
appendLine("—")
|
||||
@@ -187,16 +188,18 @@ object LocalAssistStub {
|
||||
|
||||
private fun heuristicSummary(text: String): String {
|
||||
val firstLine = text.lineSequence().firstOrNull { it.isNotBlank() }?.trim().orEmpty()
|
||||
// Do not treat "1." / "2." list markers as sentence terminators.
|
||||
val sentence = firstLine
|
||||
.split(Regex("(?<=[.!?])\\s+"))
|
||||
.split(Regex("(?<=[A-Za-z][.!?])\\s+"))
|
||||
.firstOrNull()
|
||||
?.trim()
|
||||
.orEmpty()
|
||||
|
||||
val head = when {
|
||||
sentence.isNotEmpty() && sentence.length <= 160 -> sentence
|
||||
firstLine.length <= 160 -> firstLine
|
||||
else -> firstLine.take(157) + "…"
|
||||
sentence.length in 12..160 -> sentence
|
||||
firstLine.length in 12..160 -> firstLine
|
||||
firstLine.length > 160 -> firstLine.take(157) + "…"
|
||||
else -> ""
|
||||
}
|
||||
|
||||
return head
|
||||
|
||||
@@ -7,7 +7,7 @@ import android.view.accessibility.AccessibilityNodeInfo
|
||||
import android.view.accessibility.AccessibilityWindowInfo
|
||||
|
||||
/**
|
||||
* Pulls readable text from accessibility nodes that intersect a screen-space selection.
|
||||
* Pulls readable text from accessibility nodes that fit a screen-space selection.
|
||||
*
|
||||
* Prefers the accessibility node tree over MediaProjection so the first vertical
|
||||
* slice stays permission-light and fully on-device.
|
||||
@@ -33,8 +33,8 @@ object RegionContentExtractor {
|
||||
selection: RectF
|
||||
): ExtractionResult {
|
||||
val selectionRect = Rect().also { selection.round(it) }
|
||||
val snippets = linkedSetOf<String>()
|
||||
var nodesHit = 0
|
||||
val candidates = mutableListOf<Pair<String, SelectionOverlap.Score>>()
|
||||
var nodesVisited = 0
|
||||
var packageName: String? = null
|
||||
|
||||
val roots = mutableListOf<AccessibilityNodeInfo>()
|
||||
@@ -53,61 +53,70 @@ object RegionContentExtractor {
|
||||
packageName = root.packageName?.toString()
|
||||
}
|
||||
try {
|
||||
nodesHit += collectIntersectingText(root, selectionRect, snippets)
|
||||
nodesVisited += collectCandidates(root, selectionRect, candidates)
|
||||
} finally {
|
||||
// Only recycle roots we obtained from windows; caller owns rootFallback.
|
||||
if (root !== rootFallback) {
|
||||
root.recycle()
|
||||
}
|
||||
}
|
||||
if (snippets.isNotEmpty() || nodesHit >= MAX_NODES) break
|
||||
if (candidates.isNotEmpty() || nodesVisited >= MAX_NODES) break
|
||||
}
|
||||
|
||||
val joined = snippets
|
||||
val picked = SelectionOverlap.pickBest(candidates)
|
||||
val joined = picked
|
||||
.distinct()
|
||||
.joinToString(separator = "\n")
|
||||
.trim()
|
||||
.take(MAX_CHARS)
|
||||
|
||||
Log.d(
|
||||
TAG,
|
||||
"extract: nodes=$nodesHit snippets=${snippets.size} chars=${joined.length} pkg=$packageName"
|
||||
"extract: visited=$nodesVisited candidates=${candidates.size} " +
|
||||
"picked=${picked.size} chars=${joined.length} pkg=$packageName " +
|
||||
"preview=${joined.take(100).replace('\n', '|')}"
|
||||
)
|
||||
|
||||
return ExtractionResult(
|
||||
text = joined,
|
||||
nodeCount = nodesHit,
|
||||
nodeCount = picked.size,
|
||||
bounds = RectF(selection),
|
||||
packageName = packageName
|
||||
)
|
||||
}
|
||||
|
||||
private fun collectIntersectingText(
|
||||
private fun collectCandidates(
|
||||
node: AccessibilityNodeInfo,
|
||||
selection: Rect,
|
||||
out: MutableSet<String>,
|
||||
out: MutableList<Pair<String, SelectionOverlap.Score>>,
|
||||
depth: Int = 0
|
||||
): Int {
|
||||
if (depth > 40 || out.size >= MAX_NODES) return 0
|
||||
|
||||
var hit = 0
|
||||
var visited = 1
|
||||
val bounds = Rect()
|
||||
node.getBoundsInScreen(bounds)
|
||||
val score = SelectionOverlap.score(bounds, selection)
|
||||
|
||||
if (Rect.intersects(bounds, selection)) {
|
||||
readableText(node)?.let { out.add(it) }
|
||||
hit = 1
|
||||
}
|
||||
|
||||
var childTextHits = 0
|
||||
for (i in 0 until node.childCount) {
|
||||
val child = node.getChild(i) ?: continue
|
||||
try {
|
||||
hit += collectIntersectingText(child, selection, out, depth + 1)
|
||||
val before = out.size
|
||||
visited += collectCandidates(child, selection, out, depth + 1)
|
||||
if (out.size > before) childTextHits++
|
||||
} finally {
|
||||
child.recycle()
|
||||
}
|
||||
if (out.size >= MAX_NODES) break
|
||||
}
|
||||
return hit
|
||||
|
||||
// Prefer leaf (or near-leaf) text — skip parents that only duplicate children.
|
||||
if (score.include && childTextHits == 0) {
|
||||
readableText(node)?.let { out.add(it to score) }
|
||||
}
|
||||
|
||||
return visited
|
||||
}
|
||||
|
||||
private fun readableText(node: AccessibilityNodeInfo): String? {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package com.example.crkl.vision
|
||||
|
||||
import android.graphics.Rect
|
||||
|
||||
/**
|
||||
* Scores how well an a11y node fits a circle's screen bounds.
|
||||
*
|
||||
* Loose [Rect.intersects] alone pulls in large TextViews that only graze the
|
||||
* selection (e.g. circling "How to circle" also hits the Ship demo block above).
|
||||
*
|
||||
* Geometry uses raw ints so JVM unit tests do not need a mocked [Rect].
|
||||
*/
|
||||
object SelectionOverlap {
|
||||
|
||||
data class Score(
|
||||
val include: Boolean,
|
||||
/** Higher = tighter match. */
|
||||
val rank: Float,
|
||||
val centerInSelection: Boolean,
|
||||
/** Fraction of the node covered by the selection (0..1). */
|
||||
val coverage: Float
|
||||
)
|
||||
|
||||
fun score(nodeBounds: Rect, selection: Rect): Score =
|
||||
score(
|
||||
nodeLeft = nodeBounds.left,
|
||||
nodeTop = nodeBounds.top,
|
||||
nodeRight = nodeBounds.right,
|
||||
nodeBottom = nodeBounds.bottom,
|
||||
selLeft = selection.left,
|
||||
selTop = selection.top,
|
||||
selRight = selection.right,
|
||||
selBottom = selection.bottom
|
||||
)
|
||||
|
||||
fun score(
|
||||
nodeLeft: Int,
|
||||
nodeTop: Int,
|
||||
nodeRight: Int,
|
||||
nodeBottom: Int,
|
||||
selLeft: Int,
|
||||
selTop: Int,
|
||||
selRight: Int,
|
||||
selBottom: Int
|
||||
): Score {
|
||||
val nodeW = nodeRight - nodeLeft
|
||||
val nodeH = nodeBottom - nodeTop
|
||||
val selW = selRight - selLeft
|
||||
val selH = selBottom - selTop
|
||||
if (nodeW <= 0 || nodeH <= 0 || selW <= 0 || selH <= 0) {
|
||||
return Score(include = false, rank = 0f, centerInSelection = false, coverage = 0f)
|
||||
}
|
||||
|
||||
val interLeft = maxOf(nodeLeft, selLeft)
|
||||
val interTop = maxOf(nodeTop, selTop)
|
||||
val interRight = minOf(nodeRight, selRight)
|
||||
val interBottom = minOf(nodeBottom, selBottom)
|
||||
val interW = interRight - interLeft
|
||||
val interH = interBottom - interTop
|
||||
if (interW <= 0 || interH <= 0) {
|
||||
return Score(include = false, rank = 0f, centerInSelection = false, coverage = 0f)
|
||||
}
|
||||
|
||||
val overlap = interW.toLong() * interH.toLong()
|
||||
val nodeArea = nodeW.toLong() * nodeH.toLong()
|
||||
val coverage = (overlap.toFloat() / nodeArea.toFloat()).coerceIn(0f, 1f)
|
||||
val cx = (nodeLeft + nodeRight) / 2
|
||||
val cy = (nodeTop + nodeBottom) / 2
|
||||
val centerIn = cx in selLeft until selRight && cy in selTop until selBottom
|
||||
|
||||
// Prefer nodes whose center sits in the circle; otherwise require most of
|
||||
// the node to lie inside (partial paragraph circle still works).
|
||||
val include = centerIn || coverage >= 0.45f
|
||||
val rank = (if (centerIn) 2f else 0f) + coverage +
|
||||
(1f / (1f + nodeArea / 50_000f))
|
||||
|
||||
return Score(
|
||||
include = include,
|
||||
rank = rank,
|
||||
centerInSelection = centerIn,
|
||||
coverage = coverage
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* From scored hits, keep center-in nodes when any exist; else high-coverage ones.
|
||||
*/
|
||||
fun <T> pickBest(candidates: List<Pair<T, Score>>): List<T> {
|
||||
if (candidates.isEmpty()) return emptyList()
|
||||
val included = candidates.filter { it.second.include }
|
||||
val pool = when {
|
||||
included.any { it.second.centerInSelection } ->
|
||||
included.filter { it.second.centerInSelection }
|
||||
included.isNotEmpty() -> included
|
||||
else -> candidates.filter { it.second.coverage >= 0.25f }
|
||||
}
|
||||
return pool
|
||||
.sortedByDescending { it.second.rank }
|
||||
.map { it.first }
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,21 @@ class LocalAssistStubTest {
|
||||
assertTrue(response.meta.isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun respond_numberedList_doesNotUseLoneOneAsHeadline() {
|
||||
val result = RegionContentExtractor.ExtractionResult(
|
||||
text = "1. Circle the English sentence → tap Translate [Translate]\n" +
|
||||
"2. Circle the paragraph → Copy, then Explain [Copy / Explain]",
|
||||
nodeCount = 1,
|
||||
bounds = android.graphics.RectF(),
|
||||
packageName = "com.example.crkl"
|
||||
)
|
||||
val response = LocalAssistStub.respond(result)
|
||||
assertTrue(response.body.contains("Circle the English"))
|
||||
// Old bug: summary split on "1." and showed a lone "1." above the divider.
|
||||
assertTrue(!response.body.trimStart().startsWith("1.\n"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun respond_ocrJunk_treatedAsEmpty() {
|
||||
val result = RegionContentExtractor.ExtractionResult(
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.example.crkl.vision
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
class SelectionOverlapTest {
|
||||
|
||||
@Test
|
||||
fun tightHeading_includesCenterNode_excludesGrazingBlock() {
|
||||
// Circle bbox around a heading
|
||||
val headScore = SelectionOverlap.score(
|
||||
nodeLeft = 120, nodeTop = 410, nodeRight = 360, nodeBottom = 470,
|
||||
selLeft = 100, selTop = 400, selRight = 400, selBottom = 480
|
||||
)
|
||||
// Large block above — only bottom edge grazes the selection
|
||||
val blockScore = SelectionOverlap.score(
|
||||
nodeLeft = 40, nodeTop = 100, nodeRight = 500, nodeBottom = 420,
|
||||
selLeft = 100, selTop = 400, selRight = 400, selBottom = 480
|
||||
)
|
||||
|
||||
assertTrue(headScore.include)
|
||||
assertTrue(headScore.centerInSelection)
|
||||
assertFalse(
|
||||
"grazing ship-demo block must not steal the circle",
|
||||
blockScore.include
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pickBest_prefersCenterInOverGrazing() {
|
||||
val heading = "How to circle" to SelectionOverlap.score(
|
||||
120, 410, 360, 470, 100, 400, 400, 480
|
||||
)
|
||||
val block = "1. Circle the English…" to SelectionOverlap.score(
|
||||
40, 100, 500, 420, 100, 400, 400, 480
|
||||
)
|
||||
|
||||
val picked = SelectionOverlap.pickBest(listOf(block, heading))
|
||||
assertEquals(listOf("How to circle"), picked)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pickBest_multipleCenterNodes_keepsAllRanked() {
|
||||
val a = "How to circle" to SelectionOverlap.score(
|
||||
120, 410, 360, 470, 100, 390, 400, 520
|
||||
)
|
||||
val b = "1. Tap the floating C" to SelectionOverlap.score(
|
||||
100, 480, 500, 560, 100, 390, 400, 520
|
||||
)
|
||||
val leak = "Ship demo blob" to SelectionOverlap.score(
|
||||
40, 50, 500, 400, 100, 390, 400, 520
|
||||
)
|
||||
val picked = SelectionOverlap.pickBest(listOf(leak, b, a))
|
||||
assertTrue(picked.contains("How to circle"))
|
||||
assertFalse(picked.contains("Ship demo blob"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun tinyCornerGraze_excluded() {
|
||||
val score = SelectionOverlap.score(
|
||||
nodeLeft = 0, nodeTop = 0, nodeRight = 1000, nodeBottom = 1000,
|
||||
selLeft = 900, selTop = 900, selRight = 980, selBottom = 980
|
||||
)
|
||||
assertFalse(score.centerInSelection)
|
||||
assertTrue(score.coverage < 0.45f)
|
||||
assertFalse(score.include)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user