openclaw/AXorcist
openclaw/AXorcist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Gruelsome debugging day

Browse Source
This commit is contained in:
Peter Steinberger 2025-05-27 18:50:00 +02:00
parent dda8425768
commit 1cc0706e53
7 changed files with 190 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Sources/AXorcist/Core/AccessibilityConstants.swift
View File

@ -36,6 +36,9 @@ public enum AXAttributeNames {
public static let kAXDescriptionAttribute = "AXDescription" // Often a more detailed description than title
public static let kAXHelpAttribute = "AXHelp" // Tooltip or help text
public static let kAXIdentifierAttribute = "AXIdentifier" // Developer-assigned unique ID
// DOM-specific attributes are declared in the Web-specific section below to avoid duplication
public static let kAXDOMClassListAttribute = "AXDOMClassList" // [String] or String
public static let kAXDOMIdentifierAttribute = "AXDOMIdentifier" // String (DOM id)
// State Attributes
public static let kAXEnabledAttribute = "AXEnabled" // Bool
@ -127,8 +130,6 @@ public enum AXAttributeNames {
// Web-specific (often found in WebArea roles)
public static let kAXURLAttribute = "AXURL" // URL or String
public static let kAXDocumentAttribute = "AXDocument" // String (URL or path of document)
public static let kAXDOMClassListAttribute = "AXDOMClassList" // [String] or String
// public static let kAXDOMIdentifierAttribute = "AXDOMIdentifier" // String
// public static let kAXARIADOMResourceAttribute = "AXARIADOMResource"
// public static let kAXARIADOMFunctionAttribute = "AXARIADOM-función" // Keep original as it might be specific
// public static let kAXARIADOMChildrenAttribute = "AXARIADOMChildren"
@ -185,7 +186,7 @@ public enum AXAttributeNames {
public static let kAXPathHintAttribute = "AXPathHint" // Custom attribute for path hints, if used
// Web content related
public static let kAXDOMIdentifierAttribute = "AXDOMIdentifier" // Used in web views for DOM element IDs.
// public static let kAXDOMIdentifierAttribute = "AXDOMIdentifier" // Used in web views for DOM element IDs.
// macOS 13 additions (example)
// public static let kAXCustomActionsAttribute = "AXCustomActions" // This is a guess, verify actual name
@ -348,6 +349,8 @@ public enum AXMiscConstants {
AXAttributeNames.kAXTitleAttribute,
AXAttributeNames.kAXValueAttribute,
AXAttributeNames.kAXIdentifierAttribute,
AXAttributeNames.kAXDOMClassListAttribute,
AXAttributeNames.kAXDOMIdentifierAttribute,
AXAttributeNames.kAXDescriptionAttribute,
AXAttributeNames.kAXEnabledAttribute,
AXAttributeNames.kAXFocusedAttribute,

27
Sources/AXorcist/Core/CommandEnvelope.swift
View File

@ -226,4 +226,31 @@ public struct CommandEnvelope: Codable {
self.includeChildrenInText = includeChildrenInText
self.includeIgnoredElements = includeIgnoredElements
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
commandId = try container.decode(String.self, forKey: .commandId)
command = try container.decode(CommandType.self, forKey: .command)
application = try container.decodeIfPresent(String.self, forKey: .application)
attributes = try container.decodeIfPresent([String].self, forKey: .attributes)
payload = try container.decodeIfPresent([String:String].self, forKey: .payload)
debugLogging = try container.decodeIfPresent(Bool.self, forKey: .debugLogging) ?? false
locator = try container.decodeIfPresent(Locator.self, forKey: .locator)
pathHint = try container.decodeIfPresent([String].self, forKey: .pathHint)
maxElements = try container.decodeIfPresent(Int.self, forKey: .maxElements)
maxDepth = try container.decodeIfPresent(Int.self, forKey: .maxDepth)
outputFormat = try container.decodeIfPresent(OutputFormat.self, forKey: .outputFormat)
actionName = try container.decodeIfPresent(String.self, forKey: .actionName)
actionValue = try container.decodeIfPresent(AnyCodable.self, forKey: .actionValue)
subCommands = try container.decodeIfPresent([CommandEnvelope].self, forKey: .subCommands)
point = try container.decodeIfPresent(CGPoint.self, forKey: .point)
pid = try container.decodeIfPresent(Int.self, forKey: .pid)
notifications = try container.decodeIfPresent([String].self, forKey: .notifications)
includeElementDetails = try container.decodeIfPresent([String].self, forKey: .includeElementDetails)
watchChildren = try container.decodeIfPresent(Bool.self, forKey: .watchChildren)
filterCriteria = try container.decodeIfPresent([String:String].self, forKey: .filterCriteria)
includeChildrenBrief = try container.decodeIfPresent(Bool.self, forKey: .includeChildrenBrief)
includeChildrenInText = try container.decodeIfPresent(Bool.self, forKey: .includeChildrenInText)
includeIgnoredElements = try container.decodeIfPresent(Bool.self, forKey: .includeIgnoredElements)
}
}

12
Sources/AXorcist/Core/Element+Hierarchy.swift
View File

@ -17,11 +17,19 @@ extension Element {
collectDirectChildren(collector: &childCollector)
// print("[PRINT Element.children] After collectDirectChildren, collector has: \(childCollector.collectedChildrenCount()) unique children.")
if !strict { // Only collect alternatives if not strict
// collectAlternativeChildren may be expensive, so respect `strict` flag there.
if !strict {
collectAlternativeChildren(collector: &childCollector)
collectApplicationWindows(collector: &childCollector)
}
// Always collect `AXWindows` when this element is an application. Some Electron apps only expose
// the *front-most* window via `kAXChildrenAttribute`, while all other windows are available via
// `kAXWindowsAttribute`. Not including the latter caused our searches to remain inside the first
// window (depth ≈ 37) and never reach hidden/background chat panes. Fetching `AXWindows` every
// time is cheap (<10 elements) and guarantees the walker can explore every window even during a
// brute-force scan.
collectApplicationWindows(collector: &childCollector)
// print("[PRINT Element.children] Before finalizeResults, collector has: \(childCollector.collectedChildrenCount()) unique children.")
let result = childCollector.finalizeResults()
axDebugLog("Final children count: \(result?.count ?? 0)")

75
Sources/AXorcist/Search/ElementSearch.swift
View File

@ -27,6 +27,12 @@ public func findTargetElement(
// Use criteriaDebugString in the log message
logger.info("FTE: App='\(appIdentifier)' D=\(maxDepthForSearch) C=\(criteriaDebugString.isEmpty ? "none" : criteriaDebugString) PH=\(locator.rootElementPathHint?.count ?? 0)")
// Reset per-search globals.
traversalNodeCounter = 0
// Start the global traversal timeout early, so it also covers any path navigation steps.
traversalDeadline = Date().addingTimeInterval(axorcTraversalTimeout)
defer { traversalDeadline = nil }
guard let appElement = getApplicationElement(for: appIdentifier) else {
logger.error("FTE: No app element for \(appIdentifier)")
return (nil, "Application not found or not accessible: \(appIdentifier)")
@ -41,7 +47,8 @@ public func findTargetElement(
// Convert [JSONPathHintComponent] to [PathStep]
let pathSteps: [PathStep] = jsonPathComponents.map { component in
let criterion = Criterion(attribute: component.attribute, value: component.value, matchType: component.matchType)
let attributeName = component.axAttributeName ?? component.attribute // Map aliases like ROLE/DOM to real AX attribute
let criterion = Criterion(attribute: attributeName, value: component.value, matchType: component.matchType)
return PathStep(criteria: [criterion], matchType: component.matchType, matchAllCriteria: true, maxDepthForStep: component.depth)
}
@ -89,18 +96,18 @@ public func findTargetElement(
criteria: locator.criteria,
matchType: finalSearchMatchType,
matchAllCriteria: finalSearchMatchAll,
stopAtFirstMatch: true, // For the final search, we typically want the first match.
stopAtFirstMatch: axorcStopAtFirstMatch,
maxDepth: maxDepthForSearch
)
traverseAndSearch(element: currentSearchElement, visitor: searchVisitor, currentDepth: 0, maxDepth: maxDepthForSearch)
if let foundMatch = searchVisitor.foundElement { // Changed from foundElements.first
logger.info("FindTargetEl: Found final descendant matching criteria: \(foundMatch.briefDescription(option: .smart))")
logger.info("FindTargetEl: Found final descendant matching criteria: \(foundMatch.briefDescription(option: .smart)). Nodes visited = \(traversalNodeCounter)")
return (foundMatch, nil)
} else {
let criteriaDesc = locator.criteria.map { "\($0.attribute):\($0.value)" }.joined(separator: ", ")
let finalSearchError = "FTE: Not found C=[\(criteriaDesc)] from \(searchStartingPointDescription)"
let finalSearchError = "FTE: Not found C=[\(criteriaDesc)] from \(searchStartingPointDescription). Max depth visited = \(searchVisitor.deepestDepthReached) of \(maxDepthForSearch). Nodes visited = \(traversalNodeCounter)"
logger.warning("\(finalSearchError)")
return (nil, finalSearchError)
}
@ -153,6 +160,7 @@ public func traverseAndSearch(
return
}
traversalNodeCounter += 1
let visitResult = visitor.visit(element: element, depth: currentDepth)
switch visitResult {
@ -167,11 +175,27 @@ public func traverseAndSearch(
// Continue to process children
}
if let children = element.children() {
// Maintain a static visited set per traversal to avoid cycles.
// We store the CFHash of AXUIElement to uniquely identify.
struct VisitedSet { static var set = Set<UInt>() }
if let children = element.children(strict: false), !children.isEmpty,
(axorcScanAll || (element.role().map { containerRoles.contains($0) } ?? false)) {
// Abort if we are past the deadline
if let deadline = traversalDeadline, Date() > deadline {
logger.warning("Traverse: global search timeout (\(axorcTraversalTimeout)s) reached. Aborting traversal.")
return
}
for child in children {
let hashVal: UInt = CFHash(child.underlyingElement)
if !VisitedSet.set.insert(hashVal).inserted {
continue // already visited; skip to avoid cycles
}
traverseAndSearch(element: child, visitor: visitor, currentDepth: currentDepth + 1, maxDepth: maxDepth)
// If the visitor is a SearchVisitor that stops at first match, check if it found something.
if let searchVisitor = visitor as? SearchVisitor, searchVisitor.stopAtFirstMatchInternal, searchVisitor.foundElement != nil {
if let searchVisitor = visitor as? SearchVisitor,
searchVisitor.stopAtFirstMatchInternal,
searchVisitor.foundElement != nil {
logger.debug("Traverse: SearchVisitor found match and stopAtFirstMatch is true. Stopping traversal early.")
return // Stop traversal early
}
@ -191,6 +215,7 @@ public class SearchVisitor: ElementVisitor {
private var currentMaxDepthReachedByVisitor: Int = 0
private let matchType: JSONPathHintComponent.MatchType // Added
private let matchAllCriteriaBool: Bool // Added (renamed to avoid conflict with func name)
public var deepestDepthReached: Int { currentMaxDepthReachedByVisitor }
init(
criteria: [Criterion],
@ -302,3 +327,39 @@ public class CollectAllVisitor: ElementVisitor {
// Ensure `elementMatchesAllCriteria` from SearchCriteriaUtils is accessible and synchronous.
// Ensure `Criterion` struct and `Locator` struct are defined and accessible.
// AXMiscConstants should be available. Example: public enum AXMiscConstants { public static let defaultMaxDepthSearch: Int = 10 }
// Container roles that can have meaningful descendants. Non-container roles are treated as leaves.
private let containerRoles: Set<String> = [
AXRoleNames.kAXApplicationRole,
AXRoleNames.kAXWindowRole,
AXRoleNames.kAXGroupRole,
AXRoleNames.kAXScrollAreaRole,
AXRoleNames.kAXSplitGroupRole,
AXRoleNames.kAXLayoutAreaRole,
AXRoleNames.kAXLayoutItemRole,
AXRoleNames.kAXWebAreaRole,
AXRoleNames.kAXListRole,
AXRoleNames.kAXOutlineRole,
AXRoleNames.kAXUnknownRole,
"AXGeneric","AXSection","AXArticle","AXSplitter","AXScrollBar","AXPane"
]
// MARK: - Search Timeout Handling
/// Global deadline used by `traverseAndSearch` to abort extremely long walks.
/// It is _only_ set for the duration of a single public search call and then cleared again.
private var traversalDeadline: Date?
/// Counts how many nodes have been visited during the current `findTargetElement` invocation.
private var traversalNodeCounter: Int = 0
/// Default timeout (seconds) for a full tree traversal. Override at runtime by setting `axorcTraversalTimeout`.
public var axorcTraversalTimeout: TimeInterval = 30
/// When true, traversal will ignore `containerRoles` pruning and descend into *every* child of every element.
/// Enable via CLI flag `--scan-all`.
public var axorcScanAll: Bool = false
/// Controls whether SearchVisitor should stop at the first element that satisfies the final locator criteria.
/// CLI flag `--no-stop-first` sets this to `false`.
public var axorcStopAtFirstMatch: Bool = true

34
Sources/AXorcist/Search/SearchCriteriaUtils.swift
View File

@ -193,7 +193,7 @@ private func matchAttributeByKey(
return matchIdentifierAttribute(element: element, expectedValue: expectedValue, matchType: matchType, elementDescriptionForLog: elementDescriptionForLog)
case "pid":
return matchPidCriterion(element: element, expectedValue: expectedValue, elementDescriptionForLog: elementDescriptionForLog)
case AXAttributeNames.kAXDOMClassListAttribute.lowercased(), "domclasslist", "classlist":
case AXAttributeNames.kAXDOMClassListAttribute.lowercased(), "domclasslist", "classlist", "dom":
return matchDomClassListAttribute(element: element, expectedValue: expectedValue, matchType: matchType, elementDescriptionForLog: elementDescriptionForLog)
case AXMiscConstants.isIgnoredAttributeKey.lowercased(), "isignored", "ignored":
return matchIsIgnoredCriterion(element: element, expectedValue: expectedValue, elementDescriptionForLog: elementDescriptionForLog)
@ -270,12 +270,42 @@ private func matchDomClassListAttribute(element: Element, expectedValue: String,
level: .debug,
message: "SC/MSC/DOMClassList: ActualRaw='\(String(describing: actualRaw))'"
))
return matchDomClassListCriterion(
// First try DOM class list.
if matchDomClassListCriterion(
element: element,
expectedValue: expectedValue,
matchType: matchType,
elementDescriptionForLog: elementDescriptionForLog
) {
return true
}
// Fallback 1: AXDOMIdentifier
let domIdMatch = matchSingleCriterion(
element: element,
key: AXAttributeNames.kAXDOMIdentifierAttribute,
expectedValue: expectedValue,
matchType: matchType,
elementDescriptionForLog: elementDescriptionForLog
)
if domIdMatch {
GlobalAXLogger.shared.log(AXLogEntry(level: .debug, message: "SC/DOMClass: Fallback DOMIdentifier MATCH for token '\(expectedValue)'."))
return true
}
// Fallback 2: legacy AXIdentifier
let identifierMatch = matchIdentifierAttribute(
element: element,
expectedValue: expectedValue,
matchType: matchType,
elementDescriptionForLog: elementDescriptionForLog
)
if identifierMatch {
GlobalAXLogger.shared.log(AXLogEntry(level: .debug, message: "SC/DOMClass: Fallback AXIdentifier MATCH for token '\(expectedValue)'."))
}
return identifierMatch
}
@MainActor

46
Sources/axorc/AXORCMain.swift
View File

@ -1,6 +1,6 @@
// AXORCMain.swift - Main entry point for AXORC CLI
@preconcurrency import ArgumentParser
import ArgumentParser
import AXorcist // For AXorcist instance
import CoreFoundation
import Foundation
@ -9,16 +9,20 @@ import Foundation
// let axorcVersion = "0.1.0-dev"
@main
struct AXORCCommand: @preconcurrency ParsableCommand {
struct AXORCCommand: ParsableCommand {
static let configuration: CommandConfiguration = CommandConfiguration(
commandName: "axorc",
// Use axorcVersion from AXORCModels.swift or a shared constant place
abstract: "AXORC CLI - Handles JSON commands via various input methods. Version \\(axorcVersion)"
)
@Flag(name: .long, help: "Enable debug logging for the command execution.")
// `--debug` now enables *normal* diagnostic output. Use the new `--verbose` flag for the extremely chatty logs.
@Flag(name: .long, help: "Enable debug logging (normal detail level). Use --verbose for maximum detail.")
var debug: Bool = false
@Flag(name: .long, help: "Enable *verbose* debug logging – every internal step. Produces large output.")
var verbose: Bool = false
@Flag(name: .long, help: "Read JSON payload from STDIN.")
var stdin: Bool = false
@ -28,6 +32,15 @@ struct AXORCCommand: @preconcurrency ParsableCommand {
@Option(name: .long, help: "Read JSON payload directly from this string argument, expecting a JSON string.")
var json: String?
@Option(name: .long, help: "Traversal timeout in seconds (overrides default 30).")
var timeout: Int?
@Flag(name: .long, help: "Traverse every node (ignore container role pruning). May be extremely slow.")
var scanAll: Bool = false
@Flag(name: .customLong("no-stop-first"), help: "Do not stop at first match; collect deeper matches as well.")
var noStopFirst: Bool = false
@Argument(
help: "Read JSON payload directly from this string argument. If other input flags (--stdin, --file, --json) are used, this argument is ignored."
)
@ -97,12 +110,29 @@ struct AXORCCommand: @preconcurrency ParsableCommand {
fputs("AXORCMain.run: VERY FIRST LINE EXECUTED.\n", stderr)
fflush(stderr)
GlobalAXLogger.shared.isLoggingEnabled = debug
GlobalAXLogger.shared.detailLevel = debug ? .verbose : .minimal
// Configure global logger according to flags.
if verbose {
GlobalAXLogger.shared.isLoggingEnabled = true
GlobalAXLogger.shared.detailLevel = .verbose
} else if debug {
GlobalAXLogger.shared.isLoggingEnabled = true
GlobalAXLogger.shared.detailLevel = .normal
} else {
GlobalAXLogger.shared.isLoggingEnabled = false
GlobalAXLogger.shared.detailLevel = .minimal
}
// Confirm settings
fputs("AXORCMain.run: CLI --debug flag is: \(debug). Logger enabled: \(GlobalAXLogger.shared.isLoggingEnabled).\n", stderr)
fflush(stderr)
// Set global brute-force / stop-first flags
axorcScanAll = scanAll
axorcStopAtFirstMatch = !noStopFirst
// Honour timeout override
if let timeout = timeout {
axorcTraversalTimeout = TimeInterval(timeout)
}
// For clarity in stderr output
fputs("AXORCMain.run: AXorc version \(axorcVersion) build \(axorcBuildStamp). Detail level: \(GlobalAXLogger.shared.detailLevel).\n", stderr)
// <<< TEST LOGGING START >>>
axErrorLog("AXORCMain.run: TEST ERROR LOG -- SHOULD ALWAYS APPEAR IN DEBUG OUTPUT IF LOGS ARE PRINTED")

10
Sources/axorc/Models/AXORCModels.swift
View File

@ -6,7 +6,15 @@ import AXorcist
import Foundation
// MARK: - Version and Configuration
let axorcVersion = "0.1.2a-config_fix"
let axorcVersion = "0.1.3"
/// Returns a human-readable build stamp (yyMMddHHmm) evaluated at runtime.
/// Good enough for confirming we're on the binary we just built.
var axorcBuildStamp: String {
let formatter = DateFormatter()
formatter.dateFormat = "yyMMddHHmm"
return formatter.string(from: Date())
}
// MARK: - Shared Error Detail