Class ::nsmcp::AccessDiagnosticTool (public)
::nx::Class ::nsmcp::AccessDiagnosticTool![[i]](/resources/acs-subsite/ZoomIn16.gif)
Defined in /usr/local/ns/tcl/nsmcp/traffic-tools.tcl
- Testcases:
- No testcase defined.
Source code: :property authorizer:required :property redactor:required :property peerRedactor:required :property routing:required :property {service {}} :property {defaultMode scan} :property {operation bot} :property {permission diagnostics.logs.read} :property {maxReadBytes:integer 4194304} :property {seekMinimumBytes:integer 8388608} :property {maxAggregateSources:integer 100000} :property {maxAggregateUserAgents:integer 4096} :property {inputSchema { { "type": "object", "properties": { "mode": { "type": "string", "enum": [ "scan", "page" ], "description": "Bot routing: aggregateTraffic true adds address-free source concentration and User-Agent rankings; path selects an exact queryless path. scan aggregates all pages in a background job, polled with log_results. page returns one bounded byte page. Slow requests remain paged." }, "authenticationState": { "type": "string", "enum": [ "anonymous", "authenticated", "unknown" ], "description": "Filter by logged authentication state; no identity is disclosed." }, "aggregateTraffic": { "type": "boolean", "default": false, "description": "Scan mode only: address-free source concentration and User-Agent rankings over all selected records, independent of evidence limits. No peer-disclosure permission required." }, "path": { "type": "string", "description": "Exact queryless path, e.g. /bugtracker/openacs/" }, "scanStrategy": { "type": "string", "enum": [ "window", "full" ], "default": "window", "description": "Background scans: window seeks by completion timestamps with a 300s margin and explicit ordering assumptions; full examines the entire current file." }, "lastMinutes": { "type": "integer", "minimum": 1, "maximum": 1440, "default": 60 }, "from": { "type": "number" }, "to": { "type": "number" }, "limit": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 }, "pool": { "type": "string", "default": "bots" }, "module": { "type": "string", "default": "nslog" }, "category": { "type": "string", "enum": [ "matching-rule-in-target", "matching-rule-other-pool-candidate", "bot-like-unmatched-heuristic", "no-target-rule-match", "unknown", "native-target-in-target", "native-target-other-pool-candidate", "bot-like-routed-elsewhere-heuristic", "native-other-pool" ], "description": "Bot routing only: retain evidence for this exact category. categoryTotals still covers all matching records; selectedMatches and filteredOutRecords distinguish selection from omitted evidence." }, "includePeerAddresses": { "type": "boolean", "default": false }, "beforeOffset": { "type": "integer", "minimum": 0 }, "sourceBytes": { "type": "integer", "minimum": 0 }, "sourceIdentity": { "type": "string" }, "mappingIdentity": { "type": "string" } }, "additionalProperties": false } }} :method init {} { next if {${:maxReadBytes}<1 || ${:maxReadBytes}>16777216} { error "Diagnostic read budget must be 1..16 MiB" } } :method modules {server} { return [ns_server -server $server modules] } :method logDirectory {server} { return [ns_server -server $server logdir] } :method configuredSlowPath {server module} { return [ns_config ns/server/$server/module/$module slowlogfile {}] } :method options {arguments} { if {[lindex $arguments 1] ne "object"} { error "Expected an arguments object" } set options {lastMinutes 60 limit 20 pool bots module nslog includePeerAddresses false category {} scanStrategy window aggregateTraffic false} dict set options mode [expr {${:operation} eq "bot" ? ${:defaultMode} : "page"}] set seen {} foreach {key type value} [lindex $arguments 2] { if {$key in $seen} { error "Duplicate argument: $key" } lappend seen $key switch -- $key { lastMinutes - limit - beforeOffset - sourceBytes { if {$type ne "number" || ![string is entier -strict $value] || $value<0} { error "Invalid $key" } if {$key eq "lastMinutes" && ($value<1 || $value>1440)} { error "lastMinutes must be 1..1440" } if {$key eq "limit" && ($value<1 || $value>100)} { error "limit must be 1..100" } } from - to { if {$type ne "number" || ![string is double -strict $value] || $value<0 || $value>[clock seconds]} { error "Invalid $key" } } pool - module { if {$type ne "string" || ![regexp {^[A-Za-z0-9_.-]+$} $value]} { error "Invalid $key" } } mode { if {$type ne "string" || $value ni {scan page} || (${:operation} ne "bot" && $value ne "page")} { error "Invalid diagnostic mode" } } scanStrategy { if {$type ne "string" || $value ni {window full}} { error "Invalid scanStrategy" } } category { if {${:operation} ne "bot" || $type ne "string" || $value ni { matching-rule-in-target matching-rule-other-pool-candidate bot-like-unmatched-heuristic no-target-rule-match unknown native-target-in-target native-target-other-pool-candidate bot-like-routed-elsewhere-heuristic native-other-pool }} { error "Invalid routing category" } } sourceIdentity - mappingIdentity { if {$type ne "string" || ![regexp {^[a-f0-9]{64}$} $value]} { error "Invalid $key" } } authenticationState { if {$type ne "string" || $value ni {anonymous authenticated unknown}} { error "Invalid authenticationState" } } path { if {$type ne "string" || ![string match /* $value] || [string length $value]>2048 || [string first ? $value]>=0 || [string first \x00 $value]>=0} { error "Expected exact queryless path" } } aggregateTraffic - includePeerAddresses { if {$type ne "boolean"} { error "Expected a boolean" } } default { error "Unknown argument: $key" } } dict set options $key $value } if {[dict exists $options from] != [dict exists $options to]} { error "Specify both from and to" } if {[dict exists $options from]} { if {"lastMinutes" in $seen || [dict get $options to]<=[dict get $options from] || [dict get $options to]-[dict get $options from]>86400} { error "Invalid or conflicting time selectors" } } else { dict set options to [clock seconds] dict set options from [expr {[dict get $options to]-60*[dict get $options lastMinutes]}] } set paging 0 foreach key {beforeOffset sourceBytes sourceIdentity} { incr paging [dict exists $options $key] } if {$paging ni {0 3}} { error "Paging requires beforeOffset, sourceBytes and sourceIdentity" } if {$paging && [dict get $options beforeOffset]>[dict get $options sourceBytes]} { error "Invalid cursor" } return $options } :public method validate {arguments} { # # Validate routing-diagnostic modes, filters and scan options. # # Decode the rooted JSON argument object and enforce this tool's option, # range and combination constraints. # # @param arguments Rooted ns_json typed triples for the tool's JSON argument # object. # # @return An empty result on success; invalid input raises RPC error -32602. # if {[catch { :options $arguments } message]} { return -code error -errorcode {NSMCP RPC -32602} $message } } :method source {server options} { if {${:operation} eq "bot"} { return [dict get [::nsmcp::logDiscovery discover] access path] } set module [dict get $options module] set modules [:modules $server] if {![dict exists $modules $module name] || [dict get $modules $module name] ne "nslog"} { error "Selected nslog module is not loaded for this server" } set path [:configuredSlowPath $server $module] if {$path ne "" && [file pathtype $path] ne "absolute"} { set path [file join [:logDirectory $server] $path] } return $path } # Evaluate only fully observable root URL mappings with a User-Agent # constraint. Other headers, address predicates and nested constraints are # unknown. This compares rule membership, not urlspace precedence replay. :public method classify {record maps target} { # # Classify a request using the available fallback routing rules. # # Evaluate supported map forms in order and retain unknown conditions # instead of claiming an exact match. # # @param record Recorded request fields or routing-evidence dictionary. # @param maps Ordered routing map entries to evaluate. # @param target Target connection pool used for the diagnostic comparison. # # @return The fallback classification dictionary. # set matches {} set unknown 0 set index 0 foreach map $maps { set id $index incr index if {[llength $map] ni {4 5}} { incr unknown continue } lassign $map method base glob inherit condition if {$method ne [dict get $record method]} { continue } if {$base ne "/" || $glob ne "*" || $inherit ne "inherit" || [llength $map]!=5} { incr unknown continue } if {[llength $condition]!=2 || ![string equal -nocase [lindex $condition 0] user-agent]} { incr unknown continue } if {![dict exists $record userAgent]} { incr unknown continue } if {[string match [lindex $condition 1] [dict get $record userAgent]]} { lappend matches $id } } if {[dict get $record pool] eq "unknown"} { set category unknown } elseif {[llength $matches]} { set category [expr {[dict get $record pool] eq $target ? "matching-rule-in-target" : "matching-rule-other-pool-candidate"}] } elseif {$unknown || ![dict exists $record userAgent]} { set category unknown } elseif {[regexp -nocase {bot|crawl|spider|baidu|knowledge} [dict get $record userAgent]] && [dict get $record pool] ne $target} { set category bot-like-unmatched-heuristic } else { set category no-target-rule-match } return [dict create category $category matchingRuleIndexes $matches unknownRules $unknown] } # Native lookup sees only logged User-Agent and peer address. A missing # predicate input can change precedence, so do not present a partial context # as a fully determined route. Inspect all pools, not just the bot pool. :public method contextComplete {record snapshot} { # # Check whether recorded fields suffice for the routing snapshot. # # Inspect unavailable topology data and map conditions before claiming a # complete request context. # # @param record Recorded request fields or routing-evidence dictionary. # @param snapshot Routing snapshot with availability metadata. # # @return True when the recorded context supports the required lookup, # otherwise false. # if {[dict size [dict get $snapshot availability]]>0} { return false } dict for {pool maps} [dict get $snapshot pools] { foreach map $maps { if {[llength $map] ni {4 5}} { return false } if {[lindex $map 0] ne [dict get $record method]} { continue } if {[llength $map]==4} { continue } set condition [lindex $map 4] if {[llength $condition]%2} { return false } foreach {field pattern} $condition { switch -nocase -- $field { user-agent { if {![dict exists $record userAgent] || [dict get $record userAgent] in {{} - {[REDACTED]}}} { return false } } x-ns-ip { if {![ns_ip valid [dict get $record peerAddress]]} { return false } } default { return false } } } } } return true } :public method nativeClassify {server record snapshot target} { # # Classify a request using native contextual routing. # # Combine the native mapping with context completeness and target-pool # diagnostics. # # @param server Selected NaviServer virtual-server name. # @param record Recorded request fields or routing-evidence dictionary. # @param snapshot Routing snapshot with availability metadata. # @param target Target connection pool used for the diagnostic comparison. # # @return The native classification dictionary with explicit uncertainty. # set result [dict create lookupMode native-context expectedPool unknown contextComplete [:contextComplete $record $snapshot] category unknown matchingRuleIndexes {} unknownRules 0 lookupError {}] if {![dict get $result contextComplete]} { return $result } if {[catch { ${:routing} mapped $server $record } expected]} { # A row-specific lookup error must not silently become fallback # rule membership or a claim that the default pool is correct. dict set result lookupError {Native route lookup failed for this record.} return $result } dict set result expectedPool $expected set actual [dict get $record pool] if {$actual eq "unknown"} { return $result } if {$expected eq $target} { dict set result category [expr {$actual eq $target ? "native-target-in-target" : "native-target-other-pool-candidate"}] } elseif {[dict exists $record userAgent] && [regexp -nocase {bot|crawl|spider|baidu|knowledge} [dict get $record userAgent]] && $actual ne $target} { dict set result category bot-like-routed-elsewhere-heuristic } else { dict set result category native-other-pool } return $result } # Preserve evidence for routing discrepancies before routine traffic can # consume the bounded result. Equal-priority groups retain first encounter. :method evidencePriority {category} { switch -- $category { matching-rule-other-pool-candidate - native-target-other-pool-candidate { return 0 } unknown { return 1 } default { return 2 } } } :method recordFields {record} { set fields {} dict for {key value} $record { if {$key eq "matchingRuleIndexes"} { set array {} set i 0 foreach item $value { lappend array $i number $item incr i } lappend fields $key array $array } elseif {$key in {startTime status responseBytes offset endOffset acceptTime queueTime filterTime runTime unknownRules count runTotal timingCount firstAt lastAt runMax}} { lappend fields $key number $value } elseif {$key in {redacted contextComplete requestIdentifierUnique phaseTimingAvailable}} { lappend fields $key boolean $value } else { lappend fields $key string $value } } return $fields } :method argumentTriples {options} { set fields {} dict for {key value} $options { if {$key eq "category" && $value eq ""} { continue } set type string if {$key in {includePeerAddresses aggregateTraffic}} { set type boolean } if {$key in {from to limit beforeOffset sourceBytes lastMinutes}} { set type number } lappend fields $key $type $value } return [list {} object $fields] } :method aggregateKey {record} { set key {} foreach field {category method path pool status userAgent expectedPool contextComplete lookupError matchingRuleIndexes peerAddress} { if {[dict exists $record $field]} { lappend key $field [dict get $record $field] } } return $key } # These source keys stay in private job state; only numeric distributions # leave this method. Never serialize the accumulator itself. :method accumulateTraffic {stateVar record} { upvar 1 $stateVar state set auth [dict get $record authenticationState] if {![dict exists $state authenticationStates $auth]} { dict set state authenticationStates $auth {requestCount 0 status503Count 0 successfulCount 0 successfulTimingCount 0 successfulRunTotal 0.0} } dict incr state requestCount set status [dict get $record status] set success [expr {$status>=200 && $status<300}] set timed [expr {$success && [dict exists $record runTime]}] set run [expr {$timed ? [dict get $record runTime] : 0.0}] if {$status==503} { dict incr state status503Count } if {$success} { dict incr state successfulCount } if {$timed} { dict incr state successfulTimingCount } dict set state successfulRunTotal [expr {[dict get $state successfulRunTotal]+$run}] set authRow [dict get $state authenticationStates $auth] dict incr authRow requestCount if {$status==503} { dict incr authRow status503Count } if {$success} { dict incr authRow successfulCount } if {$timed} { dict incr authRow successfulTimingCount } dict set authRow successfulRunTotal [expr {[dict get $authRow successfulRunTotal]+$run}] dict set state authenticationStates $auth $authRow set peer [dict get $record peerAddress] set valid [ns_ip valid $peer] if {$valid} { set peer [ns_crypto::md string -digest sha256 -- [list [dict get $state salt] $peer]] } if {!$valid} { dict incr state unknownPeerCount } elseif {![dict exists $state sources $peer] && [dict size [dict get $state sources]]>=${:maxAggregateSources}} { dict incr state untrackedSourceRecords } else { set count 0 if {[dict exists $state sources $peer]} { set count [dict get $state sources $peer] } dict set state sources $peer [expr {$count+1}] } set ua {} if {[dict exists $record userAgent]} { set ua [dict get $record userAgent] } if {$ua in {{} - {[REDACTED]}}} { dict incr state unknownUserAgentCount return } set ua [${:redactor} message $ua] if {[string length $ua]>2048 || (![dict exists $state agents $ua] && [dict size [dict get $state agents]]>=${:maxAggregateUserAgents})} { dict incr state untrackedUserAgentRecords return } if {![dict exists $state agents $ua]} { dict set state agents $ua {requestCount 0 status503Count 0 successfulCount 0 successfulTimingCount 0 successfulRunTotal 0.0} } set row [dict get $state agents $ua] dict incr row requestCount if {$status==503} { dict incr row status503Count } if {$success} { dict incr row successfulCount } if {$timed} { dict incr row successfulTimingCount } dict set row successfulRunTotal [expr {[dict get $row successfulRunTotal]+$run}] dict set state agents $ua $row } :method trafficTriples {state duration complete} { set fields [list scope string examined-selected-records windowComplete boolean false scanComplete boolean $complete] foreach key {requestCount status503Count successfulCount successfulTimingCount successfulRunTotal unknownPeerCount untrackedSourceRecords unknownUserAgentCount untrackedUserAgentRecords} { lappend fields $key number [dict get $state $key] } set authentication {} if {[dict exists $state authenticationStates]} { dict for {name row} [dict get $state authenticationStates] { set entry {} dict for {key value} $row { lappend entry $key number $value } lappend authentication $name object $entry } } lappend fields authenticationStates object $authentication set requests [dict get $state requestCount] lappend fields durationSeconds number $duration requestRate number [expr {$requests/double($duration)}] successfulTimingMissingCount number [expr {[dict get $state successfulCount]-[dict get $state successfulTimingCount]}] set peerReasons {} if {[dict get $state unknownPeerCount]} { lappend peerReasons 0 string missing-or-invalid-peer-fields } if {[dict get $state untrackedSourceRecords]} { lappend peerReasons 1 string source-cardinality-limit } set uaReasons {} if {[dict get $state unknownUserAgentCount]} { lappend uaReasons 0 string missing-user-agent-fields } if {[dict get $state untrackedUserAgentRecords]} { lappend uaReasons 1 string user-agent-cardinality-or-length-limit } set counts [lsort -integer -decreasing [dict values [dict get $state sources]]] set known [dict size [dict get $state sources]] lappend fields distinctKnownSources number $known observedPeerCountsComplete boolean [expr {$peerReasons eq ""}] peerCountsIncompleteReasons array $peerReasons observedUserAgentCountsComplete boolean [expr {$uaReasons eq ""}] userAgentCountsIncompleteReasons array $uaReasons sourceLimit number ${:maxAggregateSources} userAgentLimit number ${:maxAggregateUserAgents} if {$peerReasons eq ""} { lappend fields distinctSources number $known } else { lappend fields distinctSources null {} } set buckets {1 0 2-5 0 6-10 0 11-100 0 101-1000 0 1001+ 0} foreach count $counts { set bucket [expr {$count==1 ? "1" : ($count<=5 ? "2-5" : ($count<=10 ? "6-10" : ($count<=100 ? "11-100" : ($count<=1000 ? "101-1000" : "1001+"))))}] dict incr buckets $bucket } set histogram {} dict for {key count} $buckets { lappend histogram $key number $count } lappend fields sourcesByRequestCount object $histogram set top {} set rank 0 set sum 0 foreach count [lrange $counts 0 9] { incr sum $count lappend top $rank object [list rank number [expr {$rank+1}] requestCount number $count requestRate number [expr {$count/double($duration)}] share number [expr {$requests ? $count/double($requests) : 0.0}]] incr rank } lappend fields topSources array $top topTenSourceShare number [expr {$requests ? $sum/double($requests) : 0.0}] set rankings {} foreach {label metric} {byRequests requestCount by503 status503Count bySuccessfulRuntime successfulRunTotal} { set ordered {} dict for {ua row} [dict get $state agents] { lappend ordered [list [dict get $row $metric] $ua $row] } set output {} set i 0 foreach item [lrange [lsort -real -decreasing -index 0 $ordered] 0 19] { lassign $item value ua row set entry [list userAgent string $ua] dict for {key count} $row { lappend entry $key number $count } lappend output $i object $entry incr i } lappend rankings $label array $output } lappend fields userAgents object $rankings distinctKnownUserAgents number [dict size [dict get $state agents]] rankingLimit number 20 omittedRankedUserAgents number [expr {max(0,[dict size [dict get $state agents]]-20)}] limitations string {Counts cover examined records selected by window, path and category, before evidence truncation. Rankings cover tracked redacted User-Agents, not verified bots. Source keys and addresses are never disclosed. Unknown fields and cardinality limits make observed distributions partial. Seek assumptions and rotations still limit time-window coverage; observed completeness is not window completeness. Rates use the requested window duration, not a rolling burst. Successful runtime is summed 2xx handler time, not CPU time.} return $fields } # Seeking never certifies chronological order: concurrent requests and # clock changes can reorder completions. Probe failures fall back to full. :method seekSample {channel position size samplesVar bytesVar} { upvar 1 $samplesVar samples $bytesVar probeBytes seek $channel $position set block [read $channel [expr {min(65536,$size-$position)}]] incr probeBytes [string length $block] set offset $position if {$position>0} { set cut [string first \n $block] if {$cut<0} { error "No complete timestamped line at seek probe" } incr offset [expr {$cut+1}] set block [string range $block [expr {$cut+1}] end] } foreach line [lrange [split $block \n] 0 end-1] { set end [expr {$offset+[string length $line]+1}] set at [::nsmcp::accessRecordParser completionTime [encoding convertfrom utf-8 $line]] if {$at ne ""} { dict set samples $offset $at set previous -1 foreach key [lsort -integer [dict keys $samples]] { set time [dict get $samples $key] if {$time<$previous} { error "Completion timestamps are not ordered at seek probes" } set previous $time } return [list $offset $end $at] } set offset $end } error "No timestamp at seek probe" } :method scanBounds {context options} { set path [:source [$context server] $options] file stat $path before set size $before(size) set bounds [dict create from 0 to $size sourceBytes $size strategy full reason {Full scan requested or source below seek threshold.} probeBytes 0] if {[dict get $options scanStrategy] eq "full" || $size<${:seekMinimumBytes}} { return $bounds } set f [open $path rb] set samples {} set probeBytes 0 try { set prefix [read $f [expr {min(256,$size)}]] set identity [ns_crypto::md string -digest sha256 -- [list [file normalize $path] $before(dev) $before(ino) $prefix]] set first [:seekSample $f 0 $size samples probeBytes] set last [:seekSample $f [expr {max(0,$size-65536)}] $size samples probeBytes] # Spread probes across the file before assuming monotonicity. for {set i 1} {$i<16} {incr i} { :seekSample $f [expr {$size*$i/16}] $size samples probeBytes } foreach {key target} [list from [expr {[dict get $options from]-300}] to [expr {[dict get $options to]+300}]] { if {$target<=[lindex $first 2]} { dict set bounds $key 0 continue } if {$target>=[lindex $last 2]} { dict set bounds $key $size continue } set low $first set high $last for {set i 0} {$i<40 && [lindex $high 0]-[lindex $low 0]>131072} {incr i} { set middle [:seekSample $f [expr {([lindex $low 0]+[lindex $high 0])/2}] $size samples probeBytes] if {[lindex $middle 2]<$target} { set low $middle } else { set high $middle } } dict set bounds $key [expr {$key eq "from" ? [lindex $low 0] : [lindex $high 1]}] } # A lower bound beyond the final sample still examines its tail. if {[dict get $bounds from]==$size} { dict set bounds from [lindex $last 0] } file stat $path after if {$before(dev)!=$after(dev) || $before(ino)!=$after(ino) || $after(size)<$size} { error "Source changed during seek" } dict set bounds identity $identity dict set bounds strategy completion-window dict set bounds reason {Sampled completion timestamps were ordered; unsampled disorder and requests completing more than 300s outside the start-time window may be missed. Use scanStrategy full to avoid seek assumptions.} } on error {message} { dict set bounds from 0 dict set bounds to $size dict set bounds reason "Seek unavailable; full-file fallback: $message" } finally { close $f } dict set bounds probeBytes $probeBytes return $bounds } # One bounded page per ns_job invocation. Only bounded aggregate evidence # and counters cross worker boundaries, never the raw log contents. :public method scanStep {context state} { # # Run one bounded routing-diagnostic scan slice. # # Retain immutable bounds, filters, aggregation and coverage state so later # slices continue the same scan. # # @param context Authenticated request context carrying the principal, # grants and selected server. # @param state Opaque retained provider continuation or scan state. # # @return The updated scan state and diagnostic evidence. # set options [dict get $state options] if {![dict exists $state bounds]} { if {![dict exists $options scanStrategy]} { dict set options scanStrategy window } set bounds [:scanBounds $context $options] dict set state bounds $bounds if {[dict get $bounds strategy] eq "completion-window"} { dict set options beforeOffset [dict get $bounds to] dict set options sourceBytes [dict get $bounds sourceBytes] dict set options sourceIdentity [dict get $bounds identity] } } set bounds [dict get $state bounds] set aggregateTraffic [expr {[dict exists $options aggregateTraffic] && [dict get $options aggregateTraffic]}] set traffic {} if {$aggregateTraffic} { if {[dict exists $state traffic]} { set traffic [dict get $state traffic] } else { set traffic {sources {} agents {} authenticationStates {} requestCount 0 status503Count 0 successfulCount 0 successfulTimingCount 0 successfulRunTotal 0.0 unknownPeerCount 0 untrackedSourceRecords 0 unknownUserAgentCount 0 untrackedUserAgentRecords 0} dict set traffic salt [ns_crypto::randombytes -encoding hex 32] } } set page [:page $context [:argumentTriples $options] [dict get $bounds from] [expr {$aggregateTraffic ? "traffic" : ""}]] if {$aggregateTraffic} { dict set state traffic $traffic } set value [ns_json parse [ns_json value -- $page]] if {![dict get $value available]} { return [list $page {} {}] } if {![dict exists $state fields]} { dict set state fields [lindex $page 2] dict set state groups {} dict set state totals {} dict set state matches 0 dict set state selected 0 dict set state pages 0 dict set state coverage [dict get $value coverage] } else { set initial [ns_json parse [ns_json value -- [list {} object [dict get $state fields]]]] foreach key {sourceIdentity mappingIdentity from to lookupMode selectedCategory} { if {[dict get $initial $key] ne [dict get $value $key]} { error "Routing scan identity changed; start a new query" } } set coverage [dict get $state coverage] set nextCoverage [dict get $value coverage] foreach key {unrecognizedRecords oversizedRecords missingUserAgent missingPool} { dict incr coverage $key [dict get $nextCoverage $key] } dict set coverage pendingFinalRecord [expr {[dict get $coverage pendingFinalRecord] || [dict get $nextCoverage pendingFinalRecord]}] dict set coverage examinedFromOffset [dict get $nextCoverage examinedFromOffset] dict set coverage unreadEarlierBytes [dict get $nextCoverage unreadEarlierBytes] dict set state coverage $coverage } dict incr state pages dict incr state matches [dict get $value matches] dict incr state selected [dict get $value selectedMatches] set totals [dict get $state totals] dict for {category count} [dict get $value categoryTotals] { dict incr totals $category $count } dict set state totals $totals set groups [dict get $state groups] foreach record [dict get $value records] { set key [:aggregateKey $record] if {[dict exists $groups $key]} { set group [dict get $groups $key] foreach field {count timingCount} { dict incr group $field [dict get $record $field] } dict set group runTotal [expr {[dict get $group runTotal]+[dict get $record runTotal]}] dict set group runMax [expr {max([dict get $group runMax],[dict get $record runMax])}] dict set group firstAt [expr {min([dict get $group firstAt],[dict get $record firstAt])}] dict set group lastAt [expr {max([dict get $group lastAt],[dict get $record lastAt])}] dict set groups $key $group } else { if {[dict size $groups]>=[dict get $options limit]} { set worst [:evidencePriority [dict get $record category]] set victim {} dict for {otherKey otherRecord} $groups { set rank [:evidencePriority [dict get $otherRecord category]] if {$rank>$worst} { set worst $rank set victim $otherKey } } if {$victim eq ""} { continue } dict unset groups $victim } dict set groups $key $record } } # Bound state by the same evidence byte budget used for individual pages. set ordered {} dict for {key group} $groups { lappend ordered [list [:evidencePriority [dict get $group category]] $key $group] } set groups {} set entries {} set index 0 set bytes 0 set returned 0 set omitted {} dict for {category count} $totals { if {[dict get $value selectedCategory] eq "" || [dict get $value selectedCategory] eq $category} { dict set omitted $category $count } } foreach pair [lsort -integer -index 0 $ordered] { lassign $pair priority key record set fields [:recordFields $record] set size [string bytelength [ns_json value -- [list {} object $fields]]] if {$bytes+$size>65536} { continue } incr bytes $size dict set groups $key $record lappend entries $index object $fields incr index incr returned [dict get $record count] dict incr omitted [dict get $record category] [expr {-[dict get $record count]}] } dict set state groups $groups set counts {} dict for {category count} $totals { lappend counts $category number $count } set omittedCounts {} dict for {category count} $omitted { lappend omittedCounts $category number $count } set coverage [dict get $state coverage] set coverageFields {} foreach {key type original} [ns_json triples getvalue -output triples -path coverage -- $page] { lappend coverageFields $key $type [dict get $coverage $key] } lappend coverageFields scanStrategy string [dict get $bounds strategy] seekReason string [dict get $bounds reason] seekMarginSeconds number 300 seekProbeBytes number [dict get $bounds probeBytes] skippedEarlierBytes number [dict get $bounds from] skippedLaterBytes number [expr {[dict get $bounds sourceBytes]-[dict get $bounds to]}] completionOrderAssumed boolean [expr {[dict get $bounds strategy] eq "completion-window"}] lappend coverageFields pages number [dict get $state pages] aggregationScope string all-examined-pages sourceFullyExamined boolean [expr {[dict get $coverage unreadEarlierBytes]==0 && [dict get $coverage examinedToOffset]==[dict get $coverage sourceBytes]}] set fields {} foreach {key type original} [dict get $state fields] { switch -- $key { matches { set original [dict get $state matches] } selectedMatches { set original [dict get $state selected] } filteredOutRecords { set original [expr {[dict get $state matches]-[dict get $state selected]}] } categoryTotals { set original $counts } omittedRecords { set original [expr {[dict get $state selected]-$returned}] } omittedCategoryTotals { set original $omittedCounts } records { set original $entries } more { set original false } nextArguments { set original {} } coverage { set original $coverageFields } } lappend fields $key $type $original } if {$aggregateTraffic} { lappend fields trafficAggregation object [:trafficTriples $traffic [expr {[dict get $options to]-[dict get $options from]}] [expr {![dict get $value more]}]] } lappend fields scanComplete boolean [expr {![dict get $value more]}] set continuation {} if {[dict get $value more]} { set nextOptions [dict get $value nextArguments] dict set nextOptions aggregateTraffic $aggregateTraffic dict set state options $nextOptions set continuation $state } set progress [list examinedBytes number [expr {[dict get $bounds to]-[dict get $coverage unreadEarlierBytes]}] sourceBytes number [expr {[dict get $bounds to]-[dict get $bounds from]}]] return [list [list {} object $fields] $continuation $progress] } :public method execute {context arguments} { # # Run a bounded access-routing diagnostic operation. # # Validate the configured mode, require scan permissions where appropriate # and retain unknown routing evidence. # # @param context Authenticated request context carrying the principal, # grants and selected server. # @param arguments Rooted ns_json typed triples for the tool's JSON argument # object. # # @return A rooted JSON tool-result object, including availability or job # metadata as appropriate. # set options [:options $arguments] if {${:operation} eq "bot" && [dict get $options mode] eq "scan"} { ${:authorizer} require $context diagnostics.runtime.read if {[dict get $options includePeerAddresses]} { ${:authorizer} require $context diagnostics.logs.peers.read } if {${:service} eq ""} { error "Routing background service is not configured" } dict set options mode page dict unset options lastMinutes return [${:service} start $context routingSummary [dict create tool [self] options $options]] } if {[dict get $options aggregateTraffic]} { error "aggregateTraffic requires bot routing mode scan" } return [:page $context $arguments] } :public method page {context arguments {floor 0} {aggregationVar {}}} { # # Build one bounded routing-diagnostic evidence page. # # Validate options and authorize the requested evidence; optionally update # the caller's aggregation variable. # # @param context Authenticated request context carrying the principal, # grants and selected server. # @param arguments Rooted ns_json typed triples for the tool's JSON argument # object. # @param floor Minimum candidate position used by the caller's paging # operation. Defaults to '0'. # @param aggregationVar Optional caller variable receiving traffic # aggregation state. # # @return The rooted JSON evidence page. # if {$aggregationVar ne ""} { upvar 1 $aggregationVar traffic } set options [:options $arguments] set server [$context server] set peers [dict get $options includePeerAddresses] if {$peers} { ${:authorizer} require $context diagnostics.logs.peers.read } set redactor [expr {$peers ? ${:peerRedactor} : ${:redactor}}] set result [list server string $server from number [dict get $options from] to number [dict get $options to]] foreach key {path authenticationState} { if {[dict exists $options $key]} { lappend result $key string [dict get $options $key] } } set snapshot {} set maps {} set mappingIdentity {} set lookupMode fallback-rule-membership set routeCache {} if {${:operation} eq "bot"} { ${:authorizer} require $context diagnostics.runtime.read set snapshot [${:routing} snapshot $server] set target [dict get $options pool] if {![dict exists $snapshot pools $target]} { return [list {} object [concat $result [list available boolean false reason string {Effective target-pool mappings unavailable.}]]] } set maps [dict get $snapshot pools $target] if {[${:routing} contextSupported $server]} { set lookupMode native-context } set category [dict get $options category] if {$category ne "" && $category ne "unknown"} { set nativeCategory [expr {$category in { native-target-in-target native-target-other-pool-candidate bot-like-routed-elsewhere-heuristic native-other-pool }}] if {$nativeCategory != ($lookupMode eq "native-context")} { return -code error -errorcode {NSMCP RPC -32602} "Category '$category' is not produced by lookupMode '$lookupMode'; select a category for the current mode." } } set mappingIdentity [ns_crypto::md string -digest sha256 -- [list [dict get $snapshot pools] $lookupMode]] if {[dict exists $options mappingIdentity] && [dict get $options mappingIdentity] ne $mappingIdentity} { error "Effective mappings changed; start a new analysis" } lappend result pool string $target lookupMode string $lookupMode mappingIdentity string $mappingIdentity {*}[${:routing} encodeSnapshot $snapshot] } if {${:operation} eq "slow"} { set thresholds {} foreach key {slowlogconntime slowlogfiltertime slowlogqueuetime} { set value [ns_config ns/server/$server/module/[dict get $options module] $key {}] lappend thresholds $key string $value } lappend result configuredThresholds object $thresholds } set path [:source $server $options] if {$path eq "" || ![file isfile $path] || ![file readable $path]} { return [list {} object [concat $result [list available boolean false reason string {Configured log is missing, disabled or unreadable.}]]] } set path [file normalize $path] file stat $path before set channel [open $path rb] try { set prefix [read $channel [expr {min(256,$before(size))}]] set identity [ns_crypto::md string -digest sha256 -- [list $path $before(dev) $before(ino) $prefix]] set sourceBytes $before(size) set end $sourceBytes if {[dict exists $options beforeOffset]} { if {$identity ne [dict get $options sourceIdentity] || $sourceBytes<[dict get $options sourceBytes]} { error "Log rotated or truncated; start a new query" } set sourceBytes [dict get $options sourceBytes] set end [dict get $options beforeOffset] } set start [expr {max($floor,$end-${:maxReadBytes})}] seek $channel $start set bytes [read $channel [expr {$end-$start}]] set discarded 0 if {$start>0} { seek $channel [expr {$start-1}] if {[read $channel 1] ne "\n"} { set newline [string first \n $bytes] if {$newline>=0} { set discarded [expr {$newline+1}] } else { set discarded [string length $bytes] } } } file stat $path after if {[list $before(dev) $before(ino)] ne [list $after(dev) $after(ino)] || $after(size)<$sourceBytes} { error "Log changed during read; start a new query" } } finally { close $channel } set position [expr {$start+$discarded}] set cursor $position if {$cursor==$end && $start>0} { set cursor $start } set bytes [string range $bytes $discarded end] set pending [expr {$bytes ne "" && [string index $bytes end] ne "\n"}] set records {} set groups {} set totals {} set matches 0 set unrecognized 0 set oversized 0 set missingUA 0 set missingPool 0 set selectedMatches 0 foreach line [lrange [split $bytes \n] 0 end-1] { set offset $position incr position [expr {[string length $line]+1}] if {[string length $line]>65536} { incr oversized continue } set record [::nsmcp::accessRecordParser parse [encoding convertfrom utf-8 [string trimright $line \r]]] if {$record eq ""} { incr unrecognized continue } set at [dict get $record startTime] if {$at<[dict get $options from] || $at>=[dict get $options to]} { continue } if {[dict exists $options path] && [dict get $record path] ne [dict get $options path]} { continue } if {[dict exists $options authenticationState] && [dict get $record authenticationState] ne [dict get $options authenticationState]} { continue } incr matches dict set record offset $offset dict set record endOffset $position if {![dict exists $record userAgent]} { incr missingUA } if {[dict get $record pool] eq "unknown"} { incr missingPool } if {${:operation} eq "bot"} { if {$lookupMode eq "native-context"} { set routeKey [list [dict get $record method] [dict get $record path] [dict get $record peerAddress] [dict get $record pool] [dict exists $record userAgent] [:dict getdef $record userAgent ""]] if {[dict exists $routeCache $routeKey]} { set classification [dict get $routeCache $routeKey] } else { set classification [:nativeClassify $server $record $snapshot [dict get $options pool]] if {[dict size $routeCache]<4096} { dict set routeCache $routeKey $classification } } } else { set classification [:classify $record $maps [dict get $options pool]] dict set classification lookupMode fallback-rule-membership dict set classification expectedPool unknown dict set classification contextComplete false dict set classification lookupError {} } set category [dict get $classification category] dict incr totals $category if {[dict get $options category] ne "" && [dict get $options category] ne $category} { continue } incr selectedMatches if {$aggregationVar ne ""} { :accumulateTraffic traffic $record } # Group only after peer and URL redaction. Omitted groups still # contribute to category totals and omittedRecords. set record [$redactor record $record] if {[dict exists $record userAgent]} { dict set record userAgent [$redactor message [dict get $record userAgent]] } set record [dict merge $record $classification] set key [list [dict get $record category] [dict get $record method] [dict get $record path] [dict get $record pool] [dict get $record status]] lappend key [:dict getdef $record userAgent "unknown"] lappend key [dict get $record expectedPool] [dict get $record contextComplete] [dict get $record lookupError] if {$peers} { lappend key [dict get $record peerAddress] } if {![dict exists $groups $key]} { if {[dict size $groups]>=[dict get $options limit]} { set priority [:evidencePriority $category] set victim {} set worst $priority dict for {existingKey existingGroup} $groups { set rank [:evidencePriority [dict get $existingGroup category]] if {$rank>$worst} { set worst $rank set victim $existingKey } } if {$victim eq ""} { continue } dict unset groups $victim } set group [dict create method [dict get $record method] path [$redactor queryValues [dict get $record path]] pool [dict get $record pool] status [dict get $record status] userAgent [lindex $key 5] category [dict get $record category] matchingRuleIndexes [dict get $record matchingRuleIndexes] unknownRules [dict get $record unknownRules] sampleRequestIdentifier [dict get $record requestIdentifier] firstAt $at lastAt $at count 0 runTotal 0 runMax 0 timingCount 0] foreach field {lookupMode expectedPool contextComplete lookupError} { dict set group $field [dict get $record $field] } if {$peers} { dict set group peerAddress [dict get $record peerAddress] } dict set groups $key $group } set group [dict get $groups $key] dict incr group count dict set group firstAt [expr {min([dict get $group firstAt],$at)}] dict set group lastAt [expr {max([dict get $group lastAt],$at)}] if {[dict exists $record runTime]} { dict incr group timingCount dict set group runMax [expr {max([dict get $group runMax],[dict get $record runTime])}] dict set group runTotal [expr {[dict get $group runTotal]+[dict get $record runTime]}] } dict set groups $key $group } else { set record [$redactor record $record] if {[dict exists $record userAgent]} { dict set record userAgent [$redactor message [dict get $record userAgent]] } lappend records $record set records [lrange $records end-[expr {[dict get $options limit]-1}] end] } } if {${:operation} eq "bot"} { set latest [${:routing} snapshot $server] set latestIdentity [ns_crypto::md string -digest sha256 -- [list [dict get $latest pools] $lookupMode]] if {$latestIdentity ne $mappingIdentity} { error "Effective mappings changed during analysis; start a new query" } } set entries {} set i 0 if {${:operation} eq "bot"} { set ordered {} dict for {key group} $groups { lappend ordered [list [:evidencePriority [dict get $group category]] $group] } # The common output loop reverses records for slow-log chronology. set records {} foreach pair [lsort -integer -index 0 $ordered] { lappend records [lindex $pair 1] } set records [lreverse $records] } else { set selectedMatches $matches } set returnedRecords {} set resultBytes 0 foreach record [lreverse $records] { set fields [:recordFields $record] set bytes [string length [ns_json value -- [list {} object $fields]]] if {$resultBytes+$bytes>65536} { continue } incr resultBytes $bytes lappend returnedRecords $record lappend entries $i object $fields incr i } set records $returnedRecords set counts {} dict for {key value} $totals { lappend counts $key number $value } set omittedCounts {} set returnedMatches [llength $records] if {${:operation} eq "bot"} { set remaining {} dict for {category value} $totals { if {[dict get $options category] eq "" || [dict get $options category] eq $category} { dict set remaining $category $value } } set returnedMatches 0 foreach record $records { set count [dict get $record count] incr returnedMatches $count dict incr remaining [dict get $record category] [expr {-$count}] } dict for {category value} $remaining { lappend omittedCounts $category number $value } } set next {} if {$cursor>$floor} { foreach key {from to limit pool module includePeerAddresses mode} { lappend next $key [expr {$key eq "includePeerAddresses" ? "boolean" : ($key in {from to limit} ? "number" : "string")}] [dict get $options $key] } if {${:operation} eq "bot" && [dict get $options category] ne ""} { lappend next category string [dict get $options category] } foreach key {path authenticationState} { if {[dict exists $options $key]} { lappend next $key string [dict get $options $key] } } lappend next beforeOffset number $cursor sourceBytes number $sourceBytes sourceIdentity string $identity if {$mappingIdentity ne ""} { lappend next mappingIdentity string $mappingIdentity } } set evidencePolicy {Newest matching slow-request records within evidence limits.} if {${:operation} eq "bot"} { set evidencePolicy {Routing candidates first, then unknowns, then routine groups; first encounter within priority. Category filtering selects evidence only; categoryTotals still includes all observed records.} } lappend result available boolean true sourcePath string $path sourceIdentity string $identity matches number $matches categoryTotals object $counts records array $entries selectedCategory string [dict get $options category] selectedMatches number $selectedMatches filteredOutRecords number [expr {$matches-$selectedMatches}] omittedRecords number [expr {$selectedMatches-$returnedMatches}] omittedCategoryTotals object $omittedCounts evidencePolicy string $evidencePolicy more boolean [expr {$cursor>$floor}] nextArguments object $next coverage object [list sourceBytes number $sourceBytes examinedFromOffset number [expr {$start+$discarded}] examinedToOffset number $end unreadEarlierBytes number $cursor pendingFinalRecord boolean $pending unrecognizedRecords number $unrecognized oversizedRecords number $oversized missingUserAgent number $missingUA missingPool number $missingPool rotationsExamined boolean false windowComplete boolean false limitations string {Page-local counts; sum disjoint pages. Current file only. Full-window coverage is not certified. Mappings reflect query time, not historical configuration. lookupMode identifies native-context or fallback-rule-membership. Native lookup uses current URL/path precedence and logged User-Agent and valid peer IP, without disclosing peers by default. Unlogged context predicates remain unknown; completeness checks conservatively inspect all same-method rules. Older versions retain root wildcard User-Agent membership analysis, not precedence replay. Bot-like unmatched uses a separate case-insensitive heuristic, not proof of a bot. Slow-log timings are access-log timings, not SQL profiles.}] return [list {} object $result] }XQL Not present: Generic, PostgreSQL, Oracle
