Class ::nsmcp::AccessRecordParser (public)
::nx::Class ::nsmcp::AccessRecordParser![[i]](/resources/acs-subsite/ZoomIn16.gif)
Defined in /usr/local/ns/tcl/nsmcp/access-record.tcl
-1 detects common/combined layouts; -2 disables timing detection. Explicit 0 (common) or 2 (combined) can pin the configured column.
- Testcases:
- No testcase defined.
Source code: :property {timingColumn:integer -1} :method init {} { if {${:timingColumn} ni {-2 -1 0 2}} { error "timingColumn must be -2, -1, 0, or 2" } } :public method completionTime {line} { # # Extract the completion timestamp from an access-log line. # # Accept the supported epoch or common-log timestamp without interpreting # request data as executable Tcl. # # @param line Raw physical log line to inspect. # # @return Epoch seconds, or an empty result when the timestamp cannot be # parsed. # if {![regexp {^\S+\s+\S+\s+(?:"(?:[^"\\]|\\.)*"|\S+)\s+(\[[^\]\r\n]+\]|[0-9]+)\s+"} $line -> stamp]} { return {} } set stamp [string trim $stamp {[]}] if {[string is entier -strict $stamp]} { return $stamp } if {[catch { clock scan $stamp -format {%d/%b/%Y:%H:%M:%S %z} -locale C } time]} { return {} } return $time } # Called only for rejected rows, so diagnostics add no valid-row overhead. :public method failureReason {line} { # # Classify why an access-log line could not be parsed. # # This diagnostic path runs only for rejected rows and distinguishes layout, # timestamp and request failures. # # @param line Raw physical log line to inspect. # # @return The failure category: layout, timestamp or request. # if {[string first \x00 $line]>=0} { return layout } if {![regexp {^\S+\s+\S+\s+(?:"(?:[^"\\]|\\.)*"|\S+)\s+(\[[^\]\r\n]+\]|[0-9]+)\s+"((?:[^"\\\r\n]|\\.)*)"\s+([1-5][0-9]{2})\s+(-|[0-9]+)(.*)$} $line]} { return layout } if {[:completionTime $line] eq ""} { return timestamp } return request } # Apply the same plausibility and completion fallback policy to both # scanner paths. Values entering this method are seconds, not log text. :method timingValues {timestamp start phases} { set result {} if {$start ne {} && $start >= 946684800} { if {$start > [clock seconds] + 86400} { dict set result timeDependent true } else { dict set result startTime $start dict set result timingSource start foreach key {acceptTime queueTime filterTime runTime} value $phases { dict set result $key [expr {double($value)}] } } } if {![dict exists $result startTime]} { if {[string is entier -strict $timestamp]} { set time $timestamp } elseif {[catch { clock scan $timestamp -format {%d/%b/%Y:%H:%M:%S %z} -locale C } time]} { return {} } dict set result startTime $time dict set result timingSource completion } return $result } :method timingFields {timestamp fields} { set index ${:timingColumn} if {$index == -1} { set index [expr {[llength $fields] >= 2 ? 2 : 0}] } if {$index >= 0} { set field [string trim [lindex $fields $index] {"}] if {[regexp {^([0-9]+\.[0-9]+) ([0-9]+(?:\.[0-9]+)?) ([0-9]+(?:\.[0-9]+)?) ([0-9]+(?:\.[0-9]+)?) ([0-9]+(?:\.[0-9]+)?)$} $field -> start accept queue filter run]} { return [:timingValues $timestamp $start [list $accept $queue $filter $run]] } } return [:timingValues $timestamp {} {}] } # The C scanner and batch sink have already parsed and range-checked # integer microseconds. Convert units once, without rebuilding log fields. :public method timingMicroseconds {timestamp start accept queue filter run} { # # Convert C-parsed request timing values to the parser's timing fields. # # Use supplied microsecond values directly, normalizing start time and phase # durations to seconds. # # @param timestamp Parsed timestamp or timing base supplied by the parser. # @param start Request start timestamp in microseconds. # @param accept Accept-phase duration in microseconds. # @param queue Queue-phase duration in microseconds. # @param filter Filter-phase duration in microseconds, or the line-prefilter # flag for nextLine. # @param run Run-phase duration in microseconds. # # @return A timing dictionary containing the available parsed phases. # set phases {} if {$start ne {}} { set start [expr {$start / 1000000.0}] foreach value [list $accept $queue $filter $run] { lappend phases [expr {$value eq {} ? 0.0 : $value / 1000000.0}] } } return [:timingValues [string trim $timestamp {[]}] $start $phases] } :public method indexTime {line} { # # Extract a bounded timestamp suitable for access-log indexing. # # Reject NUL-bearing or unsupported lines before deriving the record's index # time. # # @param line Raw physical log line to inspect. # # @return The index timestamp, or an empty result for an unusable line. # if {[string first \x00 $line]>=0} { return {} } if {![regexp {^\S+\s+\S+\s+(?:"(?:[^"\\]|\\.)*"|\S+)\s+(\[[^\]\r\n]+\]|[0-9]+)\s+"(?:[^"\\\r\n]|\\.)*"\s+[1-5][0-9]{2}\s+(?:-|[0-9]+)(.*)$} $line -> stamp tail]} { return {} } set fields [regexp -all -inline {"(?:[^"\\]|\\.)*"} $tail] set timing [:timingFields [string trim $stamp {[]}] $fields] if {$timing eq "" || [dict exists $timing timeDependent]} { return {} } return [dict get $timing startTime] } :public method parse {line} { # # Parse one physical access-log line into a request record. # # Validate the supported layout and delegate decoded fields to the common # field normalizer. # # @param line Raw physical log line to inspect. # # @return A request dictionary, or an empty result for a rejected line. # if {[string first \x00 $line]>=0} { return {} } if {![regexp {^(\S+)\s+(\S+)\s+("(?:[^"\\]|\\.)*"|\S+)\s+(\[[^\]\r\n]+\]|[0-9]+)\s+"((?:[^"\\\r\n]|\\.)*)"\s+([1-5][0-9]{2})\s+(-|[0-9]+)(.*)$} $line -> peer identifier user stamp request status bytes tail]} { return {} } return [:fields $peer $identifier $user $stamp $request $status $bytes $tail] } # Tcl framing delivers lexical fields; decode quoted content once here. # C scanning enters requestFields directly with decoded, typed values. :public method fields {peer identifier user stamp request status bytes tail {scannedFields {}} {parsedTiming {}}} { # # Normalize decoded access-log fields into a request record. # # Combine scanned metadata and optional C-parsed timing with the same # validation used by Tcl scanning. # # @param peer Logged peer address, including the supported unknown-value # sentinel. # @param identifier Complete log-thread identifier, or the raw identifier # field when parsing access records. # @param user Logged user identifier used to derive authentication state. # @param stamp Original access-log timestamp field. # @param request Decoded HTTP request-line field. # @param status Decoded HTTP response status. # @param bytes Logged response-byte value, retaining the supported # unknown-value sentinel. # @param tail Remaining access-log fields after status and response size. # @param scannedFields Optional metadata already decoded by the C scanner. # @param parsedTiming Optional timing dictionary already decoded by the C # scanner. # # @return A normalized request dictionary, or an empty result for invalid # fields. # if {![regexp {^[1-5][0-9]{2}$} $status]} { return {} } set timestamp [string trim $stamp {[]}] set fields $scannedFields if {$fields eq {}} { set fields [regexp -all -inline {"(?:[^"\\]|\\.)*"} $tail] } set combined [expr {[llength $fields]>=2}] set agent {} set escapes [list {\\} \\ {\"} \" {\n} \n {\r} \r {\t} \t] if {$combined} { set agent [string map $escapes [string range [lindex $fields 1] 1 end-1]] } set request [string map $escapes $request] if {[string index $user 0] eq {"}} { set user [string map $escapes [string range $user 1 end-1]] } if {$parsedTiming eq {}} { set parsedTiming [:timingFields $timestamp $fields] } if {$parsedTiming eq {}} { return {} } set responseBytes 0 if {$bytes ne "-"} { set responseBytes [scan $bytes %lld] } return [:requestFields $peer $identifier $user $timestamp $request [scan $status %d] $responseBytes $combined $agent $parsedTiming] } # Shared request policy consumes decoded fields. No log quoting or lexical # numeric conversion belongs on this path. :public method requestFields {peer identifier user timestamp request status bytes combined agent timing} { # # Build a request record from validated fields and timing. # # Derive authentication and request metadata while retaining explicit # unknown-value representations. # # @param peer Logged peer address, including the supported unknown-value # sentinel. # @param identifier Complete log-thread identifier, or the raw identifier # field when parsing access records. # @param user Logged user identifier used to derive authentication state. # @param timestamp Parsed timestamp or timing base supplied by the parser. # @param request Decoded HTTP request-line field. # @param status Decoded HTTP response status. # @param bytes Logged response-byte value, retaining the supported # unknown-value sentinel. # @param combined Whether the decoded access line uses the combined format. # @param agent Decoded User-Agent value. # @param timing Normalized request-timing dictionary. # # @return The normalized request dictionary. # set authenticationState [expr {$user in {0 -} ? "anonymous" : ($user in {{} {[REDACTED]}} ? "unknown" : "authenticated")}] set result [dict create authenticationState $authenticationState timestamp $timestamp peerAddress $peer requestIdentifier {} pool unknown method unknown url {} status $status responseBytes $bytes timingSource completion] if {[::nsmcp::logRecordParser validRequestIdentifier $identifier]} { dict set result requestIdentifier $identifier } set result [dict merge $result [::nsmcp::accessRecordIdentity $identifier]] if {[dict get $result recordSource] eq "driver"} { # Request authentication was not performed by an application worker. dict set result authenticationState unknown } # Empty/binary/probe requests remain valid access records. Extract the # ends of a request line, allowing embedded spaces in the logged target. set first [string first " " $request] set last [string last " " $request] if {$first>0 && $last>$first} { set protocol [string range $request [expr {$last+1}] end] if {[string match */* $protocol]} { dict set result method [string range $request 0 [expr {$first-1}]] dict set result url [string range $request [expr {$first+1}] [expr {$last-1}]] dict set result protocol $protocol } } elseif {[string match /* $request]} { dict set result url $request } if {$combined && ${:timingColumn} in {-2 -1 2}} { dict set result userAgent $agent } set result [dict merge $result $timing] set available [expr {[dict get $timing timingSource] eq "start" && [dict get $result recordSource] ne "driver"}] dict set result phaseTimingAvailable $available dict set result phaseTimingReason [expr {$available ? "logged-worker-phases" : ([dict get $result recordSource] eq "driver" ? "driver-synthetic-phases" : "no-phase-counters")}] if {!$available} { foreach key {acceptTime queueTime filterTime runTime} { dict unset result $key } } set path [lindex [split [dict get $result url] ?] 0] if {[regexp {^[A-Za-z][A-Za-z0-9+.-]*://[^/]*(/.*)?$} $path -> suffix]} { set path [expr {$suffix eq "" ? "/" : $suffix}] } dict set result path $path return $result }XQL Not present: Generic, PostgreSQL, Oracle
