diff --git a/ModuleConfig.cfc b/ModuleConfig.cfc index 34ebb00..1f7a5ff 100644 --- a/ModuleConfig.cfc +++ b/ModuleConfig.cfc @@ -40,42 +40,115 @@ component { function configure(){ settings = { // Sentry token - "ServerSideToken" : "", + "ServerSideToken" : "", // Enable the Sentry LogBox Appender Bridge - "enableLogBoxAppender" : true, - "async" : true, + "enableLogBoxAppender" : true, + "async" : true, // Min/Max levels for appender - "levelMin" : "FATAL", - "levelMax" : "ERROR", + "levelMin" : "FATAL", + "levelMax" : "ERROR", // Enable/disable error logging - "enableExceptionLogging" : true, + "enableExceptionLogging" : true, // Sentry recommends not sending cookie and form data by default - "sendCookies" : false, - "sendPostData" : false, + "sendCookies" : false, + "sendPostData" : false, // Data sanitization, scrub fields and headers, replaced with "[Filtered]" at runtime - "scrubFields" : [], - "scrubHeaders" : [], - "release" : "", - "environment" : ( !isNull( controller ) ? controller.getSetting( "environment" ) : "" ), - "DSN" : "", - "publicKey" : "", - "privateKey" : "", - "projectID" : 0, - "sentryUrl" : "https://sentry.io", + "scrubFields" : [], + "scrubHeaders" : [], + "release" : "", + "environment" : ( !isNull( controller ) ? controller.getSetting( "environment" ) : "" ), + "DSN" : "", + "publicKey" : "", + "privateKey" : "", + "projectID" : 0, + "sentryUrl" : "https://sentry.io", // posting to "#sentryUrl#/api/#projectID#/store" is deprecated, but backward compatible // set to "envelope" to send events to modern "#sentryUrl#/api/#projectID#/envelope" - "sentryEventEndpoint" : "store", - "serverName" : cgi.server_name, - "appRoot" : expandPath( "/" ), - "sentryVersion" : 7, + "sentryEventEndpoint" : "store", + "serverName" : cgi.server_name, + "appRoot" : expandPath( "/" ), + "sentryVersion" : 7, // This is not arbitrary but must be a specific value. Leave as "cfml" // https://docs.sentry.io/development/sdk-dev/attributes/ - "platform" : "cfml", - "logger" : ( !isNull( controller ) ? controller.getSetting( "appName" ) : "sentry" ), - "userInfoUDF" : "", - "extraInfoUDFs" : {}, // these two settings may be added but they - "showJavaStackTrace" : false, - "throwOnPostError" : false + "platform" : "cfml", + "logger" : ( !isNull( controller ) ? controller.getSetting( "appName" ) : "sentry" ), + "userInfoUDF" : "", + "extraInfoUDFs" : {}, // these two settings may be added but they + "showJavaStackTrace" : false, + "throwOnPostError" : false, + // Master switch for emitted metrics. Automatic collectors and dependency metrics require it. + "enableMetrics" : false, + // Record eligible ColdBox request traces; tracesSampleRate controls trace volume. + "enableRequestTracing" : false, + // Track active eligible requests independently of trace sampling. + "enableRequestSaturation" : false, + // Maximum retained active request observations; excess starts are counted as untracked. + "maxObservedRequests" : 4096, + // Age in seconds used by the long-running-request gauge; this does not declare a timeout. + "requestLongRunningSeconds" : 30, + // Observe compatible cbq execution/publish hooks and their attempt durations. + "enableQueueTracing" : false, + // Count compatible cbq terminal completion/failure hooks separately from attempts. + "enableQueueJobMetrics" : false, + // Fixed queue destination labels allowed on attempt/job metrics. + "queueMetricDestinations" : [], + // Keep bounded, process-local envelope outcomes. Acceptance does not prove indexing. + "enableDeliveryDiagnostics" : false, + // Query tracing is off by default. auto/boxlang try native BoxLang hooks; unavailable hooks leave tracing off. + // lucee requires an application query listener. Library integrations use explicit provider-owned mappings. + // Also accepts callback(context) or a mapping implementing IQueryAdapter.configure(context). + // Example: QueryTelemetryAdapter@qb (requires a qb release containing that optional adapter). + "queryAdapter" : "none", + // Optional initial phase names/header order; measure()/begin() also accept ad hoc names without configuration. + // At most 32 distinct HTTP-safe names per instance. Automatic timing supplies handler/render. + "requestTimingNames" : [], + // Time preEvent/postEvent and preRender/postRender automatically, preserving manual phase APIs. + "enableAutomaticRequestTimings" : false, + // Publish the automatic instance's Server-Timing header when the response still permits headers. + "publishServerTiming" : false, + // Optional callback(event) authorizing header publication; requestFilter still controls collection. + "serverTimingFilter" : "", + // Fixed business workflow names allowed by the manual withOperation API. + "operationNames" : [], + // Fixed provider names allowed on manual workflow observations. + "providerNames" : [], + // Fixed names permitted by withDependency; configured dependencyAdapters add their names automatically. + "dependencyNames" : [], + // Up to 32 {name,mapping,methods:[...],classify:callback|mapping|{mapping,method}} entries. + // WireBox-created instances are intercepted; classifier mappings implement IDependencyClassifier.classify(result,failure). + "dependencyAdapters" : [], + // Writable WireBox AOP generation mapping, used only when this module registers the Mixer listener. + "dependencyAdapterGenerationPath" : "/coldbox/system/aop/tmp", + // Fixed names for the legacy manual worker-health API. + "queueWorkerNames" : [], + // Fixed names for manual queue-health snapshots; configured queues add their names automatically. + "queueNames" : [], + // Fixed executor names; automatic requestExecutors definitions add their names. + "requestExecutorNames" : [], + // Fixed pool names; automatic databasePools definitions add their names. + "databasePoolNames" : [], + "automaticCollectors" : { + // Start one background sampler per service. Disabled by default; also requires enableMetrics and a valid DSN. + "enabled" : false, + // Fixed delay between completed sampling cycles, in seconds, from 1 through 3600. No overlapping cycles. + "intervalSeconds" : 60, + // Sample available heap/thread/CPU/GC values; unsupported values stay absent. + "runtime" : true, + // Emit active/oldest/long-running request gauges; requires enableRequestSaturation. + "requestSaturation" : true, + // Poll local receipt state; never send delivery-health metrics through their own transport. + "deliveryDiagnostics" : false, + // Up to 32 {name,adapter:callback|mapping|{mapping,method},options:{...}} definitions. No default provider. + // One snapshot supplies supported backlog/depth/age/lease fields and worker activity; absent fields stay absent. + // Adapters follow sample(context); context.options is provider-specific configuration. + // cbq example: options:{workerPool:"outbound"} for workers/backlog or {connection:"database",queue:"mail"} for backlog only. + // Both require an explicit adapter:"QueueHealthAdapter@cbq" from a compatible cbq release. + "queues" : [], + // Up to 32 {name,adapter} definitions returning active/max/queued/rejected via callback(context) or ISnapshotAdapter.sample(context). + "requestExecutors" : [], + // Up to 32 {name,adapter} definitions returning active/idle/max/waiters/timeouts via callback(context) or ISnapshotAdapter.sample(context). + "databasePools" : [] + } }; // Try to look up the release based on a box.json @@ -103,8 +176,7 @@ component { ); interceptors = [ { class : "#moduleMapping#.interceptors.QueueObservability" }, - { class : "#moduleMapping#.interceptors.Observability" }, - { class : "#moduleMapping#.interceptors.QueryObservability" } + { class : "#moduleMapping#.interceptors.Observability" } ]; interceptorSettings = { customInterceptionPoints : [ "onSentryEventCapture" ] }; } @@ -113,13 +185,20 @@ component { * Fired when the module is registered and activated. */ function onLoad(){ - // Incorporate defaults into settings - if ( settings.queryAdapter == "auto" || settings.queryAdapter == "boxlang" ) { - wirebox - .getInstance( "SentryService@sentry" ) - .getObservability() - .configureQueryInstrumentation( getApplicationMetadata().name ); + // Initialize only when an explicitly configured integration needs startup work. + if ( + !isSimpleValue( settings.queryAdapter ) || settings.queryAdapter != "none" || ( + isStruct( settings.automaticCollectors ) && settings.automaticCollectors.keyExists( "enabled" ) && isBoolean( + settings.automaticCollectors.enabled + ) && settings.automaticCollectors.enabled + ) || ( isArray( settings.dependencyAdapters ) && settings.dependencyAdapters.len() ) + ) { + var telemetry = wirebox.getInstance( "SentryService@sentry" ).getObservability(); + telemetry.setAdapterInjector( wirebox ); + telemetry.configureQueryInstrumentation( getApplicationMetadata().name ); + registerDependencyAdapters( telemetry ); } + binder.mapDSL( "sentryCache", "#moduleMapping#.models.dsl.CacheDSL" ); wirebox.registerDSL( "sentryCache", "#moduleMapping#.models.dsl.CacheDSL" ); settings.scrubFields.addAll( SCRUB_FIELDS ); @@ -144,6 +223,40 @@ component { } } + private function registerDependencyAdapters( required any telemetry ){ + var bindings = arguments.telemetry.dependencyBindings(); + if ( !bindings.len() ) return; + if ( + !binder + .getListeners() + .some( function( listener ){ + return compareNoCase( listener.class, "coldbox.system.aop.Mixer" ) == 0; + } ) + ) { + binder.listener( + class = "coldbox.system.aop.Mixer", + properties = { "generationPath" : settings.dependencyAdapterGenerationPath }, + register = true + ); + } + binder + .mapAspect( "SentryDependencyInterceptor@sentry", false ) + .to( "#moduleMapping#.models.telemetry.DependencyInterceptor" ); + var mappings = []; + var methods = []; + for ( var binding in bindings ) { + if ( !arrayFindNoCase( mappings, binding.mapping ) ) mappings.append( binding.mapping ); + for ( var method in binding.methods ) { + if ( !arrayFindNoCase( methods, method ) ) methods.append( method ); + } + } + binder.bindAspect( + binder.match().mappings( mappings.toList() ), + binder.match().methods( methods.toList() ), + "SentryDependencyInterceptor@sentry" + ); + } + /** * Fired when the module is unregistered and unloaded */ diff --git a/box.json b/box.json index 8b07790..75e2d5a 100644 --- a/box.json +++ b/box.json @@ -1,7 +1,7 @@ { "name":"Sentry", "author":"Ortus Solutions ", - "version":"2.2.1", + "version":"2.2.2", "slug":"sentry", "type":"modules", "homepage":"https://github.com/coldbox-modules/sentry", diff --git a/browser/dist/esm/chunk-5LBAMSSV.js b/browser/dist/esm/chunk-OBW4BMTC.js similarity index 100% rename from browser/dist/esm/chunk-5LBAMSSV.js rename to browser/dist/esm/chunk-OBW4BMTC.js diff --git a/browser/dist/esm/feedbackReplay-6PAAGWVV.js b/browser/dist/esm/feedbackReplay-4NH5EQNJ.js similarity index 95% rename from browser/dist/esm/feedbackReplay-6PAAGWVV.js rename to browser/dist/esm/feedbackReplay-4NH5EQNJ.js index c0238f2..1057058 100644 --- a/browser/dist/esm/feedbackReplay-6PAAGWVV.js +++ b/browser/dist/esm/feedbackReplay-4NH5EQNJ.js @@ -1 +1 @@ -import{a as m,b,e as k}from"./chunk-5LBAMSSV.js";import{a as p,c as y,d as v,e as R,f as w,g}from"./chunk-UVRZCAHX.js";import{b as l,e as u}from"./chunk-QBI2JL2F.js";var t,i=!1,o=!1,r=0,d=Promise.resolve(),a;function c(){r++;let e=d,s=t?.stop({flush:!1}).catch(()=>{})??Promise.resolve();d=Promise.all([e,s]).then(()=>{})}async function f(){let e=r;await d,await Promise.resolve(),e===r&&o&&!i&&u(window.location)&&(w(document.documentElement),t?.startBuffering())}function x(){o=!0,!(t||!m())&&(g(),t=k({stickySession:!1,minReplayDuration:0,maskAllText:!l(),maskAllInputs:!l(),blockAllMedia:!l(),mask:[p],maskFn:y,maskAttributes:R,block:["script","iframe","canvas","video, audio, object, embed",p,v,'[style*="url("]',"[data-feedback-ui]"],networkCaptureBodies:!1,networkDetailAllowUrls:[],beforeAddRecordingEvent:()=>null}),b(t),window.addEventListener("popstate",c,{capture:!0}),window.addEventListener("hashchange",()=>{c(),f()}),f())}function I(){if(a)return a;i=!0;let e=t,s=e?.getReplayId(),h=r;if(!o||!e||!s||!u(window.location))return Promise.resolve(void 0);let n=(async()=>{try{return await e.flush({continueRecording:!1}),h===r&&o?s:void 0}catch{return}finally{await e.stop({flush:!1})}})();return a=n,d=n.then(()=>{}),n.finally(()=>{a===n&&(a=void 0)}),n}function E(){i=!1,c(),f()}function z(e){o=e,e?(i=!1,t?f():x()):(i=!0,c())}export{I as freezeFeedbackReplay,x as initializeFeedbackReplay,E as restartFeedbackReplay,f as resumeFeedbackReplay,z as setFeedbackReplayEnabled,c as stopFeedbackReplay}; +import{a as m,b,e as k}from"./chunk-OBW4BMTC.js";import{a as p,c as y,d as v,e as R,f as w,g}from"./chunk-UVRZCAHX.js";import{b as l,e as u}from"./chunk-QBI2JL2F.js";var t,i=!1,o=!1,r=0,d=Promise.resolve(),a;function c(){r++;let e=d,s=t?.stop({flush:!1}).catch(()=>{})??Promise.resolve();d=Promise.all([e,s]).then(()=>{})}async function f(){let e=r;await d,await Promise.resolve(),e===r&&o&&!i&&u(window.location)&&(w(document.documentElement),t?.startBuffering())}function x(){o=!0,!(t||!m())&&(g(),t=k({stickySession:!1,minReplayDuration:0,maskAllText:!l(),maskAllInputs:!l(),blockAllMedia:!l(),mask:[p],maskFn:y,maskAttributes:R,block:["script","iframe","canvas","video, audio, object, embed",p,v,'[style*="url("]',"[data-feedback-ui]"],networkCaptureBodies:!1,networkDetailAllowUrls:[],beforeAddRecordingEvent:()=>null}),b(t),window.addEventListener("popstate",c,{capture:!0}),window.addEventListener("hashchange",()=>{c(),f()}),f())}function I(){if(a)return a;i=!0;let e=t,s=e?.getReplayId(),h=r;if(!o||!e||!s||!u(window.location))return Promise.resolve(void 0);let n=(async()=>{try{return await e.flush({continueRecording:!1}),h===r&&o?s:void 0}catch{return}finally{await e.stop({flush:!1})}})();return a=n,d=n.then(()=>{}),n.finally(()=>{a===n&&(a=void 0)}),n}function E(){i=!1,c(),f()}function z(e){o=e,e?(i=!1,t?f():x()):(i=!0,c())}export{I as freezeFeedbackReplay,x as initializeFeedbackReplay,E as restartFeedbackReplay,f as resumeFeedbackReplay,z as setFeedbackReplayEnabled,c as stopFeedbackReplay}; diff --git a/browser/dist/esm/feedbackScreenshot-XN3R2P73.js b/browser/dist/esm/feedbackScreenshot-ZYNYCEWM.js similarity index 97% rename from browser/dist/esm/feedbackScreenshot-XN3R2P73.js rename to browser/dist/esm/feedbackScreenshot-ZYNYCEWM.js index fb41179..7e8a30c 100644 --- a/browser/dist/esm/feedbackScreenshot-XN3R2P73.js +++ b/browser/dist/esm/feedbackScreenshot-ZYNYCEWM.js @@ -1,2 +1,2 @@ import{a as u,b as y,d}from"./chunk-UVRZCAHX.js";import{b as f,c as b,d as m}from"./chunk-QBI2JL2F.js";function g(t){return Array.from(t.styleSheets).flatMap(n=>{if(!n.href)return[];let r=new URL(n.href,t.baseURI);if(r.origin!==new URL(t.baseURI).origin||!b(r)||r.search||r.hash)return[];try{return[Array.from(n.cssRules,o=>o.cssText).join(` -`)]}catch{return[]}})}function w(t,n){if(t.querySelectorAll(`script, iframe, canvas, video, audio, object, embed, ${u}, ${d}, [data-feedback-ui]`).forEach(e=>{e.style.visibility="hidden"}),n){t.body.style.visibility="hidden";return}let r=new Set(Array.from(t.querySelectorAll(u)).flatMap(e=>[e,...e.querySelectorAll("*")]));t.querySelectorAll(d).forEach(e=>e.removeAttribute("src"));let o=t.createTreeWalker(t.body,NodeFilter.SHOW_TEXT);for(;o.nextNode();)o.currentNode.textContent=y(o.currentNode.textContent??"",o.currentNode.parentElement);t.querySelectorAll("body, body *").forEach(e=>{let a=!f()||r.has(e),c=e,l=c.value,h=c.checked,p=e.selectedIndex;e.style.setProperty("background-image","none","important");for(let i of[...e.attributes])/^data-v-[0-9a-f]{8}$/.test(i.name)&&!i.value||(/^(?:data-|href$|srcset$|action$|formaction$|value$)/i.test(i.name)?e.removeAttribute(i.name):/^(?:title|alt|aria-label|aria-description|placeholder)$/.test(i.name)&&e.setAttribute(i.name,a?i.value.replace(/\S/g,"*"):i.value));(e.tagName==="INPUT"||e.tagName==="TEXTAREA")&&(e.value=a?"":l),e.tagName==="INPUT"&&(e.checked=a?!1:h),e.tagName==="SELECT"&&(e.selectedIndex=a?-1:p)});let s=t.createElement("style");s.textContent="*::before, *::after { content: none !important; background-image: none !important; }",t.head.append(s)}async function T(){if(m(window.location))return;let{default:t}=await import("./html2canvas-pro.esm-REK7NNFK.js"),n=window.innerWidth,r=window.innerHeight,o=g(document),s=await t(document.body,{width:n,height:r,x:window.scrollX,y:window.scrollY,scale:Math.min(1,Math.sqrt(15e5/(n*r))),backgroundColor:"#f4f8fb",logging:!1,allowTaint:!1,useCORS:!1,imageTimeout:3e3,ignoreElements:a=>a.hasAttribute("data-feedback-ui"),onclone:async a=>{for(let c of o){let l=a.createElement("style");l.textContent=c,a.head.append(l)}w(a,!1),a.body.getBoundingClientRect(),await a.fonts?.ready}}),e=await new Promise(a=>s.toBlob(a,"image/png"));if(!(!e||e.size>2e6))return{data:new Uint8Array(await e.arrayBuffer()),previewUrl:URL.createObjectURL(e)}}export{T as captureFeedbackScreenshot,g as compiledScreenshotStyles,w as sanitizeScreenshotDocument}; +`)]}catch{return[]}})}function w(t,n){if(t.querySelectorAll(`script, iframe, canvas, video, audio, object, embed, ${u}, ${d}, [data-feedback-ui]`).forEach(e=>{e.style.visibility="hidden"}),n){t.body.style.visibility="hidden";return}let r=new Set(Array.from(t.querySelectorAll(u)).flatMap(e=>[e,...e.querySelectorAll("*")]));t.querySelectorAll(d).forEach(e=>e.removeAttribute("src"));let o=t.createTreeWalker(t.body,NodeFilter.SHOW_TEXT);for(;o.nextNode();)o.currentNode.textContent=y(o.currentNode.textContent??"",o.currentNode.parentElement);t.querySelectorAll("body, body *").forEach(e=>{let a=!f()||r.has(e),c=e,l=c.value,h=c.checked,p=e.selectedIndex;e.style.setProperty("background-image","none","important");for(let i of[...e.attributes])/^data-v-[0-9a-f]{8}$/.test(i.name)&&!i.value||(/^(?:data-|href$|srcset$|action$|formaction$|value$)/i.test(i.name)?e.removeAttribute(i.name):/^(?:title|alt|aria-label|aria-description|placeholder)$/.test(i.name)&&e.setAttribute(i.name,a?i.value.replace(/\S/g,"*"):i.value));(e.tagName==="INPUT"||e.tagName==="TEXTAREA")&&(e.value=a?"":l),e.tagName==="INPUT"&&(e.checked=a?!1:h),e.tagName==="SELECT"&&(e.selectedIndex=a?-1:p)});let s=t.createElement("style");s.textContent="*::before, *::after { content: none !important; background-image: none !important; }",t.head.append(s)}async function T(){if(m(window.location))return;let{default:t}=await import("./html2canvas-pro.esm-BIHLXW6X.js"),n=window.innerWidth,r=window.innerHeight,o=g(document),s=await t(document.body,{width:n,height:r,x:window.scrollX,y:window.scrollY,scale:Math.min(1,Math.sqrt(15e5/(n*r))),backgroundColor:"#f4f8fb",logging:!1,allowTaint:!1,useCORS:!1,imageTimeout:3e3,ignoreElements:a=>a.hasAttribute("data-feedback-ui"),onclone:async a=>{for(let c of o){let l=a.createElement("style");l.textContent=c,a.head.append(l)}w(a,!1),a.body.getBoundingClientRect(),await a.fonts?.ready}}),e=await new Promise(a=>s.toBlob(a,"image/png"));if(!(!e||e.size>2e6))return{data:new Uint8Array(await e.arrayBuffer()),previewUrl:URL.createObjectURL(e)}}export{T as captureFeedbackScreenshot,g as compiledScreenshotStyles,w as sanitizeScreenshotDocument}; diff --git a/browser/dist/esm/html2canvas-pro.esm-REK7NNFK.js b/browser/dist/esm/html2canvas-pro.esm-BIHLXW6X.js similarity index 100% rename from browser/dist/esm/html2canvas-pro.esm-REK7NNFK.js rename to browser/dist/esm/html2canvas-pro.esm-BIHLXW6X.js diff --git a/browser/dist/esm/index.js b/browser/dist/esm/index.js index f39ec15..a53ebfd 100644 --- a/browser/dist/esm/index.js +++ b/browser/dist/esm/index.js @@ -1,3 +1,3 @@ -import{a as v,c as D,d as q,f as V,g as Y}from"./chunk-5LBAMSSV.js";import{a as I,d as j,f as J}from"./chunk-QBI2JL2F.js";var K=new WeakSet;function C(t){K.has(t)||(K.add(t),t.addEventProcessor((e,r)=>{if(e.type==="feedback"||e.type==="replay_event"){delete e.user,delete e.request,delete e.breadcrumbs,delete e.extra,delete e.transaction,e.type==="replay_event"&&Object.assign(e,{urls:[]});let i=r.data?.feedbackReplayId;e.type==="feedback"&&typeof i=="string"&&/^[a-f0-9]{32}$/i.test(i)&&e.contexts?.feedback&&(e.contexts.feedback.replay_id=i)}return e}))}function k(t){return t.split(/[?#]/)[0].replace(/\/(?:login|password|tokens?|invites?|kiosk|reset)(?:\/[^\s]*)?/gi,"/[private-page]").replace(/\/[A-Za-z0-9_-]{24,}(?=\/|$)/g,"/:id").replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gi,":id").replace(/\/\d+(?=\/|$)/g,"/:id").slice(0,256)}var X=new WeakSet,ie=/^(?:http\.request\.method|http\.response\.status_code|server\.address|sentry\.(?:origin|source|sample_rate|op|exclusive_time)|type|op)$/;function ae(t){try{return new URL(t).origin}catch{return"[resource]"}}function T(t){t.op?.startsWith("ui.")?t.description="interaction":t.description&&(["pageload","navigation"].includes(t.op??"")?t.description=k(t.description):/https?:\/\//i.test(t.description)?t.description=t.description.replace(/https?:\/\/\S+/gi,ae):["pageload","navigation"].includes(t.op??"")||(t.description=t.op??"operation"),t.description=t.description.split(/[?#]/)[0].replace(/\b[^\s/]+@[^\s/]+\b/g,"[private]").slice(0,256));let e={};for(let[r,i]of Object.entries(t.data??{})){if(/^browser\.web_vital\.(?:lcp|cls|inp|fcp|ttfb)\.(?:value|load_time|render_time|size)$/.test(r)&&typeof i=="number"&&Number.isFinite(i)){e[r]=i;continue}if(r==="sentry.pageload.span_id"&&typeof i=="string"&&/^[a-f0-9]{16}$/i.test(i)){e[r]=i;continue}if(["sentry.segment.name","sentry.transaction"].includes(r)&&typeof i=="string"){e[r]=k(i);continue}if(r==="user_agent.original"&&typeof i=="string"&&typeof navigator<"u"&&i===navigator.userAgent){e[r]=i.slice(0,512);continue}!ie.test(r)||!["string","number","boolean"].includes(typeof i)||(typeof i=="string"?r==="server.address"?e[r]=/^[a-z0-9.:-]+$/i.test(i)?i:"[destination]":/^[a-z0-9_. /:-]{1,128}$/i.test(i)&&(e[r]=i):e[r]=i)}return t.data=e,t}function _(t){if(X.has(t))return;X.add(t);let e=t.getOptions(),r=e.beforeSendSpan;e.beforeSendSpan=i=>T(r?r(i):i)}function Z(){return!!v()}async function G(t){let e=v();if(!e||!t.message.trim())throw new Error("Feedback is unavailable");typeof e.addEventProcessor=="function"&&C(e),await new Promise((r,i)=>{let s,o=e.on("afterSendEvent",(f,m)=>{f.event_id===s&&(clearTimeout(c),o(),m?.statusCode&&m.statusCode>=200&&m.statusCode<300?r():i(new Error("Feedback was not accepted")))}),c=setTimeout(()=>{o(),i(new Error("Feedback timed out"))},15e3);try{s=D({message:t.message.trim(),url:t.pageUrl,source:"api",tags:{page_component:t.component}},{includeReplay:!1,data:{feedbackReplayId:t.replayId},attachments:t.screenshot?[{filename:"feedback-screenshot.png",contentType:"image/png",data:t.screenshot.data}]:[]})}catch(f){clearTimeout(c),o(),i(f)}})}function Q(t,e){return new Promise(r=>{let i=!1,s=setTimeout(()=>{i=!0,r(void 0)},8e3);t.then(o=>{clearTimeout(s),i?o&&e?.(o):r(o)}).catch(()=>{clearTimeout(s),r(void 0)})})}function z(t={}){let e={message:"",busy:!1,capturing:!1,captureState:"pending",error:"",success:!1},r=0,i=!1,s={pageUrl:"",component:""},o=()=>t.changed?.({...e});function c(){e.screenshot&&(URL.revokeObjectURL(e.screenshot.previewUrl),e.captureState="removed"),e.screenshot=void 0,o()}async function f(){if(i||t.eligible?.()===!1)return;i=!0;let p=++r;e.error="",e.success=!1,e.capturing=!0,e.captureState=j(window.location)?"private":"pending",s={pageUrl:J(window.location),component:t.component?.()??"document"},o();let l=Q(import("./feedbackScreenshot-XN3R2P73.js").then(b=>b.captureFeedbackScreenshot()),b=>URL.revokeObjectURL(b.previewUrl)),R=Q(import("./feedbackReplay-6PAAGWVV.js").then(b=>b.freezeFeedbackReplay())),[g,y]=await Promise.all([l,R]);if(p!==r){g&&URL.revokeObjectURL(g.previewUrl);return}e.screenshot=g,s.replayId=y,e.capturing=!1,e.captureState!=="private"&&(e.captureState=g?"ready":"unavailable"),o()}function m(){e.busy||(w(),import("./feedbackReplay-6PAAGWVV.js").then(p=>p.restartFeedbackReplay()))}function w(){r++,i=!1,c(),e.message="",e.capturing=!1,o()}async function S(){if(e.busy||e.capturing||t.eligible?.()===!1)return;if(e.error="",!e.message.trim()){e.error="messageRequired",o();return}if(!Z()){e.error="unavailable",o();return}let p=r;e.busy=!0,o();try{await G({...s,message:e.message,screenshot:e.screenshot}),p===r&&(e.success=!0,e.message="",c())}catch{p===r&&(e.error="sendFailed")}finally{e.busy=!1,o()}}return{state:e,open:f,close:m,cancel:w,submit:S,removeScreenshot:c,setMessage(p){e.message=p,o()}}}var ee="sentry.feedback.corner";function te(){try{let t=localStorage.getItem(ee);if(["topLeft","topRight","bottomLeft","bottomRight"].includes(t??""))return t}catch{}return"bottomRight"}function re(t){try{localStorage.setItem(ee,t)}catch{}}function H(t,e){return{x:t.endsWith("Left")?e.left:e.right,y:t.startsWith("top")?e.top:e.bottom}}function ne(t,e){return(t.y<(e.top+e.bottom)/2?"top":"bottom")+(t.x<(e.left+e.right)/2?"Left":"Right")}function oe(t,e){if(e==="ArrowLeft")return t.startsWith("top")?"topLeft":"bottomLeft";if(e==="ArrowRight")return t.startsWith("top")?"topRight":"bottomRight";if(e==="ArrowUp")return t.endsWith("Left")?"topLeft":"topRight";if(e==="ArrowDown")return t.endsWith("Left")?"bottomLeft":"bottomRight"}function $(t,e){return{x:Math.max(e.left,Math.min(e.right,t.x)),y:Math.max(e.top,Math.min(e.bottom,t.y))}}var se={title:"Feedback",open:"Send feedback",close:"Close",message:"What happened?",submit:"Send",done:"Done",remove:"Remove screenshot",success:"Thank you. Your feedback was accepted.",sendFailed:"Feedback could not be sent. Please try again.",messageRequired:"Enter a message.",unavailable:"Feedback is unavailable.",disclosure:"Opening feedback uploads the recent replay buffer. Your message and screenshot are sent only when you submit.",pending:"Preparing capture\u2026",ready:"Review your screenshot",removed:"Screenshot removed",private:"Text feedback only on this page",captureUnavailable:"Capture unavailable; you can still send text feedback.",moved:"Feedback button moved"};function ce(t={}){let e=t.eligible?.()??!0;I(t.policy??{});let r={...se,...t.labels},i=new AbortController,s=document.createElement("div");s.dataset.feedbackUi="",s.dataset.sentryBlock="",s.className=t.className??"",s.innerHTML=`

`,s.hidden=!e,document.body.append(s);let o=n=>s.querySelector(n),c=o(".sentry-feedback-trigger"),f=o("dialog"),m=o("textarea"),w=o("img"),S=o("h2");S.id=`sentry-feedback-${crypto.randomUUID()}`,S.textContent=r.title,f.setAttribute("aria-labelledby",S.id),c.setAttribute("aria-label",r.open),c.title=r.open,o(".disclosure").textContent=r.disclosure,o("label span").textContent=r.message,o(".submit").textContent=r.submit,o(".remove").textContent=r.remove,o(".close").textContent=r.close,o(".success").textContent=r.success;let p=z({eligible:()=>e&&(t.eligible?.()??!0),component:t.component,changed(n){m.value=n.message,m.disabled=n.busy,o(".submit").disabled=n.busy||n.capturing,o(".close").disabled=n.busy,o(".error").textContent=n.error?r[n.error]:"",o(".capture").textContent=n.captureState==="unavailable"?r.captureUnavailable:r[n.captureState],w.hidden=!n.screenshot,n.screenshot?w.src=n.screenshot.previewUrl:w.removeAttribute("src"),o(".remove").hidden=!n.screenshot,o("form").hidden=n.success,o(".success").hidden=!n.success,n.success&&(o(".close").textContent=r.done,o(".close").focus())}}),l=(n,a,d)=>n.addEventListener(a,d,{signal:i.signal});l(c,"click",()=>{if(P){P=!1;return}f.open||(p.open(),f.showModal(),m.focus())});let R=()=>{p.state.busy||(p.close(),f.close(),c.focus())};l(o(".close"),"click",R),l(f,"cancel",n=>{n.preventDefault(),R()}),l(o("form"),"submit",n=>{n.preventDefault(),p.submit()}),l(m,"input",()=>p.setMessage(m.value)),l(o(".remove"),"click",p.removeScreenshot);let g=()=>{p.cancel(),f.close()};l(window,"popstate",g),l(window,"hashchange",g);let y=te(),b,F=0,P=!1,u,N=window.matchMedia("(prefers-reduced-motion: reduce)");function L(){let n=getComputedStyle(o(".sentry-feedback-probe")),a=U=>parseFloat(U)||0,d=window.visualViewport,x=(d?.offsetLeft??0)+16+a(n.paddingLeft),E=(d?.offsetTop??0)+16+a(n.paddingTop);return{left:x,top:E,right:Math.max(x,(d?.offsetLeft??0)+(d?.width??innerWidth)-64-a(n.paddingRight)),bottom:Math.max(E,(d?.offsetTop??0)+(d?.height??innerHeight)-64-a(n.paddingBottom))}}function B(n){b=n,c.style.transform=`translate3d(${n.x}px,${n.y}px,0)`}function h(n=!0){cancelAnimationFrame(F);let a=H(y,L()),d=b??a;if(!n||N.matches){B(a);return}let x=performance.now(),E=U=>{let A=(U-x)/700,O=1-Math.exp(-6*A)*Math.cos(7.5*A);B($({x:d.x+(a.x-d.x)*O,y:d.y+(a.y-d.y)*O},L())),A<1?F=requestAnimationFrame(E):B(a)};F=requestAnimationFrame(E)}function W(n){y=n,re(y),o('[role="status"]').textContent=r.moved,h()}l(c,"pointerdown",n=>{let a=n;!a.isPrimary||a.button!==0||u||(cancelAnimationFrame(F),u={id:a.pointerId,start:{x:a.clientX,y:a.clientY},origin:b??H(y,L()),moved:!1},P=!1,c.setPointerCapture(a.pointerId))}),l(c,"pointermove",n=>{let a=n;if(!u||u.id!==a.pointerId)return;let d=a.clientX-u.start.x,x=a.clientY-u.start.y;!u.moved&&Math.hypot(d,x)<8||(u.moved=!0,B($({x:u.origin.x+d,y:u.origin.y+x},L())))});let M=n=>{let a=n;!u||u.id!==a.pointerId||(P=u.moved||a.type==="pointercancel",u=void 0,c.hasPointerCapture(a.pointerId)&&c.releasePointerCapture(a.pointerId),P?W(ne(b,L())):h())};return l(c,"pointerup",M),l(c,"pointercancel",M),l(c,"lostpointercapture",M),l(c,"keydown",n=>{let a=n,d=oe(y,a.key);d&&(a.preventDefault(),W(d))}),l(window,"resize",()=>h(!1)),window.visualViewport&&(l(window.visualViewport,"resize",()=>h(!1)),l(window.visualViewport,"scroll",()=>h(!1))),l(N,"change",()=>h(!1)),h(!1),e&&import("./feedbackReplay-6PAAGWVV.js").then(n=>{e&&n.setFeedbackReplayEnabled(!0)}),{controller:p,setEnabled(n){e=n,n?s.hidden=!1:(g(),s.hidden=!0),import("./feedbackReplay-6PAAGWVV.js").then(a=>a.setFeedbackReplayEnabled(n))},dispose(){e=!1,i.abort(),cancelAnimationFrame(F),p.cancel(),s.remove(),import("./feedbackReplay-6PAAGWVV.js").then(n=>n.setFeedbackReplayEnabled(!1))}}}async function Pe(t,e,r){if(!e||r<=0)return!1;if(r>=100)return!0;let i=new Uint8Array(await crypto.subtle.digest("SHA-256",new TextEncoder().encode(`${t}:${e.toLowerCase()}`)));return new DataView(i.buffer).getUint32(0)%100+1<=r}function le(t){t.transaction=k(t.transaction??"document"),delete t.request,delete t.extra,delete t.breadcrumbs,t.user&&(t.user=t.user.id?{id:t.user.id}:void 0);for(let e of t.spans??[])T(e);return t}function Le(t,e){if(e)return C(e),_(e),e;let r=v();if(r)return C(r),_(r),r;let i=/^(?:localhost|127\.|0\.0\.0\.0|\[?::1\]?$)|\.(?:localhost|test)$/.test(window.location.hostname);if(!t.enabled||i&&!t.localVerification)return;q({...t,sendDefaultPii:!1,replaysSessionSampleRate:0,replaysOnErrorSampleRate:0,integrations:[V({instrumentNavigation:!t.inertia,beforeStartSpan:o=>({...o,name:k(o.name)})})],beforeSendSpan:T,beforeSendTransaction:le});let s=v();return s&&C(s),s}function Ee(t){let e=v();if(e)return Y(e,{name:k(t),op:"navigation",attributes:{"sentry.source":"component"}})}export{I as configureCapturePolicy,z as createFeedbackController,Le as initializeBrowser,ce as mountFeedbackWidget,Pe as percentageEligible,T as sanitizeSpan,le as sanitizeTransaction,k as stablePageName,Ee as startInertiaNavigation}; +

`,s.hidden=!e,document.body.append(s);let o=n=>s.querySelector(n),c=o(".sentry-feedback-trigger"),f=o("dialog"),m=o("textarea"),w=o("img"),S=o("h2");S.id=`sentry-feedback-${crypto.randomUUID()}`,S.textContent=r.title,f.setAttribute("aria-labelledby",S.id),c.setAttribute("aria-label",r.open),c.title=r.open,o(".disclosure").textContent=r.disclosure,o("label span").textContent=r.message,o(".submit").textContent=r.submit,o(".remove").textContent=r.remove,o(".close").textContent=r.close,o(".success").textContent=r.success;let p=H({eligible:()=>e&&(t.eligible?.()??!0),component:t.component,changed(n){m.value=n.message,m.disabled=n.busy,o(".submit").disabled=n.busy||n.capturing,o(".close").disabled=n.busy,o(".error").textContent=n.error?r[n.error]:"",o(".capture").textContent=n.captureState==="unavailable"?r.captureUnavailable:r[n.captureState],w.hidden=!n.screenshot,n.screenshot?w.src=n.screenshot.previewUrl:w.removeAttribute("src"),o(".remove").hidden=!n.screenshot,o("form").hidden=n.success,o(".success").hidden=!n.success,n.success&&(o(".close").textContent=r.done,o(".close").focus())}}),l=(n,a,d)=>n.addEventListener(a,d,{signal:i.signal});l(c,"click",()=>{if(P){P=!1;return}f.open||(p.open(),f.showModal(),m.focus())});let R=()=>{p.state.busy||(p.close(),f.close(),c.focus())};l(o(".close"),"click",R),l(f,"cancel",n=>{n.preventDefault(),R()}),l(o("form"),"submit",n=>{n.preventDefault(),p.submit()}),l(m,"input",()=>p.setMessage(m.value)),l(o(".remove"),"click",p.removeScreenshot);let g=()=>{p.cancel(),f.close()};l(window,"popstate",g),l(window,"hashchange",g);let y=re(),b,F=0,P=!1,u,W=window.matchMedia("(prefers-reduced-motion: reduce)");function L(){let n=getComputedStyle(o(".sentry-feedback-probe")),a=U=>parseFloat(U)||0,d=window.visualViewport,x=(d?.offsetLeft??0)+16+a(n.paddingLeft),E=(d?.offsetTop??0)+16+a(n.paddingTop);return{left:x,top:E,right:Math.max(x,(d?.offsetLeft??0)+(d?.width??innerWidth)-64-a(n.paddingRight)),bottom:Math.max(E,(d?.offsetTop??0)+(d?.height??innerHeight)-64-a(n.paddingBottom))}}function B(n){b=n,c.style.transform=`translate3d(${n.x}px,${n.y}px,0)`}function h(n=!0){cancelAnimationFrame(F);let a=$(y,L()),d=b??a;if(!n||W.matches){B(a);return}let x=performance.now(),E=U=>{let A=(U-x)/700,D=1-Math.exp(-6*A)*Math.cos(7.5*A);B(N({x:d.x+(a.x-d.x)*D,y:d.y+(a.y-d.y)*D},L())),A<1?F=requestAnimationFrame(E):B(a)};F=requestAnimationFrame(E)}function O(n){y=n,ne(y),o('[role="status"]').textContent=r.moved,h()}l(c,"pointerdown",n=>{let a=n;!a.isPrimary||a.button!==0||u||(cancelAnimationFrame(F),u={id:a.pointerId,start:{x:a.clientX,y:a.clientY},origin:b??$(y,L()),moved:!1},P=!1,c.setPointerCapture(a.pointerId))}),l(c,"pointermove",n=>{let a=n;if(!u||u.id!==a.pointerId)return;let d=a.clientX-u.start.x,x=a.clientY-u.start.y;!u.moved&&Math.hypot(d,x)<8||(u.moved=!0,B(N({x:u.origin.x+d,y:u.origin.y+x},L())))});let M=n=>{let a=n;!u||u.id!==a.pointerId||(P=u.moved||a.type==="pointercancel",u=void 0,c.hasPointerCapture(a.pointerId)&&c.releasePointerCapture(a.pointerId),P?O(oe(b,L())):h())};return l(c,"pointerup",M),l(c,"pointercancel",M),l(c,"lostpointercapture",M),l(c,"keydown",n=>{let a=n,d=ie(y,a.key);d&&(a.preventDefault(),O(d))}),l(window,"resize",()=>h(!1)),window.visualViewport&&(l(window.visualViewport,"resize",()=>h(!1)),l(window.visualViewport,"scroll",()=>h(!1))),l(W,"change",()=>h(!1)),h(!1),e&&import("./feedbackReplay-4NH5EQNJ.js").then(n=>{e&&n.setFeedbackReplayEnabled(!0)}),{controller:p,setEnabled(n){e=n,n?s.hidden=!1:(g(),s.hidden=!0),import("./feedbackReplay-4NH5EQNJ.js").then(a=>a.setFeedbackReplayEnabled(n))},dispose(){e=!1,i.abort(),cancelAnimationFrame(F),p.cancel(),s.remove(),import("./feedbackReplay-4NH5EQNJ.js").then(n=>n.setFeedbackReplayEnabled(!1))}}}async function Le(t,e,r){if(!e||r<=0)return!1;if(r>=100)return!0;let i=new Uint8Array(await crypto.subtle.digest("SHA-256",new TextEncoder().encode(`${t}:${e.toLowerCase()}`)));return new DataView(i.buffer).getUint32(0)%100+1<=r}function de(t){t.transaction=k(t.transaction??"document"),delete t.request,delete t.extra,delete t.breadcrumbs,t.user&&(t.user=t.user.id?{id:t.user.id}:void 0);for(let e of t.spans??[])T(e);return t}function Ee(t,e){if(e)return C(e),z(e),e;let r=v();if(r)return C(r),z(r),r;let i=/^(?:localhost|127\.|0\.0\.0\.0|\[?::1\]?$)|\.(?:localhost|test)$/.test(window.location.hostname);if(!t.enabled||i&&!t.localVerification)return;V({...t,sendDefaultPii:!1,replaysSessionSampleRate:0,replaysOnErrorSampleRate:0,integrations:[Y({instrumentNavigation:!t.inertia,beforeStartSpan:o=>({...o,name:k(o.name)})})],beforeSendSpan:T,beforeSendTransaction:de});let s=v();return s&&C(s),s}function Te(t){let e=v();if(e)return j(e,{name:k(t),op:"navigation",attributes:{"sentry.source":"component"}})}export{I as configureCapturePolicy,H as createFeedbackController,Ee as initializeBrowser,le as mountFeedbackWidget,Le as percentageEligible,T as sanitizeSpan,de as sanitizeTransaction,k as stablePageName,Te as startInertiaNavigation}; diff --git a/browser/src/controller.ts b/browser/src/controller.ts index 79ca4f9..89dac6e 100644 --- a/browser/src/controller.ts +++ b/browser/src/controller.ts @@ -1,5 +1,5 @@ import { feedbackAvailable, sendFeedback } from './feedback'; -import { feedbackPageUrl, isPrivateFeedbackPage } from './feedbackPrivacy'; +import { canRecordFeedbackPage, feedbackPageUrl, isPrivateFeedbackPage } from './feedbackPrivacy'; import type { FeedbackScreenshot } from './feedbackScreenshot'; export type CaptureState = 'pending' | 'ready' | 'removed' | 'unavailable' | 'private'; export interface FeedbackState { message: string; busy: boolean; capturing: boolean; screenshot?: FeedbackScreenshot; captureState: CaptureState; error: '' | 'messageRequired' | 'unavailable' | 'sendFailed'; success: boolean; } @@ -18,14 +18,15 @@ export function createFeedbackController(options: { component?: () => string; ch opened = true; const current = ++generation; state.error = ''; state.success = false; state.capturing = true; state.captureState = isPrivateFeedbackPage(window.location) ? 'private' : 'pending'; context = { pageUrl: feedbackPageUrl(window.location), component: options.component?.() ?? 'document' }; notify(); + if (state.captureState === 'private') { state.capturing = false; notify(); return; } const capture = captureWithDeadline(import('./feedbackScreenshot').then(m => m.captureFeedbackScreenshot()), image => URL.revokeObjectURL(image.previewUrl)); - const replay = captureWithDeadline(import('./feedbackReplay').then(m => m.freezeFeedbackReplay())); + const replay = canRecordFeedbackPage(window.location) ? captureWithDeadline(import('./feedbackReplay').then(m => m.freezeFeedbackReplay())) : Promise.resolve(undefined); const [image, replayId] = await Promise.all([capture, replay]); if (current !== generation) { if (image) URL.revokeObjectURL(image.previewUrl); return; } state.screenshot = image; context.replayId = replayId; state.capturing = false; if (state.captureState !== 'private') state.captureState = image ? 'ready' : 'unavailable'; notify(); } - function close() { if (state.busy) return; cancel(); void import('./feedbackReplay').then(m => m.restartFeedbackReplay()); } + function close() { if (state.busy) return; cancel(); if (canRecordFeedbackPage(window.location)) void import('./feedbackReplay').then(m => m.restartFeedbackReplay()); } function cancel() { generation++; opened = false; removeScreenshot(); state.message = ''; state.capturing = false; notify(); } async function submit() { if (state.busy || state.capturing || options.eligible?.() === false) return; diff --git a/browser/tests/controller.test.ts b/browser/tests/controller.test.ts index 33b9cb3..016c8d3 100644 --- a/browser/tests/controller.test.ts +++ b/browser/tests/controller.test.ts @@ -12,3 +12,31 @@ it('discards stale screenshot results and revokes their object URLs after naviga it('allows text feedback after screenshot capture fails', async () => { mock.capture.mockRejectedValue(Error('renderer failed')); const controller = createFeedbackController(); await controller.open(); expect(controller.state.captureState).toBe('unavailable'); controller.setMessage('Text works'); mock.send.mockResolvedValue(undefined); await controller.submit(); expect(controller.state.success).toBe(true); }); it('keeps the eight second deadline and discards a late result', async () => { vi.useFakeTimers(); let resolve!: (value: string) => void; const discard = vi.fn(); const result = captureWithDeadline(new Promise(r => resolve = r), discard); await vi.advanceTimersByTimeAsync(8000); expect(await result).toBeUndefined(); resolve('late'); await Promise.resolve(); expect(discard).toHaveBeenCalledWith('late'); }); it('does not load captures when ineligible', async () => { const controller = createFeedbackController({ eligible: () => false }); await controller.open(); expect(mock.capture).not.toHaveBeenCalled(); }); +it.each(['/login', '/email-sign-in', '/password-reset-requests/new'])('keeps private feedback text-only without starting capture or replay on %s', async (path) => { + configureCapturePolicy({ protectedPage: location => /login|email-sign-in|password-reset/.test(location.pathname) }); + history.replaceState({}, '', path); + const controller = createFeedbackController(); + await controller.open(); + expect(controller.state.captureState).toBe('private'); + expect(controller.state.capturing).toBe(false); + expect(mock.capture).not.toHaveBeenCalled(); + expect(mock.freeze).not.toHaveBeenCalled(); + controller.setMessage('Text feedback still works'); + mock.send.mockResolvedValue(undefined); + await controller.submit(); + expect(mock.send).toHaveBeenCalledWith(expect.objectContaining({ message: 'Text feedback still works', pageUrl: window.location.origin + '/[private-page]', screenshot: undefined })); + expect(mock.send.mock.calls[0][0]).not.toHaveProperty('replayId'); + controller.close(); + await Promise.resolve(); + expect(mock.restart).not.toHaveBeenCalled(); +}); +it.each(['/?search=private', '/#private'])('does not load replay when URL metadata prevents recording on %s', async (path) => { + history.replaceState({}, '', path); + const controller = createFeedbackController(); + await controller.open(); + expect(mock.capture).toHaveBeenCalledOnce(); + expect(mock.freeze).not.toHaveBeenCalled(); + controller.close(); + await Promise.resolve(); + expect(mock.restart).not.toHaveBeenCalled(); +}); diff --git a/dashboards/README.md b/dashboards/README.md new file mode 100644 index 0000000..50b5561 --- /dev/null +++ b/dashboards/README.md @@ -0,0 +1,92 @@ +# Performance dashboard recipes + +These offline recipes describe queries for Sentry's widget builder. They never +create dashboards, deploy code, change sampling, send telemetry or create alerts. +They are **not** an API import payload. Dataset and query serialization can differ +between Sentry releases: Explore calls application metrics `metrics`, while saved +dashboard widgets currently use `tracemetrics`. + +Copy `consumer.example.json` outside the module. Replace project slugs and the +readiness description, and verify the phase field against live data before use: + +```sh +node dashboards/build-recipes.mjs /path/to/consumer-config.json > /path/to/recipes.json +node --test dashboards/build-recipes.test.mjs +``` + +Organization, dashboard/project IDs, tokens and consumer routes do not belong in +module defaults. Consumers own readiness and authorization policy. The generator +uses Node built-ins without network requests. Its tests check scope isolation, +readiness exclusion, phase separation and metric sample coverage. These files +change no CFML runtime behavior. + +## Updating an existing dashboard + +1. Inspect the consumer's immutable module pin and serving release separately from + the local checkout. These recipes require observability telemetry; an older + 2.1.x checkout does not establish its availability. +2. Save the complete existing dashboard definition before editing. Preserve widget + IDs, relevant views, filters, production environment and seven-day default. + Keep consumer snapshots and raw query evidence outside the reusable module. + A connector Markdown definition is a reviewable saved-query snapshot, not a + lossless API restoration payload. +3. Prefer the Sentry connector for inspection and validation. If dashboard writes + are unavailable, apply recipes through the browser widget builder. Recipe + `fields` describe table columns; use aggregates as chart series and `groupBy` + as chart breakdowns. Preview changed queries against live data before saving. +4. Save and read the dashboard back through the connector. Compare datasets, + conditions, fields, groupings, defaults and IDs. Capture screenshots of the + saved dashboard and the affected labels. + +## Query contracts + +Set optional `suspectAssetDescriptions` to an array of exact, live-verified HTTP +descriptions when asset lifecycle timings flatten application latency. The default +is empty. Configured routes are excluded from both application percentiles and +their matching count, and retained in separate suspect-asset percentile/count +recipes. The all-route/release table remains unfiltered. Use route classification, +not a duration cutoff: genuine slow requests, uploads and failures must remain +visible. Recheck the consumer list when parameterized route names change. + +| Signal | Query | Interpretation | +| --------------- | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| HTTP | Backend `http.server`; exact readiness description split | Pair application percentiles with identical-filter counts. Retain readiness count/p95/status/release views. Security probes and failures remain; configured suspect assets stay visible separately. | +| Routes/releases | Description and release with count/p95 | Use parameterized routes. Sparse rows limit tail estimates. Label suspect asset lifecycle timing explicitly; do not silently exclude slow spans. | +| DB | `db.query`; p95/count/cumulative duration; scrubbed shape and parent `transaction` | Parent grouping separates probe queries. Cumulative span time is workload, not request latency. Coverage may be incomplete. Never include SQL bindings or remove security/readiness checks to improve charts. | +| Browser | Frontend `pageload`/`navigation` | Preserve project distinctions and sample counts. Pageload duration is not FCP or proof of first-paint experience. | +| JVM | `jvm.heap.used` gauge/byte; `jvm.threads` gauge/none | Unsupported/absent metrics remain absent. | +| Queue pools | `has:queue.name`, group `queue.name`; depth gauge/none and oldest-age gauge/second | Labels describe worker pools; a mail pool can cover several destinations. Never merge legacy `messaging.destination.name` history or sum overlapping collectors. | +| Queue coverage | Counts of each gauge with matching pool filter | Positive samples plus zero gauge means measured zero. Missing/no samples means unknown. Inspect recent hourly buckets and Explore samples for freshness; a seven-day average is not current queue state. | +| Phases | Both `app.phase`/`app.navigation`, group by operation and verified phase field | Inclusive phases overlap: never sum them. Automatic and focused instrumentation have different populations. Sort by count to keep automatic phases visible. | + +Sentry may display empty buckets at a baseline or connect neighboring samples. +A chart line alone cannot distinguish missing data from zero. Keep per-signal +sample coverage visible and explain this in widget descriptions. Never add +zero-filling formulas, use sums to disguise gaps, or infer uptime from absent +probes. Counts describe recorded telemetry, not necessarily all generated events. +Do not change sampling to improve appearance. + +Sample cadence depends on collector configuration and the number of running +instances. Higher counts can reflect concurrent instances or lifecycle overlap; +they are not an uptime percentage. Prefer per-pool averages over sums for gauges. + +## Verifying the phase field + +Raw spans can contain `description: handler` while their displayed/indexed `name` +is `app.phase`. Query both `span.name` and `span.description` with count, then +filter a known phase and verify a raw trace example. Confirm the saved widget +preview resolves the same labels. One current Sentry dataset groups handler/render +under `span.description` while `span.name` stays `app.phase`; recheck for each +consumer. `phaseField` is explicit configuration, not an inferred SDK attribute. + +If no deployed field works, retain operation-level coverage, mark the breakdown +pending deployment, and implement a bounded SDK attribute in the official module +with focused tests. A local SDK change is not evidence of live ingestion. + +## Adoption boundaries + +Dashboard edits, local checks, hosted CI, publication, consumer dependency adoption, +deployment and live ingestion are separate claims. Dashboard-only changes can use +already indexed data without deployment. These recipe checks do not require the +full CFML/browser matrix. Alerts, schedules, notification destinations, sampling, +credentials and unrelated production settings need separate task authorization. diff --git a/dashboards/build-recipes.mjs b/dashboards/build-recipes.mjs new file mode 100644 index 0000000..184b9cf --- /dev/null +++ b/dashboards/build-recipes.mjs @@ -0,0 +1,233 @@ +import { readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const slug = /^[a-z0-9][a-z0-9_-]*$/; +const quote = (value) => JSON.stringify(value); + +/** Offline query recipes for the Sentry widget builder; never writes to Sentry. */ +export function buildRecipes(config) { + for (const key of ["backendProject", "frontendProject", "environment"]) { + if (typeof config[key] !== "string" || !slug.test(config[key])) { + throw new Error(`${key} must be an explicit slug`); + } + } + if ( + typeof config.readinessDescription !== "string" || + !config.readinessDescription.trim() + ) { + throw new Error( + "readinessDescription must match a verified indexed HTTP description", + ); + } + if (!["span.description", "app.phase"].includes(config.phaseField)) { + throw new Error( + "phaseField must be verified in live data: span.description or app.phase", + ); + } + const assets = config.suspectAssetDescriptions ?? []; + if ( + !Array.isArray(assets) || + assets.length > 32 || + assets.some( + (value) => + typeof value !== "string" || + !value.trim() || + value.length > 512 || + /[\x00-\x1f*]/.test(value), + ) || + new Set(assets).size !== assets.length || + assets.includes(config.readinessDescription) + ) { + throw new Error( + "suspectAssetDescriptions must be unique exact descriptions, excluding readiness (maximum 32)", + ); + } + const backend = `project:${config.backendProject} environment:${config.environment}`; + const frontend = `project:${config.frontendProject} environment:${config.environment}`; + const http = `${backend} span.op:http.server`; + const ready = `span.description:${quote(config.readinessDescription)}`; + const nonReadiness = `${http} !${ready}`; + const application = [ + nonReadiness, + ...assets.map((value) => `!span.description:${quote(value)}`), + ].join(" "); + const queue = `${backend} has:queue.name`; + const widgets = []; + const add = ( + title, + dataset, + conditions, + fields, + groupBy = [], + description = "", + ) => { + widgets.push({ + title, + dataset, + conditions, + fields, + groupBy, + interval: "1h", + description, + }); + }; + add( + "Backend non-readiness latency", + "spans", + application, + ["p50(span.duration)", "p95(span.duration)"], + [], + assets.length + ? "Recorded ms, auto-scaled. Excludes readiness and explicitly configured suspect asset routes, which remain in separate investigation charts. Pair with matching counts. No duration or status cutoff." + : "Recorded ms, auto-scaled. Pair with matching counts. Security probes, errors and suspect asset timings remain included; duration is not necessarily user wait time.", + ); + add("Backend non-readiness count", "spans", application, ["count()"]); + if (assets.length) { + const suspect = `${nonReadiness} span.description:[${assets.map(quote).join(",")}]`; + add( + "Suspect asset recorded latency", + "spans", + suspect, + ["p50(span.duration)", "p95(span.duration)"], + [], + "Investigation only: recorded duration is not established user wait time. Exact consumer-selected routes; no duration or success filter. Pair with matching counts and retain the all-route/release table.", + ); + add("Suspect asset sample count", "spans", suspect, ["count()"]); + } + add( + "Readiness availability observations", + "spans", + `${http} ${ready}`, + [ + "span.description", + "release", + "span.status", + "count()", + "p95(span.duration)", + ], + ["span.description", "release", "span.status"], + "Recorded probes, including failures. No samples does not establish availability or zero traffic.", + ); + add( + "Backend routes and releases", + "spans", + http, + ["span.description", "release", "count()", "p95(span.duration)"], + ["span.description", "release"], + "Keep all routes and statuses visible. Investigate suspect asset lifecycle timing separately; do not silently exclude slow spans.", + ); + for (const fields of [ + ["p95(span.duration)"], + ["count()"], + ["sum(span.duration)"], + ]) { + add( + `DB ${fields[0]}`, + "spans", + `${backend} span.op:db.query`, + fields, + [], + "Observed instrumented queries only. Cumulative span time is workload, not request latency.", + ); + } + add( + "Scrubbed DB shapes by parent transaction", + "spans", + `${backend} span.op:db.query`, + [ + "span.description", + "transaction", + "count()", + "p95(span.duration)", + "sum(span.duration)", + ], + ["span.description", "transaction"], + "Separates readiness-related shapes without removing security/readiness checks. Never include SQL bindings.", + ); + for (const op of ["pageload", "navigation"]) { + add( + `Frontend ${op}`, + "spans", + `${frontend} span.op:${op}`, + ["count()", "p95(span.duration)"], + [], + "Recorded ms, auto-scaled. Count is percentile sample context; pageload duration is not FCP.", + ); + } + add( + "Frontend routes and releases", + "spans", + `${frontend} span.op:[pageload,navigation]`, + ["span.op", "transaction", "release", "count()", "p95(span.duration)"], + ["span.op", "transaction", "release"], + ); + for (const [name, unit] of [ + ["jvm.heap.used", "byte"], + ["jvm.threads", "none"], + ]) { + add( + name, + "metrics", + backend, + [`avg(value,${name},gauge,${unit})`], + [], + "Unsupported or absent metrics remain absent.", + ); + } + for (const [name, unit] of [ + ["queue.depth", "none"], + ["queue.oldest_age", "second"], + ]) { + add( + `${name} by worker pool`, + "metrics", + queue, + [`avg(value,${name},gauge,${unit})`], + ["queue.name"], + "queue.name is a worker pool. A pool may cover several destinations. Never merge legacy destination series or infer zero from absent samples.", + ); + add( + `${name} sample coverage`, + "metrics", + queue, + [`count(value,${name},gauge,${unit})`], + ["queue.name"], + "Pair with the gauge for each pool and bucket. Zero/no samples means no observation, not an empty pool.", + ); + } + add( + "Backend inclusive phases", + "spans", + `${backend} span.op:[app.phase,app.navigation]`, + ["span.op", config.phaseField, "count()", "p95(span.duration)"], + ["span.op", config.phaseField], + "Verify phaseField by grouping and filtering raw examples. Operations represent different coverage. Inclusive phases overlap: never sum them. Sort by count to keep automatic phases visible.", + ); + add( + "Backend telemetry coverage", + "spans", + `${backend} span.op:[http.server,db.query,app.phase,app.navigation]`, + ["count()"], + ["span.op"], + "Compare recent hourly buckets and releases. Span counts are not unique requests or complete unsampled traffic.", + ); + return { + schemaVersion: 1, + defaultPeriod: "7d", + environment: config.environment, + widgets, + }; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + if (!process.argv[2]) + throw new Error( + "Usage: node dashboards/build-recipes.mjs /path/to/consumer-config.json", + ); + process.stdout.write( + `${JSON.stringify(buildRecipes(JSON.parse(readFileSync(process.argv[2], "utf8"))), null, 2)}\n`, + ); +} diff --git a/dashboards/build-recipes.test.mjs b/dashboards/build-recipes.test.mjs new file mode 100644 index 0000000..cd86270 --- /dev/null +++ b/dashboards/build-recipes.test.mjs @@ -0,0 +1,120 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { buildRecipes } from "./build-recipes.mjs"; + +const config = { + backendProject: "sample-api", + frontendProject: "sample-web", + environment: "production", + readinessDescription: "GET ready/", + phaseField: "span.description", +}; + +test("requires consumer scopes and rejects query injection", () => { + for (const key of ["backendProject", "frontendProject", "environment"]) { + assert.throws(() => buildRecipes({ ...config, [key]: undefined })); + assert.throws(() => + buildRecipes({ ...config, [key]: "api OR project:other" }), + ); + } + assert.throws(() => buildRecipes({ ...config, phaseField: "span.name" })); + assert.throws(() => buildRecipes({ ...config, readinessDescription: "" })); +}); + +test("readiness and application recipes form an explicit, disjoint split", () => { + const { widgets } = buildRecipes(config); + const app = widgets.find( + (w) => w.title === "Backend non-readiness latency", + ); + const counts = widgets.find( + (w) => w.title === "Backend non-readiness count", + ); + const ready = widgets.find( + (w) => w.title === "Readiness availability observations", + ); + assert.equal(app.conditions, counts.conditions); + assert.equal( + app.conditions.replace("!span.description:", "span.description:"), + ready.conditions, + ); + assert(!ready.conditions.includes("span.status:ok")); +}); + +test("optional asset split preserves investigation data without duration or status cutoffs", () => { + const assets = ["GET media/:id/", "GET attachments/:id/"]; + const { widgets } = buildRecipes({ + ...config, + suspectAssetDescriptions: assets, + }); + const app = widgets.find( + (w) => w.title === "Backend non-readiness latency", + ); + const count = widgets.find( + (w) => w.title === "Backend non-readiness count", + ); + const suspect = widgets.find( + (w) => w.title === "Suspect asset recorded latency", + ); + const suspectCount = widgets.find( + (w) => w.title === "Suspect asset sample count", + ); + const allRoutes = widgets.find( + (w) => w.title === "Backend routes and releases", + ); + assert.equal(app.conditions, count.conditions); + assert.equal(suspect.conditions, suspectCount.conditions); + for (const asset of assets) + assert( + app.conditions.includes( + `!span.description:${JSON.stringify(asset)}`, + ), + ); + assert( + suspect.conditions.endsWith( + `span.description:[${assets.map(JSON.stringify).join(",")}]`, + ), + ); + assert(!allRoutes.conditions.includes("span.description")); + for (const widget of [app, suspect]) + assert(!/span.duration:|span.status:/.test(widget.conditions)); + for (const invalid of [ + "route", + ["GET *"], + [""], + [assets[0], assets[0]], + [config.readinessDescription], + ]) { + assert.throws(() => + buildRecipes({ ...config, suspectAssetDescriptions: invalid }), + ); + } +}); + +test("each queue gauge has per-pool sample coverage without legacy labels or zero filling", () => { + const { widgets } = buildRecipes(config); + for (const name of ["queue.depth", "queue.oldest_age"]) { + const gauge = widgets.find((w) => w.title === `${name} by worker pool`); + const samples = widgets.find( + (w) => w.title === `${name} sample coverage`, + ); + assert.deepEqual(gauge.groupBy, ["queue.name"]); + assert.equal(gauge.conditions, samples.conditions); + assert(samples.fields[0].startsWith("count(value,")); + assert(gauge.conditions.includes("has:queue.name")); + } + assert(!JSON.stringify(widgets).includes("messaging.destination.name")); +}); + +test("phase recipes retain distinct operations and verified field without summing phases", () => { + const phases = buildRecipes(config).widgets.find( + (w) => w.title === "Backend inclusive phases", + ); + assert.deepEqual(phases.groupBy, ["span.op", "span.description"]); + assert(phases.conditions.includes("span.op:[app.phase,app.navigation]")); + assert(!phases.fields.some((f) => f.startsWith("sum("))); + const custom = buildRecipes({ + ...config, + phaseField: "app.phase", + }).widgets.find((w) => w.title === phases.title); + assert.deepEqual(custom.groupBy, ["span.op", "app.phase"]); +}); diff --git a/dashboards/consumer.example.json b/dashboards/consumer.example.json new file mode 100644 index 0000000..4c64420 --- /dev/null +++ b/dashboards/consumer.example.json @@ -0,0 +1,8 @@ +{ + "backendProject": "example-backend", + "frontendProject": "example-frontend", + "environment": "production", + "readinessDescription": "GET ready/", + "phaseField": "span.description", + "suspectAssetDescriptions": [] +} diff --git a/docs/observability-upstream-audit.md b/docs/observability-upstream-audit.md new file mode 100644 index 0000000..194a772 --- /dev/null +++ b/docs/observability-upstream-audit.md @@ -0,0 +1,30 @@ +# CommuniArts upstream audit — October 22, 2026 + +Source review used the official module at `origin/development` (`ef1aaa6`), the preserved CommuniArts `codex/avatar-request-timings` worktree, and the installed adapters. ForgeBox's public entry API listed stable Sentry `2.2.1` and prerelease `3.0.0-alpha.1-snapshot`; the app currently declares Git Sentry `60112485121ce5eb276f0000287ace47e16566d6`, Git cbq `5493309dcf00c1d7485bf48cc8346e568ff4f799`, qb `14.1.0`, and browser SDK `10.74.0`. Older application prose describing `^2.2.0`/`2.2.0+9` is stale for that worktree. The same ForgeBox check listed cbq `7.0.2`, qb `14.1.2`, and funclinenums `1.1.5` as the latest stable versions; the optional qb contract here deliberately tests the app's installed `14.1.0`, not an inferred upgrade. These are separate source/version claims, not publication or deployment claims. + +| Reviewed work | Ownership and decision | +| --- | --- | +| RequestTimingService / RequestTimingInterceptor (`bb305a49`), navigation/shared-data hooks | Extract bounded monotonic phase timers, once-only completion, safe Server-Timing formatting and child-span support into `RequestTimings`. Keep route eligibility, signed-in/Inertia checks, lifecycle wiring, fixed business phases, root attribute naming and avatar behavior in CommuniArts. | +| RuntimeMetricsService (`c6c69d11`), RuntimeMetricsSpec | Optional `DatabaseQueueMetrics` extracts configured-idle/dynamic queue handling, successful-poll zeros, failure diagnostics and qb polling; add grammar-aware identifier quoting. Keep scheduler frequency, datasource/connection selection and provider eligibility in the app. Extend existing module JVM API; do not add a duplicate scheduler. | +| ObservedSentrySignals | Upstream receipt observation/freshness in `DeliveryDiagnostics`, bounded independently of payload content; fix lowercase `contexts.trace` serialization in the module error path. Keep app health-channel aggregation, stderr warning cadence and protected health endpoints in the app. | +| ObservedSentryService / TelemetryHealthService | Generic envelope diagnostics remove one reason for subclassing after adoption. ExpectedAccessDenial filtering, technical/sanitizer/upload/response diagnostics, capture exception shields, warning policy, lifecycle and frontend health remain app-specific. Legacy store capture is not covered by envelope diagnostics. Do not remove the app subclass wholesale. | +| SentryPrivacyInterceptor / SentryContextInterceptor | Keep application event sanitation, private-link rules, opaque tenant/user identity, authorization and account-change handling in the app. Existing module generic attribute/filter/propagation primitives remain; no duplicate privacy listener. | +| SentryTunnelService / TelemetryEnvelopes / backend screenshots | Keep endpoint authorization, rate/binary budgets, same-origin routing and approved readable screenshot policy in the app. Existing generic feedback/attachment/privacy APIs remain in the module. Do not add an unauthenticated relay endpoint upstream. | +| cbq attempt/context propagation | Preserve immutable execution handles and worker entry/exit hooks. Close roots before metrics can fail, omit duration when execution never began, add opt-in bounded destination labels and terminal job hook counters. Retain distinction between attempts and terminal failure. | +| DB adapters and installed-adapter contracts | Native Lucee/BoxLang hooks and qb's optional provider-owned query adapter retain de-duplication. Sentry contains no qb event interceptor and infers no query library when native hooks are unavailable. Optional pool sample boundary requires a supported provider adapter. Direct application SQL migration is separate; this task adds no wrapper or listener to legitimize that SQL. | +| Business workflows, Postmark, storage, Typesense | Add allowlisted `withOperation` outcome/duration primitives, preserving original return/error behavior and bounded classifications. Actual submission/publishing/invitation/upload/PDF logic and provider response/webhook semantics remain in the app. HTTP/provider acceptance is not delivery. | +| Browser SDK, lifecycle/resource diagnostics, replay/feedback | Already covered by existing companion/Vue integration and app diagnostic collector. Reuse them. No new browser instrumentation or wider capture policy is justified. | +| Alerts, uptime, cron, dashboards | Document setup and evidence checks. Organization thresholds, routing, endpoints, readiness dependencies and plan limits remain consumer/provider configuration. No production settings or existing weekly-review automation changed. | + +The original October 22 dashboard/queue-freshness/monitor snapshot is historical context only; this task does not claim to have rechecked current production or monitor state. It does not change the application's direct-query inventory or migrate its remaining direct SQL. + +## Release and consumer adoption + +1. Review this local module branch against development. Reconcile its release base with the app's existing `6011248` CLI/bulk-clock branch before release so adoption cannot discard those fixes. No source version or published package number was bumped here. +2. Run the supported-engine release checks when the paused full feedback loop is resumed; qualify Adobe and real database/provider adapters separately. Publish through the module's normal release process only when authorized. +3. In the existing CommuniArts worktree, adopt the published compatible range and exact lock records, following that application's package/lock rules. Reinstall dependencies rather than editing/vendoring `lib/modules` source. Do not claim this unpublished module code is installed already. +4. Replace `ObservedSentrySignals` receipt polling with the module API and `enableDeliveryDiagnostics: true` only after adapting its health aggregation and checking capture/privacy overrides. Migrate the request service to one module timing instance per eligible request while retaining lifecycle/header policy and fixed application root attributes. Replace the queue polling service's implementation with the optional adapter, configured for the actual database connection. Avoid running old/new samplers together. +5. Run the consumer's focused RequestTiming, RuntimeMetrics, SentryObservability/adapter, health, privacy and browser Vitest contracts. Confirm payload privacy, queue idle/drain/failure behavior, query fallback coverage and exactly-once completion with the installed release. +6. Deploy only on explicit authorization, then independently verify the deployed SHA/assets, ingestion acceptance, indexed metric freshness/traces and any separately authorized alert/uptime setup. Preserve the existing weekly review schedule. + +Consumer files are untouched in this task: a migration against unpublished APIs would misleadingly imply installability. [Operational setup and API examples](operational-observability.md) provide the adoption patch's reviewable contract. diff --git a/docs/observability.md b/docs/observability.md index 8df7a24..ba986a8 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -1,6 +1,6 @@ # Observability and feedback for CFML and BoxLang -All new integrations are opt-in. Existing error APIs, LogBox appender settings, standalone construction, and thread-safe WireBox initialization remain available. Version 3.0.0-alpha.1 is a preview requiring the accompanying qb/cbq hooks; it is not a published stable release. +All integrations are opt-in. Existing error APIs, LogBox appender settings, standalone construction, and thread-safe WireBox initialization remain available. Optional qb/cbq integrations require versions with the documented hooks (see the [upstream audit](observability-upstream-audit.md) for the inspected dependency versions). The operational additions described below remain unpublished until released. ## Server setup @@ -11,6 +11,7 @@ moduleSettings.sentry = { release: "my-release", tracesSampleRate: 1, enableRequestTracing: true, + // Try native engine hooks; never infer an installed query library. queryAdapter: "auto", databaseSystem: "postgresql", enableQueueTracing: true, @@ -25,7 +26,7 @@ moduleSettings.sentry = { }; ``` -The same struct works in BoxLang. Standalone usage: `new sentry.models.SentryService( settings )`. Pass the same service across operations; call `shutdown()` at application shutdown. Add `sentry.endRequest()` in Application.cfc/Application.bx `onRequestEnd` to cover early redirects and aborted requests. ColdBox postProcess finishes ordinary requests after rendering. Route-template transaction names fall back to handler/action. Response status is read from the servlet response; explicit `endRequest(statusCode=303)` is also supported. +The same struct works in BoxLang. Standalone usage: `new sentry.models.SentryService( settings )`. Pass the same service across operations; call `shutdown()` at application shutdown. Call `sentry.endRequest()` from **both** Application.cfc/Application.bx `onRequestEnd` and `onAbort`. Binary content and redirects can invoke `onAbort` while skipping `onRequestEnd`, `postRender`, and `postProcess`. ColdBox `postProcess` finishes ordinary requests after rendering. Repeated completion is safe. Route-template transaction names fall back to handler/action. Response status is read from the servlet response; explicit `endRequest(statusCode=303)` is also supported. See [response completion and timing contracts](request-completion.md) for the integration recipe, reproduction, and verification limits. Use a trustIncomingTrace callback at your ingress boundary. Reject public forged trace headers unless the request is from an approved browser/proxy boundary. Parent sampling decisions are honored; malformed/all-zero IDs are ignored. Propagation targets compare exact scheme/host/port and reject URL userinfo. Baggage carries only allowlisted technical sampling context. Use requestFilter, attributeFilter, beforeSendSignal, and beforeSendFeedback to filter signals. Application bodies, SQL bindings/results, job payloads, cached values, and credentials are never needed by instrumentation. @@ -42,6 +43,26 @@ sentry.withTraceContext( headers, function( root ) { ## Queries +Query tracing observes executions; it does not supply a query builder or choose how your application accesses the database. No query library is required. `queryAdapter` defaults to `"none"`. + +| Setting | What it enables | +| --- | --- | +| `"none"` | No automatic query instrumentation. Manual `withQuerySpan` remains available. | +| `"auto"` / `"boxlang"` | Complete native BoxLang query hooks when available; otherwise automatic query tracing stays off. There is no query-library fallback. | +| `"lucee"` | Select the native application listener configured below. This setting does not install the listener. | +| `"QueryTelemetryAdapter@qb"` | Explicitly install the optional adapter shipped by a compatible qb release. qb owns its execution listeners and cleanup. | +| Callback or WireBox mapping | Install your own execution hooks with `configure(context)`. Return `{mode:"custom",shutdown:callback}` for cleanup. | + +Custom setup receives `context.telemetry`, `context.applicationName` and `context.databaseSystem` (default `"other"`). It must install its integration; returning `"custom"` alone does not observe queries. Returning a built-in native mode selects that explicit integration instead. This setup hook is separate from dependency method advice and periodic database-pool health snapshots. + +```cfml +// Explicitly select the optional execution adapter shipped by qb. +// Requires an installed qb version containing QueryTelemetryAdapter. +moduleSettings.sentry.queryAdapter = "QueryTelemetryAdapter@qb"; +``` + +Sentry contains no qb event interceptor or built-in `"qb"` mode. Other query libraries can ship their own mapping or configure callback using the same lifecycle contract. The qb adapter creates a scrubbed span for each real execution, preserves result/exception behavior, suppresses duplicate manual/native spans, and removes only its owned listeners on shutdown after in-flight executions drain. + On Lucee 6+, compose your existing application listener: ```cfml @@ -49,14 +70,15 @@ this.query.listener = sentry.createLuceeQueryListener( listener = existingListener, databaseSystem = "postgresql" ); -// Set queryAdapter="lucee" so qb does not produce a second span. +// Select this explicitly installed native listener. +// moduleSettings.sentry.queryAdapter = "lucee"; ``` Native global listeners are officially scoped to Lucee 6+ ([Lucee documentation](https://docs.lucee.org/recipes/query-listener.html)); older engines are not automatically enabled. Existing before/after/error returns and transformations are preserved. Each thread has a nesting stack; use the manual wrapper for asynchronous query modes whose callbacks do not execute on that thread. -BoxLang auto selects application-scoped preQueryExecute/postQueryExecute/onQueryExecuteError only when the runtime exposes ON_QUERY_EXECUTE_ERROR. Earlier runtimes fall back to qb. The upstream hook includes PendingQuery identity, context, elapsed milliseconds, and the unchanged exception. Cached execution events include cached/dbtype and correlate with the same pending identity. The currently tested server runtime lacks that error event: successful native direct queries were exercised separately, but automatic configuration deliberately uses qb there. Connection acquisition/build failures occurring before native PRE remain a manual-wrapper boundary. +BoxLang auto selects application-scoped preQueryExecute/postQueryExecute/onQueryExecuteError only when the runtime exposes ON_QUERY_EXECUTE_ERROR. Earlier runtimes leave automatic query tracing off. The upstream hook includes PendingQuery identity, context, elapsed milliseconds, and the unchanged exception. Cached execution events include cached/dbtype and correlate with the same pending identity. The currently tested server runtime lacks that error event: successful native direct queries were exercised separately, but automatic configuration deliberately stays disabled there. Connection acquisition/build failures occurring before native PRE remain a manual-wrapper boundary. -qb's new onQBExecuteException closes the original preQBExecute span and preserves the database exception even if an observer fails. Pretend queries are skipped. Adobe CF and runtimes without complete native coverage use these hooks plus: +When explicitly selected, qb's onQBExecuteException closes the original preQBExecute span and preserves the database exception even if an observer fails. Pretend queries are skipped. Adobe CF and runtimes without complete native coverage can use a custom integration or this library-independent manual wrapper: ```cfml result = sentry.withQuerySpan( sql, function( span ) { @@ -66,6 +88,8 @@ result = sentry.withQuerySpan( sql, function( span ) { An explicit query wrapper suppresses native/qb span creation for its execution. SQL literals/comments/numbers are scrubbed; unsupported quoting fails closed. Cached queries and query-of-queries have distinct attributes/operations and are not reported as database round trips. Generic SQL scrubbing is conservative; use signal filters if your schema/identifiers themselves contain sensitive information. +With `databaseSystem: "postgresql"`, double-quoted table and column identifiers stay visible, including qualified names and escaped identifier quotes. String and numeric values remain redacted, and comments are removed. For example, `SELECT "users"."id" FROM "users" WHERE "email" = 'private@example.test'` is recorded as `SELECT "users"."id" FROM "users" WHERE "email" = ?`. Incomplete quotes/comments and unsupported dollar/escape quoting fail closed. Other and unknown database systems retain conservative double-quote redaction. Custom adapters may call `scrubSQL(sql, databaseSystem)`; omitting the second argument uses the configured database system. Query span attributes and manual/native listener dialect arguments override that default. + ## Queues, caches, external calls and scheduled work cbq reserves job properties.__sentry for enqueue timestamp, trace headers, and publish ID. New Scheduled/Started/Exited/Finished hooks carry an immutable executionId and attempt. Started/Exited bracket the actual worker body; Finished may run on a different thread. Handles are keyed by executionId, completion is atomic, and a late success cannot overwrite a timed-out attempt. Request/worker scopes are restored in finally; retention is bounded. Success, manual release, cancellation and failure statuses are recorded. Existing chain/batch paths dispatch through these boundaries. Their comprehensive provider/retry matrices remain deferred. @@ -126,3 +150,9 @@ Browser profiling needs a separate browser support/Document-Policy and overhead See verification.md for commands, exact versions, provider proof and deferred checks. Lucee 6 global listener cache classification requires the query result metadata to include cached (for queryExecute, an existing result option). Missing cache metadata is marked db.cache_status=unknown. Query-of-queries bypassed global listeners in 6.2.8.20; instrument those explicitly with withQuerySpan. Existing error handlers retain their handling behavior; absent one, the adapter throws the original query exception. See the [Lucee listener contract](https://docs.lucee.org/recipes/query-listener.html). + +## Operational setup + +See [operational-observability.md](operational-observability.md) for delivery freshness, bounded request timings, workflow outcomes, optional queue/pool adapters, JVM CPU/GC samples and alert/uptime/cron recipes. See [observability-upstream-audit.md](observability-upstream-audit.md) for the CommuniArts ownership audit and release/adoption boundaries. + +The [local verification report](operational-verification.md) separates focused runtime/test evidence from release, consumer and production checks. diff --git a/docs/operational-observability.md b/docs/operational-observability.md new file mode 100644 index 0000000..2348780 --- /dev/null +++ b/docs/operational-observability.md @@ -0,0 +1,289 @@ +# Operational observability + +Collectors can run through the manual APIs or the opt-in configuration-driven background sampler below. The sampler owns one scheduled executor per service and stops with that service. This module does not create uptime monitors, alert rules, or notification destinations. Existing `enableMetrics`, `enableQueueTracing`, `enableMonitors`, trace sampling, and privacy filters still apply. Configure fixed labels; never derive names from URLs, tenant IDs, email addresses, filenames, job IDs, or exception messages. + +## Delivery freshness + +Set `enableDeliveryDiagnostics: true` to collect receipt diagnostics (default false). `sentry.deliveryDiagnostics()` reports whether collection is `enabled` and returns process-local envelope outcomes: `accepted`, `rejected`, `failed`, `dropped`, `filtered`, `disabled`, and `invalid` counters, plus `pending`, `oldestPendingSeconds`, `lastAttemptAt`, `lastAcceptedAt`, and `untracked`. Timestamps are Unix seconds; zero means no observation. Acceptance is HTTP acceptance by ingestion, not indexed/searchable data. The timestamp records when acceptance was observed, potentially during a later poll. Calling this API polls pending receipts without waiting for network completion. Retention is bounded by `maxPendingEnvelopes` (diagnostic cap 4096); `untracked` explicitly reports incomplete coverage. Counters reset on service restart. No payloads, event IDs, DSNs, response bodies, or exceptions are returned. + +Poll it from an existing low-frequency scheduler or protected operator endpoint. Compare counter deltas and the last accepted timestamp across polls. Keep operator health independently accessible: sending an alarm exclusively through the broken ingestion path cannot reliably report that path's failure. Do not automatically include these diagnostics in public readiness responses. `getTransport().getDiagnostics()` remains available for transport capacity/rate-limit diagnostics. Legacy `/store` error posting is outside envelope receipt diagnostics; use `sentryEventEndpoint: "envelope"` when that coverage is required. Captures rejected before the transport boundary require application capture/lifecycle diagnostics. + +Local freshness does not prove provider retention. Independently query the latest indexed sample for each expected metric, environment, and release. A missing queue series is unknown, not zero backlog. A fresh successful sampler poll with zero pending jobs is a measured zero. Do not replace missing series with zero in dashboards or alerts. + +## Request phases and Server-Timing + +```cfml +// Create once per eligible request and keep it in PRC, never on a singleton. +// Names are registered when used; no requestTimingNames setting is required. +prc.timings = sentry.newRequestTimings(); +result = prc.timings.measure( "shared_props", function(){ return buildSharedProps(); } ); +// Or begin(name) / finish(token, status="ok") around lifecycle hooks. + +// Before response output, under the application's authorization/eligibility policy: +event.setHTTPHeader( name="Server-Timing", value=prc.timings.toServerTiming() ); +``` + +`measure(name,callback)` and `begin(name)` register a phase on first use. No upfront allowlist is required. The optional `requestTimingNames` setting seeds names and their Server-Timing header order; it does not restrict later names. Names are deduplicated without regard to case and use the first spelling in spans, metrics, snapshots and headers. For example, `render` and `Render` both record under the first spelling. + +Each instance retains at most 32 distinct names and 64 samples. Existing name validation still applies: 1–48 ASCII letters/digits/underscores, beginning with a letter; control characters are rejected. Invalid names or exhausted limits skip observation while the callback still runs once with its original return value/exception. Choose stable names for code blocks instead of generating names from user data. Durations use a monotonic clock. Tokens are opaque and belong to their creating instance; timing names, starts and span handles remain private. Foreign tokens and mutable extra token fields are ignored. Completion is atomic per token and telemetry failure preserves callback results/exceptions. `snapshot()` returns per-label duration in milliseconds and count. Child `app.phase` spans follow the current trace's sampling. Overlapping phases must not be summed. Instances are request-local; do not share an instance across parallel tasks. App-specific GET/route/authentication/Inertia eligibility and root timing attributes stay in the app. Manual instances leave header publication to the caller; the automatic instance publishes only when `publishServerTiming` is enabled. + +## Workflows and providers + +```cfml +// Empty allowlists by default; also enable metrics/tracing as desired. +operationNames: [ "invitation_send", "pdf_process" ], +providerNames: [ "mail", "storage" ] + +response = sentry.withOperation( + name="invitation_send", + provider="mail", + callback=function(){ return mailClient.send( message ); }, + classify=function( response ){ return response.wasAccepted() ? "accepted" : "failed"; } +); +``` + +The callback runs once, even when telemetry setup/completion fails. Registered names produce an `app.workflow` child span plus `workflow.operations` and `workflow.duration` metrics (milliseconds), labeled with the fixed workflow/provider/outcome. Unregistered names/providers run without observation. Allowed outcomes are `success`, `accepted`, `delivered`, `retry`, `timeout`, `cancelled`, `failed`, and `unknown`. Missing classifier means callback `success`; an invalid or failing classifier means `unknown`. A thrown callback remains the original exception with outcome `failed`. Results and exception text never enter telemetry. `accepted` does not establish `delivered`; record actual delivery only from authenticated provider delivery evidence. Use existing `withSpan`, `withHttpSpan`, and `withTraceContext` for custom spans, outbound requests and background roots; do not wrap an already instrumented provider call twice just to obtain another HTTP span. + +Audition submission, publishing, invitations, upload/PDF work and notification delivery are application workflows. Their names, business success/retry rules, compensation and provider-specific response/webhook interpretation are application policy. A workflow may include existing HTTP/storage/search spans without duplicating them. + +## Queue metrics + +Existing cbq integration uses immutable per-attempt handles, worker entry/exit scope restoration, atomic completion, receive latency, retry count, attempt outcome and processing duration. Completion callbacks may run on another thread; never replace handles with callback-thread scope lookups. A timeout wins over late success. Attempts that never started no longer emit a fictional zero processing duration. The completion timestamp is captured before exporter work, so serialization/delivery diagnostics cannot inflate processing duration. The root is completed before best-effort metric emission. + +`queueMetricDestinations: [ "mail", "pdf" ]` opts into destination labels for attempt/wait metrics. Other destinations are aggregated without a destination attribute. Optional `enableQueueJobMetrics: true` adds `queue.jobs` counters from cbq's terminal `onCBQJobComplete`/`onCBQJobFailed` callbacks, with `completed`/`failed` outcomes. Attempt `internal_error`, `released` and timeout are not proof of exhausted retries; job failure is measured at the terminal hook. Terminal counters count hook notifications and do not provide durable job-ID deduplication. No job payload, result or exception enters these metrics. Consumers already counting terminal hooks should choose one producer. + +For the cbq **database provider**, construct one sampler for each database/table being observed: + +```cfml +sampler = new sentry.models.telemetry.DatabaseQueueMetrics( + telemetry = sentry.getObservability(), + config = wirebox.getInstance( "Config@cbq" ), + queryFactory = function(){ return wirebox.getInstance( "QueryBuilder@qb" ); }, + table = "cbq_jobs", + maxQueues = 64 +); +// Existing consumer scheduler, for example once per minute: +ok = sampler.capture(); +diagnostics = sampler.getDiagnostics(); +``` + +The query factory must return a fresh qb builder configured with the correct datasource/grammar. Use a separate sampler for each connection; supply only worker definitions belonging to that connection. Redis, SQS and sync providers require their own supported queue-statistics adapters, not this table query. There is no mandatory qb/cbq dependency or automatic WireBox mapping. + +Configured idle queues emit zero immediately after a successful poll. Dynamically observed queues are remembered so they return to zero when drained. Exact destination identity is preserved, including queues that differ only in letter case. Label retention and emissions are bounded (64 by default; maximum 256 per sampler lifetime). Additional queues are omitted and counted in `omittedQueues`; fixed worker names are strongly preferred. Poll failures emit no fabricated zero samples and update `failures`/`lastFailureAt` while retaining `lastSuccessAt`. Diagnostics contain no SQL or error text. Pending depth excludes reserved/completed/failed jobs and includes delayed jobs. Oldest age is since creation, not since runnable time. The query uses qb, including grammar-aware identifier quoting; never silently change a consumer's direct-SQL policy. + +## JVM and database connections + +Schedule existing `sentry.captureRuntimeGauges()` explicitly with `enableMetrics: true`. It now returns the samples it observed: heap used/max bytes, thread count, optional process CPU load (`jvm.cpu.process`, ratio 0–1), and cumulative GC collections/time (`jvm.gc.collections.total`, `jvm.gc.duration.total`, milliseconds). Negative/unsupported CPU or GC values are omitted. GC gauges are totals since JVM startup: calculate deltas over elapsed time and handle resets; do not sum cumulative samples or interpret them as pause percentiles. CPU uses an optional JVM MXBean attribute, not a BoxLang profiler. Heap/threads remain independent of CPU/GC availability. No sampling occurs when metrics are disabled. + +Connection acquisition and pool visibility depend on the engine/provider. Query execution spans do not isolate connection waits. BoxLang automatic query tracing stays off when the native runtime lacks a caught-query failure event; no query library is inferred. qb hooks require its explicit provider-owned adapter. Native Lucee query listeners and custom/manual integrations remain supported as documented in [observability.md](observability.md). No CPU stack profiler, allocation profiler, JDBC driver patch or engine-internal datasource pool inspection is added here. + +Use a supported provider/JMX adapter with `databasePoolNames: [ "primary" ]` and `sentry.captureDatabasePool( "primary", function(){ return poolAdapter.snapshot(); } )`. The callback returns any available nonnegative numeric `active`, `idle`, `max`, `waiters`, and cumulative `timeouts`; all other fields are discarded. `db.pool.utilization` is derived only from valid active/max values. Unsupported fields remain absent; callback failure returns false. Never return connection strings, credentials, datasource/tenant IDs, SQL, or exception text. Pool counters are gauges; compare deltas and account for JVM/pool resets. This is an optional adapter boundary, not a claim that these measurements are available on every CFML/BoxLang runtime. + +If your provider exposes acquisition directly, use existing `withSpan( "primary connection", acquireCallback, "db.connection" )` strictly around its connection acquisition call, then execute queries under their normal query instrumentation. Do not label the entire query or transaction as connection wait. Do not acquire extra connections just to measure the pool. + +## Alerts, availability and scheduled work + +Use the consumer's Sentry environment/release consistently. In Sentry's [Monitors](https://docs.sentry.io/product/monitors-and-alerts/monitors/) UI, create metric monitors from the relevant Explorer query and attach the organization's chosen alert destination. Choose thresholds, minimum volume, sustained evaluation window, recovery window and notification routing based on the service's baseline and SLO. Verify the final query, filters, units, sampling, data availability and plan support before enabling it. Useful recipes: + +| Concern | Input and evaluation | +| --- | --- | +| Error regression | Errors/request ratio plus minimum request count; sustained increase relative to the prior healthy window, grouped by environment/release. An exception event count is not automatically a request error rate. | +| Latency regression | p95 `http.server.duration` or request spans for fixed route templates, plus minimum samples and a sustained window. Keep absolute SLO and relative-change thresholds separate. | +| Queue health | Fresh `queue.depth`/`queue.oldest_age`, wait/processing duration, attempt outcomes and terminal failures. Treat sample absence as telemetry failure/unknown, not an empty queue. | +| JVM/pool pressure | Sustained CPU/heap ratio, GC time deltas, pool utilization/waiters/acquisition timeouts. Unsupported fields are gaps to investigate, not zero pressure. | +| Telemetry outage | Independent polling of local receipt/sampler diagnostics plus indexed latest timestamps and provider outcome/drop counters. Fresh HTTP acceptance alone does not establish indexed samples. | + +For [external uptime monitoring](https://docs.sentry.io/product/monitors-and-alerts/monitors/uptime-monitoring/), create explicit monitors for the public liveness URL and application readiness URL. Use a cheap GET/HEAD, expected 2xx, bounded timeout and consecutive-failure/recovery thresholds. Liveness measures process reachability; readiness should return 503 when essential dependencies cannot serve traffic, without disclosing dependency details. Keep probes read-only and free of user data, credentials and session creation. Test each endpoint's actual response before saving a monitor. An external probe must run outside the application so it can detect a stopped app. The module cannot infer which dependencies define readiness. + +Existing `withMonitor`/`captureCheckIn` already support scheduled jobs. Configure `enableMonitors`, a fixed `monitorSlugs` allowlist, and per-job schedule/timezone, `checkin_margin` and `max_runtime` through `monitorConfig` (see [Cron monitors](https://docs.sentry.io/product/monitors-and-alerts/monitors/crons/)). Wrap the actual job once: `withMonitor( "nightly-summary", runSummary, monitorConfig )`. An `in_progress`/`ok` pair with the same ID measures completion; error reports execution failure; provider-side missing-check-in/runtime rules detect missed or overrun jobs even when no callback returns. Schedule syntax, thresholds, destinations and monitor limits belong to the consumer. Do not create a monitor for every fast idle poll. + +## Browser and release correlation + +The companion and official browser SDK already provide browser tracing/Web Vitals and release/environment configuration. Keep that existing integration for LCP/INP/CLS. Do not initialize a second browser SDK. Compare vitals by fixed page/component name, release, browser and sufficient sample count. Retain existing navigation/resource span privacy filters. Network failures, CSP and dynamic-import failures are distinct observations; a later successful probe cannot prove the original request succeeded. + +CommuniArts already has Vue/Inertia navigation tracing, Postmark/storage/Typesense spans, a bounded preload/resource diagnostic collector, and replay upload triggered by feedback. Those capture policies, screenshot/tunnel authorization and resource classification stay in the app. This change does not broaden replay/error capture or add redundant browser listeners. Ensure frontend and backend release IDs identify the actual deployed assets/source; package installation and accepted envelopes do not establish deployment or provider indexing. + +## Request saturation + +Enable `enableMetrics: true` and `enableRequestSaturation: true` to count eligible ColdBox requests independently of trace sampling or `enableRequestTracing`. The existing `requestFilter` applies. Request capture starts one observation; `postProcess` and explicit `endRequest()` finish it once. The exception hook finishes its saturation observation even when ColdBox skips `postProcess`, while retaining an existing trace for error correlation. Always call `sentry.endRequest()` in Application.cfc/Application.bx **both `onRequestEnd` and `onAbort`** for binary output, early redirects/aborts and final trace/scope cleanup after exceptions. A hard engine termination that bypasses all cleanup cannot be detected reliably by the module. See [completion contracts](request-completion.md). + +Poll `sentry.captureRequestSaturation()` from an existing scheduler. It emits `http.server.active_requests`, `http.server.oldest_active` (seconds), `http.server.long_running`, and cumulative `http.server.untracked_starts.total`. `sentry.requestSaturation()` returns the same local snapshot without emitting. Ages use a monotonic JVM clock; `requestLongRunningSeconds` defaults to 30. Long-running requests are not automatically declared timed out. An authenticated engine/application timeout callback can call `endRequest(timedOut=true)` or `finishRequestObservation(token,timedOut=true)` to record `http.server.timeouts` once. + +Standalone consumers can pair `token = sentry.beginRequestObservation()` with `sentry.finishRequestObservation(token)` in `finally`. Tokens have private start state and belong to their service. Retention is capped by `maxObservedRequests` (default/maximum 4096). When full, new starts are untracked and counted; observed active count is then a lower bound. Abandoned observations remain visible rather than silently expiring into a fabricated recovery. All counts reset on service restart; collection is local to that service/process. + +Engine executor information is explicit because servlet/runtime pool APIs differ: + +```cfml +// moduleSettings.sentry +requestExecutorNames: [ "http" ] + +sentry.captureRequestExecutor( "http", function(){ + return executorAdapter.snapshot(); // active, max, queued, rejected (cumulative) +} ); +health = sentry.requestExecutorDiagnostics( "http" ); +``` + +Only finite nonnegative numeric fields are emitted as `http.server.executor.active`, `.max`, `.queued`, and `.rejected.total` gauges. The module does not inspect private engine pools. Unsupported fields stay absent. A failed or empty snapshot returns false, emits no fabricated zero, and updates bounded per-name `lastSuccessAt`, `lastFailureAt`, and `failures` diagnostics. Executor and worker samplers permit up to 32 configured technical names of 1–48 ASCII letters/digits/underscore/dot/hyphen; case aliases are deduplicated and callers must use the first configured spelling. Disabled metrics skip callbacks entirely. + +## Dependency outcomes + +```cfml +// moduleSettings.sentry + dependencyNames: [ "mail", "storage", "search" ] + +result = sentry.withDependency( "mail", function(){ + return mailClient.send( message ); +}, function( result, failure ){ + if ( !isNull( arguments.failure ) ) { + return { outcome: "error" }; // Map known exception types here; do not return exception text. + } + return { + outcome: result.wasAccepted() ? "success" : "rejected", + statusCode: result.statusCode, + retryCount: result.retryCount, + circuitState: result.circuitState + }; +} ); +``` + +One callback invocation represents one logical dependency call, including any retries the callback owns. The wrapper never retries or changes circuit-breaker policy. It preserves the callback result, null/void return and original exception. The classifier receives `(result, failure)` with the unused argument null and may return only an allowlisted outcome (`success`, `error`, `timeout`, `connection_error`, `cancelled`, `rejected`, `unknown`), optional HTTP `statusCode` (100–599), nonnegative integer `retryCount`, and `circuitState` (`open`, `closed`, `half_open`). An exception cannot become `success`. Without a classifier, normal return means `success` and a thrown callback means `error`; invalid/failing classifiers produce `unknown`. The consumer must interpret provider responses—HTTP success is not necessarily business success or delivery. + +The wrapper emits `dependency.calls`, `dependency.duration` (milliseconds) and optional `dependency.retries` distributions, with fixed dependency name/outcome and optional HTTP status class/circuit state. Duration covers the callback, excluding telemetry setup, classification and export. Circuit state describes the observed call; it is not a continuously sampled breaker gauge. An `app.dependency` span follows existing sampling. Existing `withHttpSpan` remains the transport span/propagation mechanism; wrap it inside this logical dependency observation where appropriate, without adding a second HTTP span. Results, bodies, exception messages and URLs never become dependency metric labels. Unconfigured names or disabled metrics run the callback directly without classification or observation. + +## Queue health + +```cfml +// moduleSettings.sentry +queueNames: [ "mail" ] + +// Choose a provider adapter explicitly. This example uses the adapter shipped by cbq. +queueAdapter = wirebox.getInstance( "QueueHealthAdapter@cbq" ); +sentry.captureQueueHealth( "mail", function(){ + return queueAdapter.sample( { options: { workerPool: "outbound" } } ); +} ); + +// Or call another queue provider's supported snapshot API: +sentry.captureQueueHealth( "mail", function(){ + return otherQueueProvider.workerSnapshot(); +} ); +health = sentry.queueHealthDiagnostics( "mail" ); +``` + +The cbq queue adapter combines provider queue measurements with configured worker concurrency and active executions from executor statistics when the executor advertises task-method support. Its current executor capacity is not an activity count. An unavailable/uninitialized executor leaves active executions absent; these measurements do not prove that workers are alive. A provider snapshot may additionally supply actual `activeWorkers`, `lastHeartbeatAt` (JVM-compatible Unix seconds), `runnable`, `delayed`, `reserved`, `staleReservations`, and cumulative `retries`/`exhausted`. Fields map to `queue.workers.configured`, `queue.workers.active`, `queue.executions.active`, `queue.worker.heartbeat_age` (seconds), `queue.backlog.runnable`, `.delayed`, `.reserved`, `queue.reservations.stale`, `queue.retries.total`, and `queue.exhausted.total`, labeled by the configured `queue.name`. A heartbeat must come from real worker activity; zero/absent/future heartbeat timestamps are omitted. A successful poll does not manufacture a worker heartbeat. + +Only available finite nonnegative numeric fields are emitted. Cumulative retry/exhaustion gauges reset with their provider; compute deltas and handle resets. They differ from existing per-attempt and terminal-hook counters. Choose one producer for each series. Failed/empty snapshots emit no fabricated zeros and leave the last successful timestamp unchanged; read diagnostics to detect stale polling. Manual capture creates no polling threads. The optional automatic sampler is described below; neither mode mutates queues or captures job IDs/payloads. + +The cbq adapter asks its configured provider for queue health. Its database provider owns the table, datasource and aggregate query; these are not Sentry settings. Older manual integrations can still use the optional compatibility facade: + +```cfml +queueHealth = new sentry.models.telemetry.DatabaseQueueHealth( + queryFactory = function(){ return wirebox.getInstance( "QueryBuilder@qb" ); }, + table = "cbq_jobs" +); +sentry.captureQueueHealth( "mail", function(){ + return queueHealth.snapshot( "mail" ); +} ); +``` + +Use a fresh correctly configured qb builder per call. `snapshot(queue)` performs one parameterized aggregate query excluding completed/failed jobs. Unclaimed jobs are runnable when `availableDate <= now`, delayed otherwise. Reserved/claimed jobs form a separate total; a lease is stale when its `availableDate <= now`, matching the inspected cbq database-provider lease convention. These are database observations, not proof of a running worker or permission to release a lease. Configured worker definitions/connection selection and scheduler cadence belong to the consumer. Other providers need their own supported snapshot adapter. No mandatory qb/cbq dependency is added. + +### Local database and worker integration tests + +`DatabaseQueueHealthIntegrationSpec` is opt-in. Without `SENTRY_QUEUE_HEALTH_DB`, it registers no tests and the ordinary harness needs no database or cbq installation. To enable it: + +1. Create an empty disposable PostgreSQL database on **127.0.0.1:5432**, accessible as local test role `postgres` without a password. Its name must begin with `sentry_queue_health_` and contain only lowercase ASCII letters, digits and underscores. Never point this fixture at an application database. +2. Place read-only local dependency links at `test-harness/modules/qb`, `test-harness/modules/cbq` and `test-harness/modules/str` (the latter can point to cbq's nested str module). These paths are ignored. Set `SENTRY_QB_SOURCE` and `SENTRY_CBQ_SOURCE` to the matching local sources, and `SENTRY_QUEUE_HEALTH_DB` to the dedicated database name in the owned server environment. +3. Supply the engine's PostgreSQL JDBC driver. The verified Lucee runtime uses its existing `org.postgresql.jdbc` 42.6.0 bundle. The BoxLang runtime uses that driver JAR in its isolated servlet classpath. Adobe requires its `postgresql` CFPM package; copying a driver JAR alone does not satisfy its package registration. Do not change an application/server shared with other work. +4. Start the loopback envelope receiver, then run `/tests/runner.cfm?directory=&reporter=json&bundles=tests.specs.DatabaseQueueHealthIntegrationSpec` on the local harness server. The full normal runner and telemetry runner include the opt-in bundle as well. Run engine suites sequentially against the shared receiver. + +The integration fixture loads the real cbq module and registers its own database connections/pools. Jobs are serialized, persisted, fetched by cbq's real scheduled poller, deserialized and executed on actual cbq worker executors. Java latches hold a job while sampling reservation and executor activity. Tests cover empty/mixed queues, bound queue-name parameters, idle/busy/drained workers, delayed execution, retries, terminal exhaustion, expired execution leases and staged claims. Lease recovery uses deliberately expired fixture rows; it does not kill a worker process or qualify crash recovery. + +Each case creates a uniquely named table in the dedicated database and drops it afterward. Teardown stops polling, releases blocked jobs, drains/shuts down workers, removes executor/config registrations, clears shared state and closes telemetry. A forced-drain-failure regression verifies cleanup continues after an error. Synthetic failures cannot inherit an external `SENTRY_DSN`: this mode forces the module's receiver URL to loopback and disables automatic error/appender capture. No application dependency source or provider settings are modified. + + +## Configuration-driven automatic collection + +Manual APIs remain available. Enable metrics, configure a valid DSN, and enable one periodic sampler. Queue collection has **one** provider-neutral setting: `automaticCollectors.queues`. Each queue adapter supplies the fields its provider supports; worker measurements, backlog, age and lease health do not require separate collectors. + +```cfml +moduleSettings.sentry = { + DSN: getSystemSetting( "SENTRY_DSN", "" ), + enableMetrics: true, + enableRequestSaturation: true, + enableAutomaticRequestTimings: true, + publishServerTiming: false, + automaticCollectors: { + enabled: true, + intervalSeconds: 60, + runtime: true, + requestSaturation: true, + queues: [ { + // One fixed logical queue label, independent of provider identifiers. + name: "mail", + // Explicit provider-owned implementation; there is no default provider. + adapter: "QueueHealthAdapter@cbq", + // Adapter-specific selection. cbq derives connection/provider and queues from this worker pool. + options: { workerPool: "outbound" } + } ], + requestExecutors: [ { name: "http", adapter: "HttpExecutorSnapshot" } ], + databasePools: [ { name: "primary", adapter: "PrimaryPoolSnapshot" } ] + } +}; +``` + +A different queue provider supplies a different adapter or callback, with its own options. A backlog-only adapter can return `{depth:0}`; no worker fields are required. A combined snapshot can return `{depth:7,oldestAgeSeconds:15,configuredWorkers:4,activeExecutions:2,runnable:5,delayed:2,reserved:1}`. Only measured finite nonnegative fields are emitted. Supported fields include `depth`, `oldestAgeSeconds`, `configuredWorkers`, `activeWorkers`, `activeExecutions`, `lastHeartbeatAt`, `runnable`, `delayed`, `reserved`, `staleReservations`, `retries` and `exhausted`. All automatic queue series use the fixed `queue.name` label. Missing capabilities remain absent; a failed snapshot records a failed poll instead of fake zeros. + +The former automatic `queueWorkers` and `databaseQueues` settings are replaced by `queues`; they are not aliases or additional collector types. There is no SQL builder, table name, datasource, default queue provider or cbq schema in Sentry's automatic collection path. Those choices belong to the selected adapter/provider. Configured queue names extend the bounded `queueNames` allowlist. Worker and backlog data are sampled once per definition, with a shared `queueHealthDiagnostics(name)` freshness/failure record. `captureQueueHealth(name,callback)` uses the same snapshot contract for explicit manual collection. Legacy manual worker APIs and their `worker.name` labels remain available for existing integrations. + +The fixed delay starts after the first interval and follows the end of each cycle. It defaults to 60 seconds, permitting integer intervals from 1 through 3600. Cycles cannot overlap; slow adapters delay later cycles. Keep callbacks bounded and configure query/network timeouts in the provider. Shutdown prevents dispatching later adapters; an already-running callback may take time to return. ColdBox's context-aware Runnable establishes the engine application context for named datasources and releases Adobe JDBC thread state. The sampler isolates/restores trace scope, continues after adapter failures, and retains bounded process-local diagnostics through `automaticCollectorDiagnostics()`. It does not send its own failure details recursively through telemetry. + +Each group processes at most 32 definitions. Partial overrides inherit defaults. `requestSaturation` also requires `enableRequestSaturation`. `deliveryDiagnostics:true` polls local receipts. Avoid multiple producers for the same series. `collectAutomaticMetrics()` shares the non-overlap guard; `startAutomaticCollectors()` is idempotent. Restart the service after changing configuration. + +### Callback and WireBox adapter contracts + +Adapters can be callbacks, WireBox mapping IDs, compatible objects, or `{mapping,method}` descriptors. Mappings are resolved lazily. Snapshot context contains the fixed `name`, collector `kind` and a copy of the definition's `options` struct. Options never become metric attributes. + +| Adapter | Callback / interface method | Result | +| --- | --- | --- | +| Queue, executor or pool snapshot | callback(context) / ISnapshotAdapter.sample(context) | Supported numeric fields | +| Dependency classifier | callback(result,failure) / IDependencyClassifier.classify(result,failure) | Bounded outcome/status/retry/circuit fields | +| Query tracing setup | callback(context) / IQueryAdapter.configure(context) | Built-in mode or `{mode:"custom",shutdown:callback}` | + +Interfaces live under `sentry.models.telemetry.adapters`. Compatible provider adapters need not import Sentry's interface or depend on Sentry. Standalone services can use callbacks/objects; mapping IDs require an injector (ColdBox supplies it). Query tracing configuration receives `context.telemetry`, `context.applicationName` and `context.databaseSystem` (default `other`). Built-in `queryAdapter` values are `none`, `lucee`, `auto` and `boxlang`. For qb, explicitly select `QueryTelemetryAdapter@qb` from a compatible qb release; Sentry has no qb-specific interceptor or built-in `qb` mode. Custom setup installs query hooks and can return `{mode:"custom",shutdown:callback}` for lifecycle cleanup. This controls query tracing, independently of queue snapshots; there is no queue-specific query factory setting. + +### cbq adapter ownership + +The optional cbq integration lives in cbq as `QueueHealthAdapter@cbq`. Choose exactly one explicit source: `options.workerPool` or `options.connection`. The former resolves the actual worker pool each time, derives its connection/provider and queue selection, and combines worker activity with any supported `getQueueHealth(queue)` provider snapshot. The latter samples the connection/provider directly without requiring or creating a worker pool and emits no worker fields. An optional `options.queue` overrides either source's selection; otherwise the worker pool's queue name/array/wildcard or the connection's default queue is used. Ambiguous sources and the former `options.pool` spelling are rejected. cbq's database provider applies its own configured table and query options/datasource. Providers without queue-snapshot support return only supported worker measurements for a worker pool or an empty snapshot for a connection, without invented backlog or heartbeat values. + +For backlog-only collection, use the same Sentry `queues` list with connection options: + +```cfml +// Configure one logical queue snapshot without requiring cbq workers in this process. +moduleSettings.sentry.automaticCollectors.queues = [ { + // Fixed telemetry label. + name: "mail", + // Optional implementation shipped by a compatible cbq release. + adapter: "QueueHealthAdapter@cbq", + // Select the cbq connection and queue, rather than a worker pool. + options: { + // Registered cbq connection name. + connection: "database", + // Queue selection; omitted uses the connection's default queue. + queue: "mail" + } +} ]; +``` + +Worker measurements describe the entire worker pool, including when `options.queue` narrows the backlog selection. Prefer one collector per worker pool for combined measurements; use connection-only definitions for additional per-queue backlog measurements to avoid duplicating shared worker activity. + +The installed cbq version must contain the adapter; existing releases do not gain it automatically. `WorkerHealthAdapter@cbq` remains a compatibility alias. Sentry's legacy `captureCBQWorkerHealth(name,pool)` and `DatabaseQueueHealth(queryFactory,table)` entrypoints delegate to cbq's optional implementations. New integrations should use the unified adapter contract. Neither module requires the other as an installed dependency. + +### Automatic dependency and request instrumentation + +A dependencyAdapters entry binds WireBox AOP to the exact mapping and named public methods. Up to 32 entries and 16 methods per entry are retained. It observes normal calls to WireBox-created instances without a manual wrapper. Objects created with new, already-created singletons and internal calls bypassing the WireBox proxy require the manual API. Bindings are registered at module load; dependencyAdapterGenerationPath selects a writable mapping for generated AOP code when the module installs the Mixer listener. Existing Mixer configuration is reused. + +Classification follows the manual withDependency contract. A failing classifier yields unknown and cannot replace the original result/exception. The same dependency name is observed once when a manual wrapper and automatic method advice overlap or nest; the outer observation supplies its classifier. Different configured dependency names remain separate logical observations. Method arguments, results and raw exceptions never become telemetry attributes. + +With enableAutomaticRequestTimings, ColdBox preEvent/postEvent measure handler work and preRender/postRender measure response output, using a shared request-local timing instance. These are framework boundaries, not individually classified business operations; preRender occurs after view/data marshalling. Automatic finishes also emit http.server.phase.duration in milliseconds. Every open handle on that timing instance, including manual phases, is drained on exceptions and explicit endRequest; the instance then rejects new handles. Keep both Application onRequestEnd and onAbort fallbacks. Nested events can overlap, so do not sum their durations as wall time. The optional Server-Timing header is only published when publishServerTiming is true and an optional serverTimingFilter(event) permits it; responses that already committed headers cannot receive it. + +Manual newRequestTimings()/measure() and withDependency()/withOperation() remain available. sentry.getRequestTimings() returns the active automatic instance, or null outside an automatically timed request. Additional manual phases work without requestTimingNames configuration. Use distinct names instead of timing the same automatic handler/render interval again. diff --git a/docs/operational-verification.md b/docs/operational-verification.md new file mode 100644 index 0000000..78a1718 --- /dev/null +++ b/docs/operational-verification.md @@ -0,0 +1,145 @@ +# Local operational-observability verification — October 25, 2026 + +## Provider-owned query and queue adapters + +qb now owns the optional `QueryTelemetryAdapter@qb` execution listeners. Sentry resolves that explicit mapping through the generic query-setup contract, passes telemetry/application name/database system, and owns no qb interceptor or built-in `qb` mode. Manual query spans and native engine listeners remain available. Shutdown rejects new observations while active query completions drain and restore their scopes before qb unregisters its own listeners. Independent review found the original immediate-unregister race; the corrected implementation and regressions were re-reviewed. Adobe also required avoiding a listener method named `release`, which collides with engine lifecycle behavior. + +cbq's queue adapter now selects exactly one `options.workerPool` or `options.connection`. Worker pools supply combined worker/backlog data; connections supply backlog without requiring or registering workers. Queue overrides/defaults follow the selected source. Ambiguous sources and the former `options.pool` spelling are rejected. The earlier reported MySQL CI failure was superseded by the existing quoted-alias fix; it was already green before this selector follow-up. + +Final local proof on each prepared Lucee 5.4.8.2, Adobe 2023 update 25 and BoxLang 1.18.0+66 runtime: + +- **141/141 Sentry specs**, including thirteen real PostgreSQL/cbq cases. New cases prove connection-only snapshots without a worker pool and actual ColdBox/WireBox qb adapter resolution, real query success/error spans, SQL scrubbing, manual de-duplication, scope restoration, independent listener ownership and in-flight completion during shutdown. +- **14/14 cbq adapter specs**, including both selector paths, absent capabilities and failure/validation behavior. +- **16/16 focused qb adapter/failure-hook specs**, run independently by the qb implementation subagent. The complete qb suite and hosted checks are recorded separately on its draft PR. + +All local results above have zero failures/errors/skips. Evidence is `/tmp/sentry-provider-adapters-final-{lucee5,adobe2023,boxlang}.json`, `/tmp/cbq-provider-selectors-final-{lucee5,adobe2023,boxlang}.json` and `/tmp/qb-telemetry-final-{lucee5,adobe2023,boxlang}.json`. The dedicated PostgreSQL 18.1 database is `sentry_queue_health_adapters_20261025`; the final runs leave zero tables. The owned harness uses the new qb/cbq worktrees through temporary local links/settings; installed application modules remain unchanged. Formatting and whitespace checks pass. Adapter mappings require provider versions containing these additions; no release or consumer upgrade is implied by a draft PR. Prepared-runtime, provider/backend, heartbeat, timeout and delivery limitations below still apply. + +## Unified queue adapter follow-up + +The current configuration is one `automaticCollectors.queues` list, with explicit provider adapters returning whichever queue/worker fields they support. The former automatic worker/database groups are removed. Generic snapshots emit fixed `queue.name` labels with shared freshness/failure diagnostics; backlog-only providers need no worker fields. No automatic Sentry path selects cbq, constructs a SQL builder or chooses a jobs table/datasource. The cbq `QueueHealthAdapter` reads the pool/provider selection; its database provider owns the aggregate query and configured query options. Older manual APIs remain as compatibility entrypoints. + +Final local results: **137/137 Sentry specs plus 9/9 cbq adapter specs on each of Lucee 5.4.8.2, BoxLang 1.18.0+66 and Adobe 2023 update 25**, with zero failures/errors/skips. Evidence: `/tmp/sentry-unified-final-lucee5.json`, `/tmp/sentry-unified-final-boxlang.json`, `/tmp/sentry-unified-final-adobe2023.json` and the corresponding `/tmp/cbq-unified-final-*.json` files. The Sentry suite includes eleven actual PostgreSQL/cbq integration cases. One automatic definition now measures both real worker execution and database depth/age/lease state. New real queries verify provider-owned table/datasource options, multiple queues, wildcard selection and bound quote-bearing names. Native activity, unsupported fields, combined/backlog-only providers and preserved manual APIs are covered. The separate cbq hosted suite additionally tests its database snapshot against MySQL. + +Review found an optional facade test that now also needs cbq; its fixture guard is corrected. Adobe exposed a constructor/inline-callback compiler error; the provider now declares the factory and snapshot separately. After that failure, only the dedicated test database's leftover fixture tables were removed before the final reruns; final runs left zero tables. The disposable `sentry_queue_health_unified_20261024` database was removed after owned processes stopped, and local server settings/dependency links were restored. Installed application modules and the pre-existing PostgreSQL server were not changed. Formatting and whitespace checks passed. Current hosted results are recorded on the draft PRs; the earlier configurations/results below are historical. + +## Earlier provider-owned worker adapter follow-up + +Automatic `queueWorkers` now requires an explicit `adapter`, with provider configuration in `options`. Sentry no longer chooses cbq from a `pool` key. The optional cbq implementation lives in cbq as `WorkerHealthAdapter@cbq`, follows the `sample(context)` contract without a Sentry dependency, and requires `options.pool`. The existing manual cbq API delegates to that mapping; generic manual snapshot APIs remain available without cbq. + +The complete Sentry harness passed **134/134**, and cbq's new adapter bundle passed **6/6**, on each prepared Lucee 5.4.8.2, BoxLang 1.18.0+66 and Adobe 2023 update 25 runtime. All results have zero failures, errors or skips. The ten live PostgreSQL/cbq cases use cbq `next` base `fad884a` plus the new adapter, installed qb 14.1.0, and actual registered worker pools. The automatic case resolves cbq's adapter through WireBox, passes its explicit pool option, and observes an actual blocked worker with reserved work, pending depth and oldest age. Native executor activity is also checked in the adapter bundle. No installed application dependency was edited. + +Evidence: `/tmp/sentry-worker-adapter-final-lucee5.json`, `/tmp/sentry-worker-adapter-final-boxlang.json`, `/tmp/sentry-worker-adapter-final-adobe2023.json`, and the corresponding `/tmp/cbq-worker-adapter-final-*.json` files. Sentry formatting checked 63 files; cbq formatting checked its two new CFCs. Both whitespace checks passed. Independent bug/CFML/BoxLang compatibility review found a stale configuration example, now fixed, and re-review found no remaining actionable issue. The separate complete cbq suite was not run locally; its hosted workflow and the Sentry hosted matrix are reported on their draft PRs. + +The disposable database `sentry_queue_health_adapter_20261024` had no fixture tables after tests and was removed after owned processes stopped. The pre-existing PostgreSQL server remains running. Local server configuration and dependency links were restored. Using the cbq mapping requires a cbq release containing the adapter; existing releases do not acquire it automatically. Earlier evidence and limitations below retain their original scope. + +## Ad hoc timing follow-up + +Manual `measure(name,callback)` and `begin(name)` now register valid phase names on first use without `requestTimingNames` configuration. The setting remains an optional initial name/header order list. Case variants record under the first spelling. Existing 32-name/64-sample limits, token ownership, name validation and original callback outcomes are preserved. + +The updated 43-spec focused bundle reproduced the previous restriction (39 passed, two failures, two errors in `/tmp/sentry-adhoc-before.json`). After the fix, the complete ordinary harness passed **123/123 on Lucee 5.4.8.2, BoxLang 1.18.0+66 and Adobe 2023 update 25**, zero failures, errors or skips. Evidence: `/tmp/sentry-adhoc-final-lucee5.json`, `/tmp/sentry-adhoc-final-boxlang.json`, `/tmp/sentry-adhoc-final-adobe2023.json`. New coverage checks ad hoc manual and shared automatic instances, once-only tokens, case canonicalization, emitted phase metrics, limits, invalid names and preserved callback returns/exceptions. Independent source review found no remaining actionable bug or CFML/BoxLang/older-Adobe compatibility issue. Formatting checked 63 files and whitespace checks passed. + +These follow-up runs use the prepared local runtimes with optional qb compilation contracts enabled. The ten opt-in live PostgreSQL/cbq cases were not rerun for this timing-only change; their earlier 130/130 evidence below remains historical. Browser assets are unchanged. Hosted CI evidence is recorded separately on draft PR #56. + +## Earlier automatic-collector verification + +The configuration-driven automatic collectors and dependency adapters passed the complete local suite: **130/130 on each engine, zero failures, errors or skips**. Manual APIs remain covered. Each run includes 25 automatic-configuration specs and 10 live PostgreSQL/cbq cases. + +| Engine | JVM / servlet runtime | Current raw evidence | +| --- | --- | --- | +| Lucee 5.4.8.2 | Temurin 21.0.10+7, Runwar 5.2.7 | `/tmp/sentry-auto-final-lucee5.json` | +| BoxLang 1.18.0+66 | Temurin 21.0.10+7, Runwar 6.1.8 | `/tmp/sentry-auto-final-boxlang.json` | +| Adobe ColdFusion 2023 update 25 | Temurin 21.0.10+7, Runwar 5.2.7 | `/tmp/sentry-auto-final-adobe.json` | + +The new coverage verifies default-off behavior, partial settings, bounded names, callback/object/WireBox-interface contracts, real WireBox method interception, original returns/errors/null/void behavior, manual/automatic deduplication, failing observation/classifier setup, once-only query setup/cleanup, actual scheduled collection, scope isolation, no overlapping cycles, shutdown during a cycle, local receipt failures, automatic handler/render phases, early exits, invalid phase names and gated Server-Timing publication. The tenth live case runs the background sampler against a real blocked cbq job and a WireBox query factory, asserting actual execution activity, reservation, depth and oldest-age metrics. + +Review findings are fixed and re-reviewed: open timing tokens on early exits, partial nested settings at module load, dispatch after shutdown, scope lookup breaking fail-open execution, receipt-poll failures omitted from cycle diagnostics, an accidentally empty disabled test, and complex phase names reaching deduplication before validation. Full testing also found and fixed a BoxLang built-in `attempt()` name collision and missing application context on raw scheduled runnables. Automatic work now uses ColdBox's context-aware Runnable proxy when available, including Adobe JDBC thread-state cleanup. The reviewer found no remaining actionable bug or CFML/BoxLang compatibility issue. + +The existing prepared runtimes and read-only ColdBox/TestBox/qb/cbq dependencies described below were reused. Adobe's PostgreSQL package was registered, but its servlet context could not load the JDBC class after restart. The owned ignored runtime was repaired by reinstalling its CFPM PostgreSQL package and adding its existing JDBC 42.7.3 JAR to `WEB-INF/lib`, followed by restart. Both package registration and classpath availability were required; no product workaround or installed application dependency change was made. This qualifies these prepared local runtimes, not a fresh installation. + +CFFormat checked **63 files**; `git diff --check` passed. No browser code changed; the earlier **28/28** browser result below is historical and was not rerun for this backend-only change. Engine suites share a mutable loopback receiver and must run sequentially: a concurrent run interfered with one delivery-diagnostics case; the final sequential rerun passed. Raw evidence is local temporary output, not a durable CI artifact. + +The dedicated database was `sentry_queue_health_auto_20261024`, on the pre-existing local PostgreSQL 18.1 server. Fixture tables were removed, owned test processes stopped, and the disposable database dropped after verification. Remaining limits: no fresh-install or full hosted-engine matrix, Lucee 6, real MySQL/SQL Server polling, Redis/SQS adapters, worker-process crash recovery, real heartbeat producer, production profiling, real provider delivery/indexing or consumer adoption. Consumer adapters must set their own query/network timeouts; shutdown cannot force an uncooperative callback to return. WireBox interception applies to configured methods on subsequently WireBox-created instances; it cannot intercept arbitrary HTTP clients or objects created with `new`. Manual APIs cover those paths. + +## Earlier local verification — October 22, 2026 + +The following sections retain the evidence and scope of the earlier commits. Their totals and local-only statements describe those runs, before the later authorized push and draft PR update. + +After subagent review, the timestamp clock correction and the request/dependency/worker-health additions and real database/worker integration coverage, the final module harness passed **104/104** on each local engine, with zero failures, errors or skips: + +| Engine | JVM / servlet runtime | Evidence | +| --- | --- | --- | +| Lucee 5.4.8.2 | Temurin 21.0.10+7, Runwar 5.2.7 | `/tmp/sentry-queue-health-final-lucee5.json` | +| BoxLang 1.18.0+66, reported by servlet startup | Temurin 21.0.10+7, Runwar 6.1.8 | `/tmp/sentry-queue-health-final-boxlang.json` | +| Adobe ColdFusion 2023 update 25 (`2023,0,25,330977`) | Temurin 21.0.10+7, Runwar 5.2.7 | `/tmp/sentry-queue-health-final-adobe.json` | + +The BoxLang engine package is labeled `1.18.6+62`; its startup reports the runtime version above. Do not infer the running engine version from the package label or the legacy TestBox reporter. The local CommandBox initial-install hook failed before boot with missing `CFENGINE` metadata; the owned engine then used the existing compatible `bx-compat-cfml`/`bx-esapi` runtime modules and explicit Jakarta Runwar 6.1.8. These are local harness setup details, not qualified fresh-install behavior or release changes. + +The Adobe check uses an isolated copy of the previously prepared engine, including its existing Commons IO 2.4 runtime repair; it does not qualify a pristine engine installation. Native thread sampling now uses the exported JMX interface, and missing null argument slots are guarded before access. A fresh local Adobe probe returned heap, threads, process CPU, and GC count/time (`/tmp/sentry-review-jvm-adobe.json`). + +The harness uses existing ColdBox `7.5.2+27` and TestBox `3.2.0+356` dependencies through read-only links. The reused BoxLang runtime modules report `bx-compat-cfml` `1.40.0+56` and `bx-esapi` `1.10.0+14`. `SENTRY_QB_SOURCE` points to CommuniArts' installed qb `14.1.0`, read without modification. Queue tests exercise configured idle queues, dynamic queues returning to zero, failed polls, bounded lifetime labels and qb's actual PostgreSQL/MySQL/SQL Server SQL compilation. The earlier depth-sampler contracts stub database execution at qb's `get()` boundary. The new health-adapter integration below additionally executes real PostgreSQL queries and actual cbq workers. The existing cross-thread timeout/late completion/retry/scope regressions also pass. cbq's separate upstream hook suite was not rerun. + +New coverage includes opt-in receipt collection, async acceptance observed once, rejection/rate-limit drops/network failures, bounded diagnostics; fixed request labels and budgets, once-only timing tokens, original callback return/error preservation; workflow outcome privacy, classifier validation, callback-once behavior, original exception preservation; optional pool adapters; unstarted attempt cancellation, opt-in terminal counters, destination label restrictions; native JMX interface calls and unsupported CPU/GC omission; and lowercase error-event trace correlation. Existing error APIs, timestamps, request URLs and initialization concurrency remain covered. + +A local native probe additionally returned heap used/max, thread count, process CPU load and cumulative GC count/time on both engines. Raw numeric samples are in `/tmp/sentry-reusable-jvm-lucee5.json` and `/tmp/sentry-reusable-jvm-boxlang.json`. The probe template was removed. This demonstrates local MXBean support, not production profiling or provider arrival. + +Commands used the loopback envelope receiver and the module's existing runners: + +```sh +node test-harness/support/receiver.mjs +curl --fail 'http://127.0.0.1:60361/tests/runner.cfm?reporter=json' +curl --fail 'http://127.0.0.1:60362/tests/runner.cfm?reporter=json' +curl --fail 'http://127.0.0.1:60363/tests/runner.cfm?reporter=json' +box run-script format +box run-script format:check +npm --prefix browser ci --ignore-scripts +npm --prefix browser test +git diff --check +``` + +CFFormat checked 48 files successfully. The earlier unchanged browser Vitest run passed **28/28** in eight files; no browser source or assets changed. The unchanged browser development dependency tree has npm audit advisories for Vite (high), Vitest (moderate) and `@vitest/mocker` (moderate); no unrelated dependency upgrades were made. This does not assess production exploitability. + +Lucee 6, the complete hosted matrix, browser UI matrices, crash/recovery loops, real MySQL/SQL Server polling, supported vendor pool adapters, production profiling, real provider delivery/indexing and consumer adoption remain unverified here. Publication, deployment, hosted workflow dispatch, alert/uptime configuration, external messaging and automation changes were not performed. Consumer and original module checkouts remain unchanged. Owned test servers and the receiver are stopped after verification. + +The [audit and adoption sequence](observability-upstream-audit.md) records ownership and release prerequisites; [operational setup](operational-observability.md) documents configurable APIs and consumer recipes. No package/lock versions or installed application module source were changed. + +## Subagent review follow-up + +The independent review found six bugs, each reproduced by a failing regression against the initial commit: case-alias phase merging, exact queue names overwriting one another, workflow completion skipped after an earlier cleanup failure, mutable/foreign timing tokens bypassing bounds, trailing newlines accepted by the phase-name regex, and exporter work included in queue processing duration. All six are fixed. The reviewer rechecked the implementation and the BoxLang exact-name / Adobe null-array follow-ups and found no remaining actionable issue. `OperationalReviewSpec` is included in the normal and telemetry runners. Adobe also required a nonempty HTTP reset body in the receipt test fixture. + +## Timestamp clock correction + +The earlier discrepancy is now diagnosed and fixed. The local Lucee 5.4.8.2 configuration reports a time-server offset of **+72 ms**. Its `now()` implementation constructs a `DateTimeImpl` with that offset applied (confirmed from the installed runtime bytecode). Sentry's default error timestamp used `now()`, while its transport, traces and test bounds used the JVM clock. At JVM epoch millisecond `1792707331943`, adding 72 ms crosses into Unix second `1792707332` while the JVM remains in second `1792707331`. + +Default error timestamps now use the unadjusted JVM millisecond clock, consistent with the other telemetry. Explicit date arguments preserve their supplied instant. ISO and Unix values still derive from one truncated epoch second. A new deterministic boundary regression and the frozen-clock capture/post-metadata regression both failed before the implementation fix and pass afterward; the existing strict current-time bounds remain intact. + +The same local probe observed **12 future timestamps in 150 samples before** and **zero in 150 after**, with the +72 ms engine offset still present. Evidence: `/tmp/sentry-timestamp-clock-before.json`, `/tmp/sentry-timestamp-offset-before.json`, `/tmp/sentry-timestamp-clock-after.json`, and `/tmp/sentry-timestamp-regressions-before.json`. No engine time-server settings or system clocks were changed. Probe templates were removed. The timestamp correction passed its four timestamp specs and the then-complete 78-spec suite on all three local engines. The expanded final suite above includes those regressions. + + +## Request saturation, dependency outcomes and queue worker health + +The expanded suite adds **17 focused specs**, for **95/95 on each engine**, with zero failures, errors or skips. New coverage exercises bounded active-request tracking, monotonic ages, explicit once-only timeout completion, cross-thread completion, trace-disabled request lifecycle, the terminal exception path with no postProcess, retained error trace correlation, disabled collection, fixed dependency classifications and retry/status/breaker fields, original return/exception preservation, callback-only duration, observation-failure cleanup, worker heartbeat/backlog snapshots, failed/unsupported-versus-zero sampling, and fixed executor labels. + +The optional database-health query compiles against the installed qb public API with PostgreSQL, MySQL and SQL Server grammars. Its predicates distinguish runnable/delayed unclaimed jobs, claimed/reserved jobs and expired leases. Those initial contracts stub query execution at get(). The subsequent PostgreSQL integration below executes the health adapter and lease recovery against real database rows and workers; other database engines remain compilation-only. The cbq pool contract reads configured quantity and the executor's supported activity statistics. A real local ColdBox Executor wrapping a JVM fixed pool verifies idle activity as zero; unsupported executor capability leaves activity absent. This does not prove live worker heartbeats or production workload behavior. + +Independent subagent review found three bugs in the first implementation: annotation errors could skip dependency span completion; a ColdBox exception skipped postProcess and leaked an active request observation; and cbq execution capacity was mislabeled as active work. All three are fixed and covered by regressions. The capacity/activity and unsupported-executor regressions failed before the adapter fix (15 passed, 2 failed in /tmp/sentry-worker-capacity-regression-before.json). The reviewer rechecked the revised implementation and documentation and found no further actionable bug or CFML/BoxLang/Adobe compatibility issue. Dependency duration also excludes classifier/setup/export work. + +All collectors remain opt-in with bounded fixed labels. No scheduler or worker heartbeat is fabricated. Request executor details, real worker heartbeats and provider retry/exhaustion totals require supported consumer snapshots. No application adoption, dependency/package changes, browser changes, hosted checks, external provider changes, push, publication or deployment were performed. The original module and consumer worktrees remain unchanged; owned local test processes were stopped after verification. + + +## Real PostgreSQL and actual cbq worker verification + +The opt-in DatabaseQueueHealthIntegrationSpec now contributes **9 live integration specs**. Final normal-runner totals are **104/104 on Lucee 5, BoxLang and Adobe 2023**, with zero failures, errors or skips, using the exact engine/JVM versions listed above. This is real PostgreSQL 18.1 (Postgres.app) on 127.0.0.1:5432 in a newly created disposable database, sentry_queue_health_20261022. Each case creates/drops a uniquely named table; the final database has zero fixture tables. Owned test servers/receiver are stopped and the disposable database is removed after verification. The pre-existing PostgreSQL server remains running. + +The fixture uses installed qb 14.1.0 and cbq 7.0.2 package metadata with the existing local cbq hook/bulk modifications; this is not qualification of a pristine stock package installation. Read-only source fingerprints (SHA-256): DBProvider.cfc: 352a9a7703760e093c472726b94f00601125d344b41ffbe30d3a898869293d96; WorkerPool.cfc: 5bd1dc014c26f278336f307b1c0753257f1ece63bc17383997c6ca3de7da9ed8. Lucee uses its existing PostgreSQL JDBC 42.6.0 bundle; the isolated BoxLang servlet uses the same driver JAR in WEB-INF/lib. Adobe required its registered PostgreSQL CFPM package 2023.0.11.330706 and dependent JDBC 42.7.3. These are local ignored runtime setup changes, with no application dependency or package/lock changes. + +Actual cbq connections, DBProvider, scheduler polling, job persistence/serialization/deserialization, WorkerPool executors and job.handle() run without database/provider/worker mocks. The worker metric is intercepted only at the telemetry gauge boundary to assert actual public executor activity. The tests prove: + +- Empty measured zeros and mixed runnable/delayed/reserved/stale counts; completed/failed and other queues are excluded. A quote-bearing queue name verifies bound query parameters. +- Idle executor activity0, real blocked job activity1 with reserved1, followed by completion/drained zeros. +- Real delayed execution, retry/backoff returning to delayed backlog and eventual success, plus terminal exhausted failure excluded from pending counts. +- A deliberately expired execution lease and a staged reservedBy-only claim are both reported stale, then reclaimed and completed by actual cbq workers with the expected attempt counts. +- A deliberately injected drain error still cleans the real fixture table, task/pool/connection/executor registrations, shared job state and telemetry resources. + +The adapter required no product-code change: the original predicates matched observed provider behavior. New fixture issues were fixed: avoid the CFML built-in query() name; preserve exact PostgreSQL identifier spelling in qb fixture data; require positive numeric timestamps rather than null-only checks when CFML null support is disabled; retain job state through shutdown; continue independent cleanup after failures; and use a shared struct error accumulator because Adobe copies array arguments. The latter regression failed on Adobe (8 passed, 1 failed in /tmp/sentry-queue-health-final-focused-adobe.json) before the struct correction. The independent subagent reviewed the adapter and new fixture, reported the cleanup issue, and re-reviewed the cleanup and Adobe compatibility corrections with no further actionable findings. + +See operational-observability.md for the explicit local opt-in setup. The normal and telemetry runners include the integration bundle; without SENTRY_QUEUE_HEALTH_DB it registers no integration tests and requires no database. The mode forces module telemetry to loopback and disables automatic error/appender capture. Verification does not establish worker-process crash recovery, real worker heartbeats, Redis/SQS behavior, MySQL/SQL Server execution, production profiling or real provider indexing/delivery. No push, hosted workflow, publication, deployment, consumer adoption or provider setting change was performed. diff --git a/docs/request-completion.md b/docs/request-completion.md new file mode 100644 index 0000000..caa71c2 --- /dev/null +++ b/docs/request-completion.md @@ -0,0 +1,54 @@ +# Response completion and timing + +Call the same, application-owned Sentry service from both application callbacks: + +```cfml +function onRequestEnd( targetPage ) { + finishTelemetryRequest(); +} +function onAbort( targetPage ) { + finishTelemetryRequest(); +} +private function finishTelemetryRequest() { + try { + if ( !isNull( application.cbController ) ) { + application.cbController.getWireBox().getInstance( "SentryService@sentry" ).endRequest(); + } + } catch ( any telemetryUnavailable ) { + // Preserve the original response, including successful binary output. + } +} +``` + +The same callbacks work in Application.bx. Merge these calls into existing callbacks; retain their application behavior. `onAbort` does not mean the request failed: an engine can terminate successfully after writing binary content. Status still comes from the response, or an explicit `statusCode` argument. Confirmed timeout callbacks may use `endRequest(timedOut=true)`. Long requests are never automatically classified as timeouts. + +ColdBox `postProcess` completes ordinary requests. Binary output through `cfcontent` may bypass `postRender`, `postProcess`, and Application `onRequestEnd`, reaching `onAbort` instead. Finishing at `preRender` would exclude actual response output; this module does not do that. A synchronous output loop is covered until its handler/request exits. Detached asynchronous servlet work must finish its own explicit span when its actual operation completes; these application hooks are not a servlet async-completion listener. + +Request scopes are bound to an opaque request identity as well as the current thread. When a servlet worker is assigned a new request, it cannot inherit the prior request's scope. Event-driven completion also checks the originating request handle, so a late old event cannot close the worker's newer transaction. Normal completion restores a prior scope belonging to the same request. Each request and timing handle claims completion once. Timing handles are bounded (64 samples, 32 phase names), drained before the root finishes, and reject starts after closure. No new telemetry fields contain request identities or response bytes. + +Span start timestamps remain Unix epoch seconds. Span end timestamps use that start plus elapsed JVM `nanoTime`; wall-clock adjustments during a span do not become latency. Request duration metrics use the same completed span duration. Queue execution resets both start timestamp and monotonic origin when execution starts. These changes neither cap durations nor suppress slow spans. + +No hook can recover an accurate end timestamp after every lifecycle callback was bypassed, a process died, or an engine terminated a thread forcibly. The ownership guard prevents stale context reuse in that situation; it cannot reconstruct an unobserved completion. The local `getObservability().getScopeResetCount()` diagnostic counts these ownership mismatches without exporting request identities. Keep the application callbacks even with the guard. It does not send an abandoned span later with an invented duration. + +## Reproduction and local verification + +On November 2, 2026, the pinned baseline `d3062ae` was exercised in a fresh loopback CommandBox runtime with ColdBox 8 and binary `renderData`. The six bytes `0001027f80ff` reached the client in 131 ms. Hook capture recorded `onAbort`, not `onRequestEnd`; no binary transaction had completed. After an intentional 1.8-second idle interval and subsequent requests on reused workers, its root measured 1,837 ms and render measured 1,827 ms. This demonstrates instrumentation lifetime extending beyond response delivery. It does not establish the exact history of any individual production request. + +With the application abort hook and module changes, the HTTP probe checks completion **before another ColdBox request is issued**, verifies the exact bytes and security/content headers, waits through an idle interval, and verifies the timestamp never changes. It also covers normal output, a deliberately slow handler, noRender, synchronous output, explicit abort, redirect, exception, and concurrent binary/text requests. The real slow handler must retain its elapsed duration. + +Prepare the normal test-harness dependencies and a private loopback runtime using the normal server configuration as a starting point. The `request-lifecycle` fixture uses the harness's `/coldbox`, `/testbox`, and module dependencies. Its signal filter retains synthetic transaction payloads in memory and returns null, so it never ingests them into Sentry. Its report/reset endpoint is a test fixture only; do not deploy the test harness. + +```sh +node test-harness/request-lifecycle/verify.mjs \ + http://127.0.0.1:60339/request-lifecycle/ /tmp/sentry-lifecycle-after.json +``` + +Run `tests.specs.RequestCompletionSpec` through the normal TestBox runner for lifecycle ownership, repeated and late completion, two concurrent workers each executing successive requests, wall-clock jumps, queue start resets, and telemetry failures. `test-harness/tests/telemetry-runner.cfm` includes this bundle. The existing loopback envelope receiver (`node test-harness/support/receiver.mjs`) is required for transport tests in broader telemetry suites. + +The owned runtime used the `boxlang@1.18.6+62` servlet artifact (manifest `1.18.6+62`, SHA-256 `4338d7c40393c82f0c544f6e586ece9f7d1dee187ec016f421837426a6ae419b`). Its bundled core reports `1.18.0+66`, so the runtime report alone is not the servlet package version. Other engine versions, Adobe/Lucee abort semantics, remote storage behavior, servlet asynchronous output/disconnects, production ingestion, and hosted CI are separate verification boundaries. No full engine/browser matrix, hosted workflow, publication, push, or deployment is part of this check. + +Final local results: 92 passing module specs, no failures/errors, and one skipped optional qb grammar/provider fixture; all 20 HTTP responses passed. The binary response completed at the client in 17 ms and its transaction measured 5.80 ms. The intentional 250 ms handler measured 264.91 ms. Eight new lifecycle contracts are included in the 92 module passes. `cfformat check`, Node syntax validation, and `git diff --check` passed for the changed module sources. + +## Consumer adoption + +The abort-hook change can be adopted with existing `endRequest()` support. Adopt the reviewed module commit through the consumer's normal pinned dependency and lock/manifest process after its upstream work is integrated or released. Do not vendor or edit installed `lib/modules/sentry`. Keep domain authorization, storage reads, response bytes and headers in the consumer. After a separately authorized deployment, compare real affected responses with newly ingested spans; local checks do not prove production ingestion. diff --git a/interceptors/Observability.cfc b/interceptors/Observability.cfc index 69a21ad..77436c1 100644 --- a/interceptors/Observability.cfc +++ b/interceptors/Observability.cfc @@ -6,28 +6,44 @@ component { try { var telemetry = variables.sentry.getObservability(); var settings = variables.sentry.getSettings(); - if ( !settings.enableRequestTracing ) { + if ( + !settings.enableRequestTracing && !( settings.enableMetrics && settings.enableRequestSaturation ) && !settings.enableAutomaticRequestTimings + ) { return; } if ( isCustomFunction( settings.requestFilter ) && !settings.requestFilter( arguments.event ) ) { return; } - var headers = getHTTPRequestData( false ).headers; - var trusted = isCustomFunction( settings.trustIncomingTrace ) ? settings.trustIncomingTrace( - headers, - arguments.event - ) : settings.trustIncomingTrace; + + if ( arguments.event.getPrivateValue( "_sentryRequest", {} ).keyExists( "previous" ) ) return; var state = { - previous : telemetry.getScope(), - span : telemetry.startTransaction( + "previous" : telemetry.getScope(), + "id" : createUUID(), + "done" : createObject( "java", "java.util.concurrent.atomic.AtomicBoolean" ).init( false ) + }; + if ( settings.enableRequestTracing ) { + var headers = getHTTPRequestData( false ).headers; + var trusted = isCustomFunction( settings.trustIncomingTrace ) ? settings.trustIncomingTrace( + headers, + arguments.event + ) : settings.trustIncomingTrace; + state[ "span" ] = telemetry.startTransaction( requestName( arguments.event ), "http.server", headers, trusted - ) - }; + ); + } + state[ "saturationToken" ] = telemetry.beginRequestObservation(); + if ( settings.enableAutomaticRequestTimings ) { + state[ "timings" ] = telemetry.newRequestTimings( true ); + state[ "handlerTokens" ] = []; + } arguments.event.setPrivateValue( "_sentryRequest", state ); - telemetry.setScope( { span : state.span, requestState : state } ); + var next = structCopy( state.previous ); + next[ "requestState" ] = state; + if ( state.keyExists( "span" ) ) next[ "span" ] = state.span; + telemetry.setScope( next ); } catch ( any ignored ) { } } @@ -45,24 +61,101 @@ component { complete( arguments.event ); } function onException( event, interceptData ){ + var state = {}; + try { + state = arguments.event.getPrivateValue( "_sentryRequest", {} ); + if ( state.keyExists( "span" ) ) state.span.setStatus( "internal_error" ); + } catch ( any ignored ) { + } finally { + try { + finishExceptionObservation( state ); + } catch ( any ignored ) { + } + } + } + + function finishExceptionObservation( required struct state ){ + closeTimings( arguments.state, "internal_error" ); + if ( arguments.state.keyExists( "saturationToken" ) ) + variables.sentry.finishRequestObservation( arguments.state.saturationToken ); + // Keep an existing trace available to the module's exception capture. Application onRequestEnd closes it. + if ( !arguments.state.keyExists( "span" ) ) variables.sentry.endRequest( statusCode = 500 ); + } + + private function complete( required any event ){ try { var state = arguments.event.getPrivateValue( "_sentryRequest", {} ); - if ( state.keyExists( "span" ) ) { - state.span.setStatus( "internal_error" ); + if ( state.keyExists( "previous" ) ) { + try { + if ( state.keyExists( "span" ) ) state.span.rename( requestName( arguments.event ) ); + } finally { + closeTimings( state, "ok" ); + publishTimings( arguments.event, state ); + variables.sentry.endRequest( state = state ); + } } } catch ( any ignored ) { } } - private function complete( required any event ){ + function preEvent( event, interceptData ){ + phase( arguments.event, "handler", true ); + } + function postEvent( event, interceptData ){ + phase( arguments.event, "handler", false ); + } + function preRender( event, interceptData ){ + phase( arguments.event, "render", true ); + } + function postRender( event, interceptData ){ + phase( arguments.event, "render", false ); + try { + publishTimings( arguments.event, arguments.event.getPrivateValue( "_sentryRequest", {} ) ); + } catch ( any ignored ) { + } + } + private function phase( + required any event, + required string name, + required boolean begin + ){ try { var state = arguments.event.getPrivateValue( "_sentryRequest", {} ); - if ( state.keyExists( "span" ) ) { - state.span.rename( requestName( arguments.event ) ); - variables.sentry.endRequest(); + if ( !state.keyExists( "timings" ) ) return; + if ( arguments.name == "handler" ) { + if ( arguments.begin ) state.handlerTokens.append( state.timings.begin( "handler" ) ); + else if ( state.handlerTokens.len() ) { + state.timings.finish( state.handlerTokens[ state.handlerTokens.len() ] ); + state.handlerTokens.deleteAt( state.handlerTokens.len() ); + } + } else { + if ( arguments.begin && !state.keyExists( "renderToken" ) ) + state[ "renderToken" ] = state.timings.begin( "render" ); + else if ( !arguments.begin && state.keyExists( "renderToken" ) ) { + state.timings.finish( state.renderToken ); + state.delete( "renderToken" ); + } } } catch ( any ignored ) { } } + private function closeTimings( required struct state, required string status ){ + variables.sentry.getObservability().finishRequestTimings( arguments.state, arguments.status ); + } + + private function publishTimings( required any event, required struct state ){ + try { + var settings = variables.sentry.getSettings(); + if ( !settings.publishServerTiming || !arguments.state.keyExists( "timings" ) ) return; + if ( isCustomFunction( settings.serverTimingFilter ) ) { + if ( !settings.serverTimingFilter( arguments.event ) ) return; + } else if ( !isSimpleValue( settings.serverTimingFilter ) || len( trim( settings.serverTimingFilter ) ) ) + return; + var header = arguments.state.timings.toServerTiming(); + if ( len( header ) ) arguments.event.setHTTPHeader( name = "Server-Timing", value = header ); + } catch ( any ignored ) { + } + } + private function requestName( required any event ){ var route = arguments.event.getCurrentRoute(); if ( !isSimpleValue( route ) || !len( route ) ) { diff --git a/interceptors/QueryObservability.cfc b/interceptors/QueryObservability.cfc deleted file mode 100644 index 991046d..0000000 --- a/interceptors/QueryObservability.cfc +++ /dev/null @@ -1,47 +0,0 @@ -/** qb fallback; one adapter produces each execution's span. */ -component { - - property name="sentry" inject="SentryService@sentry"; - function preQBExecute( event, interceptData ){ - try { - if ( arguments.interceptData.pretend ?: false ) { - return; - } - var telemetry = variables.sentry.getObservability(); - if ( variables.sentry.getSettings().queryAdapter != "qb" ) { - return; - } - var current = telemetry.getScope(); - if ( current.keyExists( "manualQuerySpan" ) ) { - return; - } - var span = telemetry.startSpan( - telemetry.scrubSQL( arguments.interceptData.sql ), - "db.query", - { "db.system" : variables.sentry.getSettings().databaseSystem ?: "other" } - ); - arguments.interceptData[ "_sentryExecution" ] = { span : span, previous : current }; - var next = structCopy( current ); - next.span = span; - telemetry.setScope( next ); - } catch ( any ignored ) { - } - } - function postQBExecute( event, interceptData ){ - complete( arguments.interceptData ); - } - function onQBExecuteException( event, interceptData ){ - complete( arguments.interceptData, "internal_error" ); - } - private function complete( required struct data, string status = "ok" ){ - try { - var execution = arguments.data[ "_sentryExecution" ] ?: {}; - if ( execution.keyExists( "span" ) ) { - execution.span.finish( arguments.status ); - variables.sentry.getObservability().setScope( execution.previous ); - } - } catch ( any ignored ) { - } - } - -} diff --git a/interceptors/QueueObservability.cfc b/interceptors/QueueObservability.cfc index ddf14d6..7b2715b 100644 --- a/interceptors/QueueObservability.cfc +++ b/interceptors/QueueObservability.cfc @@ -166,7 +166,8 @@ component { "exited" : false, "created" : telemetry.timestamp(), "enqueuedAt" : metadata.enqueuedAt ?: telemetry.timestamp(), - "started" : 0 + "started" : 0, + "attributes" : metricAttributes( job ) } ); } catch ( any ignored ) { @@ -196,7 +197,7 @@ component { variables.sentry.distribution( "queue.wait.duration", waitTime, - {}, + attempt.attributes, "millisecond" ); } @@ -233,24 +234,36 @@ component { if ( isNull( attempt ) ) { return; } - var status = arguments.interceptData.status; + var status = arguments.interceptData.status; + var finishedAt = variables.sentry.getObservability().timestamp(); attempt.span.setAttribute( "messaging.message.released", status == "released" ); if ( attempt.span.finish( status == "released" ? "ok" : status ) ) { + // Close the root before best-effort metrics; a failed observer cannot strand it. + attempt.root.finish( status == "released" ? "ok" : status ); var telemetry = variables.sentry.getObservability(); var previous = telemetry.getScope(); try { telemetry.setScope( { "span" : attempt.root } ); - variables.sentry.counter( "queue.attempts", 1, { "outcome" : status } ); - variables.sentry.distribution( - "queue.processing.duration", - attempt.started > 0 ? ( telemetry.timestamp() - attempt.started ) * 1000 : 0, - { "outcome" : status }, - "millisecond" - ); + var labels = structCopy( attempt.attributes ); + labels[ "outcome" ] = listFind( + "ok,released,internal_error,deadline_exceeded,cancelled", + status + ) ? status : "unknown"; + variables.sentry.counter( "queue.attempts", 1, labels ); + if ( attempt.started > 0 ) { + variables.sentry.distribution( + "queue.processing.duration", + max( 0, ( finishedAt - attempt.started ) * 1000 ), + labels, + "millisecond" + ); + } } finally { telemetry.setScope( previous ); + if ( attempt.exited ) { + variables.attempts.remove( arguments.interceptData.executionId ); + } } - attempt.root.finish( status == "released" ? "ok" : status ); } if ( attempt.exited ) { variables.attempts.remove( arguments.interceptData.executionId ); @@ -262,6 +275,30 @@ component { function getPendingAttemptCount(){ return variables.attempts.size(); } + function onCBQJobComplete( event, interceptData ){ + terminalMetric( arguments.interceptData.job, "completed" ); + } + function onCBQJobFailed( event, interceptData ){ + terminalMetric( arguments.interceptData.job, "failed" ); + } + private function terminalMetric( required any job, required string outcome ){ + try { + if ( !variables.sentry.getSettings().enableQueueJobMetrics ) { + return; + } + var labels = metricAttributes( arguments.job ); + labels[ "outcome" ] = arguments.outcome; + variables.sentry.counter( "queue.jobs", 1, labels ); + } catch ( any unavailable ) { + variables.observationFailures.incrementAndGet(); + } + } + private function metricAttributes( required any job ){ + var destination = arguments.job.getQueue() ?: "default"; + return variables.sentry.getSettings().queueMetricDestinations.find( destination ) ? { + "messaging.destination.name" : destination + } : {}; + } function getObservationFailureCount(){ return variables.observationFailures.get(); } diff --git a/models/SentryService.cfc b/models/SentryService.cfc index 301458b..67f4591 100644 --- a/models/SentryService.cfc +++ b/models/SentryService.cfc @@ -57,6 +57,9 @@ component accessors=true singleton threadSafe { // Additional tags property name="tags"; + // Use the same unadjusted JVM clock as envelope and observability timestamps. + variables.timestampClock = createObject( "java", "java.lang.System" ); + /** @@ -197,6 +200,7 @@ component accessors=true singleton threadSafe { variables.observability.shutdown(); } variables.observability = new telemetry.Observability( this, variables.settings ); + if ( !isNull( variables.wirebox ) ) variables.observability.setAdapterInjector( variables.wirebox ); // in a non ColdBox context, ensure functionLineNums exists // so this service can still be used if functionLineNums @@ -287,6 +291,66 @@ component accessors=true singleton threadSafe { function captureRuntimeGauges(){ return variables.observability.captureRuntimeGauges( argumentCollection = arguments ); } + function deliveryDiagnostics(){ + return variables.observability.deliveryDiagnostics(); + } + function captureDatabasePool(){ + return variables.observability.captureDatabasePool( argumentCollection = arguments ); + } + function newRequestTimings(){ + return variables.observability.newRequestTimings( argumentCollection = arguments ); + } + function beginRequestObservation(){ + return variables.observability.beginRequestObservation( argumentCollection = arguments ); + } + function finishRequestObservation(){ + return variables.observability.finishRequestObservation( argumentCollection = arguments ); + } + function requestSaturation(){ + return variables.observability.requestSaturation( argumentCollection = arguments ); + } + function captureRequestSaturation(){ + return variables.observability.captureRequestSaturation( argumentCollection = arguments ); + } + function captureRequestExecutor(){ + return variables.observability.captureRequestExecutor( argumentCollection = arguments ); + } + function requestExecutorDiagnostics(){ + return variables.observability.requestExecutorDiagnostics( argumentCollection = arguments ); + } + function captureQueueHealth(){ + return variables.observability.captureQueueHealth( argumentCollection = arguments ); + } + function queueHealthDiagnostics(){ + return variables.observability.queueHealthDiagnostics( argumentCollection = arguments ); + } + function captureQueueWorkerHealth(){ + return variables.observability.captureQueueWorkerHealth( argumentCollection = arguments ); + } + function queueWorkerDiagnostics(){ + return variables.observability.queueWorkerDiagnostics( argumentCollection = arguments ); + } + function captureCBQWorkerHealth(){ + return variables.observability.captureCBQWorkerHealth( argumentCollection = arguments ); + } + function withDependency(){ + return variables.observability.withDependency( argumentCollection = arguments ); + } + function automaticCollectorDiagnostics(){ + return variables.observability.automaticCollectorDiagnostics( argumentCollection = arguments ); + } + function collectAutomaticMetrics(){ + return variables.observability.collectAutomaticMetrics( argumentCollection = arguments ); + } + function startAutomaticCollectors(){ + return variables.observability.startAutomaticCollectors( argumentCollection = arguments ); + } + function getRequestTimings(){ + return variables.observability.getRequestTimings( argumentCollection = arguments ); + } + function withOperation(){ + return variables.observability.withOperation( argumentCollection = arguments ); + } function getInstrumentedCache( string name = "default" ){ return new telemetry.InstrumentedCache( variables.wirebox.getInstance( "cachebox:" & arguments.name ), @@ -1209,10 +1273,14 @@ component accessors=true singleton threadSafe { var traceContext = variables.observability.getTraceContext(); if ( !traceContext.isEmpty() ) { - if ( !arguments.captureStruct.keyExists( "contexts" ) ) { - arguments.captureStruct.contexts = {}; - } - arguments.captureStruct.contexts.trace = traceContext; + // Case-insensitive structs can preserve an earlier dot-assigned uppercase key. + var captureContexts = arguments.captureStruct.keyExists( "contexts" ) ? arguments.captureStruct[ + "contexts" + ] : {}; + arguments.captureStruct.delete( "contexts" ); + captureContexts.delete( "trace" ); + captureContexts[ "trace" ] = traceContext; + arguments.captureStruct[ "contexts" ] = captureContexts; } // serialize data jsonCapture = serializeJSON( arguments.captureStruct ); @@ -1350,8 +1418,10 @@ component accessors=true singleton threadSafe { /** * Get matching UTC values from one truncated epoch second on every engine. */ - private struct function getTimeVars( date time = now() ){ - var unixSeconds = int( time.getTime() / 1000 ); + private struct function getTimeVars( date time ){ + // Lucee now() can apply a configured time-server offset to the JVM clock. + var millis = isNull( arguments.time ) ? variables.timestampClock.currentTimeMillis() : arguments.time.getTime(); + var unixSeconds = int( millis / 1000 ); var timeVars = { "unix" : toString( unixSeconds ), "iso" : createObject( "java", "java.time.Instant" ) diff --git a/models/telemetry/AdapterResolver.cfc b/models/telemetry/AdapterResolver.cfc new file mode 100644 index 0000000..f1e13e6 --- /dev/null +++ b/models/telemetry/AdapterResolver.cfc @@ -0,0 +1,40 @@ +/** Adapters are callbacks, mapping IDs, objects, or {mapping,method} descriptors. Never cache consumer instances. */ +component { + + function init( any injector = "" ){ + variables.injector = arguments.injector; + return this; + } + function setInjector( required any injector ){ + variables.injector = arguments.injector; + return this; + } + function resolve( required string mapping ){ + if ( !isObject( variables.injector ) ) throw( type = "Sentry.AdapterInjectorUnavailable" ); + return variables.injector.getInstance( arguments.mapping ); + } + function call( + required any adapter, + required string method, + struct args = {} + ){ + if ( isCustomFunction( arguments.adapter ) ) return arguments.adapter( argumentCollection = arguments.args ); + var target = arguments.adapter; + var methodName = arguments.method; + if ( isStruct( target ) && target.keyExists( "mapping" ) ) { + if ( target.keyExists( "method" ) ) methodName = target.method; + target = resolve( target.mapping ); + } else if ( isSimpleValue( target ) ) { + target = resolve( toString( target ) ); + } + if ( + !isObject( target ) || !isSimpleValue( methodName ) || !reFind( + "^[A-Za-z_][A-Za-z0-9_]*$", + methodName + ) || reFind( "[^A-Za-z0-9_]", methodName ) + ) + throw( type = "Sentry.InvalidAdapter" ); + return invoke( target, methodName, arguments.args ); + } + +} diff --git a/models/telemetry/AutomaticCollectorTask.cfc b/models/telemetry/AutomaticCollectorTask.cfc new file mode 100644 index 0000000..8f0d949 --- /dev/null +++ b/models/telemetry/AutomaticCollectorTask.cfc @@ -0,0 +1,15 @@ +/** Background entry point shields the scheduler from adapter failures. */ +component { + + function init( required any collector ){ + variables.collector = arguments.collector; + return this; + } + function run(){ + try { + variables.collector.tick(); + } catch ( any unavailable ) { + } + } + +} diff --git a/models/telemetry/AutomaticCollectors.cfc b/models/telemetry/AutomaticCollectors.cfc new file mode 100644 index 0000000..d81a335 --- /dev/null +++ b/models/telemetry/AutomaticCollectors.cfc @@ -0,0 +1,167 @@ +/** One bounded background sampler per telemetry service. No request threads, overlapping ticks or duplicate starts. */ +component { + + function init( + required any telemetry, + required struct settings, + required any resolver + ){ + variables.telemetry = arguments.telemetry; + variables.settings = arguments.settings; + variables.resolver = arguments.resolver; + variables.running = createObject( "java", "java.util.concurrent.atomic.AtomicBoolean" ).init( false ); + variables.started = createObject( "java", "java.util.concurrent.atomic.AtomicBoolean" ).init( false ); + variables.closed = createObject( "java", "java.util.concurrent.atomic.AtomicBoolean" ).init( false ); + variables.diagnostics = { + "runs" : 0, + "failures" : 0, + "lastSuccessAt" : 0, + "lastFailureAt" : 0, + "skipped" : 0 + }; + variables.id = createUUID(); + return this; + } + function start(){ + if ( !variables.settings.enabled || !variables.started.compareAndSet( false, true ) || variables.closed.get() ) + return false; + try { + var interval = variables.settings.intervalSeconds; + if ( !isNumeric( interval ) || interval < 1 || interval > 3600 || interval != int( interval ) ) + throw( type = "Sentry.InvalidCollectorInterval" ); + variables.executor = createObject( "java", "java.util.concurrent.Executors" ).newSingleThreadScheduledExecutor(); + var task = new AutomaticCollectorTask( this ); + // ColdBox proxies establish the engine application context and release Adobe JDBC thread state. + try { + task = createObject( "component", "coldbox.system.async.cbproxies.models.Runnable" ).init( task ); + } catch ( any unavailableContext ) { + } + variables.executor.scheduleWithFixedDelay( + createDynamicProxy( task, [ "java.lang.Runnable" ] ), + javacast( "long", interval ), + javacast( "long", interval ), + createObject( "java", "java.util.concurrent.TimeUnit" ).SECONDS + ); + return true; + } catch ( any unavailable ) { + variables.started.set( false ); + if ( !isNull( variables.executor ) ) variables.executor.shutdownNow(); + record( false ); + return false; + } + } + function tick(){ + if ( variables.closed.get() || !variables.settings.enabled || !variables.telemetry.collectionEnabled() ) + return; + if ( !variables.running.compareAndSet( false, true ) ) { + lock name="sentry-auto-#variables.id#" type="exclusive" timeout="2" { + variables.diagnostics.skipped++; + } + return; + } + var success = true; + var previous = {}; + try { + previous = variables.telemetry.getScope(); + variables.telemetry.clearScope(); + if ( canCollect() && variables.settings.runtime ) + success = variables.tryCollect( function(){ + variables.telemetry.captureRuntimeGauges(); + return true; + } ) && success; + if ( canCollect() && variables.settings.requestSaturation ) + success = variables.tryCollect( function(){ + variables.telemetry.captureRequestSaturation(); + return true; + } ) && success; + if ( canCollect() && variables.settings.deliveryDiagnostics ) + success = variables.tryCollect( function(){ + variables.telemetry.deliveryDiagnostics(); + return true; + } ) && success; + for ( var kind in [ "queues", "requestExecutors", "databasePools" ] ) { + var definitions = variables.settings[ kind ]; + for ( var i = 1; i <= min( 32, definitions.len() ); i++ ) { + if ( !canCollect() ) break; + var definition = definitions[ i ]; + try { + var captured = variables.collectDefinition( kind, definition ); + success = captured && success; + } catch ( any unavailable ) { + success = false; + } + } + } + } catch ( any unavailable ) { + success = false; + } finally { + try { + variables.telemetry.setScope( previous ); + } catch ( any ignored ) { + } + variables.running.set( false ); + record( success ); + } + } + private function canCollect(){ + return !variables.closed.get() && variables.telemetry.collectionEnabled(); + } + private function tryCollect( required function callback ){ + if ( !canCollect() ) return false; + try { + return arguments.callback() != false; + } catch ( any unavailable ) { + return false; + } + } + private function collectDefinition( required string kind, required struct definition ){ + var config = arguments.definition; + var collectKind = arguments.kind; + if ( !config.keyExists( "name" ) || !isSimpleValue( config.name ) ) return false; + if ( !canCollect() ) return false; + if ( config.keyExists( "options" ) && !isStruct( config.options ) ) return false; + var context = { + "name" : config.name, + "kind" : collectKind, + "options" : duplicate( config.options ?: {} ) + }; + var callback = function(){ + return variables.resolver.call( + config.adapter, + "sample", + { "context" : context } + ); + }; + if ( collectKind == "queues" ) return variables.telemetry.captureQueueHealth( config.name, callback ); + if ( collectKind == "requestExecutors" ) + return variables.telemetry.captureRequestExecutor( config.name, callback ); + if ( collectKind == "databasePools" ) + return variables.telemetry.captureDatabasePool( config.name, callback ); + return false; + } + private function record( required boolean success ){ + lock name="sentry-auto-#variables.id#" type="exclusive" timeout="2" { + var now = createObject( "java", "java.lang.System" ).currentTimeMillis() / 1000; + variables.diagnostics.runs++; + if ( arguments.success ) variables.diagnostics.lastSuccessAt = now; + else { + variables.diagnostics.failures++; + variables.diagnostics.lastFailureAt = now; + } + } + } + function diagnostics(){ + lock name="sentry-auto-#variables.id#" type="exclusive" timeout="2" { + var result = duplicate( variables.diagnostics ); + result[ "started" ] = variables.started.get() && !variables.closed.get(); + return result; + } + } + function shutdown(){ + variables.closed.set( true ); + if ( !isNull( variables.executor ) ) { + variables.executor.shutdownNow(); + } + } + +} diff --git a/models/telemetry/BoxLangQueryListener.cfc b/models/telemetry/BoxLangQueryListener.cfc index 2e3a13c..8103ba3 100644 --- a/models/telemetry/BoxLangQueryListener.cfc +++ b/models/telemetry/BoxLangQueryListener.cfc @@ -33,7 +33,7 @@ component { return; } var span = variables.telemetry.startSpan( - variables.telemetry.scrubSQL( arguments.data.sql ), + arguments.data.sql, ( arguments.data.cached ?: false ) ? "db.query.cache" : ( ( arguments.data.dbtype ?: "" ) == "query" ? "db.query.qoq" : "db.query" ), diff --git a/models/telemetry/DatabaseQueueHealth.cfc b/models/telemetry/DatabaseQueueHealth.cfc new file mode 100644 index 0000000..45b0485 --- /dev/null +++ b/models/telemetry/DatabaseQueueHealth.cfc @@ -0,0 +1,15 @@ +/** Compatibility facade. The cbq package owns its database schema and snapshot implementation. */ +component { + + function init( required function queryFactory, string table = "cbq_jobs" ){ + variables.adapter = createObject( "component", "cbq.models.Telemetry.DatabaseQueueSnapshot" ).init( + queryFactory = arguments.queryFactory, + table = arguments.table + ); + return this; + } + function snapshot( required any queue ){ + return variables.adapter.snapshot( arguments.queue ); + } + +} diff --git a/models/telemetry/DatabaseQueueMetrics.cfc b/models/telemetry/DatabaseQueueMetrics.cfc new file mode 100644 index 0000000..442edec --- /dev/null +++ b/models/telemetry/DatabaseQueueMetrics.cfc @@ -0,0 +1,108 @@ +/** Optional cbq database-provider adapter. No polling thread or installed-module mutation. */ +component { + + function init( + required any telemetry, + required any config, + required function queryFactory, + string table = "cbq_jobs", + numeric maxQueues = 64 + ){ + variables.telemetry = arguments.telemetry; + variables.config = arguments.config; + variables.queryFactory = arguments.queryFactory; + variables.table = arguments.table; + variables.maxQueues = max( 1, min( 256, arguments.maxQueues ) ); + variables.observed = {}; + variables.diagnostics = { + "lastSuccessAt" : 0, + "lastFailureAt" : 0, + "failures" : 0, + "omittedQueues" : 0 + }; + variables.id = createUUID(); + return this; + } + function capture(){ + lock name="sentry-queue-sampler-#variables.id#" type="exclusive" timeout="2" throwOnTimeout="false" { + try { + var queues = {}; + var definitions = variables.config.getWorkerPoolDefinitions(); + for ( var key in definitions ) { + var names = definitions[ key ].getQueue(); + for ( var name in isArray( names ) ? names : [ names ] ) { + queues[ queueKey( name ) ] = { "name" : name, "stats" : { "depth" : 0, "oldest" : 0 } }; + } + } + var query = variables.queryFactory(); + var rows = query + .from( variables.table ) + .select( "queue" ) + .selectRaw( "count(*) AS depth" ) + .selectRaw( "min(" & query.getGrammar().wrapValue( "createdDate" ) & ") AS oldest" ) + .whereNull( "completedDate" ) + .whereNull( "failedDate" ) + .whereNull( "reservedDate" ) + .groupBy( "queue" ) + .get(); + for ( var row in rows ) { + queues[ queueKey( row.queue ) ] = { "name" : row.queue, "stats" : row }; + } + for ( var key in variables.observed ) { + if ( !queues.keyExists( key ) ) { + queues[ key ] = { + "name" : variables.observed[ key ], + "stats" : { "depth" : 0, "oldest" : 0 } + }; + } + } + var now = createObject( "java", "java.lang.System" ).currentTimeMillis() / 1000; + var emitted = 0; + variables.diagnostics.omittedQueues = 0; + var ordered = []; + for ( var key in queues ) { + ordered.append( queues[ key ].name ); + } + arraySort( ordered, "text" ); + for ( var name in ordered ) { + var key = queueKey( name ); + if ( + emitted >= variables.maxQueues || ( + !variables.observed.keyExists( key ) && structCount( variables.observed ) >= variables.maxQueues + ) + ) { + variables.diagnostics.omittedQueues++; + continue; + } + var stats = queues[ key ].stats; + variables.telemetry.captureQueueMetrics( + name, + stats.depth, + stats.depth ? max( 0, now - stats.oldest ) : 0 + ); + if ( structCount( variables.observed ) < variables.maxQueues ) { + variables.observed[ key ] = name; + } + emitted++; + } + variables.diagnostics.lastSuccessAt = now; + return true; + } catch ( any unavailable ) { + variables.diagnostics.failures++; + variables.diagnostics.lastFailureAt = createObject( "java", "java.lang.System" ).currentTimeMillis() / 1000; + return false; + } + } + return false; + } + /** Hash exact labels internally: CFML structs and BoxLang Java-map adapters fold string key case. */ + private function queueKey( required string name ){ + return hash( arguments.name, "SHA-256", "UTF-8" ); + } + function getDiagnostics(){ + lock name="sentry-queue-sampler-#variables.id#" type="exclusive" timeout="2" { + return duplicate( variables.diagnostics ); + } + } + +} diff --git a/models/telemetry/DeliveryDiagnostics.cfc b/models/telemetry/DeliveryDiagnostics.cfc new file mode 100644 index 0000000..b492e37 --- /dev/null +++ b/models/telemetry/DeliveryDiagnostics.cfc @@ -0,0 +1,69 @@ +/** Local, content-free envelope outcomes. Never emits telemetry about its own failures. */ +component { + + function init( numeric capacity = 64 ){ + variables.capacity = max( 1, min( 4096, arguments.capacity ) ); + variables.pending = []; + variables.counts = {}; + variables.lastAccepted = 0; + variables.lastAttempt = 0; + variables.untracked = 0; + variables.id = createUUID(); + return this; + } + function record( required any receipt ){ + lock name="sentry-delivery-#variables.id#" type="exclusive" timeout="2" { + poll(); + variables.lastAttempt = epoch(); + var status = arguments.receipt.getStatus(); + if ( status == "queued" ) { + if ( variables.pending.len() < variables.capacity ) { + variables.pending.append( { "receipt" : arguments.receipt, "created" : epoch() } ); + } else { + variables.untracked++; + } + } else { + count( status ); + } + } + } + function snapshot(){ + lock name="sentry-delivery-#variables.id#" type="exclusive" timeout="2" { + poll(); + var oldest = 0; + for ( var entry in variables.pending ) { + oldest = max( oldest, epoch() - entry.created ); + } + return { + "outcomes" : duplicate( variables.counts ), + "pending" : variables.pending.len(), + "oldestPendingSeconds" : oldest, + "lastAttemptAt" : variables.lastAttempt, + "lastAcceptedAt" : variables.lastAccepted, + "untracked" : variables.untracked + }; + } + } + private function poll(){ + for ( var i = variables.pending.len(); i >= 1; i-- ) { + var status = variables.pending[ i ].receipt.getStatus(); + if ( status != "queued" ) { + count( status ); + variables.pending.deleteAt( i ); + } + } + } + private function count( required string status ){ + if ( !listFind( "accepted,rejected,dropped,failed,disabled,filtered,invalid", arguments.status ) ) { + return; + } + variables.counts[ arguments.status ] = ( variables.counts[ arguments.status ] ?: 0 ) + 1; + if ( arguments.status == "accepted" ) { + variables.lastAccepted = epoch(); + } + } + private function epoch(){ + return createObject( "java", "java.lang.System" ).currentTimeMillis() / 1000; + } + +} diff --git a/models/telemetry/Dependency.cfc b/models/telemetry/Dependency.cfc new file mode 100644 index 0000000..6224f8c --- /dev/null +++ b/models/telemetry/Dependency.cfc @@ -0,0 +1,168 @@ +/** One logical dependency call; retries and breaker policy remain in the callback. */ +component { + + function init( required any telemetry ){ + variables.telemetry = arguments.telemetry; + variables.clock = createObject( "java", "java.lang.System" ); + return this; + } + function run( + required string name, + required function callback, + any classify = "" + ){ + var started = 0; + var ended = 0; + var state = {}; + var details = { "outcome" : "success" }; + try { + state.previous = variables.telemetry.getScope(); + var next = structCopy( state.previous ); + next[ "dependencyObservations" ] = duplicate( state.previous.dependencyObservations ?: [] ); + next.dependencyObservations.append( arguments.name ); + try { + variables.telemetry.setScope( next ); + } catch ( any unavailable ) { + } + state.span = variables.telemetry.startSpan( + arguments.name, + "app.dependency", + { "dependency.name" : arguments.name } + ); + next[ "span" ] = state.span; + try { + variables.telemetry.setScope( next ); + } catch ( any unavailable ) { + } + } catch ( any unavailable ) { + } + started = variables.clock.nanoTime(); + try { + var result = arguments.callback(); + ended = variables.clock.nanoTime(); + details = classifyResult( + arguments.classify, + isNull( local.result ) ? javacast( "null", "" ) : result, + javacast( "null", "" ), + false + ); + return isNull( local.result ) ? javacast( "null", "" ) : result; + } catch ( any failure ) { + ended = variables.clock.nanoTime(); + details = classifyResult( + arguments.classify, + javacast( "null", "" ), + failure, + true + ); + rethrow; + } finally { + var duration = max( 0, ( ended - started ) / 1000000 ); + complete( arguments.name, details, duration, state ); + } + } + private function classifyResult( + required any classifier, + any result, + any failure, + boolean failed = false + ){ + var details = { "outcome" : arguments.failed ? "error" : "success" }; + if ( !isCustomFunction( arguments.classifier ) ) return details; + details.outcome = "unknown"; + try { + var raw = arguments.classifier( + isNull( arguments.result ) ? javacast( "null", "" ) : arguments.result, + isNull( arguments.failure ) ? javacast( "null", "" ) : arguments.failure + ); + if ( !isStruct( raw ) ) return details; + if ( + raw.keyExists( "outcome" ) && isSimpleValue( raw.outcome ) && listFind( + "success,error,timeout,connection_error,cancelled,rejected,unknown", + raw.outcome + ) + ) + details.outcome = raw.outcome; + // A thrown callback cannot be classified as successful. + if ( arguments.failed && details.outcome == "success" ) details.outcome = "error"; + if ( + raw.keyExists( "statusCode" ) && isNumeric( raw.statusCode ) && raw.statusCode >= 100 && raw.statusCode < 600 && raw.statusCode == int( + raw.statusCode + ) + ) + details[ "http.status_class" ] = int( raw.statusCode / 100 ) & "xx"; + if ( + raw.keyExists( "retryCount" ) && isNumeric( raw.retryCount ) && raw.retryCount >= 0 && raw.retryCount <= 2147483647 && int( + raw.retryCount + ) == raw.retryCount + ) + details[ "retries" ] = raw.retryCount; + if ( + raw.keyExists( "circuitState" ) && isSimpleValue( raw.circuitState ) && listFind( + "open,closed,half_open", + raw.circuitState + ) + ) + details[ "circuit.state" ] = raw.circuitState; + } catch ( any unavailable ) { + } + return details; + } + private function complete( + required string name, + required struct details, + required numeric duration, + required struct state + ){ + try { + if ( arguments.state.keyExists( "previous" ) ) variables.telemetry.setScope( arguments.state.previous ); + } catch ( any unavailable ) { + } + try { + if ( arguments.state.keyExists( "span" ) ) { + arguments.state.span.setAttribute( "dependency.outcome", arguments.details.outcome ); + } + } catch ( any unavailable ) { + } + try { + if ( arguments.state.keyExists( "span" ) ) { + arguments.state.span.finish( + arguments.details.outcome == "timeout" ? "deadline_exceeded" : ( + arguments.details.outcome == "success" ? "ok" : "internal_error" + ) + ); + } + } catch ( any unavailable ) { + } + var attrs = { + "dependency.name" : arguments.name, + "outcome" : arguments.details.outcome + }; + for ( var field in [ "http.status_class", "circuit.state" ] ) { + if ( arguments.details.keyExists( field ) ) attrs[ field ] = arguments.details[ field ]; + } + try { + variables.telemetry.counter( "dependency.calls", 1, attrs ); + } catch ( any unavailable ) { + } + try { + variables.telemetry.distribution( + "dependency.duration", + arguments.duration, + attrs, + "millisecond" + ); + } catch ( any unavailable ) { + } + try { + if ( arguments.details.keyExists( "retries" ) ) + variables.telemetry.distribution( + "dependency.retries", + arguments.details.retries, + attrs + ); + } catch ( any unavailable ) { + } + } + +} diff --git a/models/telemetry/DependencyInterceptor.cfc b/models/telemetry/DependencyInterceptor.cfc new file mode 100644 index 0000000..7a100fe --- /dev/null +++ b/models/telemetry/DependencyInterceptor.cfc @@ -0,0 +1,24 @@ +/** WireBox AOP advice; method arguments/results are never metric labels. */ +component implements="coldbox.system.aop.MethodInterceptor" { + + property name="wirebox" inject="wirebox"; + function invokeMethod( required invocation ) output="false"{ + var call = arguments.invocation; + var rule = {}; + try { + var telemetry = variables.wirebox.getInstance( "SentryService@sentry" ).getObservability(); + rule = telemetry.dependencyRule( call.getTargetMapping().getName(), call.getMethod() ); + } catch ( any unavailable ) { + return call.proceed(); + } + if ( !rule.keyExists( "name" ) ) return call.proceed(); + return telemetry.withDependency( + rule.name, + function(){ + return call.proceed(); + }, + rule.classify ?: "" + ); + } + +} diff --git a/models/telemetry/HealthSampler.cfc b/models/telemetry/HealthSampler.cfc new file mode 100644 index 0000000..1af8861 --- /dev/null +++ b/models/telemetry/HealthSampler.cfc @@ -0,0 +1,92 @@ +/** Explicit provider snapshots. Missing/invalid measurements stay absent; poll failures never become zeros. */ +component { + + function init( + required any telemetry, + required array names, + required struct fields, + required string label + ){ + variables.telemetry = arguments.telemetry; + variables.names = new MetricLabels( arguments.names ); + variables.fields = arguments.fields; + variables.label = arguments.label; + variables.diagnostics = {}; + variables.id = createUUID(); + return this; + } + function capture( required string name, required function sample ){ + if ( !variables.names.isAllowed( arguments.name ) ) return false; + var key = hash( arguments.name, "SHA-256" ); + var now = createObject( "java", "java.lang.System" ).currentTimeMillis() / 1000; + try { + var raw = arguments.sample(); + now = createObject( "java", "java.lang.System" ).currentTimeMillis() / 1000; + if ( !isStruct( raw ) ) throw( type = "Sentry.InvalidHealthSnapshot" ); + var values = {}; + for ( var field in variables.fields ) { + if ( + raw.keyExists( field ) && !isNull( raw[ field ] ) && isNumeric( raw[ field ] ) && raw[ field ] >= 0 && raw[ + field + ] <= 1.7976931348623157E308 + ) { + if ( field == "lastHeartbeatAt" ) { + if ( raw[ field ] > 0 && raw[ field ] <= now ) values[ field ] = now - raw[ field ]; + } else { + values[ field ] = raw[ field ]; + } + } + } + if ( !structCount( values ) ) throw( type = "Sentry.EmptyHealthSnapshot" ); + var attrs = {}; + attrs[ variables.label ] = arguments.name; + for ( var field in values ) { + var metric = variables.fields[ field ]; + variables.telemetry.gauge( + metric.name, + values[ field ], + attrs, + metric.unit + ); + } + record( key, true, now ); + return true; + } catch ( any unavailable ) { + record( key, false, now ); + return false; + } + } + private function record( + required string key, + required boolean success, + required numeric now + ){ + lock name="sentry-health-#variables.id#" type="exclusive" timeout="2" { + var state = variables.diagnostics[ arguments.key ] ?: { + "lastSuccessAt" : 0, + "lastFailureAt" : 0, + "failures" : 0 + }; + if ( arguments.success ) { + state.lastSuccessAt = arguments.now; + } else { + state.lastFailureAt = arguments.now; + state.failures++; + } + variables.diagnostics[ arguments.key ] = state; + } + } + function diagnostics( required string name ){ + if ( !variables.names.isAllowed( arguments.name ) ) return {}; + lock name="sentry-health-#variables.id#" type="exclusive" timeout="2" { + return duplicate( + variables.diagnostics[ hash( arguments.name, "SHA-256" ) ] ?: { + "lastSuccessAt" : 0, + "lastFailureAt" : 0, + "failures" : 0 + } + ); + } + } + +} diff --git a/models/telemetry/LuceeQueryListener.cfc b/models/telemetry/LuceeQueryListener.cfc index ddd5ad2..a5fdfd0 100644 --- a/models/telemetry/LuceeQueryListener.cfc +++ b/models/telemetry/LuceeQueryListener.cfc @@ -24,7 +24,7 @@ component { return result; } var span = variables.telemetry.startSpan( - variables.telemetry.scrubSQL( sql ), + sql, "db.query", { "db.system" : variables.databaseSystem, diff --git a/models/telemetry/MetricLabels.cfc b/models/telemetry/MetricLabels.cfc new file mode 100644 index 0000000..c9fe3dc --- /dev/null +++ b/models/telemetry/MetricLabels.cfc @@ -0,0 +1,20 @@ +/** Bounded, fixed technical identities. Callers must use the configured spelling. */ +component { + + function init( array names = [] ){ + variables.names = []; + for ( var name in arguments.names ) { + if ( + !isNull( name ) && isSimpleValue( name ) && variables.names.len() < 32 && len( name ) > 0 && len( + name + ) <= 48 && !reFind( "[^A-Za-z0-9_.-]", name ) && !arrayFindNoCase( variables.names, name ) + ) + variables.names.append( toString( name ) ); + } + return this; + } + function isAllowed( required string name ){ + return variables.names.find( arguments.name ) > 0; + } + +} diff --git a/models/telemetry/Observability.cfc b/models/telemetry/Observability.cfc index 60797e2..cb37256 100644 --- a/models/telemetry/Observability.cfc +++ b/models/telemetry/Observability.cfc @@ -5,7 +5,64 @@ component { variables[ "owner" ] = arguments.owner; variables[ "settings" ] = arguments.settings; variables.settings.append( defaults(), false ); + variables.postgreSQLNumberPattern = createObject( "java", "java.util.regex.Pattern" ).compile( + "(? 0 ? arguments.statusCode : responseStatus(); - span.setAttribute( "http.response.status_code", code ); - if ( code >= 500 ) { - span.setStatus( "internal_error" ); - } - counter( - "http.server.requests", - 1, - { - "route" : span.toPayload().description, - "http.response.status_code" : code + finishRequestTimings( current.requestState, arguments.timedOut ? "deadline_exceeded" : "ok" ); + if ( current.requestState.keyExists( "span" ) ) { + var span = current.requestState.span; + if ( !span.isFinished() ) { + var code = arguments.statusCode > 0 ? arguments.statusCode : responseStatus(); + span.setAttribute( "http.response.status_code", code ); + if ( code >= 500 ) { + span.setStatus( "internal_error" ); } - ); - distribution( - "http.server.duration", - max( 0, timestamp() - span.toPayload().start_timestamp ) * 1000, - { "route" : span.toPayload().description }, - "millisecond" - ); - span.finish(); - flushSignals(); + counter( + "http.server.requests", + 1, + { + "route" : span.toPayload().description, + "http.response.status_code" : code + } + ); + span.finish(); + distribution( + "http.server.duration", + span.getDurationMilliseconds(), + { "route" : span.toPayload().description }, + "millisecond" + ); + flushSignals(); + } } } finally { + try { + if ( current.requestState.keyExists( "saturationToken" ) ) + finishRequestObservation( current.requestState.saturationToken, arguments.timedOut ); + } catch ( any unavailable ) { + } setScope( current.requestState.previous ); } } function getScope(){ - return variables.scope.get() ?: {}; + var stored = variables.scope.get(); + if ( isNull( stored ) ) return {}; + if ( stored.owner != executionOwner() ) { + variables.scope.remove(); + variables.scopeResets.incrementAndGet(); + return {}; + } + return stored.value; + } + /** Local diagnostic only: a completion boundary was missed, with no known end time. */ + function getScopeResetCount(){ + return variables.scopeResets.get(); } function setScope( required struct scope ){ - variables.scope.set( arguments.scope ); + if ( arguments.scope.isEmpty() ) variables.scope.remove(); + else variables.scope.set( { "owner" : executionOwner(), "value" : arguments.scope } ); + } + /** Request scope outlives neither the HTTP request nor its worker assignment. */ + function executionOwner(){ + try { + if ( !structKeyExists( request, variables.scopeKey ) ) { + lock name="sentry-owner-#variables.scopeKey#" type="exclusive" timeout="2" { + if ( !structKeyExists( request, variables.scopeKey ) ) + request[ variables.scopeKey ] = createUUID(); + } + } + return request[ variables.scopeKey ]; + } catch ( any noRequestContext ) { + return "background"; + } } function clearScope(){ variables.scope.remove(); @@ -160,8 +453,10 @@ component { struct headers = {}, boolean trusted = false ){ - var parent = arguments.trusted ? parseHeaders( arguments.headers ) : {}; - var sampled = variables.owner.getEnabled() && ( + var started = monotonicTime(); + var startTimestamp = timestamp(); + var parent = arguments.trusted ? parseHeaders( arguments.headers ) : {}; + var sampled = variables.owner.getEnabled() && ( parent.keyExists( "sampled" ) ? parent.sampled : rand() < variables.settings.tracesSampleRate ); var data = { @@ -169,55 +464,64 @@ component { "span_id" : left( id(), 16 ), "op" : arguments.op, "description" : safeName( arguments.name ), - "start_timestamp" : timestamp(), + "start_timestamp" : startTimestamp, "status" : "ok", "data" : {} }; if ( parent.keyExists( "parent_span_id" ) ) { data[ "parent_span_id" ] = parent.parent_span_id; } - return new Span( this, data, sampled ).setBaggage( parent.baggage ?: "" ); + return new Span( + telemetry = this, + data = data, + sampled = sampled, + started = started + ).setBaggage( parent.baggage ?: "" ); } function startSpan( required string name, string op = "function", struct attributes = {} ){ + var started = monotonicTime(); + var startTimestamp = timestamp(); + var description = listFind( "db.query,db.query.cache,db.query.qoq", arguments.op ) ? scrubSQL( + arguments.name, + arguments.attributes[ "db.system" ] ?: variables.settings.databaseSystem ?: "other" + ) : safeName( arguments.name ); var current = getScope(); if ( !current.keyExists( "span" ) || current.span.isFinished() ) { return new Span( - this, - { - "trace_id" : id(), - "span_id" : left( id(), 16 ), - "op" : arguments.op, - "description" : arguments.op == "db.query" ? scrubSQL( arguments.name ) : safeName( - arguments.name - ), - "start_timestamp" : timestamp(), + telemetry = this, + data = { + "trace_id" : id(), + "span_id" : left( id(), 16 ), + "op" : arguments.op, + "description" : description, + "start_timestamp" : startTimestamp, "data" : {}, "status" : "ok" }, - false + sampled = false, + started = started ); } var parent = current.span.getContext(); return new Span( this, { - "trace_id" : parent.trace_id, - "span_id" : left( id(), 16 ), - "parent_span_id" : parent.span_id, - "op" : arguments.op, - "description" : arguments.op == "db.query" ? scrubSQL( arguments.name ) : safeName( - arguments.name - ), - "start_timestamp" : timestamp(), + "trace_id" : parent.trace_id, + "span_id" : left( id(), 16 ), + "parent_span_id" : parent.span_id, + "op" : arguments.op, + "description" : description, + "start_timestamp" : startTimestamp, "status" : "ok", "data" : sanitizeAttributes( arguments.attributes ) }, current.span.isSampled(), - current.span.getRoot() + current.span.getRoot(), + started ); } function withSpan( @@ -381,6 +685,189 @@ component { required string category, array attachments = [], boolean wait = false + ){ + var receipt = sendSignalReceipt( argumentCollection = arguments ); + if ( variables.settings.enableDeliveryDiagnostics ) { + try { + variables.delivery.record( receipt ); + } catch ( any unavailable ) { + variables.completionFailures.incrementAndGet(); + } + } + return receipt; + } + function deliveryDiagnostics(){ + var result = variables.delivery.snapshot(); + result[ "enabled" ] = variables.settings.enableDeliveryDiagnostics; + return result; + } + function newRequestTimings( boolean automatic = false ){ + var names = arguments.automatic ? [ "handler", "render" ] : []; + for ( var name in variables.settings.requestTimingNames ) { + // RequestTimings owns validation and deduplication for both manual and automatic names. + if ( !isNull( name ) ) names.append( name ); + } + return new RequestTimings( this, names, 64, arguments.automatic ); + } + + /** Consumer supplies a supported pool adapter. Never inspect engine internals or JDBC credentials. */ + function captureDatabasePool( required string name, required function sample ){ + if ( !variables.settings.enableMetrics || !variables.settings.databasePoolNames.find( arguments.name ) ) { + return false; + } + try { + var values = arguments.sample(); + var fields = { + "active" : "db.pool.active", + "idle" : "db.pool.idle", + "max" : "db.pool.max", + "waiters" : "db.pool.waiters", + "timeouts" : "db.pool.timeouts.total" + }; + for ( var field in fields ) { + if ( values.keyExists( field ) && isNumeric( values[ field ] ) && values[ field ] >= 0 ) { + gauge( + fields[ field ], + values[ field ], + { "db.pool.name" : arguments.name } + ); + } + } + if ( + values.keyExists( "active" ) && values.keyExists( "max" ) && isNumeric( values.active ) && isNumeric( + values.max + ) && values.max > 0 && values.active >= 0 && values.active <= values.max + ) { + gauge( + "db.pool.utilization", + values.active / values.max, + { "db.pool.name" : arguments.name }, + "ratio" + ); + } + return true; + } catch ( any unavailable ) { + return false; + } + } + function beginRequestObservation(){ + if ( + !variables.settings.enableMetrics || !variables.settings.enableRequestSaturation || !variables.owner.getEnabled() + ) + return {}; + try { + return variables.requests.begin(); + } catch ( any unavailable ) { + return {}; + } + } + function finishRequestObservation( required struct token, boolean timedOut = false ){ + try { + return variables.requests.finish( argumentCollection = arguments ); + } catch ( any unavailable ) { + return false; + } + } + function requestSaturation(){ + return variables.requests.snapshot(); + } + function captureRequestSaturation(){ + if ( + !variables.settings.enableMetrics || !variables.settings.enableRequestSaturation || !variables.owner.getEnabled() + ) + return {}; + try { + return variables.requests.capture(); + } catch ( any unavailable ) { + return {}; + } + } + function captureRequestExecutor( required string name, required function sample ){ + if ( !variables.settings.enableMetrics || !variables.owner.getEnabled() ) return false; + return variables.executors.capture( argumentCollection = arguments ); + } + function requestExecutorDiagnostics( required string name ){ + return variables.executors.diagnostics( arguments.name ); + } + function captureQueueHealth( required string name, required function sample ){ + if ( !variables.settings.enableMetrics || !variables.owner.getEnabled() ) return false; + return variables.queues.capture( argumentCollection = arguments ); + } + function queueHealthDiagnostics( required string name ){ + return variables.queues.diagnostics( arguments.name ); + } + function captureQueueWorkerHealth( required string name, required function sample ){ + if ( !variables.settings.enableMetrics || !variables.owner.getEnabled() ) return false; + return variables.workers.capture( argumentCollection = arguments ); + } + function queueWorkerDiagnostics( required string name ){ + return variables.workers.diagnostics( arguments.name ); + } + function captureCBQWorkerHealth( required string name, required any pool ){ + var workerPool = arguments.pool; + return captureQueueWorkerHealth( arguments.name, function(){ + // Compatibility facade only. Provider-specific sampling belongs to the installed cbq adapter. + return variables.adapterResolver.resolve( "WorkerHealthAdapter@cbq" ).samplePool( workerPool ); + } ); + } + + function withDependency( + required string name, + required function callback, + any classify = "" + ){ + if ( + !variables.settings.enableMetrics || !variables.owner.getEnabled() || !variables.dependencies.isAllowed( + arguments.name + ) + ) + return arguments.callback(); + var scope = {}; + try { + scope = getScope(); + } catch ( any unavailable ) { + return arguments.callback(); + } + if ( scope.keyExists( "dependencyObservations" ) && scope.dependencyObservations.find( arguments.name ) ) + return arguments.callback(); + var classifier = arguments.classify; + if ( !isCustomFunction( classifier ) && ( !isSimpleValue( classifier ) || len( classifier ) ) ) { + var descriptor = classifier; + classifier = function( result, failure ){ + return variables.adapterResolver.call( + descriptor, + "classify", + { + "result" : isNull( arguments.result ) ? javacast( "null", "" ) : arguments.result, + "failure" : isNull( arguments.failure ) ? javacast( "null", "" ) : arguments.failure + } + ); + }; + } + return new Dependency( this ).run( arguments.name, arguments.callback, classifier ); + } + + function withOperation( + required string name, + required function callback, + string provider = "", + any classify = "" + ){ + if ( + !variables.settings.operationNames.find( arguments.name ) || ( + len( arguments.provider ) && !variables.settings.providerNames.find( arguments.provider ) + ) + ) { + return arguments.callback(); + } + return new Operation( this ).run( argumentCollection = arguments ); + } + private function sendSignalReceipt( + required string type, + required struct payload, + required string category, + array attachments = [], + boolean wait = false ){ var data = arguments.payload; try { @@ -477,7 +964,7 @@ component { ){ var queryCallback = arguments.callback; return withSpan( - scrubSQL( arguments.sql ), + scrubSQL( arguments.sql, arguments.databaseSystem ), function( span ){ var previous = getScope(); var next = structCopy( previous ); @@ -722,14 +1209,10 @@ component { ); } function captureRuntimeGauges(){ - var memory = createObject( "java", "java.lang.management.ManagementFactory" ) - .getMemoryMXBean() - .getHeapMemoryUsage(); - gauge( "jvm.heap.used", memory.getUsed(), {}, "byte" ); - gauge( - "jvm.threads", - createObject( "java", "java.lang.management.ManagementFactory" ).getThreadMXBean().getThreadCount() - ); + if ( !variables.settings.enableMetrics ) { + return {}; + } + return new RuntimeMetrics( this ).capture(); } function getBrowserConfig(){ var browser = variables.settings.browser; @@ -850,7 +1333,17 @@ component { return ""; } } - function scrubSQL( required string sql ){ + function scrubSQL( required string sql, string databaseSystem = "" ){ + var dialect = len( arguments.databaseSystem ) ? arguments.databaseSystem : ( + variables.settings.databaseSystem ?: "other" + ); + if ( dialect == "postgresql" ) { + // Escape strings, dollar quoting and other unsupported forms fail closed. + if ( find( "$", arguments.sql ) || find( "`", arguments.sql ) || find( chr( 92 ), arguments.sql ) ) { + return "[Filtered SQL]"; + } + return scrubPostgreSQL( arguments.sql ); + } // Unsupported quoting is discarded rather than risking exporting literals. if ( find( "$", arguments.sql ) || find( "`", arguments.sql ) || find( "\\", arguments.sql ) ) { return uCase( listFirst( trim( arguments.sql ), " " ) ) & " [Filtered]"; @@ -873,6 +1366,103 @@ component { } return left( reReplace( scrubbed, "\s+", " ", "all" ), 1024 ); } + /** Scan quotes and comments together so comment markers inside values/identifiers are inert. */ + private function scrubPostgreSQL( required string sql ){ + var parts = []; + var cursor = 1; + var plainStart = 1; + var size = len( arguments.sql ); + while ( cursor <= size ) { + var character = mid( arguments.sql, cursor, 1 ); + var pair = mid( arguments.sql, cursor, 2 ); + if ( character != "'" && character != """" && pair != "--" && pair != "/*" ) { + cursor++; + continue; + } + if ( cursor > plainStart ) { + parts.append( scrubPostgreSQLNumbers( mid( arguments.sql, plainStart, cursor - plainStart ) ) ); + } + var tokenStart = cursor; + if ( character == "'" || character == """" ) { + var quote = character; + var closed = false; + cursor++; + while ( cursor <= size ) { + if ( mid( arguments.sql, cursor, 1 ) == quote ) { + if ( mid( arguments.sql, cursor + 1, 1 ) == quote ) { + cursor += 2; + continue; + } + cursor++; + closed = true; + break; + } + cursor++; + } + if ( !closed ) { + return "[Filtered SQL]"; + } + parts.append( quote == """" ? mid( arguments.sql, tokenStart, cursor - tokenStart ) : "?" ); + } else if ( pair == "--" ) { + cursor += 2; + while ( cursor <= size && !find( mid( arguments.sql, cursor, 1 ), chr( 10 ) & chr( 13 ) ) ) { + cursor++; + } + parts.append( " " ); + } else { + var depth = 1; + cursor += 2; + while ( cursor <= size && depth > 0 ) { + pair = mid( arguments.sql, cursor, 2 ); + if ( pair == "/*" ) { + depth++; + cursor += 2; + } else if ( pair == "*/" ) { + depth--; + cursor += 2; + } else { + cursor++; + } + } + if ( depth > 0 ) { + return "[Filtered SQL]"; + } + parts.append( " " ); + } + plainStart = cursor; + } + if ( plainStart <= size ) { + parts.append( + scrubPostgreSQLNumbers( + mid( + arguments.sql, + plainStart, + size - plainStart + 1 + ) + ) + ); + } + var result = ""; + for ( var part in parts ) { + var identifier = left( part, 1 ) == """"; + var text = identifier ? part : reReplace( part, "\s+", " ", "all" ); + if ( right( result, 1 ) == " " && left( text, 1 ) == " " ) { + text = removeChars( text, 1, 1 ); + } + if ( len( result ) + len( text ) > 1024 ) { + // Do not split identifiers: adapters may sanitize this description again. + if ( !identifier ) { + result &= left( text, 1024 - len( result ) ); + } + break; + } + result &= text; + } + return trim( result ); + } + private function scrubPostgreSQLNumbers( required string sql ){ + return variables.postgreSQLNumberPattern.matcher( arguments.sql ).replaceAll( "?" ); + } function getTransport(){ return variables.transport; } @@ -881,6 +1471,12 @@ component { return variables.transport.flush( arguments.timeoutMilliseconds ); } function shutdown( numeric timeoutMilliseconds = 2000 ){ + if ( !variables.closed.compareAndSet( false, true ) ) return true; + variables.collectors.shutdown(); + try { + if ( !isNull( variables.queryAdapterShutdown ) ) variables.queryAdapterShutdown(); + } catch ( any unavailable ) { + } if ( !isNull( variables.nativeQueryListener ) ) { createObject( "java", "ortus.boxlang.runtime.BoxRuntime" ) .getInstance() diff --git a/models/telemetry/Operation.cfc b/models/telemetry/Operation.cfc new file mode 100644 index 0000000..466d76c --- /dev/null +++ b/models/telemetry/Operation.cfc @@ -0,0 +1,109 @@ +/** Opt-in named workflow/provider observations; callback values and exception text never enter telemetry. */ +component { + + function init( required any telemetry ){ + variables.telemetry = arguments.telemetry; + return this; + } + function run( + required string name, + required function callback, + string provider = "", + any classify = "" + ){ + var started = createObject( "java", "java.lang.System" ).nanoTime(); + var state = {}; + var outcome = "success"; + try { + state.previous = variables.telemetry.getScope(); + state.span = variables.telemetry.startSpan( + arguments.name, + "app.workflow", + { + "workflow.name" : arguments.name, + "provider.name" : arguments.provider + } + ); + var next = structCopy( state.previous ); + next[ "span" ] = state.span; + variables.telemetry.setScope( next ); + } catch ( any unavailable ) { + } + try { + var result = arguments.callback(); + if ( isCustomFunction( arguments.classify ) ) { + outcome = "unknown"; + try { + var classified = arguments.classify( isNull( local.result ) ? javacast( "null", "" ) : result ); + if ( + isSimpleValue( classified ) && listFind( + "success,accepted,delivered,retry,timeout,cancelled,failed,unknown", + classified + ) + ) { + outcome = classified; + } + } catch ( any unavailable ) { + } + } + return isNull( local.result ) ? javacast( "null", "" ) : result; + } catch ( any failure ) { + outcome = "failed"; + rethrow; + } finally { + complete( + arguments.name, + arguments.provider, + outcome, + started, + state + ); + } + } + private function complete( + required string name, + required string provider, + required string outcome, + required numeric started, + required struct state + ){ + try { + if ( arguments.state.keyExists( "previous" ) ) { + variables.telemetry.setScope( arguments.state.previous ); + } + } catch ( any unavailable ) { + } + try { + if ( arguments.state.keyExists( "span" ) ) { + arguments.state.span.setAttribute( "workflow.outcome", arguments.outcome ); + } + } catch ( any unavailable ) { + } + try { + if ( arguments.state.keyExists( "span" ) ) { + arguments.state.span.finish( + arguments.outcome == "failed" ? "internal_error" : ( + arguments.outcome == "timeout" ? "deadline_exceeded" : "ok" + ) + ); + } + } catch ( any unavailable ) { + } + try { + var attrs = { + "workflow.name" : arguments.name, + "provider.name" : arguments.provider, + "outcome" : arguments.outcome + }; + variables.telemetry.counter( "workflow.operations", 1, attrs ); + variables.telemetry.distribution( + "workflow.duration", + max( 0, ( createObject( "java", "java.lang.System" ).nanoTime() - arguments.started ) / 1000000 ), + attrs, + "millisecond" + ); + } catch ( any unavailable ) { + } + } + +} diff --git a/models/telemetry/RequestSaturation.cfc b/models/telemetry/RequestSaturation.cfc new file mode 100644 index 0000000..984bc5b --- /dev/null +++ b/models/telemetry/RequestSaturation.cfc @@ -0,0 +1,79 @@ +/** Process-local observed requests; bounded storage and monotonic ages. */ +component { + + function init( + required any telemetry, + numeric capacity = 4096, + numeric longRunningSeconds = 30 + ){ + variables.telemetry = arguments.telemetry; + variables.capacity = max( 1, min( 4096, int( arguments.capacity ) ) ); + variables.threshold = max( 0, arguments.longRunningSeconds ); + variables.clock = createObject( "java", "java.lang.System" ); + variables.active = {}; + variables.untracked = 0; + variables.timeouts = 0; + variables.id = createUUID(); + return this; + } + function begin(){ + var token= {}; + lock name="sentry-requests-#variables.id#" type="exclusive" timeout="2" { + if ( structCount( variables.active ) >= variables.capacity ) { + variables.untracked++; + return token; + } + token = { "id" : createUUID() }; + variables.active[ hash( token.id, "SHA-256" ) ] = variables.clock.nanoTime(); + } + return token; + } + function finish( required struct token, boolean timedOut = false ){ + if ( !arguments.token.keyExists( "id" ) || !isSimpleValue( arguments.token.id ) ) return false; + var key = hash( arguments.token.id, "SHA-256" ); + lock name="sentry-requests-#variables.id#" type="exclusive" timeout="2" { + if ( !variables.active.keyExists( key ) ) return false; + variables.active.delete( key ); + if ( arguments.timedOut ) variables.timeouts++; + } + if ( arguments.timedOut ) { + try { + variables.telemetry.counter( "http.server.timeouts", 1 ); + } catch ( any unavailable ) { + } + } + return true; + } + function snapshot(){ + lock name="sentry-requests-#variables.id#" type="exclusive" timeout="2" { + var result = { + "active" : structCount( variables.active ), + "oldestSeconds" : 0, + "longRunning" : 0, + "timeouts" : variables.timeouts, + "untrackedStarts" : variables.untracked + }; + var now = variables.clock.nanoTime(); + for ( var key in variables.active ) { + var age = max( 0, ( now - variables.active[ key ] ) / 1000000000 ); + result.oldestSeconds = max( result.oldestSeconds, age ); + if ( age >= variables.threshold ) result.longRunning++; + } + return result; + } + } + function capture(){ + var result = snapshot(); + variables.telemetry.gauge( "http.server.active_requests", result.active ); + variables.telemetry.gauge( + "http.server.oldest_active", + result.oldestSeconds, + {}, + "second" + ); + variables.telemetry.gauge( "http.server.long_running", result.longRunning ); + variables.telemetry.gauge( "http.server.untracked_starts.total", result.untrackedStarts ); + return result; + } + +} diff --git a/models/telemetry/RequestTimings.cfc b/models/telemetry/RequestTimings.cfc new file mode 100644 index 0000000..3df78a5 --- /dev/null +++ b/models/telemetry/RequestTimings.cfc @@ -0,0 +1,147 @@ +/** One instance per request/operation. Caller owns eligibility and response header publication. */ +component { + + function init( + required any telemetry, + array names = [], + numeric maxSamples = 64, + boolean emitMetrics = false + ){ + variables.telemetry = arguments.telemetry; + variables.emitMetrics = arguments.emitMetrics; + variables.names = []; + for ( var name in arguments.names ) { + if ( !isNull( name ) && isSimpleValue( name ) ) registerName( toString( name ) ); + } + variables.maxSamples = max( 0, min( 64, arguments.maxSamples ) ); + variables.samples = 0; + variables.metrics = {}; + variables.tokens = createObject( "java", "java.util.concurrent.ConcurrentHashMap" ).init(); + variables.clock = createObject( "java", "java.lang.System" ); + variables.id = createUUID(); + variables.closed = createObject( "java", "java.util.concurrent.atomic.AtomicBoolean" ).init( false ); + return this; + } + private function registerName( required string name ){ + if ( + !len( arguments.name ) || len( arguments.name ) > 48 || !reFind( "^[a-zA-Z]", arguments.name ) || reFind( + "[^a-zA-Z0-9_]", + arguments.name + ) + ) + return ""; + // Use one spelling everywhere, including case-insensitive CFML snapshot structs. + var index = arrayFindNoCase( variables.names, arguments.name ); + if ( index ) return variables.names[ index ]; + if ( variables.names.len() >= 32 ) return ""; + variables.names.append( arguments.name ); + return arguments.name; + } + function begin( required string name ){ + lock name="sentry-timings-#variables.id#" type="exclusive" timeout="2" throwOnTimeout="false" { + return beginToken( arguments.name ); + } + return {}; + } + private function beginToken( required string name ){ + if ( variables.closed.get() || variables.samples >= variables.maxSamples ) { + return {}; + } + var phaseName = registerName( arguments.name ); + if ( !len( phaseName ) ) return {}; + variables.samples++; + var state = { + "name" : phaseName, + "started" : variables.clock.nanoTime() + }; + try { + state.span = variables.telemetry.startSpan( phaseName, "app.phase" ); + } catch ( any unavailable ) { + } + var token = { "id" : createUUID() }; + variables.tokens.put( token.id, state ); + return token; + } + function finish( required struct token, string status = "ok" ){ + lock name="sentry-timings-#variables.id#" type="exclusive" timeout="2" throwOnTimeout="false" { + finishToken( arguments.token, arguments.status ); + } + } + private function finishToken( required struct token, required string status ){ + var state = {}; + try { + if ( !arguments.token.keyExists( "id" ) || !isSimpleValue( arguments.token.id ) ) { + return; + } + // Atomic removal claims a private handle once, without trusting caller metadata. + state = variables.tokens.remove( toString( arguments.token.id ) ); + if ( isNull( state ) ) { + return; + } + } catch ( any unavailable ) { + return; + } + try { + var name = state.name; + var metric = variables.metrics[ name ] ?: { "duration" : 0, "count" : 0 }; + var duration = max( 0, ( variables.clock.nanoTime() - state.started ) / 1000000 ); + metric.duration += duration; + metric.count++; + variables.metrics[ name ] = metric; + } catch ( any unavailable ) { + } + try { + if ( variables.emitMetrics && !isNull( local.duration ) ) + variables.telemetry.distribution( + "http.server.phase.duration", + duration, + { "phase" : state.name }, + "millisecond" + ); + } catch ( any unavailable ) { + } + finishSpan( state, arguments.status ); + } + private function finishSpan( required struct state, required string status ){ + try { + if ( arguments.state.keyExists( "span" ) ) { + arguments.state.span.finish( arguments.status ); + } + } catch ( any unavailable ) { + } + } + function measure( required string name, required function callback ){ + var token = begin( arguments.name ); + var status = "ok"; + try { + return arguments.callback(); + } catch ( any failure ) { + status = "internal_error"; + rethrow; + } finally { + finish( token, status ); + } + } + /** Drain every owned handle, including manual phases, at the response boundary. */ + function close( string status = "ok" ){ + lock name="sentry-timings-#variables.id#" type="exclusive" timeout="2" throwOnTimeout="false" { + if ( !variables.closed.compareAndSet( false, true ) ) return; + for ( var id in variables.tokens.keySet().toArray() ) finishToken( { "id" : id }, arguments.status ); + } + } + function snapshot(){ + lock name="sentry-timings-#variables.id#" type="readonly" timeout="2" { + return duplicate( variables.metrics ); + } + } + function toServerTiming(){ + var headers = []; + for ( var name in variables.names ) { + if ( variables.metrics.keyExists( name ) ) { + headers.append( name & ";dur=" & ( round( variables.metrics[ name ].duration * 100 ) / 100 ) ); + } + } + return arrayToList( headers, ", " ); + } + +} diff --git a/models/telemetry/RuntimeMetrics.cfc b/models/telemetry/RuntimeMetrics.cfc new file mode 100644 index 0000000..e45965c --- /dev/null +++ b/models/telemetry/RuntimeMetrics.cfc @@ -0,0 +1,127 @@ +/** Inexpensive MXBean samples. Unsupported values are omitted, never invented as zero. */ +component { + + function init( required any telemetry ){ + variables.telemetry = arguments.telemetry; + return this; + } + function capture(){ + var samples = {}; + var factory = createObject( "java", "java.lang.management.ManagementFactory" ); + try { + var memory = factory.getMemoryMXBean().getHeapMemoryUsage(); + samples[ "jvm.heap.used" ] = { "value" : memory.getUsed(), "unit" : "byte" }; + if ( memory.getMax() > 0 ) { + samples[ "jvm.heap.max" ] = { "value" : memory.getMax(), "unit" : "byte" }; + } + } catch ( any unavailable ) { + } + try { + var threads = readAttribute( "java.lang:type=Threading", "ThreadCount" ); + if ( isNumeric( threads ) && threads >= 0 ) { + samples[ "jvm.threads" ] = { "value" : threads, "unit" : "none" }; + } + } catch ( any unavailable ) { + } + try { + var cpu = readAttribute( "java.lang:type=OperatingSystem", "ProcessCpuLoad" ); + if ( isNumeric( cpu ) && cpu >= 0 && cpu <= 1 ) { + samples[ "jvm.cpu.process" ] = { "value" : cpu, "unit" : "ratio" }; + } + } catch ( any unavailable ) { + } + var count = 0; + var duration = 0; + var supported = false; + try { + var beans = callMBean( + "queryNames", + [ + "javax.management.ObjectName", + "javax.management.QueryExp" + ], + [ + createObject( "java", "javax.management.ObjectName" ).init( "java.lang:type=GarbageCollector,*" ), + javacast( "null", "" ) + ] + ); + supported = beans.size() > 0; + for ( var bean in beans.toArray() ) { + var collections = readAttribute( bean.toString(), "CollectionCount" ); + var time = readAttribute( bean.toString(), "CollectionTime" ); + if ( collections >= 0 && time >= 0 ) { + count += collections; + duration += time; + } else { + supported = false; + } + } + } catch ( any unavailable ) { + supported = false; + } + if ( supported ) { + samples[ "jvm.gc.collections.total" ] = { "value" : count, "unit" : "none" }; + samples[ "jvm.gc.duration.total" ] = { "value" : duration, "unit" : "millisecond" }; + } + for ( var name in samples ) { + try { + variables.telemetry.gauge( + name, + samples[ name ].value, + {}, + samples[ name ].unit + ); + } catch ( any unavailable ) { + } + } + return samples; + } + function readAttribute( required string bean, required string attribute ){ + return callMBean( + "getAttribute", + [ + "javax.management.ObjectName", + "java.lang.String" + ], + [ + createObject( "java", "javax.management.ObjectName" ).init( arguments.bean ), + arguments.attribute + ] + ); + } + /** Call the exported interface, never a non-exported JVM implementation class. */ + private function callMBean( + required string methodName, + required array types, + required array args + ){ + var classes = []; + var javaClass = createObject( "java", "java.lang.Class" ); + for ( var type in arguments.types ) { + classes.append( javaClass.forName( type ) ); + } + var method = javaClass + .forName( "javax.management.MBeanServerConnection" ) + .getMethod( arguments.methodName, javacast( "java.lang.Class[]", classes ) ); + // Preserve null QueryExp arguments on engines whose CFML arrays omit nulls. + var reflectArray = createObject( "java", "java.lang.reflect.Array" ); + var nativeArgs = reflectArray.newInstance( + javaClass.forName( "java.lang.Object" ), + javacast( "int", arguments.types.len() ) + ); + for ( var i = 1; i <= arguments.types.len(); i++ ) { + if ( arrayIsDefined( arguments.args, i ) && !isNull( arguments.args[ i ] ) ) { + reflectArray.set( + nativeArgs, + javacast( "int", i - 1 ), + arguments.args[ i ] + ); + } + } + return method.invoke( + createObject( "java", "java.lang.management.ManagementFactory" ).getPlatformMBeanServer(), + nativeArgs + ); + } + +} diff --git a/models/telemetry/Span.cfc b/models/telemetry/Span.cfc index f50a6c3..2fe3397 100644 --- a/models/telemetry/Span.cfc +++ b/models/telemetry/Span.cfc @@ -5,7 +5,8 @@ component { required any telemetry, required struct data, boolean sampled = false, - any root + any root, + numeric started ){ variables[ "baggage" ] = ""; variables[ "telemetry" ] = arguments.telemetry; @@ -16,6 +17,7 @@ component { variables[ "done" ] = createObject( "java", "java.util.concurrent.atomic.AtomicBoolean" ).init( false ); variables[ "children" ] = createObject( "java", "java.util.concurrent.ConcurrentLinkedQueue" ).init(); variables[ "id" ] = createUUID(); + variables.started = isNull( arguments.started ) ? variables.telemetry.monotonicTime() : arguments.started; return this; } function getContext(){ @@ -73,6 +75,7 @@ component { return false; } variables.data[ "start_timestamp" ] = arguments.timestamp; + variables.started = variables.telemetry.monotonicTime(); } return true; } @@ -85,7 +88,8 @@ component { if ( len( arguments.status ) ) { variables.data[ "status" ] = arguments.status; } - variables.data[ "timestamp" ] = variables.telemetry.timestamp(); + variables.duration = ( variables.telemetry.monotonicTime() - variables.started ) / 1000000; + variables.data[ "timestamp" ] = variables.data.start_timestamp + variables.duration / 1000; if ( variables.sampled ) { if ( variables.isRoot ) { variables.telemetry.completeTransaction( this ); @@ -96,6 +100,9 @@ component { } return true; } + function getDurationMilliseconds(){ + return isFinished() ? variables.duration : ( variables.telemetry.monotonicTime() - variables.started ) / 1000000; + } function toPayload(){ var payload = duplicate( variables.data ); payload[ "data" ] = variables.telemetry.sanitizeAttributes( payload.data ); diff --git a/models/telemetry/adapters/IDependencyClassifier.cfc b/models/telemetry/adapters/IDependencyClassifier.cfc new file mode 100644 index 0000000..516aa3d --- /dev/null +++ b/models/telemetry/adapters/IDependencyClassifier.cfc @@ -0,0 +1,6 @@ +/** Return the same bounded fields accepted by withDependency. The unused result/failure argument is null. */ +interface { + + struct function classify( any result, any failure ); + +} diff --git a/models/telemetry/adapters/IQueryAdapter.cfc b/models/telemetry/adapters/IQueryAdapter.cfc new file mode 100644 index 0000000..8d2294a --- /dev/null +++ b/models/telemetry/adapters/IQueryAdapter.cfc @@ -0,0 +1,6 @@ +/** Install execution hooks using context.telemetry/applicationName/databaseSystem. Return a native mode or {mode:"custom",shutdown:callback}. */ +interface { + + any function configure( required struct context ); + +} diff --git a/models/telemetry/adapters/IQueryFactory.cfc b/models/telemetry/adapters/IQueryFactory.cfc new file mode 100644 index 0000000..7ae6b40 --- /dev/null +++ b/models/telemetry/adapters/IQueryFactory.cfc @@ -0,0 +1,6 @@ +/** Return a fresh correctly configured qb builder on every call. */ +interface { + + any function newQuery( required struct context ); + +} diff --git a/models/telemetry/adapters/ISnapshotAdapter.cfc b/models/telemetry/adapters/ISnapshotAdapter.cfc new file mode 100644 index 0000000..60b5133 --- /dev/null +++ b/models/telemetry/adapters/ISnapshotAdapter.cfc @@ -0,0 +1,6 @@ +/** Return supported numeric fields. Context contains a fixed collector name/kind and copied provider options. */ +interface { + + struct function sample( required struct context ); + +} diff --git a/readme.md b/readme.md index df96cfd..00ddeca 100644 --- a/readme.md +++ b/readme.md @@ -4,6 +4,9 @@ This module connects your CFML application to send bug reports to Sentry (https://sentry.io) +For observability-enabled consumers, see the [performance dashboard recipes](dashboards/README.md) +for configurable offline queries, sample coverage and saved-dashboard verification. + ## LICENSE Apache License, Version 2.0. diff --git a/test-harness/config/Coldbox.cfc b/test-harness/config/Coldbox.cfc index 7622afd..7a78af0 100644 --- a/test-harness/config/Coldbox.cfc +++ b/test-harness/config/Coldbox.cfc @@ -94,12 +94,30 @@ } }; + if ( len( getSystemSetting( "SENTRY_QUEUE_HEALTH_DB", "" ) ) ) { + coldbox.modulesExternalLocation.append( "/queueHealthModules" ); + modules.include.append( "qb" ); + modules.include.append( "str" ); + modules.include.append( "cbq" ); + // Synthetic failures must never use a developer's external SENTRY_DSN. + moduleSettings.sentry.DSN = "http://public@127.0.0.1:60320/1"; + moduleSettings.sentry.enableLogBoxAppender = false; + moduleSettings.sentry.enableExceptionLogging = false; + moduleSettings[ "qb" ] = { defaultGrammar : "PostgresGrammar@qb", defaultOptions : { datasource : "sentryQueueHealth" } }; + moduleSettings[ "cbq" ] = { configPath : "tests.resources.QueueHealthConfig", registerWorkers : false }; + } + } /** * Load the Module you are testing */ function afterAspectsLoad( event, interceptData, rc, prc ){ + if ( len( getSystemSetting( "SENTRY_QUEUE_HEALTH_DB", "" ) ) ) { + controller.getWireBox().getBinder().map( "QueueHealthState" ).to( "tests.resources.QueueHealthState" ).asSingleton(); + controller.getWireBox().getBinder().map( "QueueHealthJob" ).to( "tests.resources.QueueHealthJob" ); + controller.getWireBox().getBinder().map( "AutomaticLiveQueries" ).to( "tests.resources.AutomaticLiveQueries" ); + } controller.getModuleService() .registerAndActivateModule( moduleName = request.MODULE_NAME, @@ -107,4 +125,4 @@ ); } -} \ No newline at end of file +} diff --git a/test-harness/request-lifecycle/Application.cfc b/test-harness/request-lifecycle/Application.cfc new file mode 100644 index 0000000..1710be7 --- /dev/null +++ b/test-harness/request-lifecycle/Application.cfc @@ -0,0 +1,38 @@ +/** Local HTTP lifecycle fixture. All telemetry is captured in memory; nothing is ingested. */ +component { + + this.name = "sentry-lifecycle-fixed-" & hash( getCurrentTemplatePath() ); + variables.root = getDirectoryFromPath( getCurrentTemplatePath() ); + this.mappings[ "/coldbox" ] = variables.root & "../coldbox"; + this.mappings[ "/sentry" ] = variables.root & "../../"; + this.mappings[ "/moduleroot" ] = variables.root & "../../../"; + this.mappings[ "/dependencies" ] = variables.root & "../../modules"; + function onApplicationStart(){ + application.records = createObject( "java", "java.util.concurrent.ConcurrentLinkedQueue" ).init(); + application.hooks = createObject( "java", "java.util.concurrent.ConcurrentLinkedQueue" ).init(); + application.bootstrap = new coldbox.system.Bootstrap( "", variables.root, "", "/request-lifecycle" ); + application.bootstrap.loadColdBox(); + return true; + } + function onRequestStart( targetPage ){ + if ( listLast( arguments.targetPage, "/" ) == "report.cfm" ) return true; + application.hooks.add( "start" ); + return application.bootstrap.onRequestStart( arguments.targetPage ); + } + function onRequestEnd( targetPage ){ + if ( listLast( arguments.targetPage, "/" ) == "report.cfm" ) return; + application.hooks.add( "end" ); + application.cbController + .getWireBox() + .getInstance( "SentryService@sentry" ) + .endRequest(); + } + function onAbort( targetPage ){ + application.hooks.add( "abort" ); + application.cbController + .getWireBox() + .getInstance( "SentryService@sentry" ) + .endRequest(); + } + +} diff --git a/test-harness/request-lifecycle/config/Coldbox.cfc b/test-harness/request-lifecycle/config/Coldbox.cfc new file mode 100644 index 0000000..4bc57a7 --- /dev/null +++ b/test-harness/request-lifecycle/config/Coldbox.cfc @@ -0,0 +1,31 @@ +component { + + function configure(){ + coldbox = { + appName : "Sentry lifecycle fixture", + reinitPassword : "", + defaultEvent : "Responses.text", + handlersIndexAutoReload : true, + modulesExternalLocation : [ "/dependencies" ] + }; + modules = { include : [ "funclinenums" ] }; + moduleSettings = { + sentry : { + DSN : "http://public@127.0.0.1:1/1", + enableLogBoxAppender : false, + enableExceptionLogging : false, + enableRequestTracing : true, + enableAutomaticRequestTimings : true, + tracesSampleRate : 1, + beforeSendSignal : function( payload, type ){ + if ( arguments.type == "transaction" ) application.records.add( duplicate( arguments.payload ) ); + return javacast( "null", "" ); + } + } + }; + } + function afterAspectsLoad(){ + controller.getModuleService().registerAndActivateModule( "sentry", "moduleroot" ); + } + +} diff --git a/test-harness/request-lifecycle/handlers/Responses.cfc b/test-harness/request-lifecycle/handlers/Responses.cfc new file mode 100644 index 0000000..f361118 --- /dev/null +++ b/test-harness/request-lifecycle/handlers/Responses.cfc @@ -0,0 +1,49 @@ +component extends="coldbox.system.EventHandler" { + + function text( event, rc, prc ){ + return "fixture text"; + } + function binary( event, rc, prc ){ + event.setHTTPHeader( name = "Cache-Control", value = "private, no-store" ); + event.setHTTPHeader( name = "X-Content-Type-Options", value = "nosniff" ); + event.renderData( + type = "plain", + data = binaryDecode( "0001027f80ff", "hex" ), + contentType = "application/octet-stream", + isBinary = true + ); + } + function slow( event, rc, prc ){ + sleep( 250 ); + return "slow response"; + } + function noRender( event, rc, prc ){ + event.noRender(); + } + function streamed( event, rc, prc ){ + event.noRender(); + writeOutput( "stream-one" ); + sleep( 40 ); + writeOutput( "stream-two" ); + } + function aborted( event, rc, prc ){ + abort; + } + function redirect( event, rc, prc ){ + relocate( event = "Responses.text" ); + } + function failure( event, rc, prc ){ + throw( type = "SentryFixture.Original", message = "original lifecycle failure" ); + } + function report( event, rc, prc ){ + event.renderData( + type = "json", + data = { + records : application.records.toArray(), + hooks : application.hooks.toArray(), + thread : createObject( "java", "java.lang.Thread" ).currentThread().getName() + } + ); + } + +} diff --git a/test-harness/request-lifecycle/index.cfm b/test-harness/request-lifecycle/index.cfm new file mode 100644 index 0000000..b76e401 --- /dev/null +++ b/test-harness/request-lifecycle/index.cfm @@ -0,0 +1 @@ + diff --git a/test-harness/request-lifecycle/report.cfm b/test-harness/request-lifecycle/report.cfm new file mode 100644 index 0000000..c9af11d --- /dev/null +++ b/test-harness/request-lifecycle/report.cfm @@ -0,0 +1,8 @@ + +setting showDebugOutput=false; +if ( structKeyExists( url, "reset" ) ) { + application.records.clear(); + application.hooks.clear(); +} +writeOutput( serializeJSON( { records : application.records.toArray(), hooks : application.hooks.toArray() } ) ); + diff --git a/test-harness/request-lifecycle/verify.mjs b/test-harness/request-lifecycle/verify.mjs new file mode 100644 index 0000000..a6e22db --- /dev/null +++ b/test-harness/request-lifecycle/verify.mjs @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import { writeFile } from 'node:fs/promises'; + +// Only the caller's loopback fixture is accepted. No Sentry ingestion is involved. +const base = new URL(process.argv[2] || 'http://127.0.0.1:60339/request-lifecycle/'); +assert.equal(base.hostname, '127.0.0.1'); +const report = async (reset = false) => { + const result = await fetch(new URL(`report.cfm${reset ? '?reset=1' : ''}`, base)).then(r => r.json()); + return JSON.parse(JSON.stringify(result).replace(/"RECORDS":/g, '"records":').replace(/"HOOKS":/g, '"hooks":')); +}; +const request = async action => { + const start = Date.now(); + const response = await fetch(new URL(`index.cfm?event=Responses.${action}`, base), { redirect: 'manual' }); + const bytes = Buffer.from(await response.arrayBuffer()); + const end = Date.now(); + if (action === 'binary') { + assert.equal(response.status, 200); + assert.equal(bytes.toString('hex'), '0001027f80ff'); + assert.equal(response.headers.get('cache-control'), 'private, no-store'); + assert.equal(response.headers.get('x-content-type-options'), 'nosniff'); + assert.match(response.headers.get('content-type'), /^application\/octet-stream/); + } else if (action === 'streamed') { + assert.equal(response.status, 200); + assert.equal(bytes.toString(), 'stream-onestream-two'); + } else if (action === 'redirect') assert.equal(response.status, 302); + else if (action === 'failure') { + assert.equal(response.status, 500); + assert.match(bytes.toString(), /original lifecycle failure/); + } else assert.equal(response.status, 200); + return { action, start, end, responseMs: end - start }; +}; + +await request('text'); // initialize and warm up compilation before timing assertions +await report(true); +const binaryResponse = await request('binary'); +// Observe completion before allowing another ColdBox request to run on the worker. +const immediate = await report(); +assert.equal(immediate.records.length, 1, 'binary must finish without a later request'); +assert(immediate.hooks.includes('abort')); +const binary = immediate.records[0]; +assert.equal(binary.transaction, 'GET Responses.binary'); +assert(binary.timestamp * 1000 <= binaryResponse.end + 25); +assert(binary.timestamp * 1000 >= binaryResponse.start - 25); +const finishedAt = binary.timestamp; +await new Promise(resolve => setTimeout(resolve, 1200)); +const responses = [binaryResponse]; +for (const action of ['text', 'slow', 'noRender', 'streamed', 'aborted', 'redirect', 'failure']) responses.push(await request(action)); +responses.push(...await Promise.all(Array.from({ length: 12 }, (_, i) => request(i % 2 ? 'binary' : 'text')))); +const result = await report(); +assert.equal(result.records.length, responses.length, 'one transaction per response'); +assert.equal(new Set(result.records.map(r => r.contexts.trace.trace_id)).size, responses.length); +assert.equal(result.records.find(r => r.event_id === binary.event_id).timestamp, finishedAt); +const slow = result.records.find(r => r.transaction === 'GET Responses.slow'); +assert((slow.timestamp - slow.start_timestamp) * 1000 >= 240, 'real slow request must stay slow'); +assert.equal(result.records.find(r => r.transaction === 'GET Responses.failure').contexts.trace.status, 'internal_error'); +for (const record of result.records) { + for (const span of record.spans) { + assert.equal(span.trace_id, record.contexts.trace.trace_id); + assert.equal(span.parent_span_id, record.contexts.trace.span_id); + assert(span.timestamp >= span.start_timestamp); + // Millisecond wall-clock resolution can differ by <1ms from the elapsed clock. + assert(span.timestamp <= record.timestamp + 0.002); + } +} +const evidence = { binaryResponse, responses: responses.length, records: result.records.length, result }; +if (process.argv[3]) await writeFile(process.argv[3], JSON.stringify(evidence, null, 2)); +console.log(JSON.stringify({ passed: true, responses: responses.length, binaryResponseMs: binaryResponse.responseMs, + binarySpanMs: (binary.timestamp - binary.start_timestamp) * 1000, slowSpanMs: (slow.timestamp - slow.start_timestamp) * 1000 })); diff --git a/test-harness/tests/Application.cfc b/test-harness/tests/Application.cfc index dfc5603..4db0ec8 100644 --- a/test-harness/tests/Application.cfc +++ b/test-harness/tests/Application.cfc @@ -19,8 +19,9 @@ component { this.mappings[ "/tests" ] = getDirectoryFromPath( getCurrentTemplatePath() ); // The application root - rootPath = reReplaceNoCase( this.mappings[ "/tests" ], "tests(\\|/)", "" ); - this.mappings[ "/root" ] = rootPath; + rootPath = reReplaceNoCase( this.mappings[ "/tests" ], "tests(\\|/)", "" ); + this.mappings[ "/root" ] = rootPath; + this.mappings[ "/sentryAOP" ] = rootPath & ".engine/aop"; // UPDATE THE NAME OF THE MODULE IN TESTING BELOW request.MODULE_NAME = "sentry"; @@ -39,6 +40,44 @@ component { this.mappings[ "/cbq" ] = cbqSource; } this.mappings[ "/dependencies" ] = this.mappings[ "/sentry" ] & "/modules"; + qbSource = createObject( "java", "java.lang.System" ).getenv( "SENTRY_QB_SOURCE" ); + if ( !isNull( qbSource ) && len( qbSource ) ) { + this.mappings[ "/qb" ] = qbSource; + this.mappings[ "/cbpaginator" ] = qbSource & "/modules/cbpaginator"; + } + // Explicit local-only integration fixture. The dedicated database must already exist. + queueHealthDatabase = createObject( "java", "java.lang.System" ).getenv( "SENTRY_QUEUE_HEALTH_DB" ); + if ( !isNull( queueHealthDatabase ) && len( queueHealthDatabase ) ) { + if ( + !reFind( "^sentry_queue_health_[a-z0-9_]+", queueHealthDatabase ) || reFind( + "[^a-z0-9_]", + queueHealthDatabase + ) + ) + throw( type = "Sentry.UnsafeIntegrationDatabase" ); + this.mappings[ "/queueHealthModules" ] = rootPath & "modules"; + this.datasources[ "sentryQueueHealth" ] = { + "class" : "org.postgresql.Driver", + "connectionString" : "jdbc:postgresql://127.0.0.1:5432/" & queueHealthDatabase, + "username" : "postgres", + "password" : "", + "connectionLimit" : 6 + }; + if ( structKeyExists( server, "lucee" ) ) { + this.datasources.sentryQueueHealth[ "bundleName" ] = "org.postgresql.jdbc"; + this.datasources.sentryQueueHealth[ "bundleVersion" ] = "42.6.0"; + } else if ( !structKeyExists( server, "boxlang" ) ) { + this.datasources[ "sentryQueueHealth" ] = { + "driver" : "postgresql", + "host" : "127.0.0.1", + "port" : 5432, + "database" : queueHealthDatabase, + "username" : "postgres", + "password" : "" + }; + } + } + /** diff --git a/test-harness/tests/resources/AutomaticBinder.cfc b/test-harness/tests/resources/AutomaticBinder.cfc new file mode 100644 index 0000000..8e2ee65 --- /dev/null +++ b/test-harness/tests/resources/AutomaticBinder.cfc @@ -0,0 +1,12 @@ +component extends="coldbox.system.ioc.config.Binder" { + + function configure(){ + wireBox = { scopeRegistration : { enabled : false } }; + map( "SentryService@sentry" ).toValue( getProperty( "service" ) ); + map( "MailClient" ).to( "tests.resources.AutomaticMailClient" ); + map( "Classifier" ).to( "tests.resources.AutomaticClassifier" ); + map( "QueryAdapter" ).to( "tests.resources.AutomaticQueryAdapter" ); + map( "SnapshotAdapter" ).to( "tests.resources.AutomaticSnapshotAdapter" ); + } + +} diff --git a/test-harness/tests/resources/AutomaticClassifier.cfc b/test-harness/tests/resources/AutomaticClassifier.cfc new file mode 100644 index 0000000..ca757ba --- /dev/null +++ b/test-harness/tests/resources/AutomaticClassifier.cfc @@ -0,0 +1,11 @@ +component implements="sentry.models.telemetry.adapters.IDependencyClassifier" { + + struct function classify( any result, any failure ){ + return { + "outcome" : isNull( arguments.failure ) ? "rejected" : "timeout", + "statusCode" : 503, + "retryCount" : 1 + }; + } + +} diff --git a/test-harness/tests/resources/AutomaticLiveQueries.cfc b/test-harness/tests/resources/AutomaticLiveQueries.cfc new file mode 100644 index 0000000..a02c0f5 --- /dev/null +++ b/test-harness/tests/resources/AutomaticLiveQueries.cfc @@ -0,0 +1,8 @@ +component implements="sentry.models.telemetry.adapters.IQueryFactory" { + + property name="wirebox" inject="wirebox"; + any function newQuery( required struct context ){ + return variables.wirebox.getInstance( "QueryBuilder@qb" ); + } + +} diff --git a/test-harness/tests/resources/AutomaticMailClient.cfc b/test-harness/tests/resources/AutomaticMailClient.cfc new file mode 100644 index 0000000..b1c759b --- /dev/null +++ b/test-harness/tests/resources/AutomaticMailClient.cfc @@ -0,0 +1,18 @@ +component { + + property name="sentry" inject="SentryService@sentry"; + function send( required string message ){ + return { "message" : arguments.message }; + } + function fail(){ + throw( type = "OriginalDependencyFailure", message = "PRIVATE" ); + } + function nothing(){ + } + function manual( required string message ){ + return variables.sentry.withDependency( "mail", function(){ + return "PRIVATE"; + } ); + } + +} diff --git a/test-harness/tests/resources/AutomaticQueryAdapter.cfc b/test-harness/tests/resources/AutomaticQueryAdapter.cfc new file mode 100644 index 0000000..a42ecc0 --- /dev/null +++ b/test-harness/tests/resources/AutomaticQueryAdapter.cfc @@ -0,0 +1,7 @@ +component implements="sentry.models.telemetry.adapters.IQueryAdapter" { + + any function configure( required struct context ){ + return "custom"; + } + +} diff --git a/test-harness/tests/resources/AutomaticSnapshotAdapter.cfc b/test-harness/tests/resources/AutomaticSnapshotAdapter.cfc new file mode 100644 index 0000000..59993e5 --- /dev/null +++ b/test-harness/tests/resources/AutomaticSnapshotAdapter.cfc @@ -0,0 +1,13 @@ +component implements="sentry.models.telemetry.adapters.ISnapshotAdapter" { + + struct function sample( required struct context ){ + return { + "active" : 2, + "max" : 4, + "configuredWorkers" : 3, + "activeExecutions" : 1, + "password" : "PRIVATE" + }; + } + +} diff --git a/test-harness/tests/resources/QueueHealthConfig.cfc b/test-harness/tests/resources/QueueHealthConfig.cfc new file mode 100644 index 0000000..5c4dcde --- /dev/null +++ b/test-harness/tests/resources/QueueHealthConfig.cfc @@ -0,0 +1,7 @@ +/** The real cbq module starts no connections or workers until a spec registers its own isolated pool. */ +component { + + function configure(){ + } + +} diff --git a/test-harness/tests/resources/QueueHealthJob.cfc b/test-harness/tests/resources/QueueHealthJob.cfc new file mode 100644 index 0000000..d6dd6aa --- /dev/null +++ b/test-harness/tests/resources/QueueHealthJob.cfc @@ -0,0 +1,21 @@ +component extends="cbq.models.Jobs.AbstractJob" { + + function handle(){ + var state = getInstance( "QueueHealthState" ).get( getProperties().runId ); + state.handled.incrementAndGet(); + state.started.countDown(); + if ( getProperties().mode == "block" ) { + if ( + !state.release.await( + javacast( "long", 20 ), + createObject( "java", "java.util.concurrent.TimeUnit" ).SECONDS + ) + ) + throw( type = "QueueHealthFixtureTimeout" ); + } + if ( getProperties().mode == "fail" || ( getProperties().mode == "retry" && getCurrentAttempt() == 1 ) ) + throw( type = "QueueHealthSyntheticFailure" ); + return true; + } + +} diff --git a/test-harness/tests/resources/QueueHealthState.cfc b/test-harness/tests/resources/QueueHealthState.cfc new file mode 100644 index 0000000..523ab36 --- /dev/null +++ b/test-harness/tests/resources/QueueHealthState.cfc @@ -0,0 +1,30 @@ +/** Shared in-memory gates survive real job serialization and worker-thread boundaries. */ +component singleton { + + function init(){ + variables.runs = createObject( "java", "java.util.concurrent.ConcurrentHashMap" ).init(); + return this; + } + function create( required string id ){ + var state = { + "started" : createObject( "java", "java.util.concurrent.CountDownLatch" ).init( 1 ), + "release" : createObject( "java", "java.util.concurrent.CountDownLatch" ).init( 1 ), + "handled" : createObject( "java", "java.util.concurrent.atomic.AtomicInteger" ).init( 0 ) + }; + variables.runs.put( arguments.id, state ); + return state; + } + function get( required string id ){ + return variables.runs.get( arguments.id ); + } + function releaseAll(){ + for ( var id in variables.runs.keySet().toArray() ) { + variables.runs.get( id ).release.countDown(); + } + } + function clear(){ + releaseAll(); + variables.runs.clear(); + } + +} diff --git a/test-harness/tests/resources/QueueMetricsQuery.cfc b/test-harness/tests/resources/QueueMetricsQuery.cfc new file mode 100644 index 0000000..018bb8e --- /dev/null +++ b/test-harness/tests/resources/QueueMetricsQuery.cfc @@ -0,0 +1,38 @@ +/** Deterministic queue provider boundary; dialect compilation is tested with optional real qb. */ +component { + + function init( array rows = [], boolean fail = false ){ + variables.rows = arguments.rows; + variables.fail = arguments.fail; + return this; + } + function from(){ + return this; + } + function select(){ + return this; + } + function selectRaw(){ + return this; + } + function whereNull(){ + return this; + } + function groupBy(){ + return this; + } + function getGrammar(){ + return { + wrapValue : function( name ){ + return """" & name & """"; + } + }; + } + function get(){ + if ( variables.fail ) { + throw( type = "DatabaseUnavailable" ); + } + return variables.rows; + } + +} diff --git a/test-harness/tests/specs/AutomaticObservabilitySpec.cfc b/test-harness/tests/specs/AutomaticObservabilitySpec.cfc new file mode 100644 index 0000000..036f5e5 --- /dev/null +++ b/test-harness/tests/specs/AutomaticObservabilitySpec.cfc @@ -0,0 +1,793 @@ +component extends="testbox.system.BaseSpec" { + + function beforeAll(){ + // Generated AOP code lives in an ignored directory, absent in a fresh checkout. + // Lucee expandPath falls back to the web root while a mapped target does not exist. + var generationDirectory = getApplicationMetadata().mappings[ "/sentryAOP" ]; + var parentDirectory = getDirectoryFromPath( generationDirectory ); + if ( !directoryExists( parentDirectory ) ) directoryCreate( parentDirectory ); + if ( !directoryExists( generationDirectory ) ) directoryCreate( generationDirectory ); + } + + function run(){ + describe( "Automatic collectors and adapter configuration", function(){ + beforeEach( function(){ + variables.services = []; + variables.injector = javacast( "null", "" ); + } ); + afterEach( function(){ + if ( !isNull( variables.injector ) ) variables.injector.shutdown(); + for ( var service in variables.services ) service.shutdown(); + } ); + it( "invokes the callback adapter contract directly", function(){ + var resolver = new sentry.models.telemetry.AdapterResolver(); + var result = resolver.call( + function( context ){ + return { activeWorkers : context.value }; + }, + "sample", + { "context" : { "value" : 2 } } + ); + expect( result.activeWorkers ).toBe( 2 ); + } ); + + it( "records an ad hoc manual phase on the automatic request timing instance", function(){ + var service = makeService( { enableAutomaticRequestTimings : true } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "distribution" ); + var context = eventContext(); + var observer = createMock( "sentry.interceptors.Observability" ).$property( + "sentry", + "variables", + service + ); + observer.onRequestCapture( context, {} ); + var timing = service.getRequestTimings(); + expect( + timing.measure( "loadDashboard", function(){ + return 42; + } ) + ).toBe( 42 ); + expect( timing.snapshot().loadDashboard.count ).toBe( 1 ); + var calls = telemetry.$callLog( "distribution" ).distribution; + expect( calls.len() ).toBe( 1 ); + expect( calls[ 1 ][ 1 ] ).toBe( "http.server.phase.duration" ); + expect( calls[ 1 ][ 3 ].phase ).toBe( "loadDashboard" ); + observer.postProcess( context, {} ); + expect( telemetry.getScope() ).toBeEmpty(); + } ); + it( "ignores invalid manual phase names in both timing APIs", function(){ + var service = makeService( { + requestTimingNames : [ + {}, + [], + "valid", + "valid", + "bad-name", + javacast( "null", "" ) + ] + } ); + var manual = service.newRequestTimings(); + manual.finish( manual.begin( "valid" ) ); + expect( manual.snapshot().valid.count ).toBe( 1 ); + var automatic = service.newRequestTimings( true ); + automatic.finish( automatic.begin( "handler" ) ); + automatic.finish( automatic.begin( "valid" ) ); + expect( automatic.snapshot().handler.count ).toBe( 1 ); + expect( automatic.snapshot().valid.count ).toBe( 1 ); + } ); + it( "finishes open automatic phase tokens on early request termination", function(){ + var service = makeService( { + enableAutomaticRequestTimings : true, + enableRequestTracing : true, + tracesSampleRate : 1 + } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "distribution" ); + var context = eventContext(); + context.getCurrentRoute = function(){ + return "early"; + }; + context.getHTTPMethod = function(){ + return "GET"; + }; + var observer = createMock( "sentry.interceptors.Observability" ).$property( + "sentry", + "variables", + service + ); + observer.onRequestCapture( context, {} ); + observer.preEvent( context, {} ); + var timings = service.getRequestTimings(); + var root = telemetry.getScope().span; + service.endRequest( 303 ); + expect( timings.snapshot().handler.count ).toBe( 1 ); + expect( root.isFinished() ).toBeTrue(); + expect( root.getChildren()[ 1 ].isFinished() ).toBeTrue(); + expect( telemetry.getScope() ).toBeEmpty(); + } ); + it( "accepts valid partial automatic collector settings at actual module startup", function(){ + var service = makeService(); + var injector = makeInjector( service ); + var settings = duplicate( service.getSettings() ); + settings.automaticCollectors = { intervalSeconds : 30 }; + var config = createMock( "sentry.ModuleConfig" ) + .$property( "wirebox", "variables", injector ) + .$property( "binder", "variables", injector.getBinder() ) + .$property( "settings", "variables", settings ) + .$property( "moduleMapping", "variables", "sentry" ); + config.onLoad(); + expect( service.automaticCollectorDiagnostics().started ).toBeFalse(); + } ); + it( "runs application code once even when dependency scope lookup fails", function(){ + var service = makeService( { dependencyNames : [ "mail" ] } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "getScope" ).$throws( "ScopeUnavailable" ); + var calls = 0; + expect( + service.withDependency( "mail", function(){ + calls++; + return 42; + } ) + ).toBe( 42 ); + expect( calls ).toBe( 1 ); + expect( function(){ + service.withDependency( "mail", function(){ + throw( type = "OriginalFailure" ); + } ); + } ).toThrow( "OriginalFailure" ); + } ); + it( "deduplicates nested dependency observations even when span creation fails", function(){ + var service = makeService( { dependencyNames : [ "mail" ] } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "startSpan" ).$throws( "NoTracing" ); + telemetry.$( "counter" ); + expect( + service.withDependency( "mail", function(){ + return service.withDependency( "mail", function(){ + return 1; + } ); + } ) + ).toBe( 1 ); + expect( telemetry.$callLog( "counter" ).counter.len() ).toBe( 1 ); + } ); + it( "configures query instrumentation using a WireBox interface mapping", function(){ + var service = makeService( { queryAdapter : { mapping : "QueryAdapter", method : "configure" } } ); + var injector = makeInjector( service ); + service.getObservability().setAdapterInjector( injector ); + expect( service.getObservability().configureQueryInstrumentation( "fixture" ) ).toBe( "custom" ); + } ); + it( "prevents overlapping ticks and starting more adapters after shutdown", function(){ + var started = createObject( "java", "java.util.concurrent.CountDownLatch" ).init( 1 ); + var release = createObject( "java", "java.util.concurrent.CountDownLatch" ).init( 1 ); + var calls = createObject( "java", "java.util.concurrent.atomic.AtomicInteger" ).init( 0 ); + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + queues : [ + { + name : "blocked", + adapter : function( context ){ + started.countDown(); + release.await(); + return { activeWorkers : 0 }; + } + }, + { + name : "later", + adapter : function( context ){ + calls.incrementAndGet(); + return { activeWorkers : 0 }; + } + } + ] + } + } ); + var ref = createObject( "java", "java.util.concurrent.atomic.AtomicReference" ).init( service ); + var name = "auto-test-" & replace( createUUID(), "-", "", "all" ); + thread name=name action="run" reference=ref { + attributes.reference.get().collectAutomaticMetrics(); + } + try { + expect( + started.await( + javacast( "long", 5 ), + createObject( "java", "java.util.concurrent.TimeUnit" ).SECONDS + ) + ).toBeTrue(); + service.collectAutomaticMetrics(); + expect( service.automaticCollectorDiagnostics().skipped ).toBe( 1 ); + service.shutdown(); + } finally { + release.countDown(); + thread action="join" name=name timeout=5000; + } + expect( calls.get() ).toBe( 0 ); + expect( cfthread[ name ].status ).toBe( "COMPLETED" ); + } ); + it( "isolates collector work from the caller's active trace and restores it", function(){ + var observed = true; + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + queues : [ + { + name : "mail", + adapter : function( context ){ + observed = service.getTraceContext().keyExists( "span_id" ); + return { activeWorkers : 0 }; + } + } + ] + } + } ); + var root = service.startTransaction( "caller" ); + service.getObservability().setScope( { span : root } ); + service.collectAutomaticMetrics(); + expect( observed ).toBeFalse(); + expect( service.getTraceContext().span_id ).toBe( root.getContext().span_id ); + root.finish(); + } ); + + + it( "does not collect or resolve adapters when automatic collection is disabled", function(){ + var calls = 0; + var service = makeService( { + automaticCollectors : { + enabled : false, + runtime : false, + queues : [ + { + name : "mail", + adapter : function( context ){ + calls++; + return { activeWorkers : 0 }; + } + } + ] + } + } ); + service.collectAutomaticMetrics(); + expect( calls ).toBe( 0 ); + expect( service.automaticCollectorDiagnostics().started ).toBeFalse(); + } ); + it( "records local receipt snapshot failures without emitting diagnostic metrics", function(){ + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + deliveryDiagnostics : true + } + } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "deliveryDiagnostics" ).$throws( "SnapshotUnavailable" ); + telemetry.$( "gauge" ); + service.collectAutomaticMetrics(); + expect( service.automaticCollectorDiagnostics().failures ).toBe( 1 ); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 0 ); + } ); + + it( "collects supported queue and worker fields together from one provider snapshot", function(){ + var calls = 0; + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + queues : [ + { + name : "mail", + adapter : function( context ){ + expect( context.kind ).toBe( "queues" ); + calls++; + return { + depth : 7, + oldestAgeSeconds : 15, + configuredWorkers : 4, + activeExecutions : 2, + runnable : 5, + delayed : 2, + reserved : 1, + privatePayload : "PRIVATE" + }; + } + } + ] + } + } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "gauge" ); + service.collectAutomaticMetrics(); + expect( calls ).toBe( 1 ); + expect( service.queueHealthDiagnostics( "mail" ).lastSuccessAt ).toBeGT( 0 ); + var samples = telemetry.$callLog( "gauge" ).gauge; + expect( samples.len() ).toBe( 7 ); + for ( var sample in samples ) expect( sample[ 3 ][ "queue.name" ] ).toBe( "mail" ); + expect( serializeJSON( samples ) ).toInclude( "queue.depth" ); + expect( serializeJSON( samples ) ).toInclude( "queue.executions.active" ); + expect( serializeJSON( samples ) ).notToInclude( "PRIVATE" ); + } ); + it( "accepts backlog-only providers without fabricating worker or age measurements", function(){ + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + queues : [ + { + name : "remote", + adapter : function(){ + return { depth : 0 }; + } + } + ] + } + } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "gauge" ); + service.collectAutomaticMetrics(); + var samples = telemetry.$callLog( "gauge" ).gauge; + expect( samples.len() ).toBe( 1 ); + expect( samples[ 1 ][ 1 ] ).toBe( "queue.depth" ); + expect( samples[ 1 ][ 2 ] ).toBe( 0 ); + } ); + it( "requires an explicit worker adapter rather than assuming cbq from a pool setting", function(){ + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + queues : [ { name : "mail", pool : "not-a-default-provider" } ] + } + } ); + var telemetry = prepareMock( service.getObservability() ); + var injector = prepareMock( makeInjector( service ) ); + injector.$( "getInstance" ).$throws( "UnexpectedProviderLookup" ); + telemetry.setAdapterInjector( injector ); + telemetry.$( "gauge" ); + service.collectAutomaticMetrics(); + expect( service.automaticCollectorDiagnostics().failures ).toBe( 1 ); + expect( injector.$callLog( "getInstance" ).getInstance.len() ).toBe( 0 ); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 0 ); + } ); + it( "automatically samples configured callbacks and permits manual sampling too", function(){ + var calls = 0; + var service = makeService( { + enableRequestSaturation : true, + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + queues : [ + { + name : "mail", + pool : "ignored-legacy-shortcut", + options : { workerPool : "outbound", token : "PRIVATE" }, + adapter : function( context ){ + expect( context.kind ).toBe( "queues" ); + expect( context.options.workerPool ).toBe( "outbound" ); + calls++; + return { activeWorkers : 0 }; + } + } + ] + } + } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "gauge" ); + var token = service.beginRequestObservation(); + service.collectAutomaticMetrics(); + expect( calls ).toBe( 1 ); + expect( service.automaticCollectorDiagnostics().runs ).toBe( 1 ); + expect( service.automaticCollectorDiagnostics().failures ).toBe( 0 ); + var payload = serializeJSON( telemetry.$callLog( "gauge" ) ); + expect( payload ).toInclude( "http.server.active_requests" ); + expect( payload ).toInclude( "queue.workers.active" ); + expect( payload ).notToInclude( "PRIVATE" ); + expect( + service.captureQueueHealth( "mail", function(){ + return { activeWorkers : 2 }; + } ) + ).toBeTrue(); + service.finishRequestObservation( token ); + } ); + it( "continues other collectors after an adapter fails and records only bounded diagnostics", function(){ + var good = 0; + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + queues : [ + { + name : "bad", + adapter : function( context ){ + throw( type = "AdapterFailed", message = "PRIVATE" ); + } + }, + { + name : "good", + adapter : function( context ){ + good++; + return { activeWorkers : 0 }; + } + } + ] + } + } ); + service.collectAutomaticMetrics(); + expect( good ).toBe( 1 ); + expect( service.automaticCollectorDiagnostics().failures ).toBe( 1 ); + expect( serializeJSON( service.automaticCollectorDiagnostics() ) ).notToInclude( "PRIVATE" ); + } ); + it( "starts one scheduler, actually samples, and stops on shutdown", function(){ + var calls = createObject( "java", "java.util.concurrent.atomic.AtomicInteger" ).init( 0 ); + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 1, + runtime : false, + requestSaturation : false, + queues : [ + { + name : "mail", + adapter : function( context ){ + calls.incrementAndGet(); + return { activeWorkers : 0 }; + } + } + ] + } + } ); + expect( service.startAutomaticCollectors() ).toBeFalse(); + awaitCondition( function(){ + return calls.get() > 0; + } ); + service.shutdown(); + var observed = calls.get(); + sleep( 1150 ); + expect( calls.get() ).toBe( observed ); + } ); + it( "rejects invalid intervals without starting an executor", function(){ + var service = makeService( { automaticCollectors : { enabled : true, intervalSeconds : 0, runtime : false } } ); + expect( service.automaticCollectorDiagnostics().started ).toBeFalse(); + expect( service.automaticCollectorDiagnostics().failures ).toBe( 1 ); + } ); + it( "does not invoke collectors when metrics or the DSN is disabled", function(){ + var calls = 0; + var settings = { + enableMetrics : false, + automaticCollectors : { + enabled : true, + intervalSeconds : 1, + runtime : false, + queues : [ + { + name : "mail", + adapter : function( context ){ + calls++; + return { activeWorkers : 0 }; + } + } + ] + } + }; + var service = makeService( settings ); + service.collectAutomaticMetrics(); + expect( calls ).toBe( 0 ); + expect( service.automaticCollectorDiagnostics().started ).toBeFalse(); + } ); + it( "resolves snapshot mappings and explicit interface methods through actual WireBox", function(){ + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + requestExecutors : [ { name : "http", adapter : "SnapshotAdapter" } ], + databasePools : [ + { + name : "db", + adapter : { mapping : "SnapshotAdapter", method : "sample" } + } + ] + } + } ); + var injector = makeInjector( service ); + service.getObservability().setAdapterInjector( injector ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "gauge" ); + service.collectAutomaticMetrics(); + expect( service.automaticCollectorDiagnostics().failures ).toBe( 0 ); + var calls = serializeJSON( telemetry.$callLog( "gauge" ) ); + expect( calls ).toInclude( "http.server.executor.active" ); + expect( calls ).toInclude( "db.pool.utilization" ); + expect( calls ).notToInclude( "PRIVATE" ); + } ); + it( "configures query tracing with a callback once and cleans custom resources", function(){ + var calls = 0; + var stopped = 0; + var service = makeService( { + databaseSystem : "postgresql", + queryAdapter : function( context ){ + calls++; + expect( context.applicationName ).toBe( "test-app" ); + expect( context.databaseSystem ).toBe( "postgresql" ); + expect( isObject( context.telemetry ) ).toBeTrue(); + return { + mode : "custom", + shutdown : function(){ + stopped++; + } + }; + } + } ); + var telemetry = service.getObservability(); + expect( telemetry.configureQueryInstrumentation( "test-app" ) ).toBe( "custom" ); + expect( telemetry.configureQueryInstrumentation( "test-app" ) ).toBe( "custom" ); + expect( calls ).toBe( 1 ); + service.shutdown(); + expect( stopped ).toBe( 1 ); + } ); + it( "does not assume a query library when native hooks are unavailable", function(){ + for ( var mode in [ "auto", "boxlang" ] ) { + var service = makeService( { queryAdapter : mode } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( + method = "registerBoxLangQueries", + callback = function(){ + throw( type = "UnavailableNativeQueryHooks" ); + } + ); + expect( telemetry.configureQueryInstrumentation( "test" ) ).toBe( "none" ); + service.shutdown(); + } + } ); + it( "fails closed for an invalid query adapter while preserving built-in modes", function(){ + var defaultService = makeService(); + expect( defaultService.getObservability().configureQueryInstrumentation( "test" ) ).toBe( "none" ); + var service = makeService( { + queryAdapter : function( context ){ + throw( type = "PRIVATE" ); + } + } ); + expect( service.getObservability().configureQueryInstrumentation( "test" ) ).toBe( "none" ); + var lucee = makeService( { queryAdapter : "lucee" } ); + expect( lucee.getObservability().configureQueryInstrumentation( "test" ) ).toBe( "lucee" ); + } ); + it( "leaves library instrumentation to explicit provider mappings", function(){ + var service = makeService( { queryAdapter : "QueryTelemetryAdapter@qb" } ); + expect( service.getObservability().configureQueryInstrumentation( "test" ) ).toBe( "none" ); + var callback = makeService( { + queryAdapter : function( context ){ + expect( context.databaseSystem ).toBe( "other" ); + return "qb"; + } + } ); + expect( callback.getObservability().configureQueryInstrumentation( "test" ) ).toBe( "none" ); + } ); + it( "supports classifier mappings in the unchanged manual dependency API", function(){ + var service = makeService( { dependencyNames : [ "mail" ] } ); + var injector = makeInjector( service ); + service.getObservability().setAdapterInjector( injector ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "counter" ); + expect( + service.withDependency( + "mail", + function(){ + return false; + }, + "Classifier" + ) + ).toBeFalse(); + expect( telemetry.$callLog( "counter" ).counter[ 1 ][ 3 ].outcome ).toBe( "rejected" ); + } ); + it( "automatically observes actual WireBox methods, preserves results and original exceptions", function(){ + var service = makeService( { + dependencyAdapters : [ + { + name : "mail", + mapping : "MailClient", + methods : [ "send", "fail", "nothing", "manual" ], + classify : "Classifier" + } + ] + } ); + var injector = makeInjector( service ); + installModule( service, injector ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "counter" ); + telemetry.$( "distribution" ); + var gateway = injector.getInstance( "MailClient" ); + expect( gateway.send( "PRIVATE" ).message ).toBe( "PRIVATE" ); + expect( function(){ + gateway.fail(); + } ).toThrow( "OriginalDependencyFailure" ); + gateway.nothing(); + expect( gateway.manual( "PRIVATE" ) ).toBe( "PRIVATE" ); + var calls = telemetry.$callLog( "counter" ).counter; + expect( calls.len() ).toBe( 4 ); + expect( calls[ 1 ][ 3 ].outcome ).toBe( "rejected" ); + expect( calls[ 2 ][ 3 ].outcome ).toBe( "timeout" ); + expect( serializeJSON( calls ) ).notToInclude( "PRIVATE" ); + expect( telemetry.getScope() ).toBeEmpty(); + } ); + it( "keeps an outer manual dependency wrapper from double-counting an automatic method", function(){ + var service = makeService( { + dependencyAdapters : [ + { + name : "mail", + mapping : "MailClient", + methods : [ "send" ] + } + ] + } ); + var injector = makeInjector( service ); + installModule( service, injector ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "counter" ); + var gateway = injector.getInstance( "MailClient" ); + var result = service.withDependency( "mail", function(){ + return gateway.send( "PRIVATE" ); + } ); + expect( result.message ).toBe( "PRIVATE" ); + expect( telemetry.$callLog( "counter" ).counter.len() ).toBe( 1 ); + } ); + it( "bounds configured adapter names and never observes invalid names", function(){ + var entries = []; + for ( var i = 1; i <= 40; i++ ) + entries.append( { + name : "worker_" & i, + adapter : function( context ){ + return { activeWorkers : 0 }; + } + } ); + var service = makeService( { + automaticCollectors : { + enabled : true, + intervalSeconds : 3600, + runtime : false, + requestSaturation : false, + queues : entries + } + } ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "gauge" ); + service.collectAutomaticMetrics(); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 32 ); + } ); + it( "times framework phases automatically and leaves header publication opt-in", function(){ + var service = makeService( { enableAutomaticRequestTimings : true } ); + for ( var i = 1; i <= 32; i++ ) service.getSettings().requestTimingNames.append( "manual_" & i ); + var telemetry = prepareMock( service.getObservability() ); + telemetry.$( "distribution" ); + var context = eventContext(); + var observer = createMock( "sentry.interceptors.Observability" ).$property( + "sentry", + "variables", + service + ); + observer.onRequestCapture( context, {} ); + observer.preEvent( context, {} ); + observer.postEvent( context, {} ); + observer.preRender( context, {} ); + observer.postRender( context, {} ); + expect( service.getRequestTimings().snapshot().handler.count ).toBe( 1 ); + expect( service.getRequestTimings().snapshot().render.count ).toBe( 1 ); + observer.postProcess( context, {} ); + expect( context.headers ).toBeEmpty(); + expect( telemetry.$callLog( "distribution" ).distribution.len() ).toBe( 2 ); + expect( telemetry.getScope() ).toBeEmpty(); + } ); + it( "publishes Server-Timing only under configured eligibility and closes exceptional phases once", function(){ + var service = makeService( { + enableAutomaticRequestTimings : true, + publishServerTiming : true, + serverTimingFilter : function( event ){ + return event.allowed; + } + } ); + var context = eventContext(); + context.allowed = true; + var observer = createMock( "sentry.interceptors.Observability" ).$property( + "sentry", + "variables", + service + ); + observer.onRequestCapture( context, {} ); + observer.preEvent( context, {} ); + observer.onException( context, {} ); + observer.postProcess( context, {} ); + expect( context.headers[ "Server-Timing" ] ).toInclude( "handler;dur=" ); + var blocked = eventContext(); + blocked.allowed = false; + observer.onRequestCapture( blocked, {} ); + observer.preEvent( blocked, {} ); + observer.postEvent( blocked, {} ); + observer.postProcess( blocked, {} ); + expect( blocked.headers ).toBeEmpty(); + service.getSettings().serverTimingFilter = "invalid"; + var invalid = eventContext(); + observer.onRequestCapture( invalid, {} ); + observer.preEvent( invalid, {} ); + observer.postEvent( invalid, {} ); + observer.postProcess( invalid, {} ); + expect( invalid.headers ).toBeEmpty(); + } ); + } ); + } + private function makeService( struct settings = {} ){ + var config = { + DSN : "http://public@127.0.0.1:60320/1", + enableMetrics : true, + enableLogBoxAppender : false, + dependencyAdapterGenerationPath : "/sentryAOP" + }; + config.append( arguments.settings, true ); + var service = new sentry.models.SentryService( config ); + variables.services.append( service ); + return service; + } + private function makeInjector( required any service ){ + variables.injector = new coldbox.system.ioc.Injector( + binder : "tests.resources.AutomaticBinder", + properties: { service : arguments.service } + ); + return variables.injector; + } + private function installModule( required any service, required any injector ){ + var config = createMock( "sentry.ModuleConfig" ) + .$property( "wirebox", "variables", arguments.injector ) + .$property( + "binder", + "variables", + arguments.injector.getBinder() + ) + .$property( + "settings", + "variables", + arguments.service.getSettings() + ) + .$property( "moduleMapping", "variables", "sentry" ); + config.onLoad(); + } + private function eventContext(){ + var values = {}; + var headers = {}; + var event = { + headers : headers, + getPrivateValue : function( name, defaultValue ){ + return values[ name ] ?: defaultValue; + }, + setPrivateValue : function( name, value ){ + values[ name ] = value; + }, + setHTTPHeader : function( name, value ){ + headers[ name ] = value; + } + }; + return event; + } + private function awaitCondition( required function callback ){ + var deadline = getTickCount() + 5000; + while ( getTickCount() < deadline ) { + if ( arguments.callback() ) return; + sleep( 25 ); + } + throw( type = "AutomaticTelemetryTestTimeout" ); + } + +} diff --git a/test-harness/tests/specs/DatabaseQueueHealthIntegrationSpec.cfc b/test-harness/tests/specs/DatabaseQueueHealthIntegrationSpec.cfc new file mode 100644 index 0000000..5692cc1 --- /dev/null +++ b/test-harness/tests/specs/DatabaseQueueHealthIntegrationSpec.cfc @@ -0,0 +1,554 @@ +/** Opt-in, loopback PostgreSQL + real cbq DB-provider/worker integration. No mocks at the database/worker boundary. */ +component extends="testbox.system.BaseSpec" { + + function run(){ + var database = createObject( "java", "java.lang.System" ).getenv( "SENTRY_QUEUE_HEALTH_DB" ); + if ( isNull( database ) || !len( database ) ) return; + describe( "Local database queue health and actual cbq workers", function(){ + beforeEach( function(){ + variables.wirebox = application.cbController.getWireBox(); + variables.config = variables.wirebox.getInstance( "Config@cbq" ); + variables.key = "qh_" & lCase( left( replace( createUUID(), "-", "", "all" ), 20 ) ); + variables.table = "cbq_" & variables.key; + variables.pool = javacast( "null", "" ); + variables.service = javacast( "null", "" ); + variables.taskName = ""; + variables.state = variables.wirebox.getInstance( "QueueHealthState" ); + variables.state.clear(); + variables.wirebox + .getInstance( "SchemaBuilder@qb" ) + .create( variables.table, function( t ){ + t.bigIncrements( "id" ); + t.string( "queue" ); + t.longText( "payload" ); + t.bigInteger( "attempts" ); + t.bigInteger( "availableDate" ); + t.bigInteger( "createdDate" ); + t.bigInteger( "reservedDate" ).nullable(); + t.string( "reservedBy" ).nullable(); + t.bigInteger( "completedDate" ).nullable(); + t.bigInteger( "failedDate" ).nullable(); + t.bigInteger( "lastReleasedDate" ).nullable(); + t.index( "queue" ); + } ); + variables.config.registerConnection( + variables.key, + "DBProvider@cbq", + { + datasource : "sentryQueueHealth", + tableName : variables.table, + pollIntervalMilliseconds : 50 + }, + "health" + ); + variables.adapter = new sentry.models.telemetry.DatabaseQueueHealth( function(){ + return variables.wirebox.getInstance( "QueryBuilder@qb" ); + }, variables.table ); + variables.service = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60320/1", + enableMetrics : true, + queueWorkerNames : [ "health" ] + } ); + variables.service.getObservability().setAdapterInjector( variables.wirebox ); + } ); + + afterEach( function(){ + var errors = cleanupFixture(); + if ( errors.len() ) + throw( type = "QueueHealthFixtureCleanupFailed", message = errors.toList( "; " ) ); + } ); + + it( "samples worker activity and database backlog through a single provider-owned queue adapter", function(){ + startPool(); + var job = publish( "block" ); + expect( + job.state.started.await( + javacast( "long", 5 ), + createObject( "java", "java.util.concurrent.TimeUnit" ).SECONDS + ) + ).toBeTrue(); + variables.service.shutdown(); + variables.service = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60320/1", + enableMetrics : true, + automaticCollectors : { + enabled : true, + intervalSeconds : 1, + runtime : false, + requestSaturation : false, + queues : [ + { + name : "health", + adapter : "QueueHealthAdapter@cbq", + options : { workerPool : variables.key } + } + ] + } + } ); + variables.service.getObservability().setAdapterInjector( variables.wirebox ); + var telemetry = prepareMock( variables.service.getObservability() ); + telemetry.$( "gauge" ); + awaitCondition( function(){ + return variables.service.automaticCollectorDiagnostics().runs > 0; + } ); + expect( variables.service.automaticCollectorDiagnostics().failures ).toBe( 0 ); + var calls = telemetry.$callLog( "gauge" ).gauge; + var active = -1; + var reserved = -1; + for ( var call in calls ) { + if ( call[ 1 ] == "queue.executions.active" ) active = call[ 2 ]; + if ( call[ 1 ] == "queue.backlog.reserved" ) reserved = call[ 2 ]; + } + expect( active ).toBe( 1 ); + expect( reserved ).toBe( 1 ); + expect( serializeJSON( calls ) ).toInclude( "queue.depth" ); + expect( serializeJSON( calls ) ).toInclude( "queue.oldest_age" ); + job.state.release.countDown(); + awaitCondition( function(){ + return hasTimestamp( row( job.id ), "completedDate" ); + } ); + } ); + + it( "loads qb's optional adapter through WireBox and cleans only its owned listeners", function(){ + variables.service.shutdown(); + variables.service = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60320/1", + tracesSampleRate : 1, + queryAdapter : "QueryTelemetryAdapter@qb", + databaseSystem : "postgresql" + } ); + var telemetry = variables.service.getObservability(); + telemetry.setAdapterInjector( variables.wirebox ); + var interceptors = application.cbController.getInterceptorService(); + var states = interceptors.getInterceptionStates(); + var before = states.keyExists( "preQBExecute" ) ? states.preQBExecute.getInterceptors().size() : 0; + var extra = {}; + try { + variables.wirebox.getInstance( "QueryTelemetryAdapter@qb" ); + expect( telemetry.configureQueryInstrumentation( getApplicationMetadata().name ) ).toBe( "custom" ); + expect( states.preQBExecute.getInterceptors().size() ).toBe( before + 1 ); + var root = variables.service.startTransaction( "qb adapter integration" ); + telemetry.setScope( { span : root, marker : "preserved" } ); + jobQuery().where( "queue", "health" ).get(); + expect( root.getChildren().len() ).toBe( 1 ); + expect( root.getChildren()[ 1 ].toPayload().data[ "db.system" ] ).toBe( "postgresql" ); + expect( root.getChildren()[ 1 ].toPayload().status ).toBe( "ok" ); + expect( function(){ + jobQuery().select( "sentry_health_missing_column" ).get(); + } ).toThrow(); + expect( root.getChildren().len() ).toBe( 2 ); + expect( root.getChildren()[ 2 ].toPayload().status ).toBe( "internal_error" ); + variables.service.withQuerySpan( + "SELECT 'PRIVATE_QUERY_LITERAL'", + function( span ){ + return jobQuery().get(); + }, + "postgresql" + ); + expect( root.getChildren().len() ).toBe( 3 ); + expect( root.getChildren()[ 3 ].toPayload().description ).notToInclude( "PRIVATE_QUERY_LITERAL" ); + expect( telemetry.getScope().span.getContext().span_id ).toBe( root.getContext().span_id ); + expect( telemetry.getScope().marker ).toBe( "preserved" ); + expect( telemetry.getScope().keyExists( "manualQuerySpan" ) ).toBeFalse(); + extra = variables.wirebox + .getInstance( "QueryTelemetryAdapter@qb" ) + .configure( { + telemetry : telemetry, + applicationName : getApplicationMetadata().name, + databaseSystem : "postgresql" + } ); + expect( states.preQBExecute.getInterceptors().size() ).toBe( before + 2 ); + var pending = { sql : "SELECT 1", pretend : false }; + interceptors.announce( "preQBExecute", pending ); + expect( root.getChildren().len() ).toBe( 3 ); + variables.service.shutdown(); + // Active execution keeps completion hooks until it restores the original scope. + expect( states.preQBExecute.getInterceptors().size() ).toBe( before + 2 ); + interceptors.announce( "postQBExecute", pending ); + expect( root.getChildren().len() ).toBe( 4 ); + expect( telemetry.getScope().span.getContext().span_id ).toBe( root.getContext().span_id ); + expect( telemetry.getScope().keyExists( "manualQuerySpan" ) ).toBeFalse(); + expect( states.preQBExecute.getInterceptors().size() ).toBe( before + 1 ); + extra.shutdown(); + extra.shutdown(); + expect( states.preQBExecute.getInterceptors().size() ).toBe( before ); + } finally { + if ( extra.keyExists( "shutdown" ) ) extra.shutdown(); + telemetry.clearScope(); + } + } ); + it( "samples connection backlog without registering a worker pool", function(){ + seed(); + seed( { queue : "other", availableDate : epoch() + 3600 } ); + expect( variables.config.getWorkerPools().keyExists( variables.key ) ).toBeFalse(); + var adapter = variables.wirebox.getInstance( "QueueHealthAdapter@cbq" ); + var snapshot = adapter.sample( { options : { connection : variables.key } } ); + expect( snapshot.depth ).toBe( 1 ); + expect( snapshot.runnable ).toBe( 1 ); + expect( snapshot.keyExists( "configuredWorkers" ) ).toBeFalse(); + expect( snapshot.keyExists( "activeExecutions" ) ).toBeFalse(); + var allQueues = adapter.sample( { options : { connection : variables.key, queue : "*" } } ); + expect( allQueues.depth ).toBe( 2 ); + expect( allQueues.delayed ).toBe( 1 ); + expect( variables.config.getWorkerPools().keyExists( variables.key ) ).toBeFalse(); + } ); + it( "uses provider-owned table and datasource settings for multiple queues and wildcard snapshots", function(){ + seed( { availableDate : epoch() } ); + seed( { queue : "other", availableDate : epoch() + 3600 } ); + seed( { queue : "mail' OR 1=1 --", availableDate : epoch() } ); + var provider = variables.config.getConnection( variables.key ).getProvider(); + var selected = provider.getQueueHealth( [ "health", "other" ] ); + expect( selected.depth ).toBe( 2 ); + expect( selected.runnable ).toBe( 1 ); + expect( selected.delayed ).toBe( 1 ); + expect( provider.getQueueHealth( "*" ).depth ).toBe( 3 ); + expect( provider.getQueueHealth( [ "*" ] ).depth ).toBe( 3 ); + expect( provider.getQueueHealth( "mail' OR 1=1 --" ).depth ).toBe( 1 ); + } ); + it( "cleans database and worker registries even when draining reports a failure", function(){ + startPool(); + var job = publish( "block" ); + expect( + job.state.started.await( + javacast( "long", 5 ), + createObject( "java", "java.util.concurrent.TimeUnit" ).SECONDS + ) + ).toBeTrue(); + var executorName = variables.pool.getExecutor().getName(); + var errors = cleanupFixture( function(){ + throw( type = "InjectedDrainFailure", message = "synthetic drain failure" ); + } ); + expect( errors.len() ).toBe( 1 ); + expect( errors[ 1 ] ).toInclude( "synthetic drain failure" ); + expect( variables.wirebox.getInstance( "SchemaBuilder@qb" ).hasTable( variables.table ) ).toBeFalse(); + expect( variables.config.getWorkerPools().keyExists( variables.key ) ).toBeFalse(); + expect( variables.config.getConnections().keyExists( variables.key ) ).toBeFalse(); + expect( application.cbController.getAsyncManager().hasExecutor( executorName ) ).toBeFalse(); + expect( isNull( variables.state.get( job.runId ) ) ).toBeTrue(); + variables.pool = javacast( "null", "" ); + variables.service = javacast( "null", "" ); + variables.taskName = ""; + } ); + + it( "executes an empty aggregate and exports measured zeros", function(){ + assertHealth( 0, 0, 0, 0 ); + expect( + variables.service.captureQueueWorkerHealth( "health", function(){ + return variables.adapter.snapshot( "health" ); + } ) + ).toBeTrue(); + } ); + it( "separates runnable, delayed, claimed and expired rows while excluding terminal and other queues", function(){ + var now = epoch(); + seed( { "availableDate" : now } ); + seed( { "availableDate" : now + 3600 } ); + seed( { + "reservedDate" : now, + "reservedBy" : "worker", + "availableDate" : now + 3600 + } ); + seed( { + "reservedDate" : now - 10, + "reservedBy" : "dead-worker", + "availableDate" : now - 1 + } ); + seed( { + "reservedBy" : "staged-worker", + "availableDate" : now - 1 + } ); + seed( { completedDate : now } ); + seed( { failedDate : now } ); + seed( { queue : "other" } ); + assertHealth( 1, 1, 3, 2 ); + var quoted = "health' OR 1=1 --"; + seed( { queue : quoted } ); + expect( variables.adapter.snapshot( quoted ).runnable ).toBe( 1 ); + assertHealth( 1, 1, 3, 2 ); + } ); + it( "observes real idle and busy worker activity and reserved jobs until completion", function(){ + startPool( 2 ); + var samples = []; + var telemetry = prepareMock( variables.service.getObservability() ); + telemetry + .$( "gauge" ) + .$callback( function( name, value ){ + samples.append( { name : name, value : value } ); + } ); + expect( variables.service.captureCBQWorkerHealth( "health", variables.pool ) ).toBeTrue(); + expect( metric( samples, "queue.executions.active" ) ).toBe( 0 ); + expect( metric( samples, "queue.workers.configured" ) ).toBe( 2 ); + var job = publish( "block" ); + expect( + job.state.started.await( + javacast( "long", 5 ), + createObject( "java", "java.util.concurrent.TimeUnit" ).SECONDS + ) + ).toBeTrue(); + assertHealth( 0, 0, 1, 0 ); + samples = []; + expect( variables.service.captureCBQWorkerHealth( "health", variables.pool ) ).toBeTrue(); + expect( metric( samples, "queue.executions.active" ) ).toBe( 1 ); + job.state.release.countDown(); + awaitCondition( function(){ + return hasTimestamp( row( job.id ), "completedDate" ); + } ); + awaitCondition( function(){ + return variables.pool.getExecutor().getActiveCount() == 0; + } ); + assertHealth( 0, 0, 0, 0 ); + expect( job.state.handled.get() ).toBe( 1 ); + } ); + it( "keeps delayed jobs out of runnable backlog until an actual worker executes them", function(){ + var job = publish( "ok", 2 ); + assertHealth( 0, 1, 0, 0 ); + startPool(); + awaitCondition( function(){ + return hasTimestamp( row( job.id ), "completedDate" ); + } ); + assertHealth( 0, 0, 0, 0 ); + expect( job.state.handled.get() ).toBe( 1 ); + } ); + it( "reports an expired execution lease and observes a real worker reclaim it", function(){ + var job = publish( "ok", 0, 3 ); + jobQuery() + .where( "id", job.id ) + .update( { + "reservedBy" : "dead-worker", + "reservedDate" : epoch() - 30, + "availableDate" : epoch() - 1, + "attempts" : 1 + } ); + assertHealth( 0, 0, 1, 1 ); + startPool(); + awaitCondition( function(){ + return hasTimestamp( row( job.id ), "completedDate" ); + } ); + assertHealth( 0, 0, 0, 0 ); + expect( row( job.id ).attempts ).toBe( 2 ); + expect( job.state.handled.get() ).toBe( 1 ); + } ); + it( "reports and reclaims a staged claim whose execution never started", function(){ + var job = publish( "ok" ); + jobQuery() + .where( "id", job.id ) + .update( { + "reservedBy" : "dead-worker", + "availableDate" : epoch() - 1 + } ); + assertHealth( 0, 0, 1, 1 ); + startPool(); + awaitCondition( function(){ + return hasTimestamp( row( job.id ), "completedDate" ); + } ); + assertHealth( 0, 0, 0, 0 ); + expect( row( job.id ).attempts ).toBe( 1 ); + expect( job.state.handled.get() ).toBe( 1 ); + } ); + it( "observes a real retry returning to delayed backlog before success", function(){ + var job = publish( "retry", 2, 2 ); + startPool(); + awaitCondition( function(){ + var record = row( job.id ); + return record.attempts == 1 && hasTimestamp( record, "lastReleasedDate" ) && variables.adapter.snapshot( "health" ).delayed == 1; + } ); + assertHealth( 0, 1, 0, 0 ); + awaitCondition( function(){ + return hasTimestamp( row( job.id ), "completedDate" ); + } ); + assertHealth( 0, 0, 0, 0 ); + expect( row( job.id ).attempts ).toBe( 2 ); + expect( job.state.handled.get() ).toBe( 2 ); + } ); + it( "excludes an actual exhausted job from pending and reserved counts", function(){ + var job = publish( "fail", 0, 1 ); + startPool(); + awaitCondition( function(){ + return hasTimestamp( row( job.id ), "failedDate" ); + } ); + assertHealth( 0, 0, 0, 0 ); + expect( job.state.handled.get() ).toBe( 1 ); + } ); + } ); + } + private function cleanupFixture( any drain = "" ){ + var drainCallback = arguments.drain; + var errors = { "messages" : [] }; + cleanupStep( errors, function(){ + if ( !isNull( variables.service ) ) variables.service.shutdown(); + } ); + // Stop polling before releasing blocked jobs. Preserve their state until all workers exit. + cleanupStep( errors, function(){ + var scheduler = application.cbController.getSchedulerService().getSchedulers()[ "cbScheduler@cbq" ]; + if ( len( variables.taskName ) && scheduler.hasTask( variables.taskName ) ) + scheduler.removeTask( variables.taskName ); + } ); + cleanupStep( errors, function(){ + variables.state.releaseAll(); + } ); + cleanupStep( errors, function(){ + if ( !isNull( variables.pool ) ) { + try { + if ( isCustomFunction( drainCallback ) ) { + drainCallback(); + } else { + awaitCondition( function(){ + return variables.pool.getExecutor().getActiveCount() == 0 && variables.pool + .getExecutor() + .getQueue() + .isEmpty(); + }, 5000 ); + } + } finally { + variables.pool.shutdown( true, 2 ); + if ( + !variables.pool + .getExecutor() + .getNative() + .awaitTermination( + javacast( "long", 2 ), + createObject( "java", "java.util.concurrent.TimeUnit" ).SECONDS + ) + ) + throw( type = "QueueHealthWorkerShutdownTimeout" ); + } + } + } ); + // Each step runs even if draining or another cleanup step fails. + cleanupStep( errors, function(){ + if ( !isNull( variables.pool ) ) { + var name = variables.pool.getExecutor().getName(); + var async = application.cbController.getAsyncManager(); + if ( async.hasExecutor( name ) ) async.deleteExecutor( name ); + } + } ); + cleanupStep( errors, function(){ + variables.state.clear(); + } ); + cleanupStep( errors, function(){ + variables.config.getWorkerPools().delete( variables.key ); + variables.config.getConnections().delete( variables.key ); + } ); + cleanupStep( errors, function(){ + variables.wirebox.getInstance( "SchemaBuilder@qb" ).dropIfExists( variables.table ); + } ); + cleanupStep( errors, function(){ + if ( !isNull( variables.service ) ) variables.service.shutdown(); + } ); + + return errors.messages; + } + + private function cleanupStep( required struct errors, required function callback ){ + try { + arguments.callback(); + } catch ( any failure ) { + arguments.errors.messages.append( failure.message ); + } + } + private function jobQuery(){ + return variables.wirebox.getInstance( "QueryBuilder@qb" ).from( variables.table ); + } + private function epoch(){ + return createObject( "java", "java.time.Instant" ).now().getEpochSecond(); + } + private function seed( struct changes = {} ){ + var values = { + "queue" : "health", + "payload" : "{}", + "attempts" : 0, + "availableDate" : epoch(), + "createdDate" : epoch() + }; + for ( + var column in [ + "queue", + "payload", + "attempts", + "availableDate", + "createdDate", + "reservedDate", + "reservedBy", + "completedDate", + "failedDate", + "lastReleasedDate" + ] + ) { + if ( arguments.changes.keyExists( column ) ) values[ column ] = arguments.changes[ column ]; + } + jobQuery().insert( values ); + } + private function row( required any id ){ + return jobQuery().where( "id", arguments.id ).first(); + } + private function publish( + required string mode, + numeric delay = 0, + numeric maxAttempts = 2 + ){ + var runId = createUUID(); + var state = variables.state.create( runId ); + variables.wirebox + .getInstance( "cbq@cbq" ) + .dispatch( + job : "QueueHealthJob", + properties : { runId : runId, mode : arguments.mode }, + connection : variables.key, + queue : "health", + backoff : arguments.delay, + timeout : 10, + maxAttempts: arguments.maxAttempts + ); + var record = jobQuery().orderBy( "id", "desc" ).first(); + return { id : record.id, state : state, runId : runId }; + } + private function startPool( numeric quantity = 1 ){ + variables.pool = variables.config.registerWorkerPool( + name : variables.key, + connectionName : variables.key, + quantity : arguments.quantity, + queue : "health", + timeout : 10, + shutdownTimeout: 2, + maxAttempts : 2 + ); + variables.taskName = "cbq:db-watcher:" & variables.pool.getUniqueId(); + application.cbController.getSchedulerService().getSchedulers()[ "cbScheduler@cbq" ].startupTask( + variables.taskName + ); + } + private function assertHealth( + required numeric runnable, + required numeric delayed, + required numeric reserved, + required numeric stale + ){ + var snapshot = variables.adapter.snapshot( "health" ); + expect( snapshot.runnable ).toBe( arguments.runnable ); + expect( snapshot.delayed ).toBe( arguments.delayed ); + expect( snapshot.reserved ).toBe( arguments.reserved ); + expect( snapshot.staleReservations ).toBe( arguments.stale ); + } + private function awaitCondition( required function condition, numeric timeout = 12000 ){ + var end = createObject( "java", "java.lang.System" ).nanoTime() + arguments.timeout * 1000000; + while ( createObject( "java", "java.lang.System" ).nanoTime() < end ) { + if ( arguments.condition() ) return; + sleep( 25 ); + } + throw( + type = "QueueHealthConditionTimeout", + message = "Local queue condition did not become true before timeout" + ); + } + private function hasTimestamp( required struct record, required string field ){ + return arguments.record.keyExists( arguments.field ) && !isNull( arguments.record[ arguments.field ] ) && isNumeric( + arguments.record[ arguments.field ] + ) && arguments.record[ arguments.field ] > 0; + } + private function metric( required array samples, required string name ){ + for ( var sample in arguments.samples ) { + if ( sample.name == arguments.name ) return sample.value; + } + return -1; + } + +} diff --git a/test-harness/tests/specs/DatabaseQueueMetricsSpec.cfc b/test-harness/tests/specs/DatabaseQueueMetricsSpec.cfc new file mode 100644 index 0000000..a1d68c4 --- /dev/null +++ b/test-harness/tests/specs/DatabaseQueueMetricsSpec.cfc @@ -0,0 +1,123 @@ +component extends="testbox.system.BaseSpec" { + + function run(){ + describe( "Optional cbq database metrics", function(){ + it( "emits configured idle queues, retains dynamic queues, and does not fabricate zero on failure", function(){ + var samples = {}; + var rows = []; + var failed = false; + var telemetry = { + captureQueueMetrics : function( name, depth, age ){ + samples[ name ] = { "depth" : depth, "age" : age }; + } + }; + var config = { + getWorkerPoolDefinitions : function(){ + return { + "worker" : { + getQueue : function(){ + return [ "mail", "pdf" ]; + } + } + }; + } + }; + var sampler = new sentry.models.telemetry.DatabaseQueueMetrics( + telemetry, + config, + function(){ + return new tests.resources.QueueMetricsQuery( rows, failed ); + } + ); + expect( sampler.capture() ).toBeTrue(); + expect( samples.mail.depth ).toBe( 0 ); + expect( samples.pdf.age ).toBe( 0 ); + rows = [ + { + "queue" : "dynamic", + "depth" : 2, + "oldest" : createObject( "java", "java.lang.System" ).currentTimeMillis() / 1000 - 120 + } + ]; + expect( sampler.capture() ).toBeTrue(); + expect( samples.dynamic.depth ).toBe( 2 ); + expect( samples.dynamic.age ).toBeGTE( 120 ); + var lastSuccess = sampler.getDiagnostics().lastSuccessAt; + failed = true; + samples = {}; + expect( sampler.capture() ).toBeFalse(); + expect( samples ).toBeEmpty(); + expect( sampler.getDiagnostics().lastSuccessAt ).toBe( lastSuccess ); + expect( sampler.getDiagnostics().failures ).toBe( 1 ); + failed = false; + rows = []; + expect( sampler.capture() ).toBeTrue(); + expect( samples.dynamic.depth ).toBe( 0 ); + } ); + it( "bounds remembered queues and lifetime label cardinality", function(){ + var count = 0; + var queue = "first"; + var sampler = new sentry.models.telemetry.DatabaseQueueMetrics( + { + captureQueueMetrics : function(){ + count++; + } + }, + { + getWorkerPoolDefinitions : function(){ + return {}; + } + }, + function(){ + return new tests.resources.QueueMetricsQuery( [ { "queue" : queue, "depth" : 1, "oldest" : 0 } ] ); + }, + "cbq_jobs", + 1 + ); + expect( sampler.capture() ).toBeTrue(); + queue = "another"; + expect( sampler.capture() ).toBeTrue(); + expect( count ).toBe( 2 ); + expect( sampler.getDiagnostics().omittedQueues ).toBe( 1 ); + } ); + it( + title = "compiles the installed qb public API with dialect-aware identifiers", + body = function(){ + for ( var dialect in [ "Postgres", "MySQL", "SqlServer" ] ) { + var grammar = createObject( "component", "qb.models.Grammars." & dialect & "Grammar" ).init(); + var builder = prepareMock( + new qb.models.Query.QueryBuilder( grammar = grammar, paginationCollector = {} ) + ); + builder.$( "get", [] ); + var sampler = new sentry.models.telemetry.DatabaseQueueMetrics( + { + captureQueueMetrics : function(){ + } + }, + { + getWorkerPoolDefinitions : function(){ + return {}; + } + }, + function(){ + return builder; + } + ); + expect( sampler.capture() ).toBeTrue(); + var sql = builder.toSQL(); + expect( sql ).toInclude( "min(" & grammar.wrapValue( "createdDate" ) & ") AS oldest" ); + for ( var field in [ "completedDate", "failedDate", "reservedDate" ] ) { + expect( sql ).toInclude( grammar.wrapValue( field ) & " IS NULL" ); + } + expect( sql ).toInclude( "GROUP BY" ); + } + }, + skip = function(){ + var source = createObject( "java", "java.lang.System" ).getenv( "SENTRY_QB_SOURCE" ); + return isNull( source ) || !len( source ); + } + ); + } ); + } + +} diff --git a/test-harness/tests/specs/DeliveryDiagnosticsSpec.cfc b/test-harness/tests/specs/DeliveryDiagnosticsSpec.cfc new file mode 100644 index 0000000..399845b --- /dev/null +++ b/test-harness/tests/specs/DeliveryDiagnosticsSpec.cfc @@ -0,0 +1,92 @@ +component extends="testbox.system.BaseSpec" { + + function run(){ + describe( "Envelope delivery freshness", function(){ + afterEach( function(){ + cfhttp( url = "http://127.0.0.1:60320/reset", method = "post" ) { + cfhttpparam( type = "body", value = "{}" ); + } + } ); + it( "keeps receipt collection opt-in", function(){ + var service = new sentry.models.SentryService( { DSN : "http://public@127.0.0.1:60320/1" } ); + try { + service.captureFeedback( message = "test", wait = true ); + expect( service.deliveryDiagnostics().enabled ).toBeFalse(); + expect( service.deliveryDiagnostics().outcomes ).toBeEmpty(); + } finally { + service.shutdown(); + } + } ); + it( "polls asynchronous acceptance once and exposes content-free freshness", function(){ + var service = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60320/1", + enableDeliveryDiagnostics : true + } ); + try { + expect( service.deliveryDiagnostics().lastAcceptedAt ).toBe( 0 ); + var receipt = service.captureFeedback( message = "PRIVATE" ); + receipt.awaitDelivery(); + var diagnostics = service.deliveryDiagnostics(); + expect( diagnostics.outcomes.accepted ).toBe( 1 ); + expect( diagnostics.pending ).toBe( 0 ); + expect( diagnostics.lastAcceptedAt ).toBeGT( 0 ); + expect( service.deliveryDiagnostics().outcomes.accepted ).toBe( 1 ); + expect( serializeJSON( diagnostics ) ).notToInclude( "PRIVATE" ); + expect( serializeJSON( diagnostics ) ).notToInclude( receipt.getEventId() ); + } finally { + service.shutdown(); + } + } ); + it( "records rejected, failed, disabled and filtered results separately", function(){ + cfhttp( url = "http://127.0.0.1:60320/reset", method = "post" ) { + cfhttpparam( type = "body", value = "{""status"":429}" ); + } + var service = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60320/1", + enableDeliveryDiagnostics : true + } ); + try { + service.captureFeedback( message = "test", wait = true ); + service.captureFeedback( message = "test", wait = true ); + var d = service.deliveryDiagnostics(); + expect( d.outcomes.rejected ).toBe( 1 ); + expect( d.outcomes.dropped ).toBe( 1 ); + expect( d.lastAcceptedAt ).toBe( 0 ); + } finally { + service.shutdown(); + } + var offline = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60360/1", + enableDeliveryDiagnostics : true, + transportTimeout : 100 + } ); + try { + offline.captureFeedback( message = "test", wait = true ); + offline.flush(); + expect( offline.deliveryDiagnostics().outcomes.failed ).toBe( 1 ); + } finally { + offline.shutdown(); + } + var diagnostics = new sentry.models.telemetry.DeliveryDiagnostics( 1 ); + diagnostics.record( new sentry.models.telemetry.Receipt( "test", "disabled" ) ); + diagnostics.record( new sentry.models.telemetry.Receipt( "test", "filtered" ) ); + expect( diagnostics.snapshot().outcomes.disabled ).toBe( 1 ); + expect( diagnostics.snapshot().outcomes.filtered ).toBe( 1 ); + } ); + it( "bounds receipt retention and makes incomplete diagnostic coverage explicit", function(){ + var diagnostics = new sentry.models.telemetry.DeliveryDiagnostics( 1 ); + var pending = { + getStatus : function(){ + return "queued"; + } + }; + diagnostics.record( pending ); + diagnostics.record( pending ); + expect( diagnostics.snapshot().pending ).toBe( 1 ); + expect( diagnostics.snapshot().untracked ).toBe( 1 ); + expect( diagnostics.snapshot().lastAcceptedAt ).toBe( 0 ); + } ); + } ); + } + +} diff --git a/test-harness/tests/specs/OperationalReviewSpec.cfc b/test-harness/tests/specs/OperationalReviewSpec.cfc new file mode 100644 index 0000000..cb9fdcc --- /dev/null +++ b/test-harness/tests/specs/OperationalReviewSpec.cfc @@ -0,0 +1,205 @@ +component extends="testbox.system.BaseSpec" { + + function run(){ + describe( "Operational review regressions", function(){ + it( "keeps one canonical phase when configured names differ only in case", function(){ + var timing = new sentry.models.telemetry.RequestTimings( {}, [ "render", "Render" ] ); + timing.measure( "render", function(){ + return 1; + } ); + timing.measure( "Render", function(){ + return 2; + } ); + expect( timing.snapshot().render.count ).toBe( 2 ); + expect( listLen( timing.toServerTiming(), "," ) ).toBe( 1 ); + } ); + it( "rejects trailing LF and CRLF in configured HTTP timing names", function(){ + var lf = "render" & chr( 10 ); + var crlf = "render" & chr( 13 ) & chr( 10 ); + var timing = new sentry.models.telemetry.RequestTimings( {}, [ lf, crlf ] ); + expect( timing.begin( lf ) ).toBeEmpty(); + expect( timing.begin( crlf ) ).toBeEmpty(); + expect( timing.toServerTiming() ).toBe( "" ); + } ); + it( "owns timing tokens privately and ignores mutable metadata and foreign handles", function(){ + var finishes = 0; + var observer = { + startSpan : function(){ + return { + finish : function(){ + finishes++; + } + }; + } + }; + var timing = new sentry.models.telemetry.RequestTimings( observer, [ "handler" ] ); + var other = new sentry.models.telemetry.RequestTimings( observer, [ "handler" ] ); + var token = timing.begin( "handler" ); + other.finish( token ); + expect( finishes ).toBe( 0 ); + token.name = "PRIVATE"; + token.started = 0; + token.span = {}; + token.done = "malformed"; + timing.finish( token ); + timing.finish( token ); + expect( finishes ).toBe( 1 ); + expect( timing.snapshot().handler.count ).toBe( 1 ); + expect( serializeJSON( timing.snapshot() ) ).notToInclude( "PRIVATE" ); + expect( other.snapshot() ).toBeEmpty(); + timing.finish( { "done" : "malformed" } ); + } ); + it( "attempts workflow span completion even if restoration or annotation fails", function(){ + for ( var mode in [ "restore", "attribute" ] ) { + var finishes = 0; + var scopes = 0; + var telemetry = { + getScope : function(){ + return {}; + }, + setScope : function(){ + scopes++; + if ( mode == "restore" && scopes > 1 ) { + throw( type = "ObserverFailure" ); + } + }, + startSpan : function(){ + return { + setAttribute : function(){ + if ( mode == "attribute" ) { + throw( type = "ObserverFailure" ); + } + }, + finish : function(){ + finishes++; + } + }; + }, + counter : function(){ + }, + distribution : function(){ + } + }; + var operation = new sentry.models.telemetry.Operation( telemetry ); + expect( + operation.run( "workflow", function(){ + return 42; + } ) + ).toBe( 42 ); + expect( finishes ).toBe( 1 ); + expect( function(){ + operation.run( "workflow", function(){ + throw( type = "OriginalFailure" ); + } ); + } ).toThrow( "OriginalFailure" ); + expect( finishes ).toBe( 2 ); + } + } ); + it( "preserves distinct queue destinations differing only in case and reports both drained", function(){ + var samples = []; + var rows = [ + { "queue" : "mail", "depth" : 2, "oldest" : 1 }, + { "queue" : "MAIL", "depth" : 5, "oldest" : 1 } + ]; + var sampler = new sentry.models.telemetry.DatabaseQueueMetrics( + { + captureQueueMetrics : function( name, depth, age ){ + samples.append( { "name" : name, "depth" : depth } ); + } + }, + { + getWorkerPoolDefinitions : function(){ + return { + "worker" : { + getQueue : function(){ + return [ "mail", "MAIL" ]; + } + } + }; + } + }, + function(){ + return new tests.resources.QueueMetricsQuery( rows ); + } + ); + expect( sampler.capture() ).toBeTrue(); + expect( samples.len() ).toBe( 2 ); + for ( var sample in samples ) { + expect( sample.depth ).toBe( compare( sample.name, "mail" ) == 0 ? 2 : 5 ); + } + rows = []; + samples = []; + expect( sampler.capture() ).toBeTrue(); + expect( samples.len() ).toBe( 2 ); + for ( var sample in samples ) { + expect( sample.depth ).toBe( 0 ); + } + } ); + it( "excludes transaction export time from queue processing duration", function(){ + var service = prepareMock( + new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60320/1", + tracesSampleRate : 1, + enableQueueTracing : true + } ) + ); + try { + var clock = 10; + var telemetry = prepareMock( service.getObservability() ); + telemetry + .$( "timestamp" ) + .$callback( function(){ + return clock; + } ); + telemetry + .$( "completeTransaction" ) + .$callback( function(){ + clock += 5; + } ); + service.$( "counter" ); + service.$( "distribution" ); + var observer = createMock( "sentry.interceptors.QueueObservability" ).$property( + "sentry", + "variables", + service + ); + var job = { + getProperties : function(){ + return {}; + }, + getMapping : function(){ + return "TestJob"; + }, + getQueue : function(){ + return "mail"; + }, + getId : function(){ + return "id"; + }, + isBatchJob : function(){ + return false; + } + }; + var data = { + "job" : job, + "attempt" : 1, + "executionId" : "duration-test", + "status" : "ok" + }; + observer.onCBQJobAttemptScheduled( {}, data ); + observer.onCBQJobExecutionStarted( {}, data ); + clock = 11; + observer.onCBQJobExecutionExited( {}, data ); + observer.onCBQJobAttemptFinished( {}, data ); + var calls = service.$callLog( "distribution" ).distribution; + expect( calls.len() ).toBe( 2 ); + expect( calls[ 2 ][ 1 ] ).toBe( "queue.processing.duration" ); + expect( calls[ 2 ][ 2 ] ).toBe( 1000 ); + } finally { + service.shutdown(); + } + } ); + } ); + } + +} diff --git a/test-harness/tests/specs/RequestCompletionSpec.cfc b/test-harness/tests/specs/RequestCompletionSpec.cfc new file mode 100644 index 0000000..fc8ed28 --- /dev/null +++ b/test-harness/tests/specs/RequestCompletionSpec.cfc @@ -0,0 +1,222 @@ +component extends="testbox.system.BaseSpec" { + + function run(){ + describe( "request completion ownership", function(){ + beforeEach( function(){ + variables.service = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:1/1", + tracesSampleRate : 1, + enableRequestTracing : true, + enableAutomaticRequestTimings : true, + beforeSendSignal : function(){ + return javacast( "null", "" ); + } + } ); + variables.telemetry = variables.service.getObservability(); + variables.observer = createMock( "sentry.interceptors.Observability" ).$property( + "sentry", + "variables", + variables.service + ); + } ); + afterEach( function(){ + variables.service.shutdown(); + } ); + it( "drains binary render and manual handles exactly once at abort completion", function(){ + var event = context(); + variables.observer.onRequestCapture( event, {} ); + var state = event.getPrivateValue( "_sentryRequest", {} ); + var manual = state.timings.begin( "manual" ); + variables.observer.preEvent( event, {} ); + variables.observer.postEvent( event, {} ); + variables.observer.preRender( event, {} ); + variables.service.endRequest(); + var payload = state.span.toPayload(); + expect( state.span.getChildren().len() ).toBe( 3 ); + expect( state.timings.snapshot().render.count ).toBe( 1 ); + expect( state.timings.snapshot().manual.count ).toBe( 1 ); + expect( state.timings.begin( "after_response" ) ).toBeEmpty(); + state.timings.finish( manual ); + variables.observer.postRender( event, {} ); + variables.observer.postProcess( event, {} ); + variables.service.endRequest(); + expect( state.span.toPayload().timestamp ).toBe( payload.timestamp ); + expect( state.span.getChildren().len() ).toBe( 3 ); + expect( variables.telemetry.getScope() ).toBeEmpty(); + } ); + it( "does not close a new request from an old event callback on a reused worker", function(){ + var first = context(); + variables.observer.onRequestCapture( first, {} ); + var firstState = first.getPrivateValue( "_sentryRequest", {} ); + variables.service.endRequest(); + var second = context(); + variables.observer.onRequestCapture( second, {} ); + var secondState = second.getPrivateValue( "_sentryRequest", {} ); + variables.observer.postProcess( first, {} ); + expect( secondState.span.isFinished() ).toBeFalse(); + expect( variables.telemetry.getScope().requestState.id ).toBe( secondState.id ); + variables.service.endRequest(); + expect( secondState.span.isFinished() ).toBeTrue(); + expect( variables.telemetry.getScope() ).toBeEmpty(); + } ); + it( "isolates successive request contexts even if a worker missed all cleanup", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "executionOwner", "request-one" ); + telemetry.setScope( { marker : "old request" } ); + telemetry.$( "executionOwner", "request-two" ); + expect( telemetry.getScope() ).toBeEmpty(); + expect( telemetry.getScopeResetCount() ).toBe( 1 ); + telemetry.setScope( { marker : "new request" } ); + expect( telemetry.getScope().marker ).toBe( "new request" ); + } ); + it( "restores a same-request scope after completion", function(){ + variables.telemetry.setScope( { marker : "outer" } ); + var event = context(); + variables.observer.onRequestCapture( event, {} ); + variables.service.endRequest( statusCode = 303 ); + expect( variables.telemetry.getScope().marker ).toBe( "outer" ); + expect( + event.getPrivateValue( "_sentryRequest", {} ).span.toPayload().data[ "http.response.status_code" ] + ).toBe( 303 ); + } ); + it( "uses elapsed monotonic time despite a wall-clock jump and preserves epoch seconds", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "timestamp", 1700000000 ); + telemetry.$( "monotonicTime", 1000000000 ); + var span = telemetry.startTransaction( "clock" ); + telemetry.$( "timestamp", 1700027000 ); + telemetry.$( "monotonicTime", 2500000000 ); + span.finish(); + expect( span.toPayload().start_timestamp ).toBe( 1700000000 ); + expect( span.toPayload().timestamp ).toBe( 1700000001.5 ); + expect( span.getDurationMilliseconds() ).toBe( 1500 ); + expect( span.finish() ).toBeFalse(); + telemetry.$( "monotonicTime", 10000000000 ); + var backwards = telemetry.startTransaction( "backwards clock" ); + telemetry.$( "timestamp", 1700000000 ); + telemetry.$( "monotonicTime", 12000000000 ); + backwards.finish(); + expect( backwards.toPayload().timestamp ).toBe( 1700027002 ); + } ); + it( "resets the monotonic origin when queued work actually starts", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "monotonicTime", 1000000000 ); + var span = telemetry.startTransaction( "queued" ); + telemetry.$( "monotonicTime", 9000000000 ); + span.markStarted( 1700000008 ); + telemetry.$( "monotonicTime", 9250000000 ); + span.finish(); + expect( span.toPayload().timestamp ).toBe( 1700000008.25 ); + } ); + it( "preserves callback results and original exceptions with unavailable telemetry", function(){ + var timings = new sentry.models.telemetry.RequestTimings( + { + startSpan : function(){ + throw( type = "SentryFixture.Unavailable" ); + }, + distribution : function(){ + throw( type = "SentryFixture.Unavailable" ); + } + }, + [], + 64, + true + ); + expect( + timings.measure( "result", function(){ + return false; + } ) + ).toBeFalse(); + try { + timings.measure( "failure", function(){ + throw( type = "SentryFixture.Original", message = "preserve me" ); + } ); + fail( "must throw" ); + } catch ( SentryFixture.Original original ) { + expect( original.message ).toBe( "preserve me" ); + } + expect( timings.snapshot().failure.count ).toBe( 1 ); + } ); + it( "keeps concurrent request spans and restored scopes independent", function(){ + var reference = createObject( "java", "java.util.concurrent.atomic.AtomicReference" ).init( + variables.service + ); + var results = createObject( "java", "java.util.concurrent.ConcurrentHashMap" ).init(); + var gate = createObject( "java", "java.util.concurrent.CyclicBarrier" ).init( 2 ); + var names = []; + for ( var i = 1; i <= 2; i++ ) { + var name = "request-isolation-" & replace( createUUID(), "-", "", "all" ); + names.append( name ); + thread name=name action="run" service=reference results=results gate=gate marker=i { + var telemetry = attributes.service.get().getObservability(); + for ( var iteration = 1; iteration <= 2; iteration++ ) { + telemetry.setScope( { marker : attributes.marker } ); + var root = telemetry.startTransaction( "concurrent request" ); + var state = { + span : root, + previous : { marker : attributes.marker }, + done : createObject( "java", "java.util.concurrent.atomic.AtomicBoolean" ).init( + false + ) + }; + telemetry.setScope( { span : root, requestState : state } ); + attributes.gate.await( + javacast( "long", 5 ), + createObject( "java", "java.util.concurrent.TimeUnit" ).SECONDS + ); + var child = telemetry.startSpan( "owned" ); + child.finish(); + attributes.service.get().endRequest(); + attributes.results.put( + attributes.marker & "-" & iteration, + { + marker : telemetry.getScope().marker, + trace : root.getContext().trace_id, + childTrace : child.getContext().trace_id, + finished : root.isFinished(), + count : root.getChildren().len() + } + ); + telemetry.clearScope(); + } + } + } + thread action="join" name=arrayToList( names ) timeout=15000; + for ( var name in names ) expect( cfthread[ name ].status ).toBe( "COMPLETED" ); + expect( results.size() ).toBe( 4 ); + var traces = {}; + for ( var key in results.keySet() ) { + var result = results.get( key ); + expect( result.marker ).toBe( val( key ) ); + expect( result.finished ).toBeTrue(); + expect( result.count ).toBe( 1 ); + expect( result.childTrace ).toBe( result.trace ); + traces[ result.trace ] = true; + } + expect( traces.count() ).toBe( 4 ); + expect( variables.telemetry.getScope() ).toBeEmpty(); + } ); + } ); + } + private function context(){ + var state = {}; + return { + getPrivateValue : function( name, fallback ){ + return state[ arguments.name ] ?: arguments.fallback; + }, + setPrivateValue : function( name, value ){ + state[ arguments.name ] = arguments.value; + }, + getCurrentRoute : function(){ + return "binary/:id"; + }, + getCurrentEvent : function(){ + return "Responses.binary"; + }, + getHTTPMethod : function(){ + return "GET"; + } + }; + } + +} diff --git a/test-harness/tests/specs/ReusableObservabilitySpec.cfc b/test-harness/tests/specs/ReusableObservabilitySpec.cfc new file mode 100644 index 0000000..6114f8b --- /dev/null +++ b/test-harness/tests/specs/ReusableObservabilitySpec.cfc @@ -0,0 +1,382 @@ +component extends="testbox.system.BaseSpec" { + + function run(){ + describe( "Reusable observations", function(){ + beforeEach( function(){ + variables.service = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60320/1", + tracesSampleRate : 1, + enableMetrics : true, + enableQueueTracing : true, + requestTimingNames : [ "handler", "render", "bad,header" ], + operationNames : [ "invitation" ], + providerNames : [ "mail" ], + databasePoolNames : [ "primary" ] + } ); + variables.telemetry = variables.service.getObservability(); + } ); + afterEach( function(){ + variables.telemetry.clearScope(); + variables.service.shutdown(); + } ); + it( "bounds request phases, preserves callbacks and completes tokens once", function(){ + var root = variables.service.startTransaction( "request" ); + variables.telemetry.setScope( { "span" : root } ); + var timing = variables.service.newRequestTimings(); + expect( timing.begin( "bad,header" ) ).toBeEmpty(); + var token = timing.begin( "handler" ); + timing.finish( token ); + timing.finish( token, "internal_error" ); + expect( timing.snapshot().handler.count ).toBe( 1 ); + expect( + timing.measure( "render", function(){ + return 42; + } ) + ).toBe( 42 ); + expect( function(){ + timing.measure( "render", function(){ + throw( type = "OriginalFailure" ); + } ); + } ).toThrow( "OriginalFailure" ); + expect( root.getChildren().len() ).toBe( 3 ); + expect( root.getChildren()[ 3 ].toPayload().status ).toBe( "internal_error" ); + expect( timing.toServerTiming() ).toInclude( "handler;dur=" ); + expect( timing.toServerTiming() ).notToInclude( "bad" ); + expect( variables.service.newRequestTimings().snapshot() ).toBeEmpty(); + for ( var i = 1; i <= 64; i++ ) { + timing.finish( timing.begin( "handler" ) ); + } + expect( timing.snapshot().handler.count ).toBe( 62 ); + expect( timing.begin( "handler" ) ).toBeEmpty(); + root.finish(); + } ); + it( "measures ad hoc phases without configuration and preserves callback outcomes", function(){ + var timing = new sentry.models.telemetry.RequestTimings( {} ); + var calls = 0; + expect( + timing.measure( "loadDashboard", function(){ + calls++; + return false; + } ) + ).toBeFalse(); + expect( function(){ + timing.measure( "loadDashboard", function(){ + calls++; + throw( type = "DashboardFailure" ); + } ); + } ).toThrow( "DashboardFailure" ); + var token = timing.begin( "fetchData" ); + timing.finish( token ); + timing.finish( token ); + expect( calls ).toBe( 2 ); + expect( timing.snapshot().loadDashboard.count ).toBe( 2 ); + expect( timing.snapshot().fetchData.count ).toBe( 1 ); + expect( timing.toServerTiming() ).toInclude( "loadDashboard;dur=" ); + } ); + it( "bounds ad hoc names and samples while still running unobserved callbacks", function(){ + var timing = new sentry.models.telemetry.RequestTimings( {} ); + for ( var i = 1; i <= 32; i++ ) timing.measure( "phase" & i, function(){ + return 1; + } ); + expect( + timing.measure( "overflow", function(){ + return 42; + } ) + ).toBe( 42 ); + expect( structCount( timing.snapshot() ) ).toBe( 32 ); + expect( timing.snapshot() ).notToHaveKey( "overflow" ); + for ( var i = 1; i <= 32; i++ ) timing.finish( timing.begin( "phase1" ) ); + expect( timing.begin( "phase1" ) ).toBeEmpty(); + expect( timing.snapshot().phase1.count ).toBe( 33 ); + for ( + var invalid in [ + "", + "bad,header", + "bad" & chr( 10 ), + repeatString( "x", 49 ) + ] + ) { + var fresh = new sentry.models.telemetry.RequestTimings( {} ); + expect( + fresh.measure( invalid, function(){ + return 42; + } ) + ).toBe( 42 ); + expect( fresh.snapshot() ).toBeEmpty(); + expect( fresh.toServerTiming() ).toBeEmpty(); + } + } ); + it( "retains timings and original exceptions when tracing is unavailable", function(){ + var timing = new sentry.models.telemetry.RequestTimings( + { + startSpan : function(){ + throw( type = "TelemetryUnavailable" ); + } + }, + [ "handler" ] + ); + expect( + timing.measure( "handler", function(){ + return false; + } ) + ).toBeFalse(); + expect( function(){ + timing.measure( "handler", function(){ + throw( type = "OriginalFailure" ); + } ); + } ).toThrow( "OriginalFailure" ); + expect( timing.snapshot().handler.count ).toBe( 2 ); + } ); + it( "records registered workflow outcomes without collecting results or expanding capture", function(){ + var root = variables.service.startTransaction( "request" ); + variables.telemetry.setScope( { "span" : root } ); + var calls = 0; + var result = variables.service.withOperation( + "invitation", + function(){ + calls++; + return { "private" : "PRIVATE" }; + }, + "mail", + function(){ + return "accepted"; + } + ); + expect( calls ).toBe( 1 ); + expect( result.private ).toBe( "PRIVATE" ); + expect( root.getChildren()[ 1 ].toPayload().data[ "workflow.outcome" ] ).toBe( "accepted" ); + expect( serializeJSON( root.getChildren()[ 1 ].toPayload() ) ).notToInclude( "PRIVATE" ); + expect( variables.telemetry.getScope().span.getContext().span_id ).toBe( + root.getContext().span_id + ); + variables.service.withOperation( "unregistered", function(){ + calls++; + } ); + variables.service.withOperation( + "invitation", + function(){ + calls++; + }, + "unregistered" + ); + expect( calls ).toBe( 3 ); + expect( root.getChildren().len() ).toBe( 1 ); + variables.service.withOperation( + "invitation", + function(){ + return 1; + }, + "mail", + function(){ + return "PRIVATE"; + } + ); + expect( root.getChildren()[ 2 ].toPayload().data[ "workflow.outcome" ] ).toBe( "unknown" ); + expect( function(){ + variables.service.withOperation( "invitation", function(){ + throw( type = "OriginalFailure" ); + } ); + } ).toThrow( "OriginalFailure" ); + expect( root.getChildren()[ 3 ].toPayload().status ).toBe( "internal_error" ); + root.finish(); + } ); + it( "never retries application callbacks when observation fails", function(){ + var observer = new sentry.models.telemetry.Operation( { + getScope : function(){ + throw( type = "TelemetryUnavailable" ); + }, + counter : function(){ + throw( type = "TelemetryUnavailable" ); + } + } ); + var calls = 0; + expect( + observer.run( "test", function(){ + calls++; + return 0; + } ) + ).toBe( 0 ); + expect( calls ).toBe( 1 ); + observer.run( "test", function(){ + calls++; + } ); + expect( calls ).toBe( 2 ); + expect( function(){ + observer.run( "test", function(){ + throw( type = "OriginalFailure" ); + } ); + } ).toThrow( "OriginalFailure" ); + } ); + it( "keeps terminal queue outcomes separate and avoids fictional processing samples", function(){ + var service = prepareMock( variables.service ); + service.$( "counter" ); + service.$( "distribution" ); + service.getSettings().enableQueueJobMetrics = true; + service.getSettings().queueMetricDestinations = [ "mail" ]; + var observer = createMock( "sentry.interceptors.QueueObservability" ).$property( + "sentry", + "variables", + service + ); + var queue = "mail"; + var job = { + getProperties : function(){ + return {}; + }, + getMapping : function(){ + return "TestJob"; + }, + getQueue : function(){ + return queue; + }, + getId : function(){ + return "PRIVATE"; + }, + isBatchJob : function(){ + return false; + } + }; + var data = { + "job" : job, + "attempt" : 2, + "executionId" : "unstarted", + "status" : "cancelled" + }; + observer.onCBQJobAttemptScheduled( {}, data ); + observer.onCBQJobAttemptFinished( {}, data ); + observer.onCBQJobAttemptFinished( {}, data ); + expect( service.$callLog( "counter" ).counter.len() ).toBe( 1 ); + expect( service.$callLog( "distribution" ).distribution.len() ).toBe( 0 ); + observer.onCBQJobFailed( {}, data ); + queue = "PRIVATE"; + observer.onCBQJobComplete( {}, data ); + var calls = service.$callLog( "counter" ).counter; + expect( calls[ 2 ][ 1 ] ).toBe( "queue.jobs" ); + expect( calls[ 2 ][ 3 ].outcome ).toBe( "failed" ); + expect( calls[ 2 ][ 3 ][ "messaging.destination.name" ] ).toBe( "mail" ); + expect( calls[ 3 ][ 3 ].outcome ).toBe( "completed" ); + expect( serializeJSON( calls ) ).notToInclude( "PRIVATE" ); + service.getSettings().enableQueueJobMetrics = false; + observer.onCBQJobFailed( {}, data ); + expect( service.$callLog( "counter" ).counter.len() ).toBe( 3 ); + } ); + it( "samples JVM gauges and omits unsupported CPU/GC instead of substituting zero", function(){ + var samples = variables.service.captureRuntimeGauges(); + expect( samples[ "jvm.heap.used" ].value ).toBeGT( 0 ); + expect( samples[ "jvm.threads" ].value ).toBeGT( 0 ); + var sampler = prepareMock( new sentry.models.telemetry.RuntimeMetrics( variables.telemetry ) ); + expect( sampler.readAttribute( "java.lang:type=OperatingSystem", "AvailableProcessors" ) ).toBeGT( + 0 + ); + sampler + .$( "readAttribute" ) + .$callback( function( bean, attribute ){ + return attribute == "ProcessCpuLoad" ? 0.25 : 2; + } ); + var supported = sampler.capture(); + expect( supported[ "jvm.cpu.process" ].value ).toBe( 0.25 ); + if ( + createObject( "java", "java.lang.management.ManagementFactory" ) + .getGarbageCollectorMXBeans() + .size() > 0 + ) { + expect( supported[ "jvm.gc.collections.total" ].value ).toBeGT( 0 ); + expect( supported[ "jvm.gc.duration.total" ].value ).toBeGT( 0 ); + } + sampler.$( "readAttribute", -1 ); + var unsupported = sampler.capture(); + expect( unsupported ).notToHaveKey( "jvm.cpu.process" ); + expect( unsupported ).notToHaveKey( "jvm.gc.duration.total" ); + expect( unsupported ).toHaveKey( "jvm.heap.used" ); + } ); + it( "closes queue roots even if attempt metrics fail", function(){ + var service = prepareMock( variables.service ); + var observer = createMock( "sentry.interceptors.QueueObservability" ).$property( + "sentry", + "variables", + service + ); + var job = { + getProperties : function(){ + return {}; + }, + getMapping : function(){ + return "TestJob"; + }, + getQueue : function(){ + return "test"; + }, + getId : function(){ + return "test-id"; + }, + isBatchJob : function(){ + return false; + } + }; + var data = { + "job" : job, + "attempt" : 1, + "executionId" : "test-attempt", + "status" : "ok" + }; + observer.onCBQJobAttemptScheduled( {}, data ); + observer.onCBQJobExecutionStarted( {}, data ); + var span = variables.telemetry.getScope().span; + service.$( "counter" ).$throws( "TelemetryUnavailable" ); + observer.onCBQJobAttemptFinished( {}, data ); + expect( span.isFinished() ).toBeTrue(); + expect( span.getRoot().isFinished() ).toBeTrue(); + observer.onCBQJobExecutionExited( {}, data ); + expect( observer.getPendingAttemptCount() ).toBe( 0 ); + } ); + it( "requires a named supported pool adapter and sends only numeric pool fields", function(){ + var calls = 0; + expect( + variables.service.captureDatabasePool( "unregistered", function(){ + calls++; + return {}; + } ) + ).toBeFalse(); + expect( calls ).toBe( 0 ); + expect( + variables.service.captureDatabasePool( "primary", function(){ + throw( type = "UnsupportedPool" ); + } ) + ).toBeFalse(); + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "gauge" ); + expect( + variables.service.captureDatabasePool( "primary", function(){ + return { + "active" : 2, + "max" : 4, + "waiters" : -1, + "password" : "PRIVATE" + }; + } ) + ).toBeTrue(); + var calls = telemetry.$callLog( "gauge" ).gauge; + expect( calls.len() ).toBe( 3 ); + expect( serializeJSON( calls ) ).notToInclude( "PRIVATE" ); + expect( calls[ 3 ][ 1 ] ).toBe( "db.pool.utilization" ); + expect( calls[ 3 ][ 2 ] ).toBe( 0.5 ); + } ); + it( "serializes the standard lowercase trace context for error events", function(){ + var root = variables.service.startTransaction( "request" ); + variables.telemetry.setScope( { "span" : root } ); + var service = prepareMock( variables.service ).$( "post" ); + service.captureMessage( + message = "test", + cgiVars = {}, + useThread = false + ); + var json = service.$callLog( "post" ).post[ 1 ][ 4 ]; + expect( find( """trace"":", json ) ).toBeGT( 0 ); + expect( find( """TRACE"":", json ) ).toBe( 0 ); + expect( find( """contexts"":", json ) ).toBeGT( 0 ); + root.finish(); + } ); + } ); + } + +} diff --git a/test-harness/tests/specs/SQLPrivacySpec.cfc b/test-harness/tests/specs/SQLPrivacySpec.cfc new file mode 100644 index 0000000..ef7cc6e --- /dev/null +++ b/test-harness/tests/specs/SQLPrivacySpec.cfc @@ -0,0 +1,120 @@ +component extends="testbox.system.BaseSpec" { + + function run(){ + describe( "PostgreSQL query descriptions", function(){ + beforeEach( function(){ + variables.service = new sentry.models.SentryService( { DSN : "", databaseSystem : "postgresql" } ); + variables.telemetry = variables.service.getObservability(); + } ); + afterEach( function(){ + variables.service.shutdown(); + } ); + it( "keeps qualified identifiers while scrubbing strings, bindings and numbers", function(){ + var sql = "SELECT ""users"".""id"", ""users"".""column42"" FROM ""public"".""users"" WHERE ""email"" = 'private@example.test' AND ""id"" = 42 AND ""role"" = ?"; + var expected = "SELECT ""users"".""id"", ""users"".""column42"" FROM ""public"".""users"" WHERE ""email"" = ? AND ""id"" = ? AND ""role"" = ?"; + expect( variables.telemetry.scrubSQL( sql ) ).toBe( expected ); + // qb sanitizes before startSpan, which sanitizes again. + expect( variables.telemetry.scrubSQL( variables.telemetry.scrubSQL( sql ) ) ).toBe( expected ); + } ); + it( "preserves escaped identifier quotes and punctuation inside identifiers", function(){ + expect( + variables.telemetry.scrubSQL( "SELECT ""a""""b"", ""part.42"", ""--column"", ""/*column*/"", ""O'Brien"" FROM ""MixedCase""" ) + ).toBe( "SELECT ""a""""b"", ""part.42"", ""--column"", ""/*column*/"", ""O'Brien"" FROM ""MixedCase""" ); + } ); + it( "redacts escaped strings and treats comment markers inside strings as data", function(){ + expect( + variables.telemetry.scrubSQL( "SELECT 'O''Brien', '--private', '/*private*/', '""private""', ''" ) + ).toBe( "SELECT ?, ?, ?, ?, ?" ); + } ); + it( "removes line and nested block comments without exporting their contents", function(){ + var sql = "SELECT ""id"" /* outer private /* nested */ still private */ FROM ""users"" -- 'private""" & chr( + 10 + ) & "WHERE ""id"" = 42"; + expect( variables.telemetry.scrubSQL( sql ) ).toBe( "SELECT ""id"" FROM ""users"" WHERE ""id"" = ?" ); + } ); + it( "redacts decimal and exponential values without altering unquoted identifier digits", function(){ + expect( + variables.telemetry.scrubSQL( "SELECT column42 FROM ""table42"" WHERE n IN (123, -4.5, .75, 1e6, 2.5E-3)" ) + ).toBe( "SELECT column42 FROM ""table42"" WHERE n IN (?, -?, ?, ?, ?)" ); + expect( + variables.telemetry.scrubSQL( "SELECT répétition2 WHERE n IN (1_000, 1.2_5e1_0, 0xCAFE, 0o755, 0b101)" ) + ).toBe( "SELECT répétition2 WHERE n IN (?, ?, ?, ?, ?)" ); + } ); + it( "preserves identifier whitespace and bounds long descriptions without splitting quotes", function(){ + expect( variables.telemetry.scrubSQL( "SELECT ""my column"" FROM ""my table""" ) ).toBe( "SELECT ""my column"" FROM ""my table""" ); + var sql = "SELECT " & repeatString( """identifier"", ", 90 ) & """final"" FROM ""users"" WHERE ""email"" = 'private'"; + var description = variables.telemetry.scrubSQL( sql ); + expect( len( description ) ).toBeLTE( 1024 ); + expect( description ).toInclude( """identifier""" ); + expect( variables.telemetry.scrubSQL( description ) ).toBe( description ); + } ); + it( "honors the native listener dialect independently of the default", function(){ + variables.service.setEnabled( true ); + var settings = variables.service.getSettings(); + settings.tracesSampleRate = 1; + settings.databaseSystem = "other"; + var root = variables.service.startTransaction( "native query privacy" ); + variables.service.setEnabled( false ); + variables.telemetry.setScope( { span : root } ); + var listener = variables.service.createLuceeQueryListener( databaseSystem = "postgresql" ); + var args = { sql : "SELECT ""id"" FROM ""users"" WHERE ""email"" = 'private'" }; + listener.before( {}, args ); + listener.after( {}, args, {}, {} ); + expect( root.getChildren().len() ).toBe( 1 ); + expect( root.getChildren()[ 1 ].toPayload().description ).toBe( "SELECT ""id"" FROM ""users"" WHERE ""email"" = ?" ); + } ); + it( "fails closed for incomplete quotes, comments and unsupported quoting", function(){ + for ( + var sql in [ + "SELECT 'private", + "SELECT ""unfinished", + "SELECT /* private", + "SELECT $$private$$", + "SELECT $tag$private$tag$", + "SELECT `private`", + "SELECT E'private" & chr( 92 ) & "'more-private'" + ] + ) { + expect( variables.telemetry.scrubSQL( sql ) ).toBe( "[Filtered SQL]" ); + } + } ); + it( "retains conservative double-quote redaction for other and unknown dialects", function(){ + expect( variables.telemetry.scrubSQL( "SELECT ""private"" FROM ""users""", "mysql" ) ).toBe( "SELECT ? FROM ?" ); + expect( variables.telemetry.scrubSQL( "SELECT ""private"" FROM ""users""", "other" ) ).toBe( "SELECT ? FROM ?" ); + } ); + it( "uses span database attributes for all query operations, with and without parents", function(){ + var sql = "SELECT ""id"" FROM ""users"" WHERE ""email"" = 'private'"; + var expected = "SELECT ""id"" FROM ""users"" WHERE ""email"" = ?"; + for ( var parented in [ false, true ] ) { + if ( parented ) { + variables.telemetry.setScope( { span : variables.service.startTransaction( "parent" ) } ); + } + for ( var op in [ "db.query", "db.query.cache", "db.query.qoq" ] ) { + var span = variables.telemetry.startSpan( sql, op, { "db.system" : "postgresql" } ); + expect( span.toPayload().description ).toBe( expected ); + var other = variables.telemetry.startSpan( sql, op, { "db.system" : "other" } ); + expect( other.toPayload().description ).notToInclude( """users""" ); + } + } + } ); + it( "honors the manual wrapper dialect before either sanitization pass", function(){ + var sql = "SELECT ""id"" FROM ""users"" WHERE ""email"" = 'private'"; + variables.telemetry.withQuerySpan( + sql, + function( span ){ + expect( span.toPayload().description ).toBe( "SELECT ""id"" FROM ""users"" WHERE ""email"" = ?" ); + }, + "postgresql" + ); + variables.telemetry.withQuerySpan( + sql, + function( span ){ + expect( span.toPayload().description ).toBe( "SELECT ? FROM ? WHERE ? = ?" ); + }, + "other" + ); + } ); + } ); + } + +} diff --git a/test-harness/tests/specs/SaturationDependencyWorkerSpec.cfc b/test-harness/tests/specs/SaturationDependencyWorkerSpec.cfc new file mode 100644 index 0000000..bd25883 --- /dev/null +++ b/test-harness/tests/specs/SaturationDependencyWorkerSpec.cfc @@ -0,0 +1,508 @@ +component extends="testbox.system.BaseSpec" { + + function run(){ + describe( "Request saturation, dependencies and workers", function(){ + beforeEach( function(){ + variables.service = new sentry.models.SentryService( { + DSN : "http://public@127.0.0.1:60320/1", + enableMetrics : true, + enableRequestSaturation : true, + maxObservedRequests : 2, + dependencyNames : [ "mail", "MAIL", "bad,label" ], + requestExecutorNames : [ "http" ], + queueWorkerNames : [ "mail" ] + } ); + variables.telemetry = variables.service.getObservability(); + } ); + afterEach( function(){ + variables.telemetry.clearScope(); + variables.service.shutdown(); + } ); + it( "cleans exception-path request observations when postProcess does not run", function(){ + var observer = createMock( "sentry.interceptors.Observability" ).$property( + "sentry", + "variables", + variables.service + ); + var values = {}; + var event = { + getPrivateValue : function( name, defaultValue ){ + return values[ name ] ?: defaultValue; + }, + setPrivateValue : function( name, value ){ + values[ name ] = value; + }, + getCurrentRoute : function(){ + return "test"; + }, + getHTTPMethod : function(){ + return "GET"; + } + }; + observer.onRequestCapture( event, {} ); + observer.onException( event, {} ); + expect( variables.service.requestSaturation().active ).toBe( 0 ); + expect( variables.telemetry.getScope() ).toBeEmpty(); + variables.service.getSettings().enableRequestTracing = true; + values = {}; + observer.onRequestCapture( event, {} ); + var context = variables.service.getTraceContext(); + observer.onException( event, {} ); + expect( variables.service.requestSaturation().active ).toBe( 0 ); + expect( variables.service.getTraceContext().span_id ).toBe( context.span_id ); + variables.service.endRequest( statusCode = 500 ); + expect( variables.telemetry.getScope() ).toBeEmpty(); + } ); + it( "excludes classifier and exporter work from dependency duration", function(){ + var nanos = 0; + var measured = 0; + var observer = prepareMock( + new sentry.models.telemetry.Dependency( { + getScope : function(){ + nanos += 1000000000; + return {}; + }, + startSpan : function(){ + throw( type = "NoSpan" ); + }, + setScope : function(){ + }, + counter : function(){ + nanos += 1000000000; + }, + distribution : function( name, value ){ + if ( name == "dependency.duration" ) measured = value; + } + } ) + ); + observer.$property( + "clock", + "variables", + { + nanoTime : function(){ + return nanos; + } + } + ); + observer.run( + "mail", + function(){ + nanos += 10000000; + return true; + }, + function(){ + nanos += 1000000000; + return { outcome : "success" }; + } + ); + expect( measured ).toBe( 10 ); + } ); + + it( + title = "compiles database runnable, delayed and expired lease snapshots with qb grammars", + body = function(){ + for ( var dialect in [ "Postgres", "MySQL", "SqlServer" ] ) { + var grammar = createObject( "component", "qb.models.Grammars." & dialect & "Grammar" ).init(); + var builder = prepareMock( + new qb.models.Query.QueryBuilder( grammar = grammar, paginationCollector = {} ) + ); + builder.$( + "get", + [ + { + runnable : 2, + delayed : 3, + reserved : 1, + staleReservations : 1 + } + ] + ); + var adapter = new sentry.models.telemetry.DatabaseQueueHealth( function(){ + return builder; + } ); + var result = adapter.snapshot( "PRIVATE-queue" ); + expect( result.runnable ).toBe( 2 ); + var sql = builder.toSQL(); + expect( sql ).toInclude( grammar.wrapValue( "reservedBy" ) ); + expect( sql ).toInclude( grammar.wrapValue( "availableDate" ) ); + expect( sql ).toInclude( "staleReservations" ); + expect( sql ).notToInclude( "PRIVATE-queue" ); + expect( builder.getBindings().len() ).toBe( 4 ); + } + }, + skip = function(){ + var source = createObject( "java", "java.lang.System" ).getenv( "SENTRY_QB_SOURCE" ); + var cbqSource = createObject( "java", "java.lang.System" ).getenv( "SENTRY_CBQ_SOURCE" ); + return isNull( source ) || !len( source ) || isNull( cbqSource ) || !len( cbqSource ) || !fileExists( + cbqSource & "/models/Telemetry/DatabaseQueueSnapshot.cfc" + ); + } + ); + + it( "counts active requests, bounds retention and accepts completion once", function(){ + var first = variables.service.beginRequestObservation(); + var second = variables.service.beginRequestObservation(); + expect( variables.service.beginRequestObservation() ).toBeEmpty(); + var snapshot = variables.service.captureRequestSaturation(); + expect( snapshot.active ).toBe( 2 ); + expect( snapshot.untrackedStarts ).toBe( 1 ); + expect( variables.service.finishRequestObservation( { id : "foreign" }, true ) ).toBeFalse(); + expect( variables.service.finishRequestObservation( first, true ) ).toBeTrue(); + expect( variables.service.finishRequestObservation( first, true ) ).toBeFalse(); + expect( variables.service.requestSaturation().timeouts ).toBe( 1 ); + expect( variables.service.finishRequestObservation( second ) ).toBeTrue(); + snapshot = variables.service.captureRequestSaturation(); + expect( snapshot.active ).toBe( 0 ); + expect( snapshot.oldestSeconds ).toBe( 0 ); + } ); + it( "uses monotonic ages without converting long running requests into timeouts", function(){ + var nanos = 1000000000; + var observer = prepareMock( + new sentry.models.telemetry.RequestSaturation( variables.telemetry, 2, 30 ) + ); + observer.$property( + "clock", + "variables", + { + nanoTime : function(){ + return nanos; + } + } + ); + var token = observer.begin(); + nanos += 31000000000; + var snapshot = observer.snapshot(); + expect( snapshot.oldestSeconds ).toBe( 31 ); + expect( snapshot.longRunning ).toBe( 1 ); + expect( snapshot.timeouts ).toBe( 0 ); + observer.finish( token ); + expect( observer.snapshot().longRunning ).toBe( 0 ); + } ); + it( "tracks the ColdBox lifecycle with tracing disabled and cleans early endRequest once", function(){ + var observer = createMock( "sentry.interceptors.Observability" ).$property( + "sentry", + "variables", + variables.service + ); + var values = {}; + var event = { + getPrivateValue : function( name, defaultValue ){ + return values[ name ] ?: defaultValue; + }, + setPrivateValue : function( name, value ){ + values[ name ] = value; + } + }; + observer.onRequestCapture( event, {} ); + observer.onRequestCapture( event, {} ); + expect( variables.service.requestSaturation().active ).toBe( 1 ); + expect( variables.telemetry.getScope().requestState ).notToHaveKey( "span" ); + variables.service.endRequest( timedOut = true ); + observer.postProcess( event, {} ); + expect( variables.service.requestSaturation().active ).toBe( 0 ); + expect( variables.service.requestSaturation().timeouts ).toBe( 1 ); + expect( variables.telemetry.getScope() ).toBeEmpty(); + } ); + it( "completes a request token safely from another thread", function(){ + var token = variables.service.beginRequestObservation(); + var reference = createObject( "java", "java.util.concurrent.atomic.AtomicReference" ).init( + variables.service + ); + var name = "sentry-health-" & replace( createUUID(), "-", "", "all" ); + thread name=name action="run" service=reference token=token { + attributes.service.get().finishRequestObservation( attributes.token, true ); + } + thread action="join" name=name timeout=5000; + expect( cfthread[ name ].status ).toBe( "COMPLETED" ); + expect( variables.service.requestSaturation().active ).toBe( 0 ); + expect( variables.service.finishRequestObservation( token, true ) ).toBeFalse(); + expect( variables.service.requestSaturation().timeouts ).toBe( 1 ); + } ); + it( "does no collection or provider sampling when metrics are disabled", function(){ + variables.service.getSettings().enableMetrics = false; + var calls = 0; + var sample = function(){ + calls++; + return { active : 1 }; + }; + expect( variables.service.beginRequestObservation() ).toBeEmpty(); + expect( variables.service.captureRequestSaturation() ).toBeEmpty(); + expect( variables.service.captureRequestExecutor( "http", sample ) ).toBeFalse(); + expect( variables.service.captureQueueWorkerHealth( "mail", sample ) ).toBeFalse(); + expect( + variables.service.withDependency( + "mail", + function(){ + return 0; + }, + sample + ) + ).toBe( 0 ); + expect( calls ).toBe( 0 ); + } ); + it( "records bounded dependency outcomes and preserves private callback results", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "counter" ); + telemetry.$( "distribution" ); + var calls = 0; + var result = variables.service.withDependency( + "mail", + function(){ + calls++; + return { body : "PRIVATE" }; + }, + function( result, failure ){ + expect( result.body ).toBe( "PRIVATE" ); + expect( isNull( arguments.failure ) ).toBeTrue(); + return { + outcome : "rejected", + statusCode : 429, + retryCount : 2, + circuitState : "half_open", + body : result.body + }; + } + ); + expect( result.body ).toBe( "PRIVATE" ); + expect( calls ).toBe( 1 ); + var count = telemetry.$callLog( "counter" ).counter[ 1 ]; + expect( count[ 1 ] ).toBe( "dependency.calls" ); + expect( count[ 3 ].outcome ).toBe( "rejected" ); + expect( count[ 3 ][ "http.status_class" ] ).toBe( "4xx" ); + expect( count[ 3 ][ "circuit.state" ] ).toBe( "half_open" ); + var durations = telemetry.$callLog( "distribution" ).distribution; + expect( durations.len() ).toBe( 2 ); + expect( durations[ 2 ][ 1 ] ).toBe( "dependency.retries" ); + expect( durations[ 2 ][ 2 ] ).toBe( 2 ); + expect( serializeJSON( count ) ).notToInclude( "PRIVATE" ); + expect( serializeJSON( durations ) ).notToInclude( "PRIVATE" ); + } ); + it( "classifies thrown dependency failures and rethrows the original exception", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "counter" ); + telemetry.$( "distribution" ); + expect( function(){ + variables.service.withDependency( + "mail", + function(){ + throw( type = "OriginalTimeout", message = "PRIVATE" ); + }, + function( result, failure ){ + expect( isNull( arguments.result ) ).toBeTrue(); + expect( failure.type ).toBe( "OriginalTimeout" ); + return { + outcome : "timeout", + retryCount : 1, + circuitState : "open" + }; + } + ); + } ).toThrow( "OriginalTimeout" ); + expect( telemetry.$callLog( "counter" ).counter[ 1 ][ 3 ].outcome ).toBe( "timeout" ); + expect( serializeJSON( telemetry.$callLog( "counter" ) ) ).notToInclude( "PRIVATE" ); + } ); + it( "omits invalid classifier fields and never observes unconfigured case variants", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "counter" ); + telemetry.$( "distribution" ); + var classified = 0; + var callback = function(){ + return false; + }; + var classifier = function(){ + classified++; + return { + outcome : "PRIVATE", + statusCode : 999, + retryCount : -1, + circuitState : "PRIVATE" + }; + }; + expect( variables.service.withDependency( "MAIL", callback, classifier ) ).toBeFalse(); + expect( variables.service.withDependency( "bad,label", callback, classifier ) ).toBeFalse(); + expect( classified ).toBe( 0 ); + expect( variables.service.withDependency( "mail", callback, classifier ) ).toBeFalse(); + var labels = telemetry.$callLog( "counter" ).counter[ 1 ][ 3 ]; + expect( labels.outcome ).toBe( "unknown" ); + expect( labels ).notToHaveKey( "http.status_class" ); + expect( labels ).notToHaveKey( "circuit.state" ); + expect( telemetry.$callLog( "distribution" ).distribution.len() ).toBe( 1 ); + } ); + it( "preserves callbacks and cleanup even if dependency telemetry fails", function(){ + var finishes = 0; + var calls = 0; + var observer = new sentry.models.telemetry.Dependency( { + getScope : function(){ + return {}; + }, + startSpan : function(){ + return { + setAttribute : function(){ + throw( type = "ObserverFailure" ); + }, + finish : function(){ + finishes++; + } + }; + }, + setScope : function(){ + throw( type = "ObserverFailure" ); + }, + counter : function(){ + throw( type = "ObserverFailure" ); + }, + distribution : function(){ + throw( type = "ObserverFailure" ); + } + } ); + expect( + observer.run( "mail", function(){ + calls++; + return 42; + } ) + ).toBe( 42 ); + expect( calls ).toBe( 1 ); + expect( function(){ + observer.run( "mail", function(){ + throw( type = "OriginalFailure" ); + } ); + } ).toThrow( "OriginalFailure" ); + expect( finishes ).toBe( 2 ); + } ); + it( "samples worker counts, backlog and heartbeat without exporting provider contents", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "gauge" ); + var now = createObject( "java", "java.lang.System" ).currentTimeMillis() / 1000; + expect( + variables.service.captureQueueWorkerHealth( "mail", function(){ + return { + configuredWorkers : 4, + activeWorkers : 3, + activeExecutions : 2, + lastHeartbeatAt : now - 20, + runnable : 5, + delayed : 6, + reserved : 2, + staleReservations : 1, + retries : 10, + exhausted : 2, + password : "PRIVATE" + }; + } ) + ).toBeTrue(); + var calls = telemetry.$callLog( "gauge" ).gauge; + expect( calls.len() ).toBe( 10 ); + expect( serializeJSON( calls ) ).notToInclude( "PRIVATE" ); + var heartbeat = 0; + for ( var call in calls ) { + if ( call[ 1 ] == "queue.worker.heartbeat_age" ) heartbeat = call[ 2 ]; + expect( call[ 3 ][ "worker.name" ] ).toBe( "mail" ); + } + expect( heartbeat ).toBeGTE( 20 ); + expect( variables.service.queueWorkerDiagnostics( "mail" ).lastSuccessAt ).toBeGT( 0 ); + } ); + it( "keeps failed and unsupported worker snapshots distinct from measured zero", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "gauge" ); + expect( + variables.service.captureQueueWorkerHealth( "mail", function(){ + return { activeWorkers : 0, lastHeartbeatAt : 0, delayed : -1 }; + } ) + ).toBeTrue(); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 1 ); + var success = variables.service.queueWorkerDiagnostics( "mail" ).lastSuccessAt; + expect( + variables.service.captureQueueWorkerHealth( "mail", function(){ + throw( type = "ProviderUnavailable", message = "PRIVATE" ); + } ) + ).toBeFalse(); + expect( + variables.service.captureQueueWorkerHealth( "mail", function(){ + return { password : "PRIVATE" }; + } ) + ).toBeFalse(); + expect( variables.service.queueWorkerDiagnostics( "mail" ).failures ).toBe( 2 ); + expect( variables.service.queueWorkerDiagnostics( "mail" ).lastSuccessAt ).toBe( success ); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 1 ); + expect( serializeJSON( variables.service.queueWorkerDiagnostics( "mail" ) ) ).notToInclude( "PRIVATE" ); + } ); + + it( "delegates the cbq compatibility API to its optional external adapter", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "gauge" ); + var calls = 0; + telemetry.$property( + "adapterResolver", + "variables", + { + resolve : function( mapping ){ + expect( mapping ).toBe( "WorkerHealthAdapter@cbq" ); + return { + samplePool : function( pool ){ + expect( pool.marker ).toBe( "PRIVATE" ); + calls++; + return { configuredWorkers : 4, activeExecutions : 1 }; + } + }; + } + } + ); + expect( variables.service.captureCBQWorkerHealth( "mail", { marker : "PRIVATE" } ) ).toBeTrue(); + expect( calls ).toBe( 1 ); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 2 ); + expect( serializeJSON( telemetry.$callLog( "gauge" ) ) ).notToInclude( "PRIVATE" ); + } ); + it( "keeps generic manual sampling available when the optional cbq adapter is absent", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "gauge" ); + telemetry.$property( + "adapterResolver", + "variables", + { + resolve : function(){ + throw( type = "MissingOptionalAdapter" ); + } + } + ); + expect( variables.service.captureCBQWorkerHealth( "mail", {} ) ).toBeFalse(); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 0 ); + expect( + variables.service.captureQueueWorkerHealth( "mail", function(){ + return { activeWorkers : 0 }; + } ) + ).toBeTrue(); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 1 ); + } ); + + it( "restricts executor names and omits unsupported or nonnumeric fields", function(){ + var telemetry = prepareMock( variables.telemetry ); + telemetry.$( "gauge" ); + var count = 0; + var sample = function(){ + count++; + return { + active : 2, + max : 4, + queued : 0, + rejected : 3, + password : "PRIVATE", + other : 123 + }; + }; + expect( variables.service.captureRequestExecutor( "PRIVATE", sample ) ).toBeFalse(); + expect( count ).toBe( 0 ); + expect( variables.service.captureRequestExecutor( "http", sample ) ).toBeTrue(); + expect( count ).toBe( 1 ); + expect( telemetry.$callLog( "gauge" ).gauge.len() ).toBe( 4 ); + expect( serializeJSON( telemetry.$callLog( "gauge" ) ) ).notToInclude( "PRIVATE" ); + expect( + variables.service.captureRequestExecutor( "http", function(){ + return { active : -1, max : "PRIVATE" }; + } ) + ).toBeFalse(); + expect( variables.service.requestExecutorDiagnostics( "http" ).failures ).toBe( 1 ); + } ); + } ); + } + +} diff --git a/test-harness/tests/specs/SentryTimestampTests.cfc b/test-harness/tests/specs/SentryTimestampTests.cfc index 01d3889..f02a4e7 100644 --- a/test-harness/tests/specs/SentryTimestampTests.cfc +++ b/test-harness/tests/specs/SentryTimestampTests.cfc @@ -50,6 +50,28 @@ component extends="testbox.system.BaseSpec" { expect( parsed.getEpochSecond() ).toBeGTE( before ); expect( parsed.getEpochSecond() ).toBeLTE( instant.now().getEpochSecond() ); } ); + it( "uses the JVM clock at millisecond boundaries without applying an engine time-server offset", function(){ + var service = createMock( "sentry.models.SentryService" ); + makePublic( service, "getTimeVars" ); + var millis = 1792707331943; + service.$property( + "timestampClock", + "variables", + { + currentTimeMillis : function(){ + return millis; + } + } + ); + for ( var offset in [ 0, 56, 57 ] ) { + millis = 1792707331943 + offset; + var timestamps = service.getTimeVars(); + expect( timestamps.unix ).toBe( offset == 57 ? "1792707332" : "1792707331" ); + expect( timestamps.iso ).toBe( offset == 57 ? "2026-10-22T22:15:32Z" : "2026-10-22T22:15:31Z" ); + } + var reference = createObject( "java", "java.util.Date" ).init( javacast( "long", 1792707331943 ) ); + expect( service.getTimeVars( reference ).unix ).toBe( "1792707331" ); + } ); it( "uses the same valid UTC timestamp for the event and envelope", function(){ var service = createMock( "sentry.models.SentryService" ); service.$property( @@ -67,8 +89,16 @@ component extends="testbox.system.BaseSpec" { } ); service.$( "getHTTPDataForRequest", { "headers" : {}, "content" : "" } ); service.$( "post" ); + service.$property( + "timestampClock", + "variables", + { + currentTimeMillis : function(){ + return 1792707331943; + } + } + ); var instant = createObject( "java", "java.time.Instant" ); - var before = instant.now().getEpochSecond(); service.captureMessage( message = "Synthetic timestamp test", @@ -79,8 +109,8 @@ component extends="testbox.system.BaseSpec" { var captured = service.$callLog( "post" ).post[ 1 ]; var parsed = instant.parse( captured[ 3 ] ); expect( deserializeJSON( captured[ 4 ] ).timestamp ).toBe( captured[ 3 ] ); - expect( parsed.getEpochSecond() ).toBeGTE( before ); - expect( parsed.getEpochSecond() ).toBeLTE( instant.now().getEpochSecond() ); + expect( parsed.getEpochSecond() ).toBe( 1792707331 ); + expect( captured[ 3 ] ).toBe( "2026-10-22T22:15:31Z" ); } ); } ); } diff --git a/test-harness/tests/telemetry-runner.cfm b/test-harness/tests/telemetry-runner.cfm index b8cabb3..c391804 100644 --- a/test-harness/tests/telemetry-runner.cfm +++ b/test-harness/tests/telemetry-runner.cfm @@ -1,6 +1,6 @@ setting showDebugOutput=false; -bundles = "tests.specs.ObservabilitySpec,tests.specs.TransportFailureSpec,tests.specs.ExecutionIsolationSpec"; +bundles = "tests.specs.ObservabilitySpec,tests.specs.TransportFailureSpec,tests.specs.ExecutionIsolationSpec,tests.specs.RequestCompletionSpec,tests.specs.ReusableObservabilitySpec,tests.specs.DeliveryDiagnosticsSpec,tests.specs.DatabaseQueueMetricsSpec,tests.specs.OperationalReviewSpec,tests.specs.SaturationDependencyWorkerSpec,tests.specs.DatabaseQueueHealthIntegrationSpec,tests.specs.AutomaticObservabilitySpec"; if ( directoryExists( expandPath( "/cbq/tests" ) ) ) { bundles &= ",cbq.tests.specs.unit.ObservabilityHooksSpec"; } writeOutput( new testbox.system.TestBox( bundles = bundles ).run( reporter = "json" ) );