Logs
Live Cloud Logging query (nothing stored) · master ·
last 1h ·
newest first · page 1 (200/page) · 200 entries in 6576 ms
all
app
neo4j
weaviate
time
relative
absolute
7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion{tildaid:$version_id})<-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id}) MATCH...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spana71cc471d836156d
podserver-2-dev-0
query
MATCH (dv:DocumentVersion{tildaid:$version_id})<-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id})
MATCH (doc)-[:DOC_TO_ORG]->(org:Organization)
OPTIONAL MATCH (study:Study{tildaid:doc.studyId})
WITH dv, doc, org, study
RETURN dv.tildaid AS version_id,
dv.url AS url,
doc.fileName AS filename,
doc.contentType AS content_type,
dv.size AS size_bytes,
dv.linearizedPdfUrl AS linearized_pdf_url,
dv.linearizedPdfStatus AS linearized_status,
dv.languages AS languages,
dv.langDetectionStatus AS lang_detection_status,
dv.translatedPdfUrl AS translated_pdf_url,
dv.translatedPdfStatus AS translated_status,
study.tildaid AS study_id,
study.name AS study_name,
study.protocolnumber AS protocol_number,
org.tildaid AS org_id,
org.alias AS org_alias,
org.name AS org_name,
dv.uploadMethod AS upload_method,
org.orgType AS org_type,
dv.cksum AS checksum
raw payload (15 fields)
{
"allocatedBytes": 8376.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "8FE8F25D",
"id": "25411504",
"pageFaults": 0.0,
"pageHits": 18.0,
"planning": 0.0,
"query": "\n MATCH (dv:DocumentVersion{tildaid:$version_id})\u003c-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id})\n MATCH (doc)-[:DOC_TO_ORG]-\u003e(org:Organization)\n OPTIONAL MATCH (study:Study{tildaid:doc.studyId})\n WITH dv, doc, org, study\n RETURN dv.tildaid AS version_id,\n dv.url AS url,\n doc.fileName AS filename,\n doc.contentType AS content_type,\n dv.size AS size_bytes,\n dv.linearizedPdfUrl AS linearized_pdf_url,\n dv.linearizedPdfStatus AS linearized_status,\n dv.languages AS languages,\n dv.langDetectionStatus AS lang_detection_status,\n dv.translatedPdfUrl AS translated_pdf_url,\n dv.translatedPdfStatus AS translated_status,\n study.tildaid AS study_id,\n study.name AS study_name,\n study.protocolnumber AS protocol_number,\n org.tildaid AS org_id,\n org.alias AS org_alias,\n org.name AS org_name,\n dv.uploadMethod AS upload_method,\n org.orgType AS org_type,\n dv.cksum AS checksum\n ",
"queryParameters": "{document_id: \u0027f92da704-917e-49c9-9f6c-0310c9d8475c\u0027, version_id: \u002752db0899-2c0b-41c7-b389-565bf41c1987\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:53256\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.996Z",
"transactionId": 12258985.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion {tildaid: $version_id}) SET dv.ocrStatus = $status, dv.ocrResultsUrl = $url, ...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (dv:DocumentVersion {tildaid: $version_id})
SET dv.ocrStatus = $status,
dv.ocrResultsUrl = $url,
dv.ocrProcessingStartedAt = CASE
WHEN $processing_started_at IS NOT NULL THEN $processing_started_at
ELSE dv.ocrProcessingStartedAt
END,
dv.ocrProcessingEndedAt = CASE
WHEN $processing_ended_at IS NOT NULL THEN $processing_ended_at
ELSE dv.ocrProcessingEndedAt
END
WITH dv
OPTIONAL MATCH (dv)<-[:DOCUMENT_VERSION]-(doc:Document)
SET doc.ocrStatus = $status,
doc.ocrResultsUrl = $url
raw payload (15 fields)
{
"allocatedBytes": 584.0,
"cpu": 56.0,
"elapsedTimeMs": 56.0,
"executionPlanCacheKeyHash": "DC17A08F",
"id": "25411501",
"pageFaults": 0.0,
"pageHits": 6.0,
"planning": 52.0,
"query": "\n MATCH (dv:DocumentVersion {tildaid: $version_id})\n SET dv.ocrStatus = $status,\n dv.ocrResultsUrl = $url,\n dv.ocrProcessingStartedAt = CASE\n WHEN $processing_started_at IS NOT NULL THEN $processing_started_at\n ELSE dv.ocrProcessingStartedAt\n END,\n dv.ocrProcessingEndedAt = CASE\n WHEN $processing_ended_at IS NOT NULL THEN $processing_ended_at\n ELSE dv.ocrProcessingEndedAt\n END\n WITH dv\n OPTIONAL MATCH (dv)\u003c-[:DOCUMENT_VERSION]-(doc:Document)\n SET doc.ocrStatus = $status,\n doc.ocrResultsUrl = $url\n ",
"queryParameters": "{processing_started_at: \u003cnull\u003e, version_id: \u0027bdc7f2d3-251a-4baf-9357-c1f63488750c\u0027, url: \u0027gs://master-tilda-shared-storage/ocr-cache/v1/61/613a1254d3d2c7ac900b9128c6e578ec942d2b8c7d88a024be8da0ede9b6f6cd_ocr.json\u0027, processing_ended_at: \u00272026-07-27T02:24:53.923791Z\u0027, timeout: \u003cnull\u003e, status: \u0027Completed\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.984Z",
"transactionId": 12258983.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 704.0,
"cpu": 113.0,
"elapsedTimeMs": 114.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "25411483",
"pageFaults": 0.0,
"pageHits": 89941.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 21, cutoff_ms: 1785118493687}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.806Z",
"transactionId": 12258974.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion {tildaid: $version_id}) SET dv.linearizedPdfStatus = $status, dv.linearizedPdfUrl = $url, ...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (dv:DocumentVersion {tildaid: $version_id})
SET dv.linearizedPdfStatus = $status,
dv.linearizedPdfUrl = $url,
dv.pages = $pages,
dv.linearizedPdfProcessingStartedAt = CASE
WHEN $processing_started_at IS NOT NULL THEN $processing_started_at
ELSE dv.linearizedPdfProcessingStartedAt
END,
dv.linearizedPdfProcessingEndedAt = CASE
WHEN $processing_ended_at IS NOT NULL THEN $processing_ended_at
ELSE dv.linearizedPdfProcessingEndedAt
END,
dv.linearizedPdfRetryCount = CASE
WHEN $status = "Processing" THEN COALESCE(dv.linearizedPdfRetryCount, 0)
ELSE dv.linearizedPdfRetryCount
END
WITH dv
OPTIONAL MATCH (dv)<-[:DOCUMENT_VERSION]-(doc:Document)
SET doc.linearizedPdfStatus = $status,
doc.linearizedPdfUrl = $url,
doc.pages = $pages
raw payload (15 fields)
{
"allocatedBytes": 584.0,
"cpu": 47.0,
"elapsedTimeMs": 49.0,
"executionPlanCacheKeyHash": "C7CDA985",
"id": "25411490",
"pageFaults": 0.0,
"pageHits": 6.0,
"planning": 44.0,
"query": "\n MATCH (dv:DocumentVersion {tildaid: $version_id})\n SET dv.linearizedPdfStatus = $status,\n dv.linearizedPdfUrl = $url,\n dv.pages = $pages,\n dv.linearizedPdfProcessingStartedAt = CASE\n WHEN $processing_started_at IS NOT NULL THEN $processing_started_at\n ELSE dv.linearizedPdfProcessingStartedAt\n END,\n dv.linearizedPdfProcessingEndedAt = CASE\n WHEN $processing_ended_at IS NOT NULL THEN $processing_ended_at\n ELSE dv.linearizedPdfProcessingEndedAt\n END,\n dv.linearizedPdfRetryCount = CASE\n WHEN $status = \"Processing\" THEN COALESCE(dv.linearizedPdfRetryCount, 0)\n ELSE dv.linearizedPdfRetryCount\n END\n WITH dv\n OPTIONAL MATCH (dv)\u003c-[:DOCUMENT_VERSION]-(doc:Document)\n SET doc.linearizedPdfStatus = $status,\n doc.linearizedPdfUrl = $url,\n doc.pages = $pages\n ",
"queryParameters": "{pages: 0, processing_started_at: \u00272026-07-27T02:24:53.728215Z\u0027, version_id: \u0027bdc7f2d3-251a-4baf-9357-c1f63488750c\u0027, url: \u0027\u0027, processing_ended_at: \u003cnull\u003e, timeout: \u003cnull\u003e, status: \u0027Processing\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.784Z",
"transactionId": 12258979.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion {tildaid: $version_id}) SET dv.linearizedPdfStatus = $status, dv.linearizedPdfUrl = $url, ...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (dv:DocumentVersion {tildaid: $version_id})
SET dv.linearizedPdfStatus = $status,
dv.linearizedPdfUrl = $url,
dv.pages = $pages,
dv.linearizedPdfProcessingStartedAt = CASE
WHEN $processing_started_at IS NOT NULL THEN $processing_started_at
ELSE dv.linearizedPdfProcessingStartedAt
END,
dv.linearizedPdfProcessingEndedAt = CASE
WHEN $processing_ended_at IS NOT NULL THEN $processing_ended_at
ELSE dv.linearizedPdfProcessingEndedAt
END,
dv.linearizedPdfRetryCount = CASE
WHEN $status = "Processing" THEN COALESCE(dv.linearizedPdfRetryCount, 0)
ELSE dv.linearizedPdfRetryCount
END
WITH dv
OPTIONAL MATCH (dv)<-[:DOCUMENT_VERSION]-(doc:Document)
SET doc.linearizedPdfStatus = $status,
doc.linearizedPdfUrl = $url,
doc.pages = $pages
raw payload (15 fields)
{
"allocatedBytes": 584.0,
"cpu": 4.0,
"elapsedTimeMs": 9.0,
"executionPlanCacheKeyHash": "C7CDA985",
"id": "25411494",
"pageFaults": 0.0,
"pageHits": 7.0,
"planning": 4.0,
"query": "\n MATCH (dv:DocumentVersion {tildaid: $version_id})\n SET dv.linearizedPdfStatus = $status,\n dv.linearizedPdfUrl = $url,\n dv.pages = $pages,\n dv.linearizedPdfProcessingStartedAt = CASE\n WHEN $processing_started_at IS NOT NULL THEN $processing_started_at\n ELSE dv.linearizedPdfProcessingStartedAt\n END,\n dv.linearizedPdfProcessingEndedAt = CASE\n WHEN $processing_ended_at IS NOT NULL THEN $processing_ended_at\n ELSE dv.linearizedPdfProcessingEndedAt\n END,\n dv.linearizedPdfRetryCount = CASE\n WHEN $status = \"Processing\" THEN COALESCE(dv.linearizedPdfRetryCount, 0)\n ELSE dv.linearizedPdfRetryCount\n END\n WITH dv\n OPTIONAL MATCH (dv)\u003c-[:DOCUMENT_VERSION]-(doc:Document)\n SET doc.linearizedPdfStatus = $status,\n doc.linearizedPdfUrl = $url,\n doc.pages = $pages\n ",
"queryParameters": "{pages: 0, processing_started_at: \u00272026-07-27T02:24:53.768524Z\u0027, version_id: \u002765ac9b5f-2b28-4cf8-a86b-496962cc36ef\u0027, url: \u0027\u0027, processing_ended_at: \u003cnull\u003e, timeout: \u003cnull\u003e, status: \u0027Processing\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:53256\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.784Z",
"transactionId": 12258982.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion {tildaid: $version_id}) RETURN dv.linearizedPdfStatus AS status, dv.linearizedPdfUrl AS url, ...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (dv:DocumentVersion {tildaid: $version_id})
RETURN dv.linearizedPdfStatus AS status,
dv.linearizedPdfUrl AS url,
dv.pages AS pages
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "1D11D835",
"id": "25411492",
"pageFaults": 0.0,
"pageHits": 5.0,
"planning": 0.0,
"query": "\n MATCH (dv:DocumentVersion {tildaid: $version_id})\n RETURN dv.linearizedPdfStatus AS status,\n dv.linearizedPdfUrl AS url,\n dv.pages AS pages\n ",
"queryParameters": "{version_id: \u002765ac9b5f-2b28-4cf8-a86b-496962cc36ef\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:53256\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.765Z",
"transactionId": 12258981.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Ocr Cache Mode
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Ocr Cache Mode
raw payload (17 fields)
{
"behavior": "check_and_save",
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"enabled": true,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Ocr Cache Mode",
"org_alias": "sandboxIcon",
"processor": "ocr",
"service": "docproc",
"step": "ocr_cache_mode",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.761Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Ocr started
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Ocr started
raw payload (17 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/65ac/65ac9b5f-2b28-4cf8-a86b-496962cc36ef",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Ocr started",
"org_alias": "sandboxIcon",
"pdf_size_mb": 0.18,
"processor": "ocr",
"service": "docproc",
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.761Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 2120.9,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "ocr",
"org_alias": "sandboxIcon",
"reserved_mb": 1000.6,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.761Z",
"version": "0.1.0",
"wait_seconds": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 3699.6,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 406.0,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "ocr",
"org_alias": "sandboxIcon",
"requested_mb": 1000.6,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.761Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Linearization started
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Linearization started
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/65ac/65ac9b5f-2b28-4cf8-a86b-496962cc36ef",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Linearization started",
"org_alias": "sandboxIcon",
"processor": "linearizer",
"service": "docproc",
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.759Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 1120.4,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "linearize",
"org_alias": "sandboxIcon",
"reserved_mb": 40.4,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.759Z",
"version": "0.1.0",
"wait_seconds": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 3740.0,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 406.0,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "linearize",
"org_alias": "sandboxIcon",
"requested_mb": 40.4,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.759Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Image metrics computed: 1 total pages, 1 selected for rendering
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceprocessors/doc_converter.py:388
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Image metrics computed: 1 total pages, 1 selected for rendering
raw payload (9 fields)
{
"caller": "processors/doc_converter.py:388",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Image metrics computed: 1 total pages, 1 selected for rendering",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.758Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Step 7/7 | translation disabled skipped (translation disabled)
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Step 7/7 | translation disabled skipped (translation disabled)
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 4.48,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Step 7/7 | translation disabled skipped (translation disabled)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"skip_reason": "translation disabled",
"status": "skipped",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.756Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [5/7] Settings ...................... OK 0.01s (Disabled)
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [5/7] Settings ...................... OK 0.01s (Disabled)
raw payload (18 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.01,
"elapsed_s": 4.48,
"environment": "DEVELOPMENT",
"lang_detect_threshold": 0.0,
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [5/7] Settings ...................... OK 0.01s (Disabled)",
"mode": "Disabled",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"step": "translation_settings_check",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.756Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (d:Document{tildaid: $document_id})-[:DOC_TO_ORG]->(org:Organization) MATCH (d)-[:DOCUMENT_VERSION]->(dv:DocumentVersion{tildaid:...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (d:Document{tildaid: $document_id})-[:DOC_TO_ORG]->(org:Organization)
MATCH (d)-[:DOCUMENT_VERSION]->(dv:DocumentVersion{tildaid: $version_id})
OPTIONAL MATCH (org)-[:USES]->(fm:FolderManager{name: "StudyInstance", scopeId: d.studyId})
RETURN org.orgType AS org_type,
COALESCE(dv.langDetectionStatus, "") AS lang_detection_status,
COALESCE(fm.docTranslationMode, org.docTranslationMode, "Disabled") AS translation_mode,
COALESCE(fm.langDetectMinThresholdPct, 0) AS lang_detect_min_threshold_pct
raw payload (15 fields)
{
"allocatedBytes": 336.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "543EF88F",
"id": "25411491",
"pageFaults": 0.0,
"pageHits": 29.0,
"planning": 0.0,
"query": "\n MATCH (d:Document{tildaid: $document_id})-[:DOC_TO_ORG]-\u003e(org:Organization)\n MATCH (d)-[:DOCUMENT_VERSION]-\u003e(dv:DocumentVersion{tildaid: $version_id})\n OPTIONAL MATCH (org)-[:USES]-\u003e(fm:FolderManager{name: \"StudyInstance\", scopeId: d.studyId})\n RETURN org.orgType AS org_type,\n COALESCE(dv.langDetectionStatus, \"\") AS lang_detection_status,\n COALESCE(fm.docTranslationMode, org.docTranslationMode, \"Disabled\") AS translation_mode,\n COALESCE(fm.langDetectMinThresholdPct, 0) AS lang_detect_min_threshold_pct\n ",
"queryParameters": "{document_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027, version_id: \u002765ac9b5f-2b28-4cf8-a86b-496962cc36ef\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:53256\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.753Z",
"transactionId": 12258980.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "annotation_detection", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id":...
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "annotation_detection", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef", "content_type": "application/pdf", "document_size_mb": 0.1843891143798828, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 80.4609727859497, "actual_mb": 7.265625, "memory_error_pct": 90.97000104718005, "duration_seconds": 0.9961531162261963, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119093.747001, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"annotation_detection\", \"document_id\": \"a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\", \"version_id\": \"65ac9b5f-2b28-4cf8-a86b-496962cc36ef\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.1843891143798828, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 80.4609727859497, \"actual_mb\": 7.265625, \"memory_error_pct\": 90.97000104718005, \"duration_seconds\": 0.9961531162261963, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119093.747001, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.747Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [3/7] Annotations ................... OK 1.0s (none found)
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [3/7] Annotations ................... OK 1.0s (none found)
raw payload (20 fields)
{
"annotation_types": null,
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 1.0,
"elapsed_s": 4.47,
"environment": "DEVELOPMENT",
"has_annotations": false,
"has_track_changes": false,
"is_redacted": false,
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [3/7] Annotations ................... OK 1.0s (none found)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"step": "annotation_detection",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.746Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Annotation detection complete
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Annotation detection complete
raw payload (21 fields)
{
"annotation_types": [],
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 1.0,
"environment": "DEVELOPMENT",
"has_annotations": false,
"has_track_changes": false,
"is_redacted": false,
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Annotation detection complete",
"org_alias": "sandboxIcon",
"processor": "annotation_detector",
"service": "docproc",
"status": "success",
"step_times": {
"detect_pdf_annotations": 0.97,
"update_neo4j": 0.02
},
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.746Z",
"total_s": 1.0,
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Update Neo4J OK 0.02s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Update Neo4J OK 0.02s
raw payload (19 fields)
{
"annotations": "not present",
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.02,
"elapsed_s": 1.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Update Neo4J OK 0.02s",
"org_alias": "sandboxIcon",
"processor": "annotation_detector",
"redacted": "no",
"service": "docproc",
"step": "update_neo4j",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.746Z",
"track_changes": "not present",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Message processed successfully
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:576
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span34d06c040e19fa6e
poddocproc-7bc76669-ljh8h
message
Message processed successfully
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:576",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Message processed successfully",
"message_id": "21226334429042141",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.730Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion {tildaid: $version_id}) SET dv.annotations = $annotations_status, dv.annotationTypes =...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (dv:DocumentVersion {tildaid: $version_id})
SET dv.annotations = $annotations_status,
dv.annotationTypes = $annotation_types,
dv.redacted = $redacted_status,
dv.trackChanges = $track_changes_status
WITH dv
OPTIONAL MATCH (dv)<-[:DOCUMENT_VERSION]-(doc:Document)
SET doc.annotations = $annotations_status,
doc.annotationTypes = $annotation_types,
doc.redacted = $redacted_status,
doc.trackChanges = $track_changes_status
RETURN dv.tildaid AS updated_version_id
raw payload (15 fields)
{
"allocatedBytes": 692.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "8D7C202D",
"id": "25411488",
"pageFaults": 0.0,
"pageHits": 6.0,
"planning": 0.0,
"query": "\n MATCH (dv:DocumentVersion {tildaid: $version_id})\n SET dv.annotations = $annotations_status,\n dv.annotationTypes = $annotation_types,\n dv.redacted = $redacted_status,\n dv.trackChanges = $track_changes_status\n WITH dv\n OPTIONAL MATCH (dv)\u003c-[:DOCUMENT_VERSION]-(doc:Document)\n SET doc.annotations = $annotations_status,\n doc.annotationTypes = $annotation_types,\n doc.redacted = $redacted_status,\n doc.trackChanges = $track_changes_status\n RETURN dv.tildaid AS updated_version_id\n ",
"queryParameters": "{annotations_status: \u0027not present\u0027, track_changes_status: \u0027not present\u0027, annotation_types: [], version_id: \u002765ac9b5f-2b28-4cf8-a86b-496962cc36ef\u0027, redacted_status: \u0027no\u0027, timeout: \u003cnull\u003e}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:53256\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.729Z",
"transactionId": 12258978.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion {tildaid: $version_id}) RETURN dv.linearizedPdfStatus AS status, dv.linearizedPdfUrl AS url, ...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (dv:DocumentVersion {tildaid: $version_id})
RETURN dv.linearizedPdfStatus AS status,
dv.linearizedPdfUrl AS url,
dv.pages AS pages
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 23.0,
"elapsedTimeMs": 28.0,
"executionPlanCacheKeyHash": "1D11D835",
"id": "25411485",
"pageFaults": 0.0,
"pageHits": 5.0,
"planning": 26.0,
"query": "\n MATCH (dv:DocumentVersion {tildaid: $version_id})\n RETURN dv.linearizedPdfStatus AS status,\n dv.linearizedPdfUrl AS url,\n dv.pages AS pages\n ",
"queryParameters": "{version_id: \u0027bdc7f2d3-251a-4baf-9357-c1f63488750c\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.726Z",
"transactionId": 12258976.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Detect Pdf Annotations OK 0.97s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Detect Pdf Annotations OK 0.97s
raw payload (24 fields)
{
"annotation_types": [],
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"comments": 0.0,
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.97,
"elapsed_s": 0.97,
"environment": "DEVELOPMENT",
"has_annotations": false,
"is_redacted": false,
"markups": 0.0,
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Detect Pdf Annotations OK 0.97s",
"org_alias": "sandboxIcon",
"processor": "annotation_detector",
"redactions": 0.0,
"service": "docproc",
"shapes": 0.0,
"step": "detect_pdf_annotations",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.721Z",
"total_count": 0.0,
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Annotations are not present
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/pdf_detector.py:175
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Annotations are not present
raw payload (15 fields)
{
"annotations_present": false,
"caller": "utils/pdf_detector.py:175",
"cluster": "master",
"comments": 0.0,
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_rtn3no26.pdf",
"is_redacted": false,
"markups": 0.0,
"message": "Annotations are not present",
"redactions": 0.0,
"service": "docproc",
"shapes": 0.0,
"timestamp": "2026-07-27T02:24:53.720Z",
"total": 0.0,
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc No applied redactions detected
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceutils/pdf_detector.py:347
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
No applied redactions detected
raw payload (8 fields)
{
"caller": "utils/pdf_detector.py:347",
"cluster": "master",
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_rtn3no26.pdf",
"message": "No applied redactions detected",
"service": "docproc",
"timestamp": "2026-07-27T02:24:53.718Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Document already has an in-progress AIRun; skipping (another path is processing)
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:3574
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span34d06c040e19fa6e
poddocproc-7bc76669-ljh8h
message
Document already has an in-progress AIRun; skipping (another path is processing)
raw payload (13 fields)
{
"caller": "orchestration/document_orchestrator.py:3574",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"in_progress_ai_run_id": "6f162abd-960f-4479-82e6-e0f0f909fd63",
"message": "Document already has an in-progress AIRun; skipping (another path is processing)",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.709Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (d:Document {tildaid: $document_id}) MATCH (d)-[:DOC_TO_ORG]-(o:Organization) MATCH...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span34d06c040e19fa6e
podserver-2-dev-0
query
MATCH (d:Document {tildaid: $document_id})
MATCH (d)-[:DOC_TO_ORG]-(o:Organization)
MATCH (d)-[:DOCUMENT_VERSION]->(dv:DocumentVersion)
WHERE dv.tildaid = COALESCE($version_id, d.primaryVersionId)
AND (dv.uploadMethod IS NULL OR dv.uploadMethod <> 'Internally Modified')
AND d.delpending = false
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
AND (
(o.orgType = "CRO"
AND (EXISTS { (d)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] }
OR EXISTS { (:GoldenDataset)-[:CONTAINS_DOCUMENT]->(d) }))
OR (o.orgType <> "CRO")
)
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH d, o, ai,
CASE WHEN ai IS NOT NULL AND ai.status IN ['in_progress', 'aborted'] THEN ai.tildaid END AS in_progress_ai_run_id,
CASE WHEN ai IS NOT NULL AND ai.status IN ['in_progress', 'aborted'] THEN ai.run_type END AS in_progress_ai_run_run_type,
CASE WHEN ai IS NOT NULL AND ai.status = 'complete' THEN ai.tildaid END AS completed_ai_run_id,
CASE WHEN ai IS NOT NULL AND ai.status = 'complete' THEN ai.run_type END AS completed_ai_run_run_type,
CASE WHEN ai IS NOT NULL AND ai.status = 'failed' THEN COALESCE(ai.auto_retry_count, 0) END AS failed_latest_auto_retry_count
RETURN d.tildaid AS document_id,
COALESCE($version_id, d.primaryVersionId) AS version_id,
d.studyId AS study_id,
o.alias AS org_alias,
o.tildaid AS org_id,
in_progress_ai_run_id,
in_progress_ai_run_run_type,
completed_ai_run_id,
completed_ai_run_run_type,
failed_latest_auto_retry_count,
COALESCE(d.aiEnabled, false) AS ai_enabled,
o.orgType AS org_type,
EXISTS { (d)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] } AS in_staging,
EXISTS { (:GoldenDataset)-[:CONTAINS_DOCUMENT]->(d) } AS is_golden_dataset
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 28416.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "73550B79",
"id": "25411486",
"pageFaults": 0.0,
"pageHits": 21.0,
"planning": 0.0,
"query": "\n MATCH (d:Document {tildaid: $document_id})\n MATCH (d)-[:DOC_TO_ORG]-(o:Organization)\n MATCH (d)-[:DOCUMENT_VERSION]-\u003e(dv:DocumentVersion)\n WHERE dv.tildaid = COALESCE($version_id, d.primaryVersionId)\n AND (dv.uploadMethod IS NULL OR dv.uploadMethod \u003c\u003e \u0027Internally Modified\u0027)\n AND d.delpending = false\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n AND (\n (o.orgType = \"CRO\"\n AND (EXISTS { (d)-[:INCLUDES]-(f:Folder) WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027] }\n OR EXISTS { (:GoldenDataset)-[:CONTAINS_DOCUMENT]-\u003e(d) }))\n OR (o.orgType \u003c\u003e \"CRO\")\n )\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH d, o, ai,\n CASE WHEN ai IS NOT NULL AND ai.status IN [\u0027in_progress\u0027, \u0027aborted\u0027] THEN ai.tildaid END AS in_progress_ai_run_id,\n CASE WHEN ai IS NOT NULL AND ai.status IN [\u0027in_progress\u0027, \u0027aborted\u0027] THEN ai.run_type END AS in_progress_ai_run_run_type,\n CASE WHEN ai IS NOT NULL AND ai.status = \u0027complete\u0027 THEN ai.tildaid END AS completed_ai_run_id,\n CASE WHEN ai IS NOT NULL AND ai.status = \u0027complete\u0027 THEN ai.run_type END AS completed_ai_run_run_type,\n CASE WHEN ai IS NOT NULL AND ai.status = \u0027failed\u0027 THEN COALESCE(ai.auto_retry_count, 0) END AS failed_latest_auto_retry_count\n RETURN d.tildaid AS document_id,\n COALESCE($version_id, d.primaryVersionId) AS version_id,\n d.studyId AS study_id,\n o.alias AS org_alias,\n o.tildaid AS org_id,\n in_progress_ai_run_id,\n in_progress_ai_run_run_type,\n completed_ai_run_id,\n completed_ai_run_run_type,\n failed_latest_auto_retry_count,\n COALESCE(d.aiEnabled, false) AS ai_enabled,\n o.orgType AS org_type,\n EXISTS { (d)-[:INCLUDES]-(f:Folder) WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027] } AS in_staging,\n EXISTS { (:GoldenDataset)-[:CONTAINS_DOCUMENT]-\u003e(d) } AS is_golden_dataset\n LIMIT 1\n ",
"queryParameters": "{document_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027, version_id: \u002765ac9b5f-2b28-4cf8-a86b-496962cc36ef\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:53256\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.705Z",
"transactionId": 12258977.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Pubsub:Attribute Update started
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span34d06c040e19fa6e
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Pubsub:Attribute Update started
raw payload (17 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "pubsub",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Pubsub:Attribute Update started",
"operation": "attribute_update",
"org_alias": "sandboxIcon",
"processor": "pubsub:attribute_update",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.700Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Ocr Cache Mode
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Ocr Cache Mode
raw payload (17 fields)
{
"behavior": "check_and_save",
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"enabled": true,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Ocr Cache Mode",
"org_alias": "sandboxIcon",
"processor": "ocr",
"service": "docproc",
"step": "ocr_cache_mode",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.698Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Ocr started
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Ocr started
raw payload (17 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/bdc7/bdc7f2d3-251a-4baf-9357-c1f63488750c",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Ocr started",
"org_alias": "sandboxIcon",
"pdf_size_mb": 0.0,
"processor": "ocr",
"service": "docproc",
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.698Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 1160.5,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "ocr",
"org_alias": "sandboxIcon",
"reserved_mb": 1000.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.698Z",
"version": "0.1.0",
"wait_seconds": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4659.5,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 406.5,
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "ocr",
"org_alias": "sandboxIcon",
"requested_mb": 1000.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.697Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Linearization started
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Linearization started
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/bdc7/bdc7f2d3-251a-4baf-9357-c1f63488750c",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Linearization started",
"org_alias": "sandboxIcon",
"processor": "linearizer",
"service": "docproc",
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.695Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 160.5,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "linearize",
"org_alias": "sandboxIcon",
"reserved_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.695Z",
"version": "0.1.0",
"wait_seconds": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (d:Document {tildaid: $document_id}) MATCH (d)-[:DOC_TO_ORG]-(o:Organization) MATCH...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span34d06c040e19fa6e
podserver-2-dev-0
query
MATCH (d:Document {tildaid: $document_id})
MATCH (d)-[:DOC_TO_ORG]-(o:Organization)
MATCH (d)-[:DOCUMENT_VERSION]->(dv:DocumentVersion)
WHERE dv.tildaid = COALESCE($version_id, d.primaryVersionId)
AND (dv.uploadMethod IS NULL OR dv.uploadMethod <> 'Internally Modified')
AND d.delpending = false
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
AND (
(o.orgType = "CRO"
AND (EXISTS { (d)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] }
OR EXISTS { (:GoldenDataset)-[:CONTAINS_DOCUMENT]->(d) }))
OR (o.orgType <> "CRO")
)
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH d, o, ai,
CASE WHEN ai IS NOT NULL AND ai.status IN ['in_progress', 'aborted'] THEN ai.tildaid END AS in_progress_ai_run_id,
CASE WHEN ai IS NOT NULL AND ai.status IN ['in_progress', 'aborted'] THEN ai.run_type END AS in_progress_ai_run_run_type,
CASE WHEN ai IS NOT NULL AND ai.status = 'complete' THEN ai.tildaid END AS completed_ai_run_id,
CASE WHEN ai IS NOT NULL AND ai.status = 'complete' THEN ai.run_type END AS completed_ai_run_run_type,
CASE WHEN ai IS NOT NULL AND ai.status = 'failed' THEN COALESCE(ai.auto_retry_count, 0) END AS failed_latest_auto_retry_count
RETURN d.tildaid AS document_id,
COALESCE($version_id, d.primaryVersionId) AS version_id,
d.studyId AS study_id,
o.alias AS org_alias,
o.tildaid AS org_id,
in_progress_ai_run_id,
in_progress_ai_run_run_type,
completed_ai_run_id,
completed_ai_run_run_type,
failed_latest_auto_retry_count,
COALESCE(d.aiEnabled, false) AS ai_enabled,
o.orgType AS org_type,
EXISTS { (d)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] } AS in_staging,
EXISTS { (:GoldenDataset)-[:CONTAINS_DOCUMENT]->(d) } AS is_golden_dataset
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 28416.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "73550B79",
"id": "25411484",
"pageFaults": 0.0,
"pageHits": 21.0,
"planning": 0.0,
"query": "\n MATCH (d:Document {tildaid: $document_id})\n MATCH (d)-[:DOC_TO_ORG]-(o:Organization)\n MATCH (d)-[:DOCUMENT_VERSION]-\u003e(dv:DocumentVersion)\n WHERE dv.tildaid = COALESCE($version_id, d.primaryVersionId)\n AND (dv.uploadMethod IS NULL OR dv.uploadMethod \u003c\u003e \u0027Internally Modified\u0027)\n AND d.delpending = false\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n AND (\n (o.orgType = \"CRO\"\n AND (EXISTS { (d)-[:INCLUDES]-(f:Folder) WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027] }\n OR EXISTS { (:GoldenDataset)-[:CONTAINS_DOCUMENT]-\u003e(d) }))\n OR (o.orgType \u003c\u003e \"CRO\")\n )\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH d, o, ai,\n CASE WHEN ai IS NOT NULL AND ai.status IN [\u0027in_progress\u0027, \u0027aborted\u0027] THEN ai.tildaid END AS in_progress_ai_run_id,\n CASE WHEN ai IS NOT NULL AND ai.status IN [\u0027in_progress\u0027, \u0027aborted\u0027] THEN ai.run_type END AS in_progress_ai_run_run_type,\n CASE WHEN ai IS NOT NULL AND ai.status = \u0027complete\u0027 THEN ai.tildaid END AS completed_ai_run_id,\n CASE WHEN ai IS NOT NULL AND ai.status = \u0027complete\u0027 THEN ai.run_type END AS completed_ai_run_run_type,\n CASE WHEN ai IS NOT NULL AND ai.status = \u0027failed\u0027 THEN COALESCE(ai.auto_retry_count, 0) END AS failed_latest_auto_retry_count\n RETURN d.tildaid AS document_id,\n COALESCE($version_id, d.primaryVersionId) AS version_id,\n d.studyId AS study_id,\n o.alias AS org_alias,\n o.tildaid AS org_id,\n in_progress_ai_run_id,\n in_progress_ai_run_run_type,\n completed_ai_run_id,\n completed_ai_run_run_type,\n failed_latest_auto_retry_count,\n COALESCE(d.aiEnabled, false) AS ai_enabled,\n o.orgType AS org_type,\n EXISTS { (d)-[:INCLUDES]-(f:Folder) WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027] } AS in_staging,\n EXISTS { (:GoldenDataset)-[:CONTAINS_DOCUMENT]-\u003e(d) } AS is_golden_dataset\n LIMIT 1\n ",
"queryParameters": "{document_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027, version_id: \u002765ac9b5f-2b28-4cf8-a86b-496962cc36ef\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:53256\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.695Z",
"transactionId": 12258975.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4699.5,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 406.5,
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "linearize",
"org_alias": "sandboxIcon",
"requested_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.695Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Image metrics computed: 1 total pages, 1 selected for rendering
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceprocessors/doc_converter.py:388
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Image metrics computed: 1 total pages, 1 selected for rendering
raw payload (9 fields)
{
"caller": "processors/doc_converter.py:388",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Image metrics computed: 1 total pages, 1 selected for rendering",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.694Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Step 7/7 | translation disabled skipped (translation disabled)
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Step 7/7 | translation disabled skipped (translation disabled)
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 4.42,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Step 7/7 | translation disabled skipped (translation disabled)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"skip_reason": "translation disabled",
"status": "skipped",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.691Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] [5/7] Settings ...................... OK 0.01s (Disabled)
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [5/7] Settings ...................... OK 0.01s (Disabled)
raw payload (18 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.01,
"elapsed_s": 4.42,
"environment": "DEVELOPMENT",
"lang_detect_threshold": 0.0,
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [5/7] Settings ...................... OK 0.01s (Disabled)",
"mode": "Disabled",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"step": "translation_settings_check",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.691Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Processing document event
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:915
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span34d06c040e19fa6e
poddocproc-7bc76669-ljh8h
message
Processing document event
raw payload (14 fields)
{
"caller": "handlers/pubsub_handler.py:915",
"cluster": "master",
"component": "pubsub",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Processing document event",
"operation": "Update",
"org_alias": "sandboxIcon",
"service": "docproc",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.689Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Processing message
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:549
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span34d06c040e19fa6e
poddocproc-7bc76669-ljh8h
message
Processing message
raw payload (14 fields)
{
"caller": "handlers/pubsub_handler.py:549",
"cluster": "master",
"component": "pubsub",
"delivery_attempt": 1.0,
"environment": "DEVELOPMENT",
"message": "Processing message",
"message_id": "21226334429042141",
"operation": "Update",
"org_alias": "sandboxIcon",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.689Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Received messages
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:258
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Received messages
raw payload (10 fields)
{
"caller": "handlers/pubsub_handler.py:258",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Received messages",
"message_count": 1.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T02:24:53.687Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (d:Document{tildaid: $document_id})-[:DOC_TO_ORG]->(org:Organization) MATCH (d)-[:DOCUMENT_VERSION]->(dv:DocumentVersion{tildaid:...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (d:Document{tildaid: $document_id})-[:DOC_TO_ORG]->(org:Organization)
MATCH (d)-[:DOCUMENT_VERSION]->(dv:DocumentVersion{tildaid: $version_id})
OPTIONAL MATCH (org)-[:USES]->(fm:FolderManager{name: "StudyInstance", scopeId: d.studyId})
RETURN org.orgType AS org_type,
COALESCE(dv.langDetectionStatus, "") AS lang_detection_status,
COALESCE(fm.docTranslationMode, org.docTranslationMode, "Disabled") AS translation_mode,
COALESCE(fm.langDetectMinThresholdPct, 0) AS lang_detect_min_threshold_pct
raw payload (15 fields)
{
"allocatedBytes": 336.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "543EF88F",
"id": "25411482",
"pageFaults": 0.0,
"pageHits": 45.0,
"planning": 0.0,
"query": "\n MATCH (d:Document{tildaid: $document_id})-[:DOC_TO_ORG]-\u003e(org:Organization)\n MATCH (d)-[:DOCUMENT_VERSION]-\u003e(dv:DocumentVersion{tildaid: $version_id})\n OPTIONAL MATCH (org)-[:USES]-\u003e(fm:FolderManager{name: \"StudyInstance\", scopeId: d.studyId})\n RETURN org.orgType AS org_type,\n COALESCE(dv.langDetectionStatus, \"\") AS lang_detection_status,\n COALESCE(fm.docTranslationMode, org.docTranslationMode, \"Disabled\") AS translation_mode,\n COALESCE(fm.langDetectMinThresholdPct, 0) AS lang_detect_min_threshold_pct\n ",
"queryParameters": "{document_id: \u0027d0d3e988-1c04-483d-b6c6-27fbdef67c85\u0027, version_id: \u0027bdc7f2d3-251a-4baf-9357-c1f63488750c\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.686Z",
"transactionId": 12258973.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "annotation_detection", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id":...
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "annotation_detection", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c", "content_type": "application/pdf", "document_size_mb": 0.0023679733276367188, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 80.00591993331909, "actual_mb": 7.7265625, "memory_error_pct": 90.34251152109782, "duration_seconds": 0.9734628200531006, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119093.6811347, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"annotation_detection\", \"document_id\": \"d0d3e988-1c04-483d-b6c6-27fbdef67c85\", \"version_id\": \"bdc7f2d3-251a-4baf-9357-c1f63488750c\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.0023679733276367188, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 80.00591993331909, \"actual_mb\": 7.7265625, \"memory_error_pct\": 90.34251152109782, \"duration_seconds\": 0.9734628200531006, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119093.6811347, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.681Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] [3/7] Annotations ................... OK 0.97s (none found)
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [3/7] Annotations ................... OK 0.97s (none found)
raw payload (20 fields)
{
"annotation_types": null,
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.97,
"elapsed_s": 4.41,
"environment": "DEVELOPMENT",
"has_annotations": false,
"has_track_changes": false,
"is_redacted": false,
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [3/7] Annotations ................... OK 0.97s (none found)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"step": "annotation_detection",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.680Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Annotation detection complete
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Annotation detection complete
raw payload (21 fields)
{
"annotation_types": [],
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.97,
"environment": "DEVELOPMENT",
"has_annotations": false,
"has_track_changes": false,
"is_redacted": false,
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Annotation detection complete",
"org_alias": "sandboxIcon",
"processor": "annotation_detector",
"service": "docproc",
"status": "success",
"step_times": {
"detect_pdf_annotations": 0.9,
"update_neo4j": 0.07
},
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.680Z",
"total_s": 0.97,
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Update Neo4J OK 0.07s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Update Neo4J OK 0.07s
raw payload (19 fields)
{
"annotations": "not present",
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.07,
"elapsed_s": 0.97,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Update Neo4J OK 0.07s",
"org_alias": "sandboxIcon",
"processor": "annotation_detector",
"redacted": "no",
"service": "docproc",
"step": "update_neo4j",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.680Z",
"track_changes": "not present",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion {tildaid: $version_id}) SET dv.annotations = $annotations_status, dv.annotationTypes =...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (dv:DocumentVersion {tildaid: $version_id})
SET dv.annotations = $annotations_status,
dv.annotationTypes = $annotation_types,
dv.redacted = $redacted_status,
dv.trackChanges = $track_changes_status
WITH dv
OPTIONAL MATCH (dv)<-[:DOCUMENT_VERSION]-(doc:Document)
SET doc.annotations = $annotations_status,
doc.annotationTypes = $annotation_types,
doc.redacted = $redacted_status,
doc.trackChanges = $track_changes_status
RETURN dv.tildaid AS updated_version_id
raw payload (15 fields)
{
"allocatedBytes": 692.0,
"cpu": 45.0,
"elapsedTimeMs": 46.0,
"executionPlanCacheKeyHash": "8D7C202D",
"id": "25411480",
"pageFaults": 0.0,
"pageHits": 6.0,
"planning": 42.0,
"query": "\n MATCH (dv:DocumentVersion {tildaid: $version_id})\n SET dv.annotations = $annotations_status,\n dv.annotationTypes = $annotation_types,\n dv.redacted = $redacted_status,\n dv.trackChanges = $track_changes_status\n WITH dv\n OPTIONAL MATCH (dv)\u003c-[:DOCUMENT_VERSION]-(doc:Document)\n SET doc.annotations = $annotations_status,\n doc.annotationTypes = $annotation_types,\n doc.redacted = $redacted_status,\n doc.trackChanges = $track_changes_status\n RETURN dv.tildaid AS updated_version_id\n ",
"queryParameters": "{annotations_status: \u0027not present\u0027, track_changes_status: \u0027not present\u0027, annotation_types: [], version_id: \u0027bdc7f2d3-251a-4baf-9357-c1f63488750c\u0027, redacted_status: \u0027no\u0027, timeout: \u003cnull\u003e}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.665Z",
"transactionId": 12258972.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Detect Pdf Annotations OK 0.9s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Detect Pdf Annotations OK 0.9s
raw payload (24 fields)
{
"annotation_types": [],
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"comments": 0.0,
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.9,
"elapsed_s": 0.91,
"environment": "DEVELOPMENT",
"has_annotations": false,
"is_redacted": false,
"markups": 0.0,
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Detect Pdf Annotations OK 0.9s",
"org_alias": "sandboxIcon",
"processor": "annotation_detector",
"redactions": 0.0,
"service": "docproc",
"shapes": 0.0,
"step": "detect_pdf_annotations",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.613Z",
"total_count": 0.0,
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Annotations are not present
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/pdf_detector.py:175
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Annotations are not present
raw payload (15 fields)
{
"annotations_present": false,
"caller": "utils/pdf_detector.py:175",
"cluster": "master",
"comments": 0.0,
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_g0ly2icp.pdf",
"is_redacted": false,
"markups": 0.0,
"message": "Annotations are not present",
"redactions": 0.0,
"service": "docproc",
"shapes": 0.0,
"timestamp": "2026-07-27T02:24:53.612Z",
"total": 0.0,
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc No applied redactions detected
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceutils/pdf_detector.py:347
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
No applied redactions detected
raw payload (8 fields)
{
"caller": "utils/pdf_detector.py:347",
"cluster": "master",
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_g0ly2icp.pdf",
"message": "No applied redactions detected",
"service": "docproc",
"timestamp": "2026-07-27T02:24:53.612Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]->(doc:Document {tildaid: $doc_id}) RETURN tracker
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]->(doc:Document {tildaid: $doc_id})
RETURN tracker
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 0.0,
"elapsedTimeMs": 0.0,
"executionPlanCacheKeyHash": "B8141227",
"id": "25411478",
"pageFaults": 0.0,
"pageHits": 5.0,
"planning": 0.0,
"query": "\n MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]-\u003e(doc:Document {tildaid: $doc_id})\n RETURN tracker\n ",
"queryParameters": "{doc_id: \u0027f92da704-917e-49c9-9f6c-0310c9d8475c\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.512Z",
"transactionId": 12258971.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC INFO docproc [extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=3074 bytes
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourcecommon/docparse.py:310
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=3074 bytes
raw payload (9 fields)
{
"caller": "common/docparse.py:310",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=3074 bytes",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.498Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc PDF type detected for TMF
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:911
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
PDF type detected for TMF
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:911",
"cluster": "master",
"component": "orchestrator_certification",
"environment": "DEVELOPMENT",
"is_scanned": false,
"is_text_based": true,
"message": "PDF type detected for TMF",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.497Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Extracted document pages for TMF
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:893
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Extracted document pages for TMF
raw payload (13 fields)
{
"caller": "orchestration/document_orchestrator.py:893",
"cluster": "master",
"component": "orchestrator_certification",
"environment": "DEVELOPMENT",
"extracted_pages": 1.0,
"is_full_document": true,
"message": "Extracted document pages for TMF",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.490Z",
"total_pages": 1.0,
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=3074 bytes
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourcecommon/docparse.py:310
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=3074 bytes
raw payload (9 fields)
{
"caller": "common/docparse.py:310",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=3074 bytes",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.487Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 200.5,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "tmf_preprocess",
"org_alias": "sandboxIcon",
"reserved_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.481Z",
"version": "0.1.0",
"wait_seconds": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4659.5,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 405.5,
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "tmf_preprocess",
"org_alias": "sandboxIcon",
"requested_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.480Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "download_file", "document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c", "version_id":...
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "download_file", "document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c", "version_id": "52db0899-2c0b-41c7-b389-565bf41c1987", "content_type": "application/pdf", "document_size_mb": 0.0029315948486328125, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 40.00439739227295, "actual_mb": 0.515625, "memory_error_pct": 98.71107919725947, "duration_seconds": 0.08275651931762695, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119093.4779897, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"download_file\", \"document_id\": \"f92da704-917e-49c9-9f6c-0310c9d8475c\", \"version_id\": \"52db0899-2c0b-41c7-b389-565bf41c1987\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.0029315948486328125, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 40.00439739227295, \"actual_mb\": 0.515625, \"memory_error_pct\": 98.71107919725947, \"duration_seconds\": 0.08275651931762695, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119093.4779897, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.478Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [f92da704-917e-49c9-9f6c-0310c9d8475c] [2/7] Download ...................... OK 0.08s (0.00 MB,...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[f92da704-917e-49c9-9f6c-0310c9d8475c] [2/7] Download ...................... OK 0.08s (0.00 MB, gs://master-tilda-shared-storage/master/sandboxIcon/52db/52db0899-2c0b-41c7-b389-565bf41c1987)
raw payload (20 fields)
{
"caller": "utils/processing_logger.py:194",
"checksum_sync_skipped": true,
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"duration_s": 0.08,
"elapsed_s": 0.18,
"environment": "DEVELOPMENT",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/52db/52db0899-2c0b-41c7-b389-565bf41c1987",
"message": "[f92da704-917e-49c9-9f6c-0310c9d8475c] [2/7] Download ...................... OK 0.08s (0.00 MB, gs://master-tilda-shared-storage/master/sandboxIcon/52db/52db0899-2c0b-41c7-b389-565bf41c1987)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"size_mb": "0.00 MB",
"step": "download_original",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.477Z",
"used_linearized": false,
"version": "0.1.0",
"version_id": "52db0899-2c0b-41c7-b389-565bf41c1987"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceorchestration/document_orchestrator.py:701
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:701",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"environment": "DEVELOPMENT",
"message": "Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.474Z",
"version": "0.1.0",
"version_id": "52db0899-2c0b-41c7-b389-565bf41c1987"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Downloaded bytes from GCS
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_gcs/client.py:270
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Downloaded bytes from GCS
raw payload (12 fields)
{
"caller": "tilda_gcs/client.py:270",
"cluster": "master",
"component": "gcs",
"environment": "DEVELOPMENT",
"message": "Downloaded bytes from GCS",
"org_alias": "sandboxIcon",
"path": "master/sandboxIcon/52db/52db0899-2c0b-41c7-b389-565bf41c1987",
"service": "docproc",
"size_bytes": 3074.0,
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.473Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 200.5,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "download_file",
"org_alias": "sandboxIcon",
"reserved_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.394Z",
"version": "0.1.0",
"wait_seconds": 0.02
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4659.5,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 404.9,
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "download_file",
"org_alias": "sandboxIcon",
"requested_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.393Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [f92da704-917e-49c9-9f6c-0310c9d8475c] [1/7] Context ....................... OK 0.08s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[f92da704-917e-49c9-9f6c-0310c9d8475c] [1/7] Context ....................... OK 0.08s
raw payload (19 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"duration_s": 0.08,
"elapsed_s": 0.04,
"environment": "DEVELOPMENT",
"filename": "Site101_NorthAmerican_NewPI_MedicalLicense(8).pdf",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/52db/52db0899-2c0b-41c7-b389-565bf41c1987",
"message": "[f92da704-917e-49c9-9f6c-0310c9d8475c] [1/7] Context ....................... OK 0.08s",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"size_mb": 0.0,
"step": "create_context",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.341Z",
"version": "0.1.0",
"version_id": "52db0899-2c0b-41c7-b389-565bf41c1987"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:592
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:592",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"environment": "DEVELOPMENT",
"message": "Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.341Z",
"version": "0.1.0",
"work_order_id": "c4efcc9f-9847-4099-a208-de401b1ac83f"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc [ASYNC_WO_QUERY] ✅ Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=f92da704-917e-49c9-9f6c-0310c9d8475c
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetmf/work_order_queries.py:83
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[ASYNC_WO_QUERY] ✅ Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=f92da704-917e-49c9-9f6c-0310c9d8475c
raw payload (9 fields)
{
"caller": "tmf/work_order_queries.py:83",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[ASYNC_WO_QUERY] \u2705 Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=f92da704-917e-49c9-9f6c-0310c9d8475c",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.330Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id}) WHERE w.delpending = false RETURN w.tildaid AS...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id})
WHERE w.delpending = false
RETURN w.tildaid AS work_order_id, coalesce(w.title, '') AS work_order_name
ORDER BY coalesce(w.updatedat, w.createdat) DESC
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 684.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "1D6BF7F0",
"id": "25411470",
"pageFaults": 0.0,
"pageHits": 8.0,
"planning": 0.0,
"query": "\n MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]-\u003e(d:Document {tildaid: $doc_id})\n WHERE w.delpending = false\n RETURN w.tildaid AS work_order_id, coalesce(w.title, \u0027\u0027) AS work_order_name\n ORDER BY coalesce(w.updatedat, w.createdat) DESC\n LIMIT 1\n ",
"queryParameters": "{doc_id: \u0027f92da704-917e-49c9-9f6c-0310c9d8475c\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.322Z",
"transactionId": 12258970.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc [ASYNC_WO_QUERY] Finding work order for doc_id=f92da704-917e-49c9-9f6c-0310c9d8475c
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetmf/work_order_queries.py:78
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[ASYNC_WO_QUERY] Finding work order for doc_id=f92da704-917e-49c9-9f6c-0310c9d8475c
raw payload (9 fields)
{
"caller": "tmf/work_order_queries.py:78",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[ASYNC_WO_QUERY] Finding work order for doc_id=f92da704-917e-49c9-9f6c-0310c9d8475c",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.318Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [f92da704-917e-49c9-9f6c-0310c9d8475c] ══════════════════════════════════════════════════════
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[f92da704-917e-49c9-9f6c-0310c9d8475c] ══════════════════════════════════════════════════════
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"elapsed_s": 0.01,
"environment": "DEVELOPMENT",
"message": "[f92da704-917e-49c9-9f6c-0310c9d8475c] \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.312Z",
"version": "0.1.0",
"version_id": "52db0899-2c0b-41c7-b389-565bf41c1987"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [f92da704-917e-49c9-9f6c-0310c9d8475c] 3/10 slots in use
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[f92da704-917e-49c9-9f6c-0310c9d8475c] 3/10 slots in use
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"elapsed_s": 0.01,
"environment": "DEVELOPMENT",
"message": "[f92da704-917e-49c9-9f6c-0310c9d8475c] 3/10 slots in use",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.312Z",
"version": "0.1.0",
"version_id": "52db0899-2c0b-41c7-b389-565bf41c1987"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [f92da704-917e-49c9-9f6c-0310c9d8475c] PIPELINE START — Site101_NorthAmerican_NewPI_MedicalLicense(8).pdf (0.00 MB)
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[f92da704-917e-49c9-9f6c-0310c9d8475c] PIPELINE START — Site101_NorthAmerican_NewPI_MedicalLicense(8).pdf (0.00 MB)
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[f92da704-917e-49c9-9f6c-0310c9d8475c] PIPELINE START \u2014 Site101_NorthAmerican_NewPI_MedicalLicense(8).pdf (0.00 MB)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.301Z",
"version": "0.1.0",
"version_id": "52db0899-2c0b-41c7-b389-565bf41c1987"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [f92da704-917e-49c9-9f6c-0310c9d8475c] ══════════════════════════════════════════════════════
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
[f92da704-917e-49c9-9f6c-0310c9d8475c] ══════════════════════════════════════════════════════
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[f92da704-917e-49c9-9f6c-0310c9d8475c] \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.301Z",
"version": "0.1.0",
"version_id": "52db0899-2c0b-41c7-b389-565bf41c1987"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (o:Organization {tildaid: $org_id}) RETURN o.orgType AS org_type
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (o:Organization {tildaid: $org_id})
RETURN o.orgType AS org_type
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 0.0,
"elapsedTimeMs": 0.0,
"executionPlanCacheKeyHash": "0393DF4C",
"id": "25411468",
"pageFaults": 0.0,
"pageHits": 4.0,
"planning": 0.0,
"query": "MATCH (o:Organization {tildaid: $org_id}) RETURN o.orgType AS org_type",
"queryParameters": "{org_id: \u00279f996610-f74a-4cd5-9052-cb5405d78c6f\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.292Z",
"transactionId": 12258969.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id}) WHERE w.delpending = false RETURN w.tildaid AS...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id})
WHERE w.delpending = false
RETURN w.tildaid AS work_order_id, coalesce(w.title, '') AS work_order_name
ORDER BY coalesce(w.updatedat, w.createdat) DESC
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 684.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "1D6BF7F0",
"id": "25411466",
"pageFaults": 0.0,
"pageHits": 8.0,
"planning": 0.0,
"query": "\n MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]-\u003e(d:Document {tildaid: $doc_id})\n WHERE w.delpending = false\n RETURN w.tildaid AS work_order_id, coalesce(w.title, \u0027\u0027) AS work_order_name\n ORDER BY coalesce(w.updatedat, w.createdat) DESC\n LIMIT 1\n ",
"queryParameters": "{doc_id: \u0027f92da704-917e-49c9-9f6c-0310c9d8475c\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.279Z",
"transactionId": 12258968.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (dv:DocumentVersion{tildaid:$version_id})<-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id}) MATCH...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (dv:DocumentVersion{tildaid:$version_id})<-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id})
MATCH (doc)-[:DOC_TO_ORG]->(org:Organization)
OPTIONAL MATCH (study:Study{tildaid:doc.studyId})
WITH dv, doc, org, study
RETURN dv.tildaid AS version_id,
dv.url AS url,
doc.fileName AS filename,
doc.contentType AS content_type,
dv.size AS size_bytes,
dv.linearizedPdfUrl AS linearized_pdf_url,
dv.linearizedPdfStatus AS linearized_status,
dv.languages AS languages,
dv.langDetectionStatus AS lang_detection_status,
dv.translatedPdfUrl AS translated_pdf_url,
dv.translatedPdfStatus AS translated_status,
study.tildaid AS study_id,
study.name AS study_name,
study.protocolnumber AS protocol_number,
org.tildaid AS org_id,
org.alias AS org_alias,
org.name AS org_name,
dv.uploadMethod AS upload_method,
org.orgType AS org_type,
dv.cksum AS checksum
raw payload (15 fields)
{
"allocatedBytes": 8376.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "8FE8F25D",
"id": "25411465",
"pageFaults": 0.0,
"pageHits": 18.0,
"planning": 0.0,
"query": "\n MATCH (dv:DocumentVersion{tildaid:$version_id})\u003c-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id})\n MATCH (doc)-[:DOC_TO_ORG]-\u003e(org:Organization)\n OPTIONAL MATCH (study:Study{tildaid:doc.studyId})\n WITH dv, doc, org, study\n RETURN dv.tildaid AS version_id,\n dv.url AS url,\n doc.fileName AS filename,\n doc.contentType AS content_type,\n dv.size AS size_bytes,\n dv.linearizedPdfUrl AS linearized_pdf_url,\n dv.linearizedPdfStatus AS linearized_status,\n dv.languages AS languages,\n dv.langDetectionStatus AS lang_detection_status,\n dv.translatedPdfUrl AS translated_pdf_url,\n dv.translatedPdfStatus AS translated_status,\n study.tildaid AS study_id,\n study.name AS study_name,\n study.protocolnumber AS protocol_number,\n org.tildaid AS org_id,\n org.alias AS org_alias,\n org.name AS org_name,\n dv.uploadMethod AS upload_method,\n org.orgType AS org_type,\n dv.cksum AS checksum\n ",
"queryParameters": "{document_id: \u0027f92da704-917e-49c9-9f6c-0310c9d8475c\u0027, version_id: \u002752db0899-2c0b-41c7-b389-565bf41c1987\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.267Z",
"transactionId": 12258967.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (ds:GoldenDataset)-[:CONTAINS_DOCUMENT]->(:Document {tildaid: $document_id}) RETURN ds.tildaid AS golden_dataset_id LIMIT 1
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (ds:GoldenDataset)-[:CONTAINS_DOCUMENT]->(:Document {tildaid: $document_id})
RETURN ds.tildaid AS golden_dataset_id
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 328.0,
"cpu": 0.0,
"elapsedTimeMs": 0.0,
"executionPlanCacheKeyHash": "D4A26122",
"id": "25411464",
"pageFaults": 0.0,
"pageHits": 5.0,
"planning": 0.0,
"query": "\n MATCH (ds:GoldenDataset)-[:CONTAINS_DOCUMENT]-\u003e(:Document {tildaid: $document_id})\n RETURN ds.tildaid AS golden_dataset_id\n LIMIT 1\n ",
"queryParameters": "{document_id: \u0027f92da704-917e-49c9-9f6c-0310c9d8475c\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.253Z",
"transactionId": 12258966.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (n:AIRun {tildaid: $node_id}) RETURN n
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (n:AIRun {tildaid: $node_id})
RETURN n
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "316B40A9",
"id": "25411463",
"pageFaults": 0.0,
"pageHits": 6.0,
"planning": 0.0,
"query": "MATCH (n:AIRun {tildaid: $node_id}) RETURN n",
"queryParameters": "{node_id: \u00277e91c56b-7877-4193-a001-2d1530047235\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.240Z",
"transactionId": 12258965.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (d:Document {tildaid: $doc_id}) SET d.cbrRunStatus = $status, d.cbrRunStatusAt = $ts
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (d:Document {tildaid: $doc_id})
SET d.cbrRunStatus = $status, d.cbrRunStatusAt = $ts
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "532B3015",
"id": "25411462",
"pageFaults": 0.0,
"pageHits": 5.0,
"planning": 0.0,
"query": "\n MATCH (d:Document {tildaid: $doc_id})\n SET d.cbrRunStatus = $status, d.cbrRunStatusAt = $ts\n ",
"queryParameters": "{doc_id: \u0027f92da704-917e-49c9-9f6c-0310c9d8475c\u0027, timeout: \u003cnull\u003e, status: \u003cnull\u003e, ts: 1785119093}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:53.212Z",
"transactionId": 12258964.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC INFO docproc Created document processing AIRun
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceorchestration/document_orchestrator.py:3737
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Created document processing AIRun
raw payload (12 fields)
{
"ai_run_id": "7e91c56b-7877-4193-a001-2d1530047235",
"caller": "orchestration/document_orchestrator.py:3737",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "f92da704-917e-49c9-9f6c-0310c9d8475c",
"environment": "DEVELOPMENT",
"message": "Created document processing AIRun",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.201Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Transaction completed in 0.13s
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_neo4j/client.py:261
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Transaction completed in 0.13s
raw payload (10 fields)
{
"caller": "tilda_neo4j/client.py:261",
"cluster": "master",
"component": "neo4j",
"environment": "DEVELOPMENT",
"message": "Transaction completed in 0.13s",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.196Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "property_extraction", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id":...
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "property_extraction", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef", "content_type": "application/pdf", "document_size_mb": 0.1843891143798828, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 0.9219455718994141, "actual_mb": 3.3125, "memory_error_pct": 259.2945289791358, "duration_seconds": 0.33315253257751465, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119093.195436, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"property_extraction\", \"document_id\": \"a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\", \"version_id\": \"65ac9b5f-2b28-4cf8-a86b-496962cc36ef\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.1843891143798828, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 0.9219455718994141, \"actual_mb\": 3.3125, \"memory_error_pct\": 259.2945289791358, \"duration_seconds\": 0.33315253257751465, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119093.195436, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.195Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Properties OK 0.3s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Properties OK 0.3s
raw payload (22 fields)
{
"author": null,
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"created_by": null,
"creation_date": null,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.3,
"elapsed_s": 3.92,
"environment": "DEVELOPMENT",
"last_modified_by": "pypdf",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Properties OK 0.3s",
"modified_date": null,
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"security": "No Protection",
"service": "docproc",
"step": "property_extraction",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.195Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Property extraction complete
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Property extraction complete
raw payload (23 fields)
{
"author": null,
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"created_by": null,
"creation_date": null,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.29,
"environment": "DEVELOPMENT",
"last_modified_by": "pypdf",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Property extraction complete",
"modified_date": null,
"org_alias": "sandboxIcon",
"processor": "property_extractor",
"security": "No Protection",
"service": "docproc",
"status": "success",
"step_times": {
"extract_properties": 0.15,
"update_neo4j": 0.1
},
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.184Z",
"total_s": 0.29,
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Update Neo4J OK 0.1s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Update Neo4J OK 0.1s
raw payload (17 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.1,
"elapsed_s": 0.29,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Update Neo4J OK 0.1s",
"org_alias": "sandboxIcon",
"processor": "property_extractor",
"properties_written": true,
"service": "docproc",
"step": "update_neo4j",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.184Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC neo4j docproc MATCH (a:AIRun {tildaid: $from_id}) MATCH (b:Document {tildaid: $to_id}) MERGE (a)-[r:AI_RUN_TO_DOCUMENT_LATEST]->(b) ON CREATE SET r =...
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
podserver-2-dev-0
query
MATCH (a:AIRun {tildaid: $from_id})
MATCH (b:Document {tildaid: $to_id})
MERGE (a)-[r:AI_RUN_TO_DOCUMENT_LATEST]->(b)
ON CREATE SET r = $properties, r.createdat = timestamp()
RETURN r
raw payload (15 fields)
{
"allocatedBytes": 3920.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "8E40208B",
"id": "25411455",
"pageFaults": 0.0,
"pageHits": 6.0,
"planning": 0.0,
"query": "MATCH (a:AIRun {tildaid: $from_id}) MATCH (b:Document {tildaid: $to_id}) MERGE (a)-[r:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(b) ON CREATE SET r = $properties, r.createdat = timestamp() RETURN r",
"queryParameters": "{to_id: \u0027f92da704-917e-49c9-9f6c-0310c9d8475c\u0027, from_id: \u00277e91c56b-7877-4193-a001-2d1530047235\u0027, properties: {}}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:53.159Z",
"transactionId": 12258962.0,
"waiting": 0.0
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "property_extraction", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id":...
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "property_extraction", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c", "content_type": "application/pdf", "document_size_mb": 0.0023679733276367188, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 0.011839866638183594, "actual_mb": 6.75, "memory_error_pct": 56910.77728554168, "duration_seconds": 0.41480469703674316, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119093.1503875, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"property_extraction\", \"document_id\": \"d0d3e988-1c04-483d-b6c6-27fbdef67c85\", \"version_id\": \"bdc7f2d3-251a-4baf-9357-c1f63488750c\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.0023679733276367188, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 0.011839866638183594, \"actual_mb\": 6.75, \"memory_error_pct\": 56910.77728554168, \"duration_seconds\": 0.41480469703674316, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119093.1503875, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.150Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Properties OK 0.41s (by anonymous)
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Properties OK 0.41s (by anonymous)
raw payload (22 fields)
{
"author": "anonymous",
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"created_by": "anonymous",
"creation_date": "2026-07-03T12:52:50+05:00",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.41,
"elapsed_s": 3.86,
"environment": "DEVELOPMENT",
"last_modified_by": "ReportLab PDF Library - (opensource)",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Properties OK 0.41s (by anonymous)",
"modified_date": "2026-07-03T12:52:50+05:00",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"security": "No Protection",
"service": "docproc",
"step": "property_extraction",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.129Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Property extraction complete
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Property extraction complete
raw payload (23 fields)
{
"author": "anonymous",
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"created_by": "anonymous",
"creation_date": "2026-07-03T12:52:50+05:00",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.41,
"environment": "DEVELOPMENT",
"last_modified_by": "ReportLab PDF Library - (opensource)",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Property extraction complete",
"modified_date": "2026-07-03T12:52:50+05:00",
"org_alias": "sandboxIcon",
"processor": "property_extractor",
"security": "No Protection",
"service": "docproc",
"status": "success",
"step_times": {
"extract_properties": 0.01,
"update_neo4j": 0.39
},
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.129Z",
"total_s": 0.41,
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Update Neo4J OK 0.39s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Update Neo4J OK 0.39s
raw payload (17 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.39,
"elapsed_s": 0.4,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Update Neo4J OK 0.39s",
"org_alias": "sandboxIcon",
"processor": "property_extractor",
"properties_written": true,
"service": "docproc",
"step": "update_neo4j",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.118Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}7s ago2026-07-27 02:24:53 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Extract Properties OK 0.15s
time2026-07-27 02:24:53 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Extract Properties OK 0.15s
raw payload (22 fields)
{
"author": null,
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"created_by": null,
"creation_date": null,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.15,
"elapsed_s": 0.18,
"environment": "DEVELOPMENT",
"last_modified_by": "pypdf",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Extract Properties OK 0.15s",
"modified_date": null,
"org_alias": "sandboxIcon",
"processor": "property_extractor",
"security": "No Protection",
"service": "docproc",
"step": "extract_properties",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.073Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc Starting execute_in_transaction
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourcetilda_neo4j/client.py:248
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
Starting execute_in_transaction
raw payload (10 fields)
{
"caller": "tilda_neo4j/client.py:248",
"cluster": "master",
"component": "neo4j",
"environment": "DEVELOPMENT",
"message": "Starting execute_in_transaction",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.066Z",
"version": "0.1.0"
}7s ago2026-07-27 02:24:53 UTC DEBUG docproc docsvc PATCH succeeded for doc f92da704-917e-49c9-9f6c-0310c9d8475c: status=207
time2026-07-27 02:24:53 UTC
severityDEBUG
servicedocproc
sourceapi/recommendation_updater.py:323
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace03455c3fe795e64c7a5f9e9ee5e06c4f
span4eb087f6059c372b
poddocproc-7bc76669-ljh8h
message
docsvc PATCH succeeded for doc f92da704-917e-49c9-9f6c-0310c9d8475c: status=207
raw payload (9 fields)
{
"caller": "api/recommendation_updater.py:323",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "docsvc PATCH succeeded for doc f92da704-917e-49c9-9f6c-0310c9d8475c: status=207",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:53.036Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Property extraction started
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Property extraction started
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Property extraction started",
"org_alias": "sandboxIcon",
"processor": "property_extractor",
"service": "docproc",
"size_bytes": 193346.0,
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.897Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}8s ago2026-07-27 02:24:52 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 161.4,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "property_extraction",
"org_alias": "sandboxIcon",
"reserved_mb": 0.9,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.861Z",
"version": "0.1.0",
"wait_seconds": 0.07
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "convert_to_pdf", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id":...
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "convert_to_pdf", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef", "content_type": "application/pdf", "document_size_mb": 0.1843891143798828, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 0.9219455718994141, "actual_mb": 0.00390625, "memory_error_pct": 99.57630362148687, "duration_seconds": 0.0025975704193115234, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119092.7911024, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"convert_to_pdf\", \"document_id\": \"a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\", \"version_id\": \"65ac9b5f-2b28-4cf8-a86b-496962cc36ef\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.1843891143798828, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 0.9219455718994141, \"actual_mb\": 0.00390625, \"memory_error_pct\": 99.57630362148687, \"duration_seconds\": 0.0025975704193115234, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119092.7911024, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.791Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [4a/7] Convert PDF ................... OK 0.0s (0.18 MB)
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [4a/7] Convert PDF ................... OK 0.0s (0.18 MB)
raw payload (19 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"converted_from": null,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.0,
"elapsed_s": 3.51,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [4a/7] Convert PDF ................... OK 0.0s (0.18 MB)",
"org_alias": "sandboxIcon",
"pages": 0.0,
"pdf_mb": "0.18 MB",
"processor": "orchestrator",
"service": "docproc",
"step": "convert_to_pdf",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.790Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Doc Converter complete
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Doc Converter complete
raw payload (18 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Doc Converter complete",
"org_alias": "sandboxIcon",
"output_type": "pdf",
"processor": "doc_converter",
"service": "docproc",
"status": "success",
"step_times": null,
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.790Z",
"total_s": 0.0,
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Already Pdf
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Already Pdf
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"extension": ".pdf",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Already Pdf",
"org_alias": "sandboxIcon",
"processor": "doc_converter",
"service": "docproc",
"step": "already_pdf",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.790Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Doc Converter started
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Doc Converter started
raw payload (15 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Doc Converter started",
"org_alias": "sandboxIcon",
"processor": "doc_converter",
"service": "docproc",
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.790Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}8s ago2026-07-27 02:24:52 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 161.4,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "convert_to_pdf",
"org_alias": "sandboxIcon",
"reserved_mb": 0.9,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.779Z",
"version": "0.1.0",
"wait_seconds": 0.03
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4659.5,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 399.4,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "convert_to_pdf",
"org_alias": "sandboxIcon",
"requested_mb": 0.9,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.779Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc PDF opened
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourceutils/pdf_detector.py:142
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
PDF opened
raw payload (10 fields)
{
"caller": "utils/pdf_detector.py:142",
"cluster": "master",
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_rtn3no26.pdf",
"message": "PDF opened",
"pages_to_scan": 1.0,
"service": "docproc",
"timestamp": "2026-07-27T02:24:52.754Z",
"total_pages": 1.0,
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC INFO docproc Starting annotation detection
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/pdf_detector.py:134
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Starting annotation detection
raw payload (8 fields)
{
"caller": "utils/pdf_detector.py:134",
"cluster": "master",
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_rtn3no26.pdf",
"message": "Starting annotation detection",
"service": "docproc",
"timestamp": "2026-07-27T02:24:52.753Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc PDF magic bytes check
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourceutils/pdf_detector.py:74
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
PDF magic bytes check
raw payload (11 fields)
{
"caller": "utils/pdf_detector.py:74",
"cluster": "master",
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_rtn3no26.pdf",
"is_pdf": true,
"message": "PDF magic bytes check",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.752Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Annotation detection started
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Annotation detection started
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] Annotation detection started",
"org_alias": "sandboxIcon",
"processor": "annotation_detector",
"service": "docproc",
"size_bytes": 193346.0,
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.750Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}8s ago2026-07-27 02:24:52 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 160.5,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "annotation_detection",
"org_alias": "sandboxIcon",
"reserved_mb": 80.5,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.750Z",
"version": "0.1.0",
"wait_seconds": 0.0
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4740.0,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 398.7,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "annotation_detection",
"org_alias": "sandboxIcon",
"requested_mb": 80.5,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.750Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "tmf_preprocess", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id":...
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "tmf_preprocess", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef", "content_type": "application/pdf", "document_size_mb": 0.1843891143798828, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 40.368778228759766, "actual_mb": 0.0, "memory_error_pct": 100.0, "duration_seconds": 3.213874340057373, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119092.748067, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"tmf_preprocess\", \"document_id\": \"a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\", \"version_id\": \"65ac9b5f-2b28-4cf8-a86b-496962cc36ef\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.1843891143798828, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 40.368778228759766, \"actual_mb\": 0.0, \"memory_error_pct\": 100.0, \"duration_seconds\": 3.213874340057373, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119092.748067, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.748Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc OpenAI file_id created for TMF
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:954
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
OpenAI file_id created for TMF
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:954",
"cluster": "master",
"component": "orchestrator_certification",
"environment": "DEVELOPMENT",
"file_id": "file-S56jqG2PsNTEAkY...",
"full_file_id": "file-S56jqG2PsNTEAkYffDpZ5M",
"message": "OpenAI file_id created for TMF",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.747Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Transaction completed in 0.06s
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_neo4j/client.py:261
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Transaction completed in 0.06s
raw payload (10 fields)
{
"caller": "tilda_neo4j/client.py:261",
"cluster": "master",
"component": "neo4j",
"environment": "DEVELOPMENT",
"message": "Transaction completed in 0.06s",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.747Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Extract Properties OK 0.01s
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Extract Properties OK 0.01s
raw payload (22 fields)
{
"author": "anonymous",
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"created_by": "anonymous",
"creation_date": "2026-07-03T12:52:50+05:00",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.01,
"elapsed_s": 0.01,
"environment": "DEVELOPMENT",
"last_modified_by": "ReportLab PDF Library - (opensource)",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Extract Properties OK 0.01s",
"modified_date": "2026-07-03T12:52:50+05:00",
"org_alias": "sandboxIcon",
"processor": "property_extractor",
"security": "No Protection",
"service": "docproc",
"step": "extract_properties",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.728Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Property extraction started
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Property extraction started
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Property extraction started",
"org_alias": "sandboxIcon",
"processor": "property_extractor",
"service": "docproc",
"size_bytes": 2483.0,
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.720Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}8s ago2026-07-27 02:24:52 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 120.4,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "property_extraction",
"org_alias": "sandboxIcon",
"reserved_mb": 0.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.719Z",
"version": "0.1.0",
"wait_seconds": 0.0
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4699.6,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 398.8,
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "property_extraction",
"org_alias": "sandboxIcon",
"requested_mb": 0.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.719Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "convert_to_pdf", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id":...
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "convert_to_pdf", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c", "content_type": "application/pdf", "document_size_mb": 0.0023679733276367188, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 0.011839866638183594, "actual_mb": 0.0, "memory_error_pct": 100.0, "duration_seconds": 0.0011401176452636719, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119092.7192128, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"convert_to_pdf\", \"document_id\": \"d0d3e988-1c04-483d-b6c6-27fbdef67c85\", \"version_id\": \"bdc7f2d3-251a-4baf-9357-c1f63488750c\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.0023679733276367188, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 0.011839866638183594, \"actual_mb\": 0.0, \"memory_error_pct\": 100.0, \"duration_seconds\": 0.0011401176452636719, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119092.7192128, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.719Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] [4a/7] Convert PDF ................... OK 0.0s (0.00 MB)
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [4a/7] Convert PDF ................... OK 0.0s (0.00 MB)
raw payload (19 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"converted_from": null,
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.0,
"elapsed_s": 3.44,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [4a/7] Convert PDF ................... OK 0.0s (0.00 MB)",
"org_alias": "sandboxIcon",
"pages": 0.0,
"pdf_mb": "0.00 MB",
"processor": "orchestrator",
"service": "docproc",
"step": "convert_to_pdf",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.718Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Doc Converter complete
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Doc Converter complete
raw payload (18 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Doc Converter complete",
"org_alias": "sandboxIcon",
"output_type": "pdf",
"processor": "doc_converter",
"service": "docproc",
"status": "success",
"step_times": null,
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.718Z",
"total_s": 0.0,
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Already Pdf
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Already Pdf
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"extension": ".pdf",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Already Pdf",
"org_alias": "sandboxIcon",
"processor": "doc_converter",
"service": "docproc",
"step": "already_pdf",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.718Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Doc Converter started
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Doc Converter started
raw payload (15 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Doc Converter started",
"org_alias": "sandboxIcon",
"processor": "doc_converter",
"service": "docproc",
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.718Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}8s ago2026-07-27 02:24:52 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 120.4,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "convert_to_pdf",
"org_alias": "sandboxIcon",
"reserved_mb": 0.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.717Z",
"version": "0.1.0",
"wait_seconds": 0.01
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4699.6,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 398.7,
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "convert_to_pdf",
"org_alias": "sandboxIcon",
"requested_mb": 0.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.712Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc PDF opened
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourceutils/pdf_detector.py:142
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
PDF opened
raw payload (10 fields)
{
"caller": "utils/pdf_detector.py:142",
"cluster": "master",
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_g0ly2icp.pdf",
"message": "PDF opened",
"pages_to_scan": 1.0,
"service": "docproc",
"timestamp": "2026-07-27T02:24:52.711Z",
"total_pages": 1.0,
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC INFO docproc Starting annotation detection
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/pdf_detector.py:134
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Starting annotation detection
raw payload (8 fields)
{
"caller": "utils/pdf_detector.py:134",
"cluster": "master",
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_g0ly2icp.pdf",
"message": "Starting annotation detection",
"service": "docproc",
"timestamp": "2026-07-27T02:24:52.709Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc PDF magic bytes check
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourceutils/pdf_detector.py:74
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
PDF magic bytes check
raw payload (11 fields)
{
"caller": "utils/pdf_detector.py:74",
"cluster": "master",
"environment": "DEVELOPMENT",
"file_path": "/tmp/docproc_annot_g0ly2icp.pdf",
"is_pdf": true,
"message": "PDF magic bytes check",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.709Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] Annotation detection started
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Annotation detection started
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] Annotation detection started",
"org_alias": "sandboxIcon",
"processor": "annotation_detector",
"service": "docproc",
"size_bytes": 2483.0,
"status": "running",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.707Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}8s ago2026-07-27 02:24:52 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:52 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 120.4,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "annotation_detection",
"org_alias": "sandboxIcon",
"reserved_mb": 80.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.707Z",
"version": "0.1.0",
"wait_seconds": 0.0
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4779.6,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 398.7,
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "annotation_detection",
"org_alias": "sandboxIcon",
"requested_mb": 80.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.707Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "tmf_preprocess", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id":...
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "tmf_preprocess", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c", "content_type": "application/pdf", "document_size_mb": 0.0023679733276367188, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 40.00473594665527, "actual_mb": 0.0390625, "memory_error_pct": 99.90235531100096, "duration_seconds": 3.22308087348938, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119092.7039962, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"tmf_preprocess\", \"document_id\": \"d0d3e988-1c04-483d-b6c6-27fbdef67c85\", \"version_id\": \"bdc7f2d3-251a-4baf-9357-c1f63488750c\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.0023679733276367188, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 40.00473594665527, \"actual_mb\": 0.0390625, \"memory_error_pct\": 99.90235531100096, \"duration_seconds\": 3.22308087348938, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119092.7039962, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.704Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc OpenAI file_id created for TMF
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:954
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
OpenAI file_id created for TMF
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:954",
"cluster": "master",
"component": "orchestrator_certification",
"environment": "DEVELOPMENT",
"file_id": "file-E75RgFsTdyss5tK...",
"full_file_id": "file-E75RgFsTdyss5tKDB6udRC",
"message": "OpenAI file_id created for TMF",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.703Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Transaction completed in 0.03s
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_neo4j/client.py:261
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Transaction completed in 0.03s
raw payload (10 fields)
{
"caller": "tilda_neo4j/client.py:261",
"cluster": "master",
"component": "neo4j",
"environment": "DEVELOPMENT",
"message": "Transaction completed in 0.03s",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.703Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Starting execute_in_transaction
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_neo4j/client.py:248
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Starting execute_in_transaction
raw payload (10 fields)
{
"caller": "tilda_neo4j/client.py:248",
"cluster": "master",
"component": "neo4j",
"environment": "DEVELOPMENT",
"message": "Starting execute_in_transaction",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.690Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Starting execute_in_transaction
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcetilda_neo4j/client.py:248
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Starting execute_in_transaction
raw payload (10 fields)
{
"caller": "tilda_neo4j/client.py:248",
"cluster": "master",
"component": "neo4j",
"environment": "DEVELOPMENT",
"message": "Starting execute_in_transaction",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:52.670Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Pulling messages
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T02:24:52.102Z",
"version": "0.1.0"
}8s ago2026-07-27 02:24:52 UTC DEBUG docproc Recovery fill
time2026-07-27 02:24:52 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 1.0,
"backlog_started": 1.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 8.0,
"service": "docproc",
"timestamp": "2026-07-27T02:24:51.999Z",
"version": "0.1.0"
}9s ago2026-07-27 02:24:51 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 02:24:51 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9613244.0,
"cpu": 171.0,
"elapsedTimeMs": 171.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "25411384",
"pageFaults": 0.0,
"pageHits": 156403.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785118191823}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:51.996Z",
"transactionId": 12258951.0,
"waiting": 0.0
}9s ago2026-07-27 02:24:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...
time2026-07-27 02:24:51 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (r:TmfEvaluationRun {status: $running})
WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
ORDER BY coalesce(r.updatedat, r.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 800.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "95899445",
"id": "25411386",
"pageFaults": 0.0,
"pageHits": 21.0,
"planning": 0.0,
"query": "\n MATCH (r:TmfEvaluationRun {status: $running})\n WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n ORDER BY coalesce(r.updatedat, r.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785118791935}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:51.941Z",
"transactionId": 12258952.0,
"waiting": 0.0
}9s ago2026-07-27 02:24:51 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 02:24:51 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 712.0,
"cpu": 114.0,
"elapsedTimeMs": 114.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "25411376",
"pageFaults": 0.0,
"pageHits": 89940.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 24, cutoff_ms: 1785118491548}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:51.667Z",
"transactionId": 12258950.0,
"waiting": 0.0
}9s ago2026-07-27 02:24:51 UTC DEBUG docproc Message processed successfully
time2026-07-27 02:24:51 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:576
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanc181534941f3bf03
poddocproc-7bc76669-ljh8h
message
Message processed successfully
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:576",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Message processed successfully",
"message_id": "21217978059053556",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:51.574Z",
"version": "0.1.0"
}9s ago2026-07-27 02:24:51 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PubSub upload complete
time2026-07-27 02:24:51 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanc181534941f3bf03
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PubSub upload complete
raw payload (17 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "pubsub",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PubSub upload complete",
"org_alias": "sandboxIcon",
"processor": "pubsub:upload",
"service": "docproc",
"status": "success",
"step_times": null,
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:51.551Z",
"total_s": 0.0,
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}9s ago2026-07-27 02:24:51 UTC DEBUG docproc Document upload event received (no processing - file not yet in GCS)
time2026-07-27 02:24:51 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:3442
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanc181534941f3bf03
poddocproc-7bc76669-ljh8h
message
Document upload event received (no processing - file not yet in GCS)
raw payload (13 fields)
{
"caller": "orchestration/document_orchestrator.py:3442",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Document upload event received (no processing - file not yet in GCS)",
"org_alias": "sandboxIcon",
"service": "docproc",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:51.550Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}9s ago2026-07-27 02:24:51 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PubSub upload started
time2026-07-27 02:24:51 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanc181534941f3bf03
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PubSub upload started
raw payload (17 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "pubsub",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PubSub upload started",
"operation": "upload",
"org_alias": "sandboxIcon",
"processor": "pubsub:upload",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:51.550Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}9s ago2026-07-27 02:24:51 UTC DEBUG docproc Processing document event
time2026-07-27 02:24:51 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:915
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanc181534941f3bf03
poddocproc-7bc76669-ljh8h
message
Processing document event
raw payload (14 fields)
{
"caller": "handlers/pubsub_handler.py:915",
"cluster": "master",
"component": "pubsub",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Processing document event",
"operation": "Upload",
"org_alias": "sandboxIcon",
"service": "docproc",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:51.550Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}9s ago2026-07-27 02:24:51 UTC DEBUG docproc Processing message
time2026-07-27 02:24:51 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:549
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanc181534941f3bf03
poddocproc-7bc76669-ljh8h
message
Processing message
raw payload (14 fields)
{
"caller": "handlers/pubsub_handler.py:549",
"cluster": "master",
"component": "pubsub",
"delivery_attempt": 1.0,
"environment": "DEVELOPMENT",
"message": "Processing message",
"message_id": "21217978059053556",
"operation": "Upload",
"org_alias": "sandboxIcon",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:51.550Z",
"version": "0.1.0"
}9s ago2026-07-27 02:24:51 UTC neo4j docproc MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]->(doc:Document {tildaid: $doc_id}) RETURN tracker
time2026-07-27 02:24:51 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]->(doc:Document {tildaid: $doc_id})
RETURN tracker
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 0.0,
"elapsedTimeMs": 0.0,
"executionPlanCacheKeyHash": "B8141227",
"id": "25411375",
"pageFaults": 0.0,
"pageHits": 5.0,
"planning": 0.0,
"query": "\n MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]-\u003e(doc:Document {tildaid: $doc_id})\n RETURN tracker\n ",
"queryParameters": "{doc_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:51.549Z",
"transactionId": 12258949.0,
"waiting": 0.0
}9s ago2026-07-27 02:24:51 UTC DEBUG docproc Received messages
time2026-07-27 02:24:51 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:258
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Received messages
raw payload (10 fields)
{
"caller": "handlers/pubsub_handler.py:258",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Received messages",
"message_count": 1.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T02:24:51.547Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC INFO docproc [extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=193346 bytes
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourcecommon/docparse.py:310
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=193346 bytes
raw payload (9 fields)
{
"caller": "common/docparse.py:310",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=193346 bytes",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.566Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC DEBUG docproc PDF type detected for TMF
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:911
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
PDF type detected for TMF
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:911",
"cluster": "master",
"component": "orchestrator_certification",
"environment": "DEVELOPMENT",
"is_scanned": false,
"is_text_based": true,
"message": "PDF type detected for TMF",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.563Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]->(doc:Document {tildaid: $doc_id}) RETURN tracker
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]->(doc:Document {tildaid: $doc_id})
RETURN tracker
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 1.0,
"elapsedTimeMs": 4.0,
"executionPlanCacheKeyHash": "B8141227",
"id": "25411275",
"pageFaults": 0.0,
"pageHits": 17.0,
"planning": 1.0,
"query": "\n MATCH (tracker:DocAITracker)-[:DOC_AI_TRACKER_TO_DOCUMENT]-\u003e(doc:Document {tildaid: $doc_id})\n RETURN tracker\n ",
"queryParameters": "{doc_id: \u0027d0d3e988-1c04-483d-b6c6-27fbdef67c85\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:49.538Z",
"transactionId": 12258943.0,
"waiting": 0.0
}11s ago2026-07-27 02:24:49 UTC DEBUG docproc Extracted document pages for TMF
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:893
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Extracted document pages for TMF
raw payload (13 fields)
{
"caller": "orchestration/document_orchestrator.py:893",
"cluster": "master",
"component": "orchestrator_certification",
"environment": "DEVELOPMENT",
"extracted_pages": 1.0,
"is_full_document": true,
"message": "Extracted document pages for TMF",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.535Z",
"total_pages": 1.0,
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC INFO docproc [extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=193346 bytes
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourcecommon/docparse.py:310
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=193346 bytes
raw payload (9 fields)
{
"caller": "common/docparse.py:310",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=193346 bytes",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.534Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 80.4,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "tmf_preprocess",
"org_alias": "sandboxIcon",
"reserved_mb": 40.4,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.533Z",
"version": "0.1.0",
"wait_seconds": 0.0
}11s ago2026-07-27 02:24:49 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4780.0,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 402.9,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "tmf_preprocess",
"org_alias": "sandboxIcon",
"requested_mb": 40.4,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.533Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "download_file", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id":...
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "download_file", "document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea", "version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef", "content_type": "application/pdf", "document_size_mb": 0.1843891143798828, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 40.276583671569824, "actual_mb": 4.3984375, "memory_error_pct": 89.07941761926362, "duration_seconds": 0.24262571334838867, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119089.5318651, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"download_file\", \"document_id\": \"a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\", \"version_id\": \"65ac9b5f-2b28-4cf8-a86b-496962cc36ef\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.1843891143798828, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 40.276583671569824, \"actual_mb\": 4.3984375, \"memory_error_pct\": 89.07941761926362, \"duration_seconds\": 0.24262571334838867, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119089.5318651, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.531Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [2/7] Download ...................... OK 0.24s (0.18 MB,...
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [2/7] Download ...................... OK 0.24s (0.18 MB, gs://master-tilda-shared-storage/master/sandboxIcon/65ac/65ac9b5f-2b28-4cf8-a86b-496962cc36ef)
raw payload (20 fields)
{
"caller": "utils/processing_logger.py:194",
"checksum_sync_skipped": true,
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.24,
"elapsed_s": 0.25,
"environment": "DEVELOPMENT",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/65ac/65ac9b5f-2b28-4cf8-a86b-496962cc36ef",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [2/7] Download ...................... OK 0.24s (0.18 MB, gs://master-tilda-shared-storage/master/sandboxIcon/65ac/65ac9b5f-2b28-4cf8-a86b-496962cc36ef)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"size_mb": "0.18 MB",
"step": "download_original",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.531Z",
"used_linearized": false,
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}11s ago2026-07-27 02:24:49 UTC INFO docproc Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceorchestration/document_orchestrator.py:701
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:701",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.530Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}11s ago2026-07-27 02:24:49 UTC DEBUG docproc Downloaded bytes from GCS
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetilda_gcs/client.py:270
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Downloaded bytes from GCS
raw payload (12 fields)
{
"caller": "tilda_gcs/client.py:270",
"cluster": "master",
"component": "gcs",
"environment": "DEVELOPMENT",
"message": "Downloaded bytes from GCS",
"org_alias": "sandboxIcon",
"path": "master/sandboxIcon/65ac/65ac9b5f-2b28-4cf8-a86b-496962cc36ef",
"service": "docproc",
"size_bytes": 193346.0,
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.530Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC INFO docproc [extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=2483 bytes
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourcecommon/docparse.py:310
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=2483 bytes
raw payload (9 fields)
{
"caller": "common/docparse.py:310",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=2483 bytes",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.524Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC DEBUG docproc PDF type detected for TMF
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:911
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
PDF type detected for TMF
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:911",
"cluster": "master",
"component": "orchestrator_certification",
"environment": "DEVELOPMENT",
"is_scanned": false,
"is_text_based": true,
"message": "PDF type detected for TMF",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.523Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC DEBUG docproc Extracted document pages for TMF
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:893
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Extracted document pages for TMF
raw payload (13 fields)
{
"caller": "orchestration/document_orchestrator.py:893",
"cluster": "master",
"component": "orchestrator_certification",
"environment": "DEVELOPMENT",
"extracted_pages": 1.0,
"is_full_document": true,
"message": "Extracted document pages for TMF",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.493Z",
"total_pages": 1.0,
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC INFO docproc [extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=2483 bytes
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourcecommon/docparse.py:310
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=2483 bytes
raw payload (9 fields)
{
"caller": "common/docparse.py:310",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[extract-start] 1 total pages, first_n_pages=10, last_m_pages=10, document_size=2483 bytes",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.492Z",
"version": "0.1.0"
}11s ago2026-07-27 02:24:49 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 80.3,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "tmf_preprocess",
"org_alias": "sandboxIcon",
"reserved_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.480Z",
"version": "0.1.0",
"wait_seconds": 0.0
}11s ago2026-07-27 02:24:49 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4779.7,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 398.7,
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "tmf_preprocess",
"org_alias": "sandboxIcon",
"requested_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.480Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc RESOURCE_METRICS: {"operation": "download_file", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id":...
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/estimator.py:426
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
RESOURCE_METRICS: {"operation": "download_file", "document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85", "version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c", "content_type": "application/pdf", "document_size_mb": 0.0023679733276367188, "document_pages": null, "num_images": null, "total_image_size_mb": null, "estimated_mb": 40.003551959991455, "actual_mb": 0.14453125, "memory_error_pct": 99.63870395772719, "duration_seconds": 0.19373273849487305, "total_input_tokens": 0, "total_output_tokens": 0, "total_tokens": 0, "tokens_by_model": "{}", "timestamp": 1785119089.477004, "pod_name": "docproc-7bc76669-ljh8h"}
raw payload (9 fields)
{
"caller": "tilda_resource_scheduler/estimator.py:426",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "RESOURCE_METRICS: {\"operation\": \"download_file\", \"document_id\": \"d0d3e988-1c04-483d-b6c6-27fbdef67c85\", \"version_id\": \"bdc7f2d3-251a-4baf-9357-c1f63488750c\", \"content_type\": \"application/pdf\", \"document_size_mb\": 0.0023679733276367188, \"document_pages\": null, \"num_images\": null, \"total_image_size_mb\": null, \"estimated_mb\": 40.003551959991455, \"actual_mb\": 0.14453125, \"memory_error_pct\": 99.63870395772719, \"duration_seconds\": 0.19373273849487305, \"total_input_tokens\": 0, \"total_output_tokens\": 0, \"total_tokens\": 0, \"tokens_by_model\": \"{}\", \"timestamp\": 1785119089.477004, \"pod_name\": \"docproc-7bc76669-ljh8h\"}",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.477Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] [2/7] Download ...................... OK 0.19s (0.00 MB,...
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [2/7] Download ...................... OK 0.19s (0.00 MB, gs://master-tilda-shared-storage/master/sandboxIcon/bdc7/bdc7f2d3-251a-4baf-9357-c1f63488750c)
raw payload (20 fields)
{
"caller": "utils/processing_logger.py:194",
"checksum_sync_skipped": true,
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.19,
"elapsed_s": 0.2,
"environment": "DEVELOPMENT",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/bdc7/bdc7f2d3-251a-4baf-9357-c1f63488750c",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [2/7] Download ...................... OK 0.19s (0.00 MB, gs://master-tilda-shared-storage/master/sandboxIcon/bdc7/bdc7f2d3-251a-4baf-9357-c1f63488750c)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"size_mb": "0.00 MB",
"step": "download_original",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.476Z",
"used_linearized": false,
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}12s ago2026-07-27 02:24:49 UTC INFO docproc Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceorchestration/document_orchestrator.py:701
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:701",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Skipping SHA256 and docsvc checksum sync; DocumentVersion.cksum already set",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.471Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc Downloaded bytes from GCS
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetilda_gcs/client.py:270
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Downloaded bytes from GCS
raw payload (12 fields)
{
"caller": "tilda_gcs/client.py:270",
"cluster": "master",
"component": "gcs",
"environment": "DEVELOPMENT",
"message": "Downloaded bytes from GCS",
"org_alias": "sandboxIcon",
"path": "master/sandboxIcon/bdc7/bdc7f2d3-251a-4baf-9357-c1f63488750c",
"service": "docproc",
"size_bytes": 2483.0,
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.471Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 80.3,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "download_file",
"org_alias": "sandboxIcon",
"reserved_mb": 40.3,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.288Z",
"version": "0.1.0",
"wait_seconds": 0.0
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4780.0,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 398.5,
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "download_file",
"org_alias": "sandboxIcon",
"requested_mb": 40.3,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.288Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [1/7] Context ....................... OK 0.09s
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [1/7] Context ....................... OK 0.09s
raw payload (19 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"duration_s": 0.09,
"elapsed_s": 0.01,
"environment": "DEVELOPMENT",
"filename": "Site101_NorthAmerican_NewPI_FinancialDisclosureForm_signed(14).pdf",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/65ac/65ac9b5f-2b28-4cf8-a86b-496962cc36ef",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] [1/7] Context ....................... OK 0.09s",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"size_mb": 0.18,
"step": "create_context",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.286Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:592
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:592",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"environment": "DEVELOPMENT",
"message": "Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.286Z",
"version": "0.1.0",
"work_order_id": "c4efcc9f-9847-4099-a208-de401b1ac83f"
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc [ASYNC_WO_QUERY] ✅ Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=a3e0b2b1-816f-4215-a2d6-9e985e76e2ea
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetmf/work_order_queries.py:83
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[ASYNC_WO_QUERY] ✅ Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=a3e0b2b1-816f-4215-a2d6-9e985e76e2ea
raw payload (9 fields)
{
"caller": "tmf/work_order_queries.py:83",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[ASYNC_WO_QUERY] \u2705 Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.286Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC INFO docproc Stage memory budget reserved
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:504
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory budget reserved
raw payload (15 fields)
{
"allocated_after_mb": 40.0,
"caller": "tilda_resource_scheduler/scheduler.py:504",
"cluster": "master",
"component": "resource_scheduler",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory budget reserved",
"operation": "download_file",
"org_alias": "sandboxIcon",
"reserved_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.283Z",
"version": "0.1.0",
"wait_seconds": 0.0
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc Stage memory reservation: requested vs available
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetilda_resource_scheduler/scheduler.py:452
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Stage memory reservation: requested vs available
raw payload (15 fields)
{
"available_mb": 4820.0,
"caller": "tilda_resource_scheduler/scheduler.py:452",
"cluster": "master",
"component": "resource_scheduler",
"current_rss_mb": 398.5,
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Stage memory reservation: requested vs available",
"operation": "download_file",
"org_alias": "sandboxIcon",
"requested_mb": 40.0,
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.282Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id}) WHERE w.delpending = false RETURN w.tildaid AS...
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id})
WHERE w.delpending = false
RETURN w.tildaid AS work_order_id, coalesce(w.title, '') AS work_order_name
ORDER BY coalesce(w.updatedat, w.createdat) DESC
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 684.0,
"cpu": 0.0,
"elapsedTimeMs": 0.0,
"executionPlanCacheKeyHash": "1D6BF7F0",
"id": "25411273",
"pageFaults": 0.0,
"pageHits": 8.0,
"planning": 0.0,
"query": "\n MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]-\u003e(d:Document {tildaid: $doc_id})\n WHERE w.delpending = false\n RETURN w.tildaid AS work_order_id, coalesce(w.title, \u0027\u0027) AS work_order_name\n ORDER BY coalesce(w.updatedat, w.createdat) DESC\n LIMIT 1\n ",
"queryParameters": "{doc_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:49.282Z",
"transactionId": 12258941.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] [1/7] Context ....................... OK 0.13s
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [1/7] Context ....................... OK 0.13s
raw payload (19 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"duration_s": 0.13,
"elapsed_s": 0.01,
"environment": "DEVELOPMENT",
"filename": "Site101_NorthAmerican_NewPI_GCP_Certificate(12).pdf",
"gcs_url": "gs://master-tilda-shared-storage/master/sandboxIcon/bdc7/bdc7f2d3-251a-4baf-9357-c1f63488750c",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] [1/7] Context ....................... OK 0.13s",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"size_mb": 0.0,
"step": "create_context",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.280Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourceorchestration/document_orchestrator.py:592
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)
raw payload (12 fields)
{
"caller": "orchestration/document_orchestrator.py:592",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"environment": "DEVELOPMENT",
"message": "Resolved work_order_id from Neo4j in create_context (before tmf_site_info_extract / TMF LLM)",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.279Z",
"version": "0.1.0",
"work_order_id": "c4efcc9f-9847-4099-a208-de401b1ac83f"
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc [ASYNC_WO_QUERY] ✅ Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=d0d3e988-1c04-483d-b6c6-27fbdef67c85
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetmf/work_order_queries.py:83
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[ASYNC_WO_QUERY] ✅ Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=d0d3e988-1c04-483d-b6c6-27fbdef67c85
raw payload (9 fields)
{
"caller": "tmf/work_order_queries.py:83",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[ASYNC_WO_QUERY] \u2705 Found work_order_id=c4efcc9f-9847-4099-a208-de401b1ac83f for doc_id=d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.279Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc [ASYNC_WO_QUERY] Finding work order for doc_id=a3e0b2b1-816f-4215-a2d6-9e985e76e2ea
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetmf/work_order_queries.py:78
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[ASYNC_WO_QUERY] Finding work order for doc_id=a3e0b2b1-816f-4215-a2d6-9e985e76e2ea
raw payload (9 fields)
{
"caller": "tmf/work_order_queries.py:78",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[ASYNC_WO_QUERY] Finding work order for doc_id=a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.277Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] ══════════════════════════════════════════════════════
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] ══════════════════════════════════════════════════════
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.277Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] 2/10 slots in use
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] 2/10 slots in use
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] 2/10 slots in use",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.277Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PIPELINE START — Site101_NorthAmerican_NewPI_FinancialDisclosureForm_signed(14).pdf (0.18 MB)
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PIPELINE START — Site101_NorthAmerican_NewPI_FinancialDisclosureForm_signed(14).pdf (0.18 MB)
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] PIPELINE START \u2014 Site101_NorthAmerican_NewPI_FinancialDisclosureForm_signed(14).pdf (0.18 MB)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.277Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id}) WHERE w.delpending = false RETURN w.tildaid AS...
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id})
WHERE w.delpending = false
RETURN w.tildaid AS work_order_id, coalesce(w.title, '') AS work_order_name
ORDER BY coalesce(w.updatedat, w.createdat) DESC
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 684.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "1D6BF7F0",
"id": "25411270",
"pageFaults": 0.0,
"pageHits": 8.0,
"planning": 0.0,
"query": "\n MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]-\u003e(d:Document {tildaid: $doc_id})\n WHERE w.delpending = false\n RETURN w.tildaid AS work_order_id, coalesce(w.title, \u0027\u0027) AS work_order_name\n ORDER BY coalesce(w.updatedat, w.createdat) DESC\n LIMIT 1\n ",
"queryParameters": "{doc_id: \u0027d0d3e988-1c04-483d-b6c6-27fbdef67c85\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:49.278Z",
"transactionId": 12258940.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC INFO docproc [a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] ══════════════════════════════════════════════════════
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
poddocproc-7bc76669-ljh8h
message
[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] ══════════════════════════════════════════════════════
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "a3e0b2b1-816f-4215-a2d6-9e985e76e2ea",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[a3e0b2b1-816f-4215-a2d6-9e985e76e2ea] \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.276Z",
"version": "0.1.0",
"version_id": "65ac9b5f-2b28-4cf8-a86b-496962cc36ef"
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc [ASYNC_WO_QUERY] Finding work order for doc_id=d0d3e988-1c04-483d-b6c6-27fbdef67c85
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcetmf/work_order_queries.py:78
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[ASYNC_WO_QUERY] Finding work order for doc_id=d0d3e988-1c04-483d-b6c6-27fbdef67c85
raw payload (9 fields)
{
"caller": "tmf/work_order_queries.py:78",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "[ASYNC_WO_QUERY] Finding work order for doc_id=d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"org_alias": "sandboxIcon",
"service": "docproc",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.275Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] ══════════════════════════════════════════════════════
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] ══════════════════════════════════════════════════════
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.274Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] 2/10 slots in use
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] 2/10 slots in use
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] 2/10 slots in use",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.274Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] PIPELINE START — Site101_NorthAmerican_NewPI_GCP_Certificate(12).pdf (0.00 MB)
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] PIPELINE START — Site101_NorthAmerican_NewPI_GCP_Certificate(12).pdf (0.00 MB)
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] PIPELINE START \u2014 Site101_NorthAmerican_NewPI_GCP_Certificate(12).pdf (0.00 MB)",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.274Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}12s ago2026-07-27 02:24:49 UTC INFO docproc [d0d3e988-1c04-483d-b6c6-27fbdef67c85] ══════════════════════════════════════════════════════
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
sourceutils/processing_logger.py:194
clustermaster
orgsandboxIcon
tenantsandboxIcon
version0.1.0
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
poddocproc-7bc76669-ljh8h
message
[d0d3e988-1c04-483d-b6c6-27fbdef67c85] ══════════════════════════════════════════════════════
raw payload (16 fields)
{
"caller": "utils/processing_logger.py:194",
"cluster": "master",
"component": "orchestrator_certification",
"document_id": "d0d3e988-1c04-483d-b6c6-27fbdef67c85",
"elapsed_s": 0.0,
"environment": "DEVELOPMENT",
"message": "[d0d3e988-1c04-483d-b6c6-27fbdef67c85] \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550",
"org_alias": "sandboxIcon",
"processor": "orchestrator",
"service": "docproc",
"status": "running",
"study_id": "c4defff9-d109-4da9-8662-5e0452d70540",
"tenant": "sandboxIcon",
"timestamp": "2026-07-27T02:24:49.274Z",
"version": "0.1.0",
"version_id": "bdc7f2d3-251a-4baf-9357-c1f63488750c"
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (o:Organization {tildaid: $org_id}) RETURN o.orgType AS org_type
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (o:Organization {tildaid: $org_id})
RETURN o.orgType AS org_type
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 0.0,
"elapsedTimeMs": 38.0,
"executionPlanCacheKeyHash": "0393DF4C",
"id": "25411260",
"pageFaults": 0.0,
"pageHits": 4.0,
"planning": 20.0,
"query": "MATCH (o:Organization {tildaid: $org_id}) RETURN o.orgType AS org_type",
"queryParameters": "{org_id: \u00279f996610-f74a-4cd5-9052-cb5405d78c6f\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:49.273Z",
"transactionId": 12258939.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (o:Organization {tildaid: $org_id}) RETURN o.orgType AS org_type
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (o:Organization {tildaid: $org_id})
RETURN o.orgType AS org_type
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 27.0,
"elapsedTimeMs": 42.0,
"executionPlanCacheKeyHash": "0393DF4C",
"id": "25411258",
"pageFaults": 0.0,
"pageHits": 48.0,
"planning": 25.0,
"query": "MATCH (o:Organization {tildaid: $org_id}) RETURN o.orgType AS org_type",
"queryParameters": "{org_id: \u00279f996610-f74a-4cd5-9052-cb5405d78c6f\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:49.272Z",
"transactionId": 12258938.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id}) WHERE w.delpending = false RETURN w.tildaid AS...
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace30a387c51cfc9077a5742b6cc96d86dc
spanec7c622facf35143
podserver-2-dev-0
query
MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id})
WHERE w.delpending = false
RETURN w.tildaid AS work_order_id, coalesce(w.title, '') AS work_order_name
ORDER BY coalesce(w.updatedat, w.createdat) DESC
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 684.0,
"cpu": 62.0,
"elapsedTimeMs": 64.0,
"executionPlanCacheKeyHash": "1D6BF7F0",
"id": "25411251",
"pageFaults": 0.0,
"pageHits": 8.0,
"planning": 60.0,
"query": "\n MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]-\u003e(d:Document {tildaid: $doc_id})\n WHERE w.delpending = false\n RETURN w.tildaid AS work_order_id, coalesce(w.title, \u0027\u0027) AS work_order_name\n ORDER BY coalesce(w.updatedat, w.createdat) DESC\n LIMIT 1\n ",
"queryParameters": "{doc_id: \u0027d0d3e988-1c04-483d-b6c6-27fbdef67c85\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T02:24:49.226Z",
"transactionId": 12258933.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id}) WHERE w.delpending = false RETURN w.tildaid AS...
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]->(d:Document {tildaid: $doc_id})
WHERE w.delpending = false
RETURN w.tildaid AS work_order_id, coalesce(w.title, '') AS work_order_name
ORDER BY coalesce(w.updatedat, w.createdat) DESC
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 684.0,
"cpu": 3.0,
"elapsedTimeMs": 17.0,
"executionPlanCacheKeyHash": "1D6BF7F0",
"id": "25411256",
"pageFaults": 0.0,
"pageHits": 8.0,
"planning": 13.0,
"query": "\n MATCH (w:WorkOrder)-[:WORK_ORDER_TO_DOCUMENT]-\u003e(d:Document {tildaid: $doc_id})\n WHERE w.delpending = false\n RETURN w.tildaid AS work_order_id, coalesce(w.title, \u0027\u0027) AS work_order_name\n ORDER BY coalesce(w.updatedat, w.createdat) DESC\n LIMIT 1\n ",
"queryParameters": "{doc_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:49.226Z",
"transactionId": 12258937.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (dv:DocumentVersion{tildaid:$version_id})<-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id}) MATCH...
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (dv:DocumentVersion{tildaid:$version_id})<-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id})
MATCH (doc)-[:DOC_TO_ORG]->(org:Organization)
OPTIONAL MATCH (study:Study{tildaid:doc.studyId})
WITH dv, doc, org, study
RETURN dv.tildaid AS version_id,
dv.url AS url,
doc.fileName AS filename,
doc.contentType AS content_type,
dv.size AS size_bytes,
dv.linearizedPdfUrl AS linearized_pdf_url,
dv.linearizedPdfStatus AS linearized_status,
dv.languages AS languages,
dv.langDetectionStatus AS lang_detection_status,
dv.translatedPdfUrl AS translated_pdf_url,
dv.translatedPdfStatus AS translated_status,
study.tildaid AS study_id,
study.name AS study_name,
study.protocolnumber AS protocol_number,
org.tildaid AS org_id,
org.alias AS org_alias,
org.name AS org_name,
dv.uploadMethod AS upload_method,
org.orgType AS org_type,
dv.cksum AS checksum
raw payload (15 fields)
{
"allocatedBytes": 8376.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "8FE8F25D",
"id": "25411255",
"pageFaults": 0.0,
"pageHits": 17.0,
"planning": 0.0,
"query": "\n MATCH (dv:DocumentVersion{tildaid:$version_id})\u003c-[:DOCUMENT_VERSION]-(doc:Document{tildaid:$document_id})\n MATCH (doc)-[:DOC_TO_ORG]-\u003e(org:Organization)\n OPTIONAL MATCH (study:Study{tildaid:doc.studyId})\n WITH dv, doc, org, study\n RETURN dv.tildaid AS version_id,\n dv.url AS url,\n doc.fileName AS filename,\n doc.contentType AS content_type,\n dv.size AS size_bytes,\n dv.linearizedPdfUrl AS linearized_pdf_url,\n dv.linearizedPdfStatus AS linearized_status,\n dv.languages AS languages,\n dv.langDetectionStatus AS lang_detection_status,\n dv.translatedPdfUrl AS translated_pdf_url,\n dv.translatedPdfStatus AS translated_status,\n study.tildaid AS study_id,\n study.name AS study_name,\n study.protocolnumber AS protocol_number,\n org.tildaid AS org_id,\n org.alias AS org_alias,\n org.name AS org_name,\n dv.uploadMethod AS upload_method,\n org.orgType AS org_type,\n dv.cksum AS checksum\n ",
"queryParameters": "{document_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027, version_id: \u002765ac9b5f-2b28-4cf8-a86b-496962cc36ef\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:49.201Z",
"transactionId": 12258936.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (ds:GoldenDataset)-[:CONTAINS_DOCUMENT]->(:Document {tildaid: $document_id}) RETURN ds.tildaid AS golden_dataset_id LIMIT 1
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (ds:GoldenDataset)-[:CONTAINS_DOCUMENT]->(:Document {tildaid: $document_id})
RETURN ds.tildaid AS golden_dataset_id
LIMIT 1
raw payload (15 fields)
{
"allocatedBytes": 328.0,
"cpu": 0.0,
"elapsedTimeMs": 0.0,
"executionPlanCacheKeyHash": "D4A26122",
"id": "25411254",
"pageFaults": 0.0,
"pageHits": 5.0,
"planning": 0.0,
"query": "\n MATCH (ds:GoldenDataset)-[:CONTAINS_DOCUMENT]-\u003e(:Document {tildaid: $document_id})\n RETURN ds.tildaid AS golden_dataset_id\n LIMIT 1\n ",
"queryParameters": "{document_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:49.191Z",
"transactionId": 12258935.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC DEBUG docproc Pulling messages
time2026-07-27 02:24:49 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T02:24:49.184Z",
"version": "0.1.0"
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (n:AIRun {tildaid: $node_id}) RETURN n
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (n:AIRun {tildaid: $node_id})
RETURN n
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 0.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "316B40A9",
"id": "25411253",
"pageFaults": 0.0,
"pageHits": 6.0,
"planning": 0.0,
"query": "MATCH (n:AIRun {tildaid: $node_id}) RETURN n",
"queryParameters": "{node_id: \u00276f162abd-960f-4479-82e6-e0f0f909fd63\u0027}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:49.182Z",
"transactionId": 12258934.0,
"waiting": 0.0
}12s ago2026-07-27 02:24:49 UTC neo4j docproc MATCH (d:Document {tildaid: $doc_id}) SET d.cbrRunStatus = $status, d.cbrRunStatusAt = $ts
time2026-07-27 02:24:49 UTC
severityINFO
servicedocproc
clustermaster
orgsandboxIcon
trace8a3ecc554e673b74ca7c7a8faa58979f
span2e3909f36864e9d6
podserver-2-dev-0
query
MATCH (d:Document {tildaid: $doc_id})
SET d.cbrRunStatus = $status, d.cbrRunStatusAt = $ts
raw payload (15 fields)
{
"allocatedBytes": 312.0,
"cpu": 0.0,
"elapsedTimeMs": 0.0,
"executionPlanCacheKeyHash": "532B3015",
"id": "25411252",
"pageFaults": 0.0,
"pageHits": 5.0,
"planning": 0.0,
"query": "\n MATCH (d:Document {tildaid: $doc_id})\n SET d.cbrRunStatus = $status, d.cbrRunStatusAt = $ts\n ",
"queryParameters": "{doc_id: \u0027a3e0b2b1-816f-4215-a2d6-9e985e76e2ea\u0027, timeout: \u003cnull\u003e, status: \u003cnull\u003e, ts: 1785119089}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.0.201:47754\tserver/10.173.0.8:7688\u003e",
"timestamp": "2026-07-27T02:24:49.163Z",
"transactionId": 12258932.0,
"waiting": 0.0
}Paginated with Cloud Logging page cursors, so you can walk the whole 1h window — not just the newest page. Click any filter by value to scope the view (resets to page 1). Paste a trace id to see every log line across all services for that one request.