Logs
Live Cloud Logging query (nothing stored) · master ·
last 6h ·
newest first · page 1 (200/page) · 200 entries in 1594 ms
all
app
neo4j
weaviate
time
relative
absolute
8s ago2026-07-27 00:10:07 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:10:07 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:10:07.193Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}8s ago2026-07-27 00:10:07 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:10:07 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:10:07.118Z",
"version": "0.1.0"
}8s ago2026-07-27 00:10:07 UTC DEBUG docproc Pulling messages
time2026-07-27 00:10:07 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:10:07.070Z",
"version": "0.1.0"
}9s ago2026-07-27 00:10:06 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:10:06 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 62.0,
"elapsedTimeMs": 108.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24651509",
"pageFaults": 0.0,
"pageHits": 66849.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:10:06.675Z",
"transactionId": 12029947.0,
"waiting": 0.0
}9s ago2026-07-27 00:10:06 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:10:06 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:10:06.067Z",
"version": "0.1.0"
}9s ago2026-07-27 00:10:06 UTC DEBUG docproc Recovery fill
time2026-07-27 00:10:06 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:10:06.067Z",
"version": "0.1.0"
}24s ago2026-07-27 00:09:51 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:09:51 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:09:51.559Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}24s ago2026-07-27 00:09:51 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:09:51 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:09:51.490Z",
"version": "0.1.0"
}28s ago2026-07-27 00:09:47 UTC DEBUG docproc Pulling messages
time2026-07-27 00:09:47 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:09:47.544Z",
"version": "0.1.0"
}29s ago2026-07-27 00:09:46 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:09:46 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:09:46.540Z",
"version": "0.1.0"
}29s ago2026-07-27 00:09:46 UTC DEBUG docproc Recovery fill
time2026-07-27 00:09:46 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:09:46.540Z",
"version": "0.1.0"
}29s ago2026-07-27 00:09:46 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:09:46 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 169.0,
"elapsedTimeMs": 169.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24649290",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785110086366}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:46.538Z",
"transactionId": 12028773.0,
"waiting": 0.0
}29s ago2026-07-27 00:09:46 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:09:46 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 97.0,
"elapsedTimeMs": 97.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24649284",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110386264}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:46.364Z",
"transactionId": 12028772.0,
"waiting": 0.0
}39s ago2026-07-27 00:09:36 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:09:36 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:09:36.078Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}40s ago2026-07-27 00:09:35 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:09:35 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:09:35.996Z",
"version": "0.1.0"
}40s ago2026-07-27 00:09:35 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:09:35 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 212.0,
"elapsedTimeMs": 228.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24648871",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785110075558}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:35.788Z",
"transactionId": 12028763.0,
"waiting": 0.0
}40s ago2026-07-27 00:09:35 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:09:35 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 80.0,
"elapsedTimeMs": 80.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24648864",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:35.556Z",
"transactionId": 12028762.0,
"waiting": 0.0
}48s ago2026-07-27 00:09:27 UTC DEBUG docproc Pulling messages
time2026-07-27 00:09:27 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:09:27.002Z",
"version": "0.1.0"
}50s ago2026-07-27 00:09:25 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:09:25 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:09:25.997Z",
"version": "0.1.0"
}50s ago2026-07-27 00:09:25 UTC DEBUG docproc Recovery fill
time2026-07-27 00:09:25 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:09:25.997Z",
"version": "0.1.0"
}50s ago2026-07-27 00:09:25 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:09:25 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 164.0,
"elapsedTimeMs": 164.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24648508",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785110065828}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:25.995Z",
"transactionId": 12028757.0,
"waiting": 0.0
}50s ago2026-07-27 00:09:25 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:09:25 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 90.0,
"elapsedTimeMs": 90.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24648506",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110365732}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:25.826Z",
"transactionId": 12028756.0,
"waiting": 0.0
}55s ago2026-07-27 00:09:20 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:09:20 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:09:20.468Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}55s ago2026-07-27 00:09:20 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:09:20 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:09:20.389Z",
"version": "0.1.0"
}55s ago2026-07-27 00:09:20 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:09:20 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 177.0,
"elapsedTimeMs": 177.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24648289",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785110060003}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:20.183Z",
"transactionId": 12028746.0,
"waiting": 0.0
}56s ago2026-07-27 00:09:19 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:09:19 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 65.0,
"elapsedTimeMs": 65.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24648285",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:19.996Z",
"transactionId": 12028745.0,
"waiting": 0.0
}1m ago2026-07-27 00:09:06 UTC DEBUG docproc Pulling messages
time2026-07-27 00:09:06 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:09:06.496Z",
"version": "0.1.0"
}1m ago2026-07-27 00:09:05 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:09:05 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:09:05.494Z",
"version": "0.1.0"
}1m ago2026-07-27 00:09:05 UTC DEBUG docproc Recovery fill
time2026-07-27 00:09:05 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:09:05.493Z",
"version": "0.1.0"
}1m ago2026-07-27 00:09:05 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:09:05 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 176.0,
"elapsedTimeMs": 176.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24646892",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785110045313}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:05.492Z",
"transactionId": 12028131.0,
"waiting": 0.0
}1m ago2026-07-27 00:09:05 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:09:05 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 107.0,
"elapsedTimeMs": 108.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24646872",
"pageFaults": 0.0,
"pageHits": 89963.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110345201}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:05.311Z",
"transactionId": 12028117.0,
"waiting": 0.0
}1m ago2026-07-27 00:09:04 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:09:04 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:09:04.922Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}1m ago2026-07-27 00:09:04 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:09:04 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:09:04.851Z",
"version": "0.1.0"
}1m ago2026-07-27 00:09:04 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:09:04 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 182.0,
"elapsedTimeMs": 182.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24646737",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785110044464}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:04.649Z",
"transactionId": 12028036.0,
"waiting": 0.0
}1m ago2026-07-27 00:09:04 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:09:04 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 85.0,
"elapsedTimeMs": 85.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24646722",
"pageFaults": 0.0,
"pageHits": 66849.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:09:04.462Z",
"transactionId": 12028025.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...
time2026-07-27 00:08:51 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (r:TmfEvaluationRun {status: $running})
WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
ORDER BY coalesce(r.updatedat, r.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 800.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "95899445",
"id": "24645172",
"pageFaults": 0.0,
"pageHits": 21.0,
"planning": 0.0,
"query": "\n MATCH (r:TmfEvaluationRun {status: $running})\n WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n ORDER BY coalesce(r.updatedat, r.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785110631234}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:51.238Z",
"transactionId": 12027079.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:49 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:08:49 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:08:49.369Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}1m ago2026-07-27 00:08:49 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:08:49 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:08:49.299Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:49 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:08:49 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 183.0,
"elapsedTimeMs": 183.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24644961",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785110028886}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:49.072Z",
"transactionId": 12027076.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:48 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:08:48 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 67.0,
"elapsedTimeMs": 67.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24644954",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:48.885Z",
"transactionId": 12027075.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:46 UTC DEBUG docproc Pulling messages
time2026-07-27 00:08:46 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:08:46.955Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:45 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:08:45 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:08:45.951Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:45 UTC DEBUG docproc Recovery fill
time2026-07-27 00:08:45 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:08:45.951Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:45 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:08:45 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 174.0,
"elapsedTimeMs": 174.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24644777",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785110025771}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:45.949Z",
"transactionId": 12027070.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:45 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:08:45 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 94.0,
"elapsedTimeMs": 94.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24644773",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110325672}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:45.769Z",
"transactionId": 12027069.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:33 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:08:33 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:08:33.812Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}1m ago2026-07-27 00:08:33 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:08:33 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:08:33.740Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:33 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:08:33 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 182.0,
"elapsedTimeMs": 182.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24644453",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785110013348}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:33.533Z",
"transactionId": 12027058.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:33 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:08:33 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 96.0,
"elapsedTimeMs": 96.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24644451",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:33.346Z",
"transactionId": 12027057.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:26 UTC DEBUG docproc Pulling messages
time2026-07-27 00:08:26 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:08:26.410Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:25 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:08:25 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:08:25.406Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:25 UTC DEBUG docproc Recovery fill
time2026-07-27 00:08:25 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:08:25.405Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:25 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:08:25 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 166.0,
"elapsedTimeMs": 167.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24644439",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785110005234}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:25.404Z",
"transactionId": 12027053.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:25 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:08:25 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 88.0,
"elapsedTimeMs": 88.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24644438",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110305141}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:25.233Z",
"transactionId": 12027052.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:18 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:08:18 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:08:18.241Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}1m ago2026-07-27 00:08:18 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:08:18 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:08:18.172Z",
"version": "0.1.0"
}1m ago2026-07-27 00:08:17 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:08:17 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 163.0,
"elapsedTimeMs": 163.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24644384",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109997794}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:17.960Z",
"transactionId": 12027041.0,
"waiting": 0.0
}1m ago2026-07-27 00:08:17 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:08:17 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 61.0,
"elapsedTimeMs": 61.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24644383",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:17.792Z",
"transactionId": 12027040.0,
"waiting": 0.0
}2m ago2026-07-27 00:08:05 UTC DEBUG docproc Pulling messages
time2026-07-27 00:08:05 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:08:05.897Z",
"version": "0.1.0"
}2m ago2026-07-27 00:08:04 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:08:04 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:08:04.894Z",
"version": "0.1.0"
}2m ago2026-07-27 00:08:04 UTC DEBUG docproc Recovery fill
time2026-07-27 00:08:04 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:08:04.894Z",
"version": "0.1.0"
}2m ago2026-07-27 00:08:04 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:08:04 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 172.0,
"elapsedTimeMs": 173.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24643467",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109984717}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:04.892Z",
"transactionId": 12026365.0,
"waiting": 0.0
}2m ago2026-07-27 00:08:04 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:08:04 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 95.0,
"elapsedTimeMs": 95.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24643451",
"pageFaults": 0.0,
"pageHits": 89963.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110284617}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:04.716Z",
"transactionId": 12026352.0,
"waiting": 0.0
}2m ago2026-07-27 00:08:02 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:08:02 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:08:02.723Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}2m ago2026-07-27 00:08:02 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:08:02 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:08:02.653Z",
"version": "0.1.0"
}2m ago2026-07-27 00:08:02 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:08:02 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 169.0,
"elapsedTimeMs": 169.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24643104",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109982284}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:02.456Z",
"transactionId": 12026075.0,
"waiting": 0.0
}2m ago2026-07-27 00:08:02 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:08:02 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 65.0,
"elapsedTimeMs": 65.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24643093",
"pageFaults": 0.0,
"pageHits": 66849.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:08:02.283Z",
"transactionId": 12026066.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:47 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:07:47 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:07:47.209Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}2m ago2026-07-27 00:07:47 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:07:47 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:07:47.137Z",
"version": "0.1.0"
}2m ago2026-07-27 00:07:46 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:07:46 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 161.0,
"elapsedTimeMs": 161.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24642170",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109966783}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:46.947Z",
"transactionId": 12025372.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:46 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:07:46 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 60.0,
"elapsedTimeMs": 60.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24642169",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:46.781Z",
"transactionId": 12025371.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:45 UTC DEBUG docproc Pulling messages
time2026-07-27 00:07:45 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:07:45.393Z",
"version": "0.1.0"
}2m ago2026-07-27 00:07:44 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:07:44 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:07:44.390Z",
"version": "0.1.0"
}2m ago2026-07-27 00:07:44 UTC DEBUG docproc Recovery fill
time2026-07-27 00:07:44 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:07:44.390Z",
"version": "0.1.0"
}2m ago2026-07-27 00:07:44 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:07:44 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 162.0,
"elapsedTimeMs": 162.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24642164",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109964223}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:44.388Z",
"transactionId": 12025368.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:44 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:07:44 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 105.0,
"elapsedTimeMs": 105.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24642160",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110264078}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:44.221Z",
"transactionId": 12025364.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:31 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:07:31 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:07:31.714Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}2m ago2026-07-27 00:07:31 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:07:31 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:07:31.641Z",
"version": "0.1.0"
}2m ago2026-07-27 00:07:31 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:07:31 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 163.0,
"elapsedTimeMs": 163.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24641309",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109951236}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:31.401Z",
"transactionId": 12025355.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:31 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:07:31 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 63.0,
"elapsedTimeMs": 63.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24641308",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:31.235Z",
"transactionId": 12025354.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:24 UTC DEBUG docproc Pulling messages
time2026-07-27 00:07:24 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:07:24.813Z",
"version": "0.1.0"
}2m ago2026-07-27 00:07:23 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:07:23 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:07:23.808Z",
"version": "0.1.0"
}2m ago2026-07-27 00:07:23 UTC DEBUG docproc Recovery fill
time2026-07-27 00:07:23 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:07:23.808Z",
"version": "0.1.0"
}2m ago2026-07-27 00:07:23 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:07:23 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 163.0,
"elapsedTimeMs": 163.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24641293",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109943640}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:23.806Z",
"transactionId": 12025346.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:23 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:07:23 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 88.0,
"elapsedTimeMs": 88.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24641292",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110243547}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:23.638Z",
"transactionId": 12025345.0,
"waiting": 0.0
}2m ago2026-07-27 00:07:16 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:07:16 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:07:16.163Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}2m ago2026-07-27 00:07:16 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:07:16 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:07:16.097Z",
"version": "0.1.0"
}3m ago2026-07-27 00:07:15 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:07:15 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 171.0,
"elapsedTimeMs": 171.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24641275",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109935708}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:15.882Z",
"transactionId": 12025338.0,
"waiting": 0.0
}3m ago2026-07-27 00:07:15 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:07:15 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 64.0,
"elapsedTimeMs": 64.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24641274",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:15.706Z",
"transactionId": 12025337.0,
"waiting": 0.0
}3m ago2026-07-27 00:07:05 UTC DEBUG docproc Pulling messages
time2026-07-27 00:07:05 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:07:05.349Z",
"version": "0.1.0"
}3m ago2026-07-27 00:07:04 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:07:04 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:07:04.347Z",
"version": "0.1.0"
}3m ago2026-07-27 00:07:04 UTC DEBUG docproc Recovery fill
time2026-07-27 00:07:04 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:07:04.347Z",
"version": "0.1.0"
}3m ago2026-07-27 00:07:04 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:07:04 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 206.0,
"elapsedTimeMs": 207.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24640359",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109924136}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:04.345Z",
"transactionId": 12024625.0,
"waiting": 0.0
}3m ago2026-07-27 00:07:04 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:07:04 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 94.0,
"elapsedTimeMs": 94.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24640341",
"pageFaults": 0.0,
"pageHits": 89963.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110224017}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:04.114Z",
"transactionId": 12024610.0,
"waiting": 0.0
}3m ago2026-07-27 00:07:00 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:07:00 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:07:00.627Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}3m ago2026-07-27 00:07:00 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:07:00 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:07:00.554Z",
"version": "0.1.0"
}3m ago2026-07-27 00:07:00 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:07:00 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 161.0,
"elapsedTimeMs": 161.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24639790",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109920205}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:00.374Z",
"transactionId": 12024174.0,
"waiting": 0.0
}3m ago2026-07-27 00:07:00 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:07:00 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 64.0,
"elapsedTimeMs": 65.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24639778",
"pageFaults": 0.0,
"pageHits": 66849.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:07:00.204Z",
"transactionId": 12024164.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...
time2026-07-27 00:06:51 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (r:TmfEvaluationRun {status: $running})
WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
ORDER BY coalesce(r.updatedat, r.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 800.0,
"cpu": 12.0,
"elapsedTimeMs": 12.0,
"executionPlanCacheKeyHash": "95899445",
"id": "24639158",
"pageFaults": 0.0,
"pageHits": 21.0,
"planning": 12.0,
"query": "\n MATCH (r:TmfEvaluationRun {status: $running})\n WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n ORDER BY coalesce(r.updatedat, r.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785110511216}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:51.232Z",
"transactionId": 12023674.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:45 UTC DEBUG docproc Pulling messages
time2026-07-27 00:06:45 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:06:45.251Z",
"version": "0.1.0"
}3m ago2026-07-27 00:06:45 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:06:45 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:06:45.131Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}3m ago2026-07-27 00:06:45 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:06:45 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:06:45.054Z",
"version": "0.1.0"
}3m ago2026-07-27 00:06:44 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:06:44 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 162.0,
"elapsedTimeMs": 163.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24639105",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109904732}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:44.897Z",
"transactionId": 12023666.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:44 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:06:44 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 62.0,
"elapsedTimeMs": 62.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24639104",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:44.730Z",
"transactionId": 12023665.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:44 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:06:44 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:06:44.248Z",
"version": "0.1.0"
}3m ago2026-07-27 00:06:44 UTC DEBUG docproc Recovery fill
time2026-07-27 00:06:44 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:06:44.248Z",
"version": "0.1.0"
}3m ago2026-07-27 00:06:44 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:06:44 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 162.0,
"elapsedTimeMs": 163.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24639099",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109904081}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:44.246Z",
"transactionId": 12023660.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:44 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:06:44 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 87.0,
"elapsedTimeMs": 88.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24639098",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110203987}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:44.079Z",
"transactionId": 12023659.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:29 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:06:29 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:06:29.660Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}3m ago2026-07-27 00:06:29 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:06:29 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:06:29.591Z",
"version": "0.1.0"
}3m ago2026-07-27 00:06:29 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:06:29 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 172.0,
"elapsedTimeMs": 172.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24638892",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109889202}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:29.377Z",
"transactionId": 12023651.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:29 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:06:29 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 63.0,
"elapsedTimeMs": 63.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24638888",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:29.201Z",
"transactionId": 12023650.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:25 UTC DEBUG docproc Pulling messages
time2026-07-27 00:06:25 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:06:25.780Z",
"version": "0.1.0"
}3m ago2026-07-27 00:06:24 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:06:24 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:06:24.775Z",
"version": "0.1.0"
}3m ago2026-07-27 00:06:24 UTC DEBUG docproc Recovery fill
time2026-07-27 00:06:24 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:06:24.775Z",
"version": "0.1.0"
}3m ago2026-07-27 00:06:24 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:06:24 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 202.0,
"elapsedTimeMs": 203.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24638718",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109884567}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:24.773Z",
"transactionId": 12023648.0,
"waiting": 0.0
}3m ago2026-07-27 00:06:24 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:06:24 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 105.0,
"elapsedTimeMs": 105.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24638707",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110184456}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:24.565Z",
"transactionId": 12023647.0,
"waiting": 0.0
}4m ago2026-07-27 00:06:14 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:06:14 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:06:14.131Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}4m ago2026-07-27 00:06:14 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:06:14 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:06:14.055Z",
"version": "0.1.0"
}4m ago2026-07-27 00:06:13 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:06:13 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 169.0,
"elapsedTimeMs": 169.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24638288",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109873712}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:13.884Z",
"transactionId": 12023633.0,
"waiting": 0.0
}4m ago2026-07-27 00:06:13 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:06:13 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 61.0,
"elapsedTimeMs": 62.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24638284",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:13.683Z",
"transactionId": 12023632.0,
"waiting": 0.0
}4m ago2026-07-27 00:06:05 UTC DEBUG docproc Pulling messages
time2026-07-27 00:06:05 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:06:05.220Z",
"version": "0.1.0"
}4m ago2026-07-27 00:06:04 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:06:04 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:06:04.216Z",
"version": "0.1.0"
}4m ago2026-07-27 00:06:04 UTC DEBUG docproc Recovery fill
time2026-07-27 00:06:04 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:06:04.216Z",
"version": "0.1.0"
}4m ago2026-07-27 00:06:04 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:06:04 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 190.0,
"elapsedTimeMs": 190.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24637252",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109864022}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:04.214Z",
"transactionId": 12022874.0,
"waiting": 0.0
}4m ago2026-07-27 00:06:04 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:06:04 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 91.0,
"elapsedTimeMs": 91.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24637237",
"pageFaults": 0.0,
"pageHits": 89963.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110163924}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:06:04.020Z",
"transactionId": 12022863.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:58 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:05:58 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:05:58.613Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}4m ago2026-07-27 00:05:58 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:05:58 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:05:58.541Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:58 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:05:58 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 178.0,
"elapsedTimeMs": 178.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24636415",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109858163}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:58.348Z",
"transactionId": 12022212.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:58 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:05:58 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 70.0,
"elapsedTimeMs": 70.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24636404",
"pageFaults": 0.0,
"pageHits": 66849.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:58.162Z",
"transactionId": 12022202.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:44 UTC DEBUG docproc Pulling messages
time2026-07-27 00:05:44 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:05:44.698Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:43 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:05:43 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:05:43.695Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:43 UTC DEBUG docproc Recovery fill
time2026-07-27 00:05:43 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:05:43.695Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:43 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:05:43 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 191.0,
"elapsedTimeMs": 191.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24636052",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 21.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109843499}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:43.693Z",
"transactionId": 12021959.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:43 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:05:43 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 102.0,
"elapsedTimeMs": 102.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24636051",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 14.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110143392}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:43.497Z",
"transactionId": 12021958.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:43 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:05:43 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:05:43.083Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}4m ago2026-07-27 00:05:43 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:05:43 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:05:43.010Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:42 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:05:42 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 183.0,
"elapsedTimeMs": 183.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24636050",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 18.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109842592}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:42.777Z",
"transactionId": 12021957.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:42 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:05:42 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 70.0,
"elapsedTimeMs": 71.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24636049",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 8.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:42.590Z",
"transactionId": 12021956.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:27 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:05:27 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:05:27.511Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}4m ago2026-07-27 00:05:27 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:05:27 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:05:27.441Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:27 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:05:27 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 167.0,
"elapsedTimeMs": 167.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24636010",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109827073}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:27.247Z",
"transactionId": 12021948.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:27 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:05:27 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 62.0,
"elapsedTimeMs": 62.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24636009",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:27.071Z",
"transactionId": 12021947.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:24 UTC DEBUG docproc Pulling messages
time2026-07-27 00:05:24 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:05:24.175Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:23 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:05:23 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:05:23.172Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:23 UTC DEBUG docproc Recovery fill
time2026-07-27 00:05:23 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:05:23.172Z",
"version": "0.1.0"
}4m ago2026-07-27 00:05:23 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:05:23 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 199.0,
"elapsedTimeMs": 199.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24636001",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109822967}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:23.170Z",
"transactionId": 12021941.0,
"waiting": 0.0
}4m ago2026-07-27 00:05:22 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:05:22 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 102.0,
"elapsedTimeMs": 102.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24636000",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110122860}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:22.965Z",
"transactionId": 12021940.0,
"waiting": 0.0
}5m ago2026-07-27 00:05:12 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:05:12 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:05:12.002Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}5m ago2026-07-27 00:05:11 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:05:11 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:05:11.934Z",
"version": "0.1.0"
}5m ago2026-07-27 00:05:11 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:05:11 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 183.0,
"elapsedTimeMs": 183.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24635972",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109811548}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:11.734Z",
"transactionId": 12021930.0,
"waiting": 0.0
}5m ago2026-07-27 00:05:11 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:05:11 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 62.0,
"elapsedTimeMs": 63.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24635961",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:11.547Z",
"transactionId": 12021922.0,
"waiting": 0.0
}5m ago2026-07-27 00:05:04 UTC DEBUG docproc Pulling messages
time2026-07-27 00:05:04 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:05:04.630Z",
"version": "0.1.0"
}5m ago2026-07-27 00:05:03 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:05:03 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:05:03.625Z",
"version": "0.1.0"
}5m ago2026-07-27 00:05:03 UTC DEBUG docproc Recovery fill
time2026-07-27 00:05:03 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:05:03.625Z",
"version": "0.1.0"
}5m ago2026-07-27 00:05:03 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:05:03 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 184.0,
"elapsedTimeMs": 184.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24634949",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109803436}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:03.623Z",
"transactionId": 12021124.0,
"waiting": 0.0
}5m ago2026-07-27 00:05:03 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:05:03 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 102.0,
"elapsedTimeMs": 102.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24634932",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110103329}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:05:03.434Z",
"transactionId": 12021111.0,
"waiting": 0.0
}5m ago2026-07-27 00:04:56 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:04:56 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:04:56.475Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}5m ago2026-07-27 00:04:56 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:04:56 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:04:56.407Z",
"version": "0.1.0"
}5m ago2026-07-27 00:04:56 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:04:56 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 166.0,
"elapsedTimeMs": 166.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24633881",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109796053}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:56.222Z",
"transactionId": 12020276.0,
"waiting": 0.0
}5m ago2026-07-27 00:04:56 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:04:56 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 63.0,
"elapsedTimeMs": 63.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24633880",
"pageFaults": 0.0,
"pageHits": 66849.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:56.051Z",
"transactionId": 12020275.0,
"waiting": 0.0
}5m ago2026-07-27 00:04:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...
time2026-07-27 00:04:51 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (r:TmfEvaluationRun {status: $running})
WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
ORDER BY coalesce(r.updatedat, r.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 800.0,
"cpu": 1.0,
"elapsedTimeMs": 1.0,
"executionPlanCacheKeyHash": "95899445",
"id": "24633872",
"pageFaults": 0.0,
"pageHits": 21.0,
"planning": 0.0,
"query": "\n MATCH (r:TmfEvaluationRun {status: $running})\n WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n ORDER BY coalesce(r.updatedat, r.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785110391207}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:51.213Z",
"transactionId": 12020271.0,
"waiting": 0.0
}5m ago2026-07-27 00:04:45 UTC DEBUG docproc Pulling messages
time2026-07-27 00:04:45 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:04:45.068Z",
"version": "0.1.0"
}5m ago2026-07-27 00:04:44 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:04:44 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:04:44.065Z",
"version": "0.1.0"
}5m ago2026-07-27 00:04:44 UTC DEBUG docproc Recovery fill
time2026-07-27 00:04:44 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:04:44.065Z",
"version": "0.1.0"
}5m ago2026-07-27 00:04:44 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:04:44 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 167.0,
"elapsedTimeMs": 167.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24633847",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109783893}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:44.063Z",
"transactionId": 12020259.0,
"waiting": 0.0
}5m ago2026-07-27 00:04:43 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:04:43 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 88.0,
"elapsedTimeMs": 89.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24633846",
"pageFaults": 0.0,
"pageHits": 89963.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110083800}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:43.892Z",
"transactionId": 12020258.0,
"waiting": 0.0
}5m ago2026-07-27 00:04:40 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:04:40 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:04:40.979Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}5m ago2026-07-27 00:04:40 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:04:40 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:04:40.902Z",
"version": "0.1.0"
}5m ago2026-07-27 00:04:40 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:04:40 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 161.0,
"elapsedTimeMs": 162.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24633843",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109780528}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:40.692Z",
"transactionId": 12020256.0,
"waiting": 0.0
}5m ago2026-07-27 00:04:40 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:04:40 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 64.0,
"elapsedTimeMs": 64.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24633841",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:40.526Z",
"transactionId": 12020255.0,
"waiting": 0.0
}5m ago2026-07-27 00:04:25 UTC DEBUG docproc Pulling messages
time2026-07-27 00:04:25 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:04:25.545Z",
"version": "0.1.0"
}5m ago2026-07-27 00:04:25 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:04:25 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:04:25.454Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}5m ago2026-07-27 00:04:25 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:04:25 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:04:25.375Z",
"version": "0.1.0"
}5m ago2026-07-27 00:04:24 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:04:24 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:04:24.543Z",
"version": "0.1.0"
}5m ago2026-07-27 00:04:24 UTC DEBUG docproc Recovery fill
time2026-07-27 00:04:24 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:04:24.542Z",
"version": "0.1.0"
}6m ago2026-07-27 00:04:09 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:04:09 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:04:09.890Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}6m ago2026-07-27 00:04:09 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:04:09 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:04:09.816Z",
"version": "0.1.0"
}6m ago2026-07-27 00:04:09 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:04:09 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 163.0,
"elapsedTimeMs": 163.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24633600",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109749463}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:09.627Z",
"transactionId": 12020147.0,
"waiting": 0.0
}6m ago2026-07-27 00:04:09 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:04:09 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 63.0,
"elapsedTimeMs": 63.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24633586",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:09.461Z",
"transactionId": 12020138.0,
"waiting": 0.0
}6m ago2026-07-27 00:04:05 UTC DEBUG docproc Pulling messages
time2026-07-27 00:04:05 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:04:05.001Z",
"version": "0.1.0"
}6m ago2026-07-27 00:04:03 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:04:03 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:04:03.997Z",
"version": "0.1.0"
}6m ago2026-07-27 00:04:03 UTC DEBUG docproc Recovery fill
time2026-07-27 00:04:03 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:04:03.997Z",
"version": "0.1.0"
}6m ago2026-07-27 00:04:03 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:04:03 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 159.0,
"elapsedTimeMs": 160.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24632728",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109743833}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:03.995Z",
"transactionId": 12019486.0,
"waiting": 0.0
}6m ago2026-07-27 00:04:03 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:04:03 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 91.0,
"elapsedTimeMs": 91.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24632711",
"pageFaults": 0.0,
"pageHits": 89927.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110043737}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:04:03.831Z",
"transactionId": 12019474.0,
"waiting": 0.0
}6m ago2026-07-27 00:03:54 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:03:54 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:03:54.391Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}6m ago2026-07-27 00:03:54 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:03:54 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:03:54.328Z",
"version": "0.1.0"
}6m ago2026-07-27 00:03:54 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:03:54 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 162.0,
"elapsedTimeMs": 163.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24631535",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109733959}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:03:54.126Z",
"transactionId": 12018571.0,
"waiting": 0.0
}6m ago2026-07-27 00:03:53 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:03:53 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 60.0,
"elapsedTimeMs": 60.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24631534",
"pageFaults": 0.0,
"pageHits": 66849.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:03:53.957Z",
"transactionId": 12018570.0,
"waiting": 0.0
}6m ago2026-07-27 00:03:44 UTC DEBUG docproc Pulling messages
time2026-07-27 00:03:44 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:03:44.474Z",
"version": "0.1.0"
}6m ago2026-07-27 00:03:43 UTC DEBUG docproc Pull returned 0 messages
time2026-07-27 00:03:43 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:284
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pull returned 0 messages
raw payload (9 fields)
{
"caller": "handlers/pubsub_handler.py:284",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Pull returned 0 messages",
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:03:43.471Z",
"version": "0.1.0"
}6m ago2026-07-27 00:03:43 UTC DEBUG docproc Recovery fill
time2026-07-27 00:03:43 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:429
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Recovery fill
raw payload (13 fields)
{
"adopt_list_len": 0.0,
"adopt_started": 0.0,
"backlog_list_len": 0.0,
"backlog_started": 0.0,
"caller": "handlers/pubsub_handler.py:429",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"message": "Recovery fill",
"remaining_slots": 10.0,
"service": "docproc",
"timestamp": "2026-07-27T00:03:43.471Z",
"version": "0.1.0"
}6m ago2026-07-27 00:03:43 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:03:43 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
RETURN doc_id, org_alias, version_id, study_id, current_retry_count
ORDER BY doc_id
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 9610664.0,
"cpu": 165.0,
"elapsedTimeMs": 166.0,
"executionPlanCacheKeyHash": "7644B06C",
"id": "24631455",
"pageFaults": 0.0,
"pageHits": 156448.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n ORDER BY doc_id\n LIMIT $limit\n ",
"queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785109723300}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:03:43.469Z",
"transactionId": 12018558.0,
"waiting": 0.0
}6m ago2026-07-27 00:03:43 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:03:43 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
o.tildaid AS org_id, o.alias AS org_alias
ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
LIMIT $limit
raw payload (15 fields)
{
"allocatedBytes": 736.0,
"cpu": 90.0,
"elapsedTimeMs": 90.0,
"executionPlanCacheKeyHash": "E5FA14C6",
"id": "24631454",
"pageFaults": 0.0,
"pageHits": 89963.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n o.tildaid AS org_id, o.alias AS org_alias\n ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n LIMIT $limit\n ",
"queryParameters": "{limit: 30, cutoff_ms: 1785110023206}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:03:43.299Z",
"transactionId": 12018557.0,
"waiting": 0.0
}6m ago2026-07-27 00:03:38 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:03:38 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:03:38.888Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}6m ago2026-07-27 00:03:38 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
time2026-07-27 00:03:38 UTC
severityDEBUG
servicedocproc
sourcejobs/cloud_monitoring_metrics.py:296
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0
raw payload (7 fields)
{
"caller": "jobs/cloud_monitoring_metrics.py:296",
"cluster": "master",
"environment": "DEVELOPMENT",
"message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
"service": "docproc",
"timestamp": "2026-07-27T00:03:38.821Z",
"version": "0.1.0"
}6m ago2026-07-27 00:03:38 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...
time2026-07-27 00:03:38 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
WHERE f.name IN ['Staging', 'Pre-Staging']
AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
WITH o, d, ai
ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
WITH o, d, collect(ai)[0] AS latest_ai
WHERE (
latest_ai IS NULL
OR latest_ai.status = "aborted"
OR (
latest_ai.status = "failed"
AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
)
)
WITH d
RETURN count(d) AS total
raw payload (15 fields)
{
"allocatedBytes": 7666360.0,
"cpu": 163.0,
"elapsedTimeMs": 164.0,
"executionPlanCacheKeyHash": "590286C5",
"id": "24631441",
"pageFaults": 0.0,
"pageHits": 156362.0,
"planning": 0.0,
"query": "\n MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n WITH o, d, ai\n ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n WITH o, d, collect(ai)[0] AS latest_ai\n WHERE (\n latest_ai IS NULL\n OR latest_ai.status = \"aborted\"\n OR (\n latest_ai.status = \"failed\"\n AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n )\n )\n WITH d\n RETURN count(d) AS total\n ",
"queryParameters": "{max_retries: 5, threshold_ms: 1785109718459}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:03:38.630Z",
"transactionId": 12018554.0,
"waiting": 0.0
}6m ago2026-07-27 00:03:38 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...
time2026-07-27 00:03:38 UTC
severityINFO
servicedocproc
clustermaster
podserver-2-dev-0
query
MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
WHERE d.aiEnabled = true
AND (
(ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
OR ai.status = "aborted"
)
RETURN count(ai) AS total
raw payload (15 fields)
{
"allocatedBytes": 488.0,
"cpu": 63.0,
"elapsedTimeMs": 63.0,
"executionPlanCacheKeyHash": "E4907905",
"id": "24631438",
"pageFaults": 0.0,
"pageHits": 66825.0,
"planning": 0.0,
"query": "\n MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n WHERE d.aiEnabled = true\n AND (\n (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n OR ai.status = \"aborted\"\n )\n RETURN count(ai) AS total\n ",
"queryParameters": "{cutoff_ms: 0}",
"runtime": "pipelined",
"source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:59818\tserver/10.173.0.8:7687\u003e",
"timestamp": "2026-07-27T00:03:38.457Z",
"transactionId": 12018553.0,
"waiting": 0.0
}6m ago2026-07-27 00:03:23 UTC DEBUG docproc Pulling messages
time2026-07-27 00:03:23 UTC
severityDEBUG
servicedocproc
sourcehandlers/pubsub_handler.py:240
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Pulling messages
raw payload (11 fields)
{
"caller": "handlers/pubsub_handler.py:240",
"cluster": "master",
"component": "pubsub",
"environment": "DEVELOPMENT",
"max_messages": 1.0,
"message": "Pulling messages",
"pull_timeout_s": 60.0,
"service": "docproc",
"subscription_name": "tilda-document-master-docproc-sub",
"timestamp": "2026-07-27T00:03:23.949Z",
"version": "0.1.0"
}6m ago2026-07-27 00:03:23 UTC DEBUG docproc Reported pending workload
time2026-07-27 00:03:23 UTC
severityDEBUG
servicedocproc
sourcejobs/hpa_pending_workload.py:176
clustermaster
version0.1.0
poddocproc-7bc76669-ljh8h
message
Reported pending workload
raw payload (15 fields)
{
"backlog_capped": 0.0,
"caller": "jobs/hpa_pending_workload.py:176",
"cluster": "master",
"component": "hpa_pending_workload",
"environment": "DEVELOPMENT",
"in_progress_count": 0.0,
"message": "Reported pending workload",
"num_replicas": 1.0,
"recovery_workload": 0.0,
"service": "docproc",
"signal": 0.0,
"timestamp": "2026-07-27T00:03:23.385Z",
"total_workload": 0.0,
"unacked": 0.0,
"version": "0.1.0"
}Paginated with Cloud Logging page cursors, so you can walk the whole 6h window — not just the newest page. Click any filter by value to scope the view (resets to page 1). Paste a trace id to see every log line across all services for that one request.